enkly-ai/pkg/officekit/pdf.go

// pdf.go adds best-effort PDF text extraction: read_pdf's only job is
// "give me back the words", not full-fidelity layout, so this
// deliberately stays a few hundred lines of pure stdlib (no PDF
// library, no cgo, nothing to go get in an offline deployment) rather
// than a complete PDF renderer. It walks the file's raw bytes for
// `stream ... endstream` blocks, inflates the ones declared
// FlateDecode (the overwhelming majority of PDFs produced by real
// tools — Word, LibreOffice, browsers, Pandoc, Pages — all compress
// their content streams this way), and scans the decompressed
// PostScript-like content for the handful of text-showing operators
// (Tj, TJ, ', ") that actually place characters on the page.
// Everything else in the file — fonts, images, vector graphics,
// annotations, page geometry — is ignored.
//
// What this can't do, and won't pretend to: scanned/image-only PDFs
// (no embedded text at all — that needs OCR, out of scope here),
// PDFs whose content streams use a filter other than FlateDecode
// (rare in practice), and any PDF that encodes its text with a custom
// font encoding and no usable mapping back to plain characters — those
// come out as empty or as mangled text, and ReadPdfText says so rather
// than silently returning garbage.
package officekit

import (
	"bytes"
	"compress/zlib"
	"fmt"
	"io"
	"regexp"
	"strconv"
	"strings"
)

// pdfStreamRe finds every `stream\r?\n ... \r?\nendstream` block in
// the file, capturing the body separately from the "stream" keyword
// that precedes it — ReadPdfText looks backward from the keyword for
// the object dictionary's /Filter entry rather than actually parsing
// PDF's object/dictionary syntax.
var pdfStreamRe = regexp.MustCompile(`(?s)stream\r?\n(.*?)\r?\nendstream`)

// pdfDictLookback is how far back from a `stream` keyword ReadPdfText
// looks for that stream object's dictionary (and in particular its
// /Filter entry) — generous enough for any realistic content-stream
// dictionary (/Length, /Filter, /Length1, a handful of other keys)
// without risking bleeding into the previous object.
const pdfDictLookback = 600

// ReadPdfText extracts a PDF's plain text content, best-effort. See
// the package doc above for exactly what it does and doesn't handle.
// Returns an error only when the input isn't recognizable as a PDF,
// or no extractable text was found anywhere in it — a PDF that fails
// to extract from some streams but not others still returns whatever
// text it did recover from the rest.
func ReadPdfText(data []byte) (string, error) {
	head := data
	if len(head) > 1024 {
		head = head[:1024]
	}
	if !bytes.Contains(head, []byte("%PDF-")) {
		return "", fmt.Errorf("officekit: not a PDF file (missing %%PDF- header)")
	}

	var out strings.Builder
	for _, loc := range pdfStreamRe.FindAllSubmatchIndex(data, -1) {
		streamStart, bodyStart, bodyEnd := loc[0], loc[2], loc[3]
		dictStart := streamStart - pdfDictLookback
		if dictStart < 0 {
			dictStart = 0
		}
		dict := data[dictStart:streamStart]
		if !bytes.Contains(dict, []byte("FlateDecode")) {
			continue // not a stream this package knows how to decompress
		}
		inflated, err := inflateZlib(data[bodyStart:bodyEnd])
		if err != nil {
			continue // FlateDecode-tagged but not plain DEFLATE (e.g. an embedded font program); skip it
		}
		if text := extractShownText(inflated); text != "" {
			if out.Len() > 0 {
				out.WriteString("\n\n")
			}
			out.WriteString(text)
		}
	}
	result := strings.TrimSpace(out.String())
	if result == "" {
		return "", fmt.Errorf("officekit: no extractable text found (the PDF may be scanned/image-only, or use an encoding this reader doesn't support)")
	}
	return result, nil
}

func inflateZlib(b []byte) ([]byte, error) {
	r, err := zlib.NewReader(bytes.NewReader(b))
	if err != nil {
		return nil, err
	}
	defer r.Close()
	return io.ReadAll(r)
}

// pdfTextOpRe matches one text-showing operator call together with
// its operand(s): one or more PDF "literal strings" — parenthesized,
// with their own backslash-escape syntax decoded by
// pdfUnescapeLiteral — followed by Tj, TJ, ' or ". A TJ call's operand
// is actually an array, e.g. `[(Hel)-20(lo)] TJ`, which is why this
// matches "one or more" consecutive literals rather than exactly one.
var pdfTextOpRe = regexp.MustCompile(`(?s)((?:\((?:\\.|[^()\\])*\)[^()]*?)+)(Tj|TJ|'|")`)

// pdfLiteralRe pulls each individual (...) literal string out of one
// operator call's operand span.
var pdfLiteralRe = regexp.MustCompile(`(?s)\((?:\\.|[^()\\])*\)`)

// pdfNewlineOpRe matches the content-stream operators that move the
// text cursor to a new line (Td, TD, T*) — used only as a heuristic
// for where extractShownText inserts a newline into the recovered
// text, since the stream itself carries no notion of "paragraph" or
// "line" beyond cursor movement.
var pdfNewlineOpRe = regexp.MustCompile(`\bT[dD*]\b`)

// extractShownText walks one decompressed content stream in document
// order and concatenates the string(s) passed to each text-showing
// operator call, inserting a newline wherever a line-positioning
// operator (Td/TD/T*) appeared since the previous call and a plain
// space otherwise — an approximation of the stream's original line
// breaks, not a layout engine. A single TJ call's several literals
// (PDF splits one word across more than one literal to carry kerning
// adjustments, e.g. `[(Hel)-20(lo)] TJ`) are joined with nothing
// between them, since the gap is a spacing nudge, not a real word
// break — any actual space in the shown text already appears as a
// literal " " character the way it does in a Tj call.
func extractShownText(stream []byte) string {
	var sb strings.Builder
	pos := 0
	pendingNewline := false
	for pos < len(stream) {
		loc := pdfTextOpRe.FindSubmatchIndex(stream[pos:])
		if loc == nil {
			break
		}
		matchStart, matchEnd := pos+loc[0], pos+loc[1]
		if pdfNewlineOpRe.Match(stream[pos:matchStart]) {
			pendingNewline = true
		}
		operand := stream[pos+loc[2] : pos+loc[3]]
		var call strings.Builder
		for _, lit := range pdfLiteralRe.FindAll(operand, -1) {
			call.WriteString(pdfUnescapeLiteral(lit[1 : len(lit)-1]))
		}
		if text := call.String(); text != "" {
			if sb.Len() > 0 {
				if pendingNewline {
					sb.WriteByte('\n')
				} else {
					sb.WriteByte(' ')
				}
			}
			pendingNewline = false
			sb.WriteString(text)
		}
		pos = matchEnd
	}
	return strings.TrimSpace(sb.String())
}

// pdfUnescapeLiteral decodes a PDF literal string's backslash escapes
// (\n \r \t \( \) \\ and octal \ddd) and treats every other byte as
// raw Latin-1 — correct for the common case (a simple, non-custom
// font encoding) and the best this package can do without parsing the
// PDF's font /Encoding and /ToUnicode tables.
func pdfUnescapeLiteral(b []byte) string {
	var sb strings.Builder
	for i := 0; i < len(b); i++ {
		c := b[i]
		if c != '\\' || i == len(b)-1 {
			sb.WriteByte(c)
			continue
		}
		i++
		switch n := b[i]; {
		case n == 'n':
			sb.WriteByte('\n')
		case n == 'r':
			sb.WriteByte('\r')
		case n == 't':
			sb.WriteByte('\t')
		case n == 'b' || n == 'f':
			// backspace/form-feed: not meaningful in extracted text
		case n == '(' || n == ')' || n == '\\':
			sb.WriteByte(n)
		case n == '\n':
			// backslash-newline is a line continuation, not a literal char
		case n >= '0' && n <= '7':
			j := i
			for j < len(b) && j < i+3 && b[j] >= '0' && b[j] <= '7' {
				j++
			}
			if v, err := strconv.ParseInt(string(b[i:j]), 8, 32); err == nil {
				sb.WriteByte(byte(v))
			}
			i = j - 1
		default:
			sb.WriteByte(n)
		}
	}
	return sb.String()
}