Init Project

tools/indexer/Dockerfile

@@ -0,0 +1,11 @@
+FROM golang:1.23-bookworm AS build
+WORKDIR /src
+COPY go.mod go.sum* ./
+COPY gen/ gen/
+COPY tools/indexer/ tools/indexer/
+RUN go mod tidy
+RUN CGO_ENABLED=0 go build -o /out/indexer ./tools/indexer/cmd/indexer
+
+FROM gcr.io/distroless/static-debian12:nonroot
+COPY --from=build /out/indexer /indexer
+ENTRYPOINT ["/indexer"]

tools/indexer/cmd/indexer/main.go

@@ -0,0 +1,176 @@
+// Command indexer replaces cmd/indexer: a one-shot (or cron'd) batch
+// job that walks a directory and feeds every recognized text file to
+// the knowledge service's IndexFile RPC, then PruneMissing for
+// whatever it didn't see. Unlike the monolith's version, it no longer
+// talks to Postgres directly (knowledge owns that database now) — it's
+// a pure gRPC client, so it can run anywhere with network access to
+// the knowledge service, pointed at any local checkout: typically the
+// same volume mounted into the workspace service (docker compose run
+// --rm -v $(pwd):/src indexer -path /src), but it works equally well
+// against an ad hoc local directory for local development.
+package main
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io/fs"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+
+	knowledgev1 "enkly/gen/knowledge/v1"
+)
+
+var skipDirs = map[string]bool{
+	".git": true, "node_modules": true, "vendor": true, ".venv": true,
+	"venv": true, "__pycache__": true, "dist": true, "build": true,
+	".idea": true, ".vscode": true, "target": true, ".next": true,
+	"bin": true, ".cache": true,
+}
+
+var indexedExts = map[string]bool{
+	".go": true, ".py": true, ".js": true, ".mjs": true, ".cjs": true, ".ts": true,
+	".tsx": true, ".jsx": true, ".java": true, ".kt": true, ".kts": true, ".scala": true,
+	".rs": true, ".c": true, ".h": true, ".cc": true, ".cpp": true, ".hpp": true,
+	".cs": true, ".rb": true, ".php": true, ".swift": true, ".dart": true, ".lua": true,
+	".sh": true, ".bash": true, ".zsh": true, ".sql": true, ".proto": true,
+	".html": true, ".css": true, ".scss": true, ".vue": true, ".svelte": true,
+	".md": true, ".txt": true, ".rst": true,
+	".yaml": true, ".yml": true, ".toml": true, ".json": true, ".ini": true,
+	".gradle": true, ".tf": true,
+}
+
+var indexedNames = map[string]bool{
+	"Dockerfile": true, "Makefile": true, "Rakefile": true, "Gemfile": true,
+	"Procfile": true, "CMakeLists.txt": true, ".env.example": true,
+}
+
+var skipFiles = map[string]bool{
+	"package-lock.json": true, "yarn.lock": true, "pnpm-lock.yaml": true,
+	"go.sum": true, "Cargo.lock": true, "poetry.lock": true, "composer.lock": true,
+}
+
+const maxIndexFileSize = 512 * 1024
+
+func shouldIndex(name string) bool {
+	if skipFiles[name] || strings.HasSuffix(name, ".min.js") || strings.HasSuffix(name, ".min.css") {
+		return false
+	}
+	if indexedNames[name] {
+		return true
+	}
+	return indexedExts[strings.ToLower(filepath.Ext(name))]
+}
+
+func looksLikeText(data []byte) bool {
+	head := data
+	if len(head) > 8192 {
+		head = head[:8192]
+	}
+	if bytes.IndexByte(head, 0) >= 0 {
+		return false
+	}
+	return utf8.Valid(data)
+}
+
+func main() {
+	logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
+
+	var (
+		root          = flag.String("path", ".", "directory to index")
+		knowledgeAddr = flag.String("knowledge-addr", envOr("ENKLY_KNOWLEDGE_GRPC_ADDR", "localhost:9005"), "knowledge service gRPC address")
+	)
+	flag.Parse()
+
+	absRoot, err := filepath.Abs(*root)
+	if err != nil {
+		logger.Error("resolve path", "err", err)
+		os.Exit(1)
+	}
+	if info, err := os.Stat(absRoot); err != nil || !info.IsDir() {
+		logger.Error("not a readable directory", "path", absRoot)
+		os.Exit(1)
+	}
+
+	conn, err := grpc.NewClient(*knowledgeAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		logger.Error("dial knowledge service", "err", err)
+		os.Exit(1)
+	}
+	defer conn.Close()
+	client := knowledgev1.NewKnowledgeServiceClient(conn)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+
+	var (
+		count int
+		seen  []string
+	)
+	err = filepath.WalkDir(absRoot, func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if p == absRoot {
+			return nil
+		}
+		if d.IsDir() {
+			if skipDirs[d.Name()] {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		if !shouldIndex(d.Name()) {
+			return nil
+		}
+		info, err := d.Info()
+		if err != nil || info.Size() > maxIndexFileSize {
+			return nil
+		}
+		data, err := os.ReadFile(p)
+		if err != nil || !looksLikeText(data) {
+			return nil
+		}
+		rel, err := filepath.Rel(absRoot, p)
+		if err != nil {
+			return nil
+		}
+		rel = filepath.ToSlash(rel)
+
+		resp, err := client.IndexFile(ctx, &knowledgev1.IndexFileRequest{FilePath: rel, Content: string(data)})
+		if err != nil {
+			return fmt.Errorf("index %s: %w", rel, err)
+		}
+		logger.Info("indexed", "file", rel, "chunks_indexed", resp.GetChunksIndexed(), "chunks_skipped", resp.GetChunksSkipped())
+		seen = append(seen, rel)
+		count++
+		return nil
+	})
+	if err != nil {
+		logger.Error("walk", "err", err)
+		os.Exit(1)
+	}
+
+	pruned, err := client.PruneMissing(ctx, &knowledgev1.PruneMissingRequest{KeepFilePaths: seen})
+	if err != nil {
+		logger.Error("prune missing", "err", err)
+		os.Exit(1)
+	}
+
+	logger.Info("done", "files_indexed", count, "rows_pruned", pruned.GetRowsDeleted())
+}
+
+func envOr(key, fallback string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return fallback
+}