430 lines
16 KiB
Go
430 lines
16 KiB
Go
// workflow.go implements the dispatch_workflow tool: enkly's answer to
|
|
// "dynamic workflows... orchestration of hundreds of subagents in
|
|
// parallel in a single session, with automatic verification of results
|
|
// and real-time plan adjustment." Where dispatch_subagent (subagent.go)
|
|
// hands off ONE investigation and waits for it, dispatch_workflow takes
|
|
// a whole batch of them — typically one per file, service, or call site
|
|
// in a large mechanical job like a codebase migration — and runs as
|
|
// many as the deployment's configured (and plan-entitled) parallelism
|
|
// allows at once, respects any depends_on ordering between them, checks
|
|
// every report against its own task before handing results back, and
|
|
// retries a step once if that check fails, instead of silently treating
|
|
// an empty or off-topic report as done.
|
|
//
|
|
// Every step is, under the hood, exactly the same kind of run
|
|
// dispatch_subagent launches — e.Subagent.Run against a fresh,
|
|
// read-only sub-agent — just many of them in flight at once instead of
|
|
// one. That reuse is deliberate: it's the same nested agentrun.Agent,
|
|
// the same restrictedExecutor read-only boundary, the same
|
|
// SubagentSystemPrompt, so a workflow step can never write or run
|
|
// anything any more than a lone dispatch_subagent call could. What
|
|
// *is* new here is the scheduling, the bounded concurrency, and the
|
|
// verification pass — everything below this point in the file.
|
|
package toolexec
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
|
|
"enkly/pkg/entitlements"
|
|
"enkly/pkg/vllmkit"
|
|
"enkly/services/agent/internal/infrastructure/agentrun"
|
|
)
|
|
|
|
// ---------------------------------------------------------------------
|
|
// Entitlements: whether dispatch_workflow is usable this turn
|
|
// ---------------------------------------------------------------------
|
|
|
|
// entitlementsKey is WithEntitlements' context key.
|
|
type entitlementsKey struct{}
|
|
|
|
// WithEntitlements attaches the caller's resolved capability Set (see
|
|
// pkg/entitlements) to ctx for dispatchWorkflow to read back — set once
|
|
// per turn by application.Service.StartTurn from the plan name
|
|
// billing's CheckAndIncrementUsage already returned, the same place
|
|
// agentrun.WithPreferredLanguage and agentrun.WithEffort get attached
|
|
// to the same run context.
|
|
func WithEntitlements(ctx context.Context, set entitlements.Set) context.Context {
|
|
return context.WithValue(ctx, entitlementsKey{}, set)
|
|
}
|
|
|
|
// entitlementsFrom reads back what WithEntitlements stored. The zero
|
|
// Set (DynamicWorkflows: false) is the correct default for a context
|
|
// nothing ever attached one to — see entitlements.Catalog.Resolve's
|
|
// doc comment on failing closed.
|
|
func entitlementsFrom(ctx context.Context) entitlements.Set {
|
|
set, _ := ctx.Value(entitlementsKey{}).(entitlements.Set)
|
|
return set
|
|
}
|
|
|
|
// ---------------------------------------------------------------------
|
|
// dispatch_workflow argument parsing
|
|
// ---------------------------------------------------------------------
|
|
|
|
type workflowStep struct {
|
|
ID string
|
|
Task string
|
|
DependsOn []string
|
|
}
|
|
|
|
// maxWorkflowSteps bounds one dispatch_workflow call regardless of
|
|
// configured parallelism — "hundreds" of concurrent sub-agents is the
|
|
// target, not an unbounded number typed into one tool call; well past
|
|
// this a migration should be split into more than one dispatch_workflow
|
|
// round anyway (the model can always call it again).
|
|
const maxWorkflowSteps = 500
|
|
|
|
func parseWorkflowSteps(raw []any) ([]workflowStep, error) {
|
|
if len(raw) == 0 {
|
|
return nil, fmt.Errorf("dispatch_workflow: \"steps\" must be a non-empty array")
|
|
}
|
|
if len(raw) > maxWorkflowSteps {
|
|
return nil, fmt.Errorf("dispatch_workflow: %d steps exceeds the %d-step limit for one call; split into more than one dispatch_workflow", len(raw), maxWorkflowSteps)
|
|
}
|
|
seen := make(map[string]bool, len(raw))
|
|
steps := make([]workflowStep, 0, len(raw))
|
|
for i, item := range raw {
|
|
m, ok := item.(map[string]any)
|
|
if !ok {
|
|
return nil, fmt.Errorf("dispatch_workflow: step %d is not an object", i)
|
|
}
|
|
id := strings.TrimSpace(argStr(m, "id"))
|
|
task := strings.TrimSpace(argStr(m, "task"))
|
|
if id == "" {
|
|
return nil, fmt.Errorf("dispatch_workflow: step %d is missing \"id\"", i)
|
|
}
|
|
if task == "" {
|
|
return nil, fmt.Errorf("dispatch_workflow: step %q is missing \"task\"", id)
|
|
}
|
|
if seen[id] {
|
|
return nil, fmt.Errorf("dispatch_workflow: duplicate step id %q", id)
|
|
}
|
|
seen[id] = true
|
|
|
|
var deps []string
|
|
if rawDeps, ok := m["depends_on"].([]any); ok {
|
|
for _, d := range rawDeps {
|
|
if s, ok := d.(string); ok && strings.TrimSpace(s) != "" {
|
|
deps = append(deps, strings.TrimSpace(s))
|
|
}
|
|
}
|
|
}
|
|
steps = append(steps, workflowStep{ID: id, Task: task, DependsOn: deps})
|
|
}
|
|
return steps, nil
|
|
}
|
|
|
|
// validateDependencies checks that every depends_on name refers to a
|
|
// real step in the same batch and that the dependency graph has no
|
|
// cycle (Kahn's algorithm — the same technique runWorkflow's own
|
|
// scheduler uses, run here purely as an up-front check so a cyclic or
|
|
// malformed workflow fails loudly as a single clear tool error instead
|
|
// of silently deadlocking the scheduler).
|
|
func validateDependencies(steps []workflowStep) error {
|
|
index := make(map[string]int, len(steps))
|
|
for i, s := range steps {
|
|
index[s.ID] = i
|
|
}
|
|
indegree := make([]int, len(steps))
|
|
adj := make([][]int, len(steps))
|
|
for i, s := range steps {
|
|
for _, dep := range s.DependsOn {
|
|
j, ok := index[dep]
|
|
if !ok {
|
|
return fmt.Errorf("dispatch_workflow: step %q depends_on unknown step %q", s.ID, dep)
|
|
}
|
|
if j == i {
|
|
return fmt.Errorf("dispatch_workflow: step %q cannot depend on itself", s.ID)
|
|
}
|
|
indegree[i]++
|
|
adj[j] = append(adj[j], i)
|
|
}
|
|
}
|
|
queue := make([]int, 0, len(steps))
|
|
for i, d := range indegree {
|
|
if d == 0 {
|
|
queue = append(queue, i)
|
|
}
|
|
}
|
|
visited := 0
|
|
for len(queue) > 0 {
|
|
n := queue[0]
|
|
queue = queue[1:]
|
|
visited++
|
|
for _, m := range adj[n] {
|
|
indegree[m]--
|
|
if indegree[m] == 0 {
|
|
queue = append(queue, m)
|
|
}
|
|
}
|
|
}
|
|
if visited != len(steps) {
|
|
return fmt.Errorf("dispatch_workflow: depends_on forms a cycle — every step must be reachable without looping back")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ---------------------------------------------------------------------
|
|
// Verification
|
|
// ---------------------------------------------------------------------
|
|
|
|
// WorkflowVerifier checks whether a step's report actually answers its
|
|
// task, returning ok plus a short human-readable reason when it
|
|
// doesn't. toolexec.Executor.WorkflowVerifier defaults to
|
|
// defaultWorkflowVerifier; a deployment can swap in something richer
|
|
// (e.g. one extra, cheap LLM call judging relevance) by setting that
|
|
// field — see Executor's doc comment.
|
|
type WorkflowVerifier func(task, report string) (ok bool, reason string)
|
|
|
|
// defaultWorkflowVerifier is a deliberately cheap, deterministic check
|
|
// — no second model round-trip per step, consistent with this
|
|
// codebase's preference (see pkg/mathkit) for dependency-free,
|
|
// inspectable logic wherever a real engine isn't needed. It catches the
|
|
// two failure shapes a stalled or confused sub-agent actually produces
|
|
// in practice (an empty report, or the canned "finished without
|
|
// writing a report" fallback from subagent.go's own dispatchSubagent)
|
|
// plus an implausibly short report, without trying to judge the report's
|
|
// substance — that judgment call is exactly what's handed back to the
|
|
// dispatching model via each step's Verified/Reason, per "real-time
|
|
// plan adjustment": the model, not this function, decides what a thin
|
|
// report means for the overall task.
|
|
func defaultWorkflowVerifier(task, report string) (bool, string) {
|
|
r := strings.TrimSpace(report)
|
|
if r == "" {
|
|
return false, "empty report"
|
|
}
|
|
if strings.Contains(r, "finished without writing a report") {
|
|
return false, "sub-agent produced no findings"
|
|
}
|
|
if strings.HasPrefix(r, "[step failed to run:") {
|
|
return false, "sub-agent run did not complete"
|
|
}
|
|
const minPlausibleReportChars = 40
|
|
if len(r) < minPlausibleReportChars {
|
|
return false, "report too short to plausibly answer the task"
|
|
}
|
|
_ = task // reserved for a future content-aware verifier (e.g. keyword/file-path overlap with the task)
|
|
return true, ""
|
|
}
|
|
|
|
// ---------------------------------------------------------------------
|
|
// Scheduling + execution
|
|
// ---------------------------------------------------------------------
|
|
|
|
// stepOutcome is one step's final result, after up to two attempts.
|
|
type stepOutcome struct {
|
|
ID string
|
|
Report string
|
|
Verified bool
|
|
Reason string
|
|
Attempts int
|
|
}
|
|
|
|
// maxStepAttempts bounds the "plan adjustment" retry per step: one
|
|
// retry, with the verifier's own reason folded into the retry's task
|
|
// as a concrete note on what was missing, then accept whatever comes
|
|
// back — a workflow step is meant to be a bounded, mechanical unit of
|
|
// work, not an open-ended negotiation with the model.
|
|
const maxStepAttempts = 2
|
|
|
|
// runStepWithVerification runs one step (possibly twice) against sub,
|
|
// verifying the report each time and amending the task with the
|
|
// verifier's reason before a retry.
|
|
func runStepWithVerification(ctx context.Context, sub *agentrun.Agent, task string, verify WorkflowVerifier) (report string, attempts int, verified bool, reason string) {
|
|
for attempts = 1; attempts <= maxStepAttempts; attempts++ {
|
|
raw, _, err := sub.Run(ctx, []vllmkit.Message{{Role: vllmkit.RoleUser, Content: task}}, agentrun.Callbacks{})
|
|
raw = strings.TrimSpace(raw)
|
|
switch {
|
|
case raw != "":
|
|
report = raw
|
|
case err != nil:
|
|
report = fmt.Sprintf("[step failed to run: %v]", err)
|
|
default:
|
|
report = ""
|
|
}
|
|
|
|
ok, why := verify(task, report)
|
|
if ok {
|
|
return report, attempts, true, ""
|
|
}
|
|
reason = why
|
|
if attempts < maxStepAttempts && ctx.Err() == nil {
|
|
task = fmt.Sprintf("%s\n\n[Your previous attempt's report was judged insufficient: %s. Investigate more "+
|
|
"thoroughly — check files you may have skipped, and make sure your report cites specific evidence — "+
|
|
"then try again.]", task, why)
|
|
}
|
|
}
|
|
return report, attempts, false, reason
|
|
}
|
|
|
|
// runWorkflow executes every step, honoring depends_on and bounded by
|
|
// maxParallel concurrent sub-agent runs at any moment — not maxParallel
|
|
// per "wave", the whole run: a step becomes eligible the instant its
|
|
// last dependency finishes and immediately competes for the same
|
|
// semaphore every other ready step is drawing from, so a workflow with
|
|
// an uneven dependency shape (a few early steps gating a large fan-out)
|
|
// still keeps the configured number of sub-agents busy throughout,
|
|
// rather than only during a step's own "wave".
|
|
func runWorkflow(ctx context.Context, sub *agentrun.Agent, goal string, steps []workflowStep, maxParallel int, verify WorkflowVerifier) []stepOutcome {
|
|
if maxParallel < 1 {
|
|
maxParallel = 1
|
|
}
|
|
n := len(steps)
|
|
index := make(map[string]int, n)
|
|
for i, s := range steps {
|
|
index[s.ID] = i
|
|
}
|
|
indegree := make([]int, n)
|
|
dependents := make([][]int, n)
|
|
for i, s := range steps {
|
|
for _, dep := range s.DependsOn {
|
|
j := index[dep]
|
|
indegree[i]++
|
|
dependents[j] = append(dependents[j], i)
|
|
}
|
|
}
|
|
|
|
outcomes := make([]stepOutcome, n)
|
|
sem := make(chan struct{}, maxParallel)
|
|
var mu sync.Mutex
|
|
var wg sync.WaitGroup
|
|
|
|
var schedule func(i int)
|
|
schedule = func(i int) {
|
|
defer wg.Done()
|
|
|
|
select {
|
|
case sem <- struct{}{}:
|
|
case <-ctx.Done():
|
|
mu.Lock()
|
|
outcomes[i] = stepOutcome{ID: steps[i].ID, Report: "[skipped: session stopped before this step started]", Attempts: 0}
|
|
mu.Unlock()
|
|
return
|
|
}
|
|
task := fmt.Sprintf("Overall workflow goal: %s\n\nYour step (id %q): %s", goal, steps[i].ID, steps[i].Task)
|
|
report, attempts, verified, reason := runStepWithVerification(ctx, sub, task, verify)
|
|
<-sem
|
|
|
|
mu.Lock()
|
|
outcomes[i] = stepOutcome{ID: steps[i].ID, Report: report, Verified: verified, Reason: reason, Attempts: attempts}
|
|
var newlyReady []int
|
|
for _, j := range dependents[i] {
|
|
indegree[j]--
|
|
if indegree[j] == 0 {
|
|
newlyReady = append(newlyReady, j)
|
|
}
|
|
}
|
|
mu.Unlock()
|
|
|
|
for _, j := range newlyReady {
|
|
wg.Add(1)
|
|
go schedule(j)
|
|
}
|
|
}
|
|
|
|
for i := 0; i < n; i++ {
|
|
if indegree[i] == 0 {
|
|
wg.Add(1)
|
|
go schedule(i)
|
|
}
|
|
}
|
|
wg.Wait()
|
|
|
|
return outcomes
|
|
}
|
|
|
|
// renderWorkflowSummary turns every step's outcome into the single
|
|
// string handed back as the dispatch_workflow tool result — the "plan
|
|
// adjustment" input the dispatching model actually reads: it sees at a
|
|
// glance which steps verified and which didn't, by id, so it can decide
|
|
// what to do about the shortfall (re-dispatch, handle manually, or
|
|
// proceed if the unverified step turns out not to matter) rather than
|
|
// that distinction being silently lost.
|
|
func renderWorkflowSummary(goal string, outcomes []stepOutcome) string {
|
|
verifiedN := 0
|
|
for _, o := range outcomes {
|
|
if o.Verified {
|
|
verifiedN++
|
|
}
|
|
}
|
|
var b strings.Builder
|
|
fmt.Fprintf(&b, "Workflow complete: %d/%d steps verified. Goal: %s\n", verifiedN, len(outcomes), goal)
|
|
if verifiedN < len(outcomes) {
|
|
fmt.Fprintf(&b, "%d step(s) need attention — see the ⚠️ entries below before treating this workflow as done.\n", len(outcomes)-verifiedN)
|
|
}
|
|
b.WriteString("\n")
|
|
for _, o := range outcomes {
|
|
status := "✅ verified"
|
|
if !o.Verified {
|
|
status = fmt.Sprintf("⚠️ needs attention — %s", o.Reason)
|
|
}
|
|
fmt.Fprintf(&b, "### Step %q — %s (%d attempt(s))\n%s\n\n", o.ID, status, o.Attempts, o.Report)
|
|
}
|
|
return strings.TrimSpace(b.String())
|
|
}
|
|
|
|
// ---------------------------------------------------------------------
|
|
// The tool call itself
|
|
// ---------------------------------------------------------------------
|
|
|
|
// dispatchWorkflow runs the dispatch_workflow tool call: gated by
|
|
// entitlementsFrom(ctx).DynamicWorkflows, bounded by
|
|
// Executor.WorkflowMaxParallel and the resolved effort level's own
|
|
// MaxParallelSubagents cap (whichever is smaller — see
|
|
// agentrun.EffortFromContext), scheduled by runWorkflow, and rendered
|
|
// by renderWorkflowSummary.
|
|
func (e *Executor) dispatchWorkflow(ctx context.Context, call vllmkit.ToolCallRequest) vllmkit.ToolResult {
|
|
if e.Subagent == nil {
|
|
return vllmkit.ToolResult{Name: call.Name, Content: "dispatch_workflow: sub-agents are not configured on this deployment", IsError: true}
|
|
}
|
|
|
|
ent := entitlementsFrom(ctx)
|
|
if !ent.DynamicWorkflows {
|
|
plan := ent.PlanName
|
|
if plan == "" {
|
|
plan = "your current plan"
|
|
}
|
|
return vllmkit.ToolResult{
|
|
Name: call.Name,
|
|
Content: fmt.Sprintf("dispatch_workflow: dynamic workflows aren't available on %s (this feature is part of "+
|
|
"the Enterprise/Team/Max tier). Use dispatch_subagent for the same investigative work one step at a "+
|
|
"time instead, or do it directly.", plan),
|
|
IsError: true,
|
|
}
|
|
}
|
|
|
|
goal := strings.TrimSpace(argStr(call.Arguments, "goal"))
|
|
if goal == "" {
|
|
return vllmkit.ToolResult{Name: call.Name, Content: `dispatch_workflow: "goal" is required`, IsError: true}
|
|
}
|
|
rawSteps, _ := call.Arguments["steps"].([]any)
|
|
steps, err := parseWorkflowSteps(rawSteps)
|
|
if err != nil {
|
|
return vllmkit.ToolResult{Name: call.Name, Content: err.Error(), IsError: true}
|
|
}
|
|
if err := validateDependencies(steps); err != nil {
|
|
return vllmkit.ToolResult{Name: call.Name, Content: err.Error(), IsError: true}
|
|
}
|
|
|
|
maxParallel := e.WorkflowMaxParallel
|
|
if maxParallel <= 0 {
|
|
maxParallel = defaultWorkflowMaxParallel
|
|
}
|
|
if levelCap := agentrun.EffortFromContext(ctx).Resolve().MaxParallelSubagents; levelCap > 0 && levelCap < maxParallel {
|
|
maxParallel = levelCap
|
|
}
|
|
if maxParallel > len(steps) {
|
|
maxParallel = len(steps)
|
|
}
|
|
|
|
verify := e.WorkflowVerifier
|
|
if verify == nil {
|
|
verify = defaultWorkflowVerifier
|
|
}
|
|
|
|
outcomes := runWorkflow(ctx, e.Subagent, goal, steps, maxParallel, verify)
|
|
return vllmkit.ToolResult{Name: call.Name, Content: renderWorkflowSummary(goal, outcomes)}
|
|
}
|