From 697e91f8e69b0ccc89dd05eef66f35e0d1974bee Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Fri, 16 Oct 2026 13:55:21 -0500 Subject: [PATCH 1/6] build: re-pin forge v1.12.9 and grove v1.8.0 Moves forge to v1.12.9 and grove plus its mongo, pg and sqlite drivers to v1.8.0. go-utils comes along to v1.4.1 because forge needs it. With this pin, grove records a db.client span and the operation duration for every store query, so you get database telemetry without any change in shield. Build, vet and the race tests pass unchanged. --- go.mod | 21 +++++++++++++-------- go.sum | 50 ++++++++++++++++++++++++++++---------------------- 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index ea07433..1035e94 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,12 @@ module github.com/xraph/shield go 1.26.0 require ( - github.com/xraph/forge v1.12.7 - github.com/xraph/go-utils v1.3.0 - github.com/xraph/grove v1.7.3 - github.com/xraph/grove/drivers/mongodriver v1.7.3 - github.com/xraph/grove/drivers/pgdriver v1.7.3 - github.com/xraph/grove/drivers/sqlitedriver v1.7.3 + github.com/xraph/forge v1.12.9 + github.com/xraph/go-utils v1.4.1 + github.com/xraph/grove v1.8.0 + github.com/xraph/grove/drivers/mongodriver v1.8.0 + github.com/xraph/grove/drivers/pgdriver v1.8.0 + github.com/xraph/grove/drivers/sqlitedriver v1.8.0 github.com/xraph/vessel v1.0.4 go.jetify.com/typeid/v2 v2.0.0-alpha.3 go.mongodb.org/mongo-driver/v2 v2.5.0 @@ -25,6 +25,7 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/fatih/color v1.19.0 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/fxamacker/cbor/v2 v2.9.2 // indirect github.com/gabriel-vasile/mimetype v1.4.15 // indirect @@ -54,6 +55,8 @@ require ( github.com/gofrs/uuid/v5 v5.3.2 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grafana/pyroscope-go v1.4.3 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.11 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/hashicorp/consul/api v1.34.4 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -94,15 +97,17 @@ require ( github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.2.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/xraph/confy v1.0.3 // indirect + github.com/xraph/confy v1.1.0 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect go.opentelemetry.io/otel v1.45.0 // indirect - go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.45.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 // indirect go.opentelemetry.io/otel/metric v1.45.0 // indirect go.opentelemetry.io/otel/sdk v1.45.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect go.opentelemetry.io/otel/trace v1.45.0 // indirect go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.uber.org/atomic v1.11.0 // indirect diff --git a/go.sum b/go.sum index fb82a0f..bd82677 100644 --- a/go.sum +++ b/go.sum @@ -61,8 +61,8 @@ github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRr github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/fxamacker/cbor/v2 v2.9.2 h1:X4Ksno9+x3cz0TZv69ec1hxP/+tymuR8PXQJyDwfh78= @@ -152,6 +152,10 @@ github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17k github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grafana/pyroscope-go v1.4.3 h1:XAfYZe5ie8eRTsKAMHQG60ESJjHBV5XUijk8vDvqalw= +github.com/grafana/pyroscope-go v1.4.3/go.mod h1:enNhwzbML7+hMzJHTvKAIqTGqIaOOF1rgF+2au+NOwg= +github.com/grafana/pyroscope-go/godeltaprof v0.1.11 h1:el5LYpXissAiCKZ5/6yjlr6mhYVV6Cp5lahTocxraXM= +github.com/grafana/pyroscope-go/godeltaprof v0.1.11/go.mod h1:jl1V8M4cWsXciROCPIDDG7CtjSjT/ECbp6eLVuMxYRI= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/hashicorp/consul/api v1.34.4 h1:0U4YZ1Yp7K9WK9ex0gTJraFim26l02wCvsmf2ukalVE= @@ -334,8 +338,8 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -362,20 +366,20 @@ github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs= github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/xraph/confy v1.0.3 h1:mY0kIo9+dU+XcGGN7cm4IfOhvk4SDj5TrnccpGi21fs= -github.com/xraph/confy v1.0.3/go.mod h1:DcYB+N0yKpr3t1yhhkdIIiCfch30Kh5wW6GRntUMOGk= -github.com/xraph/forge v1.12.7 h1:hsmw7bm1u82bW0I1sBenk1vEUYF2xV3dujeFt9iJkos= -github.com/xraph/forge v1.12.7/go.mod h1:O1JOH9fxDhKRyvBbIl2UpZ+aFCjv7vO/IbNfAwnYSEI= -github.com/xraph/go-utils v1.3.0 h1:bMZxwH9Xi/6UqhHSotKHB3a+/FnqnlUnEKf+aHZaw5c= -github.com/xraph/go-utils v1.3.0/go.mod h1:NfSn00r74ZjcIhVujIDPdKV25Ktghy2yz0DQmIKEARk= -github.com/xraph/grove v1.7.3 h1:zGwoXhxPYfxyZMpp3MYyXyEo8QqIJeJvnDKJZMpFj3Q= -github.com/xraph/grove v1.7.3/go.mod h1:PFMxsJ2mFagV3ghdbufuBXn2ImdXSC27MZDpUsePKHM= -github.com/xraph/grove/drivers/mongodriver v1.7.3 h1:qo7aQEKPDFjkjSZJbmdbLUgeRE8bjQCGLQcBwgSTGjA= -github.com/xraph/grove/drivers/mongodriver v1.7.3/go.mod h1:p+HhIx2Ii8dTfCCX/OaqEcIfPxSQxMWHorjXDqDWYlU= -github.com/xraph/grove/drivers/pgdriver v1.7.3 h1:QlyfKTDsijG4Rpcr21y09eEVXtoBipk2521O9VmKWrg= -github.com/xraph/grove/drivers/pgdriver v1.7.3/go.mod h1:KdmMNyhsATlkIr7CEZjSXpVFveSJt1eYO2lgPFaWTrE= -github.com/xraph/grove/drivers/sqlitedriver v1.7.3 h1:nnw6yafmdGf5091jiQxAkljBsSA7qpGUGo5VSCvn66I= -github.com/xraph/grove/drivers/sqlitedriver v1.7.3/go.mod h1:6tokGLpt5pI3ZvXDYlSkqwT+h2k/ky8qDvaUnbTmzeI= +github.com/xraph/confy v1.1.0 h1:xkWaujezyxi+8AcSgmkWarB6MacGlrhozJi+QcOD5P0= +github.com/xraph/confy v1.1.0/go.mod h1:HZwQ/RixbBNtBMT7IcULVYbAkyFjeXcJDNJDYj6m5oI= +github.com/xraph/forge v1.12.9 h1:hZ/qqPpXzTQm17aFdMsc7oINDuFrJAwIyidkphQT/yw= +github.com/xraph/forge v1.12.9/go.mod h1:atngBzJc3w2wDz/PYOZroYE7K5Xm5BpCDLIh5wLm9Oc= +github.com/xraph/go-utils v1.4.1 h1:uP4CD+AaCKThbfR8kdQUh0kKHCFFaGb300fKMEJoSJY= +github.com/xraph/go-utils v1.4.1/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= +github.com/xraph/grove v1.8.0 h1:j1QsA/sqnc3WlVRSqvkRXrksC1ckv17hPvEEMjinwAc= +github.com/xraph/grove v1.8.0/go.mod h1:pZsulQT7WO3Owv9f2bxjW/qZxaz8y75SwdtVTPWKod8= +github.com/xraph/grove/drivers/mongodriver v1.8.0 h1:DzT2vRThDm02CkP9P37x5ECHXF3koqm9Rlcs9TYYc0k= +github.com/xraph/grove/drivers/mongodriver v1.8.0/go.mod h1:vp9jfR08GBmh3a42CGAdC87Z2vy8JurEO2VT4c0X17k= +github.com/xraph/grove/drivers/pgdriver v1.8.0 h1:UxYJKdObN9Qd0kQD8JrD9AX4AeDRAV8M5WlOKdQBdJE= +github.com/xraph/grove/drivers/pgdriver v1.8.0/go.mod h1:V9vYMFGIWdtkTOrOEB+rR5kXlhkuQeWjjCKRa0btTdg= +github.com/xraph/grove/drivers/sqlitedriver v1.8.0 h1:CKTCDEN7t/DQbTuZEuvcZx5JYe+rvxLw0oRXaBeyAOo= +github.com/xraph/grove/drivers/sqlitedriver v1.8.0/go.mod h1:ZC2rVcW+ZfWDIiTl1BZ2F9jeIlhl5LayBMyENJoXZSk= github.com/xraph/vessel v1.0.4 h1:YVG80hm7bAKTwogwLXZMlp46tMHHJThUcvCUOWao9Yk= github.com/xraph/vessel v1.0.4/go.mod h1:5hgrMbuczxu2kRIM3iVJ3wPSb6HOvbMhV4nkRFaPNqs= github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= @@ -391,18 +395,20 @@ go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= -go.opentelemetry.io/otel/exporters/jaeger v1.17.0 h1:D7UpUy2Xc2wsi1Ras6V40q806WM07rqoCWzXu7Sqy+4= -go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.45.0 h1:pnxy6c/kvNBWdNNFzqpjuJLm9Hjhgk/Q0nY221rwuk0= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.45.0/go.mod h1:qw6YsFapotRwoDhXRZvljzaOvCQB7UfnafEJagpN2TA= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8= go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns= +go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI= go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= From b20c0099d35e56884b9ce3d3d56cdb1a7d9c37ce Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Fri, 16 Oct 2026 14:09:09 -0500 Subject: [PATCH 2/6] feat(engine): trace scans and plugin hooks Every ScanInput and ScanOutput call now runs under a "scan content" span from the github.com/xraph/shield tracer. It carries the scan ID, direction, decision, whether the scan blocked, the finding count and, when a store is configured, whether the result was persisted. A failed store write keeps the scan's verdict but marks the span persist_failed, so a scan the dashboard will never show stops being a log line nobody reads. Plugin hooks run under a "run plugin hooks" child span with the hook name and the number of plugins called. No span starts for a hook nobody implements. A hook that returns an error marks that span plugin_hook_failed and adds an exception event with the error's Go type. The scanned text, findings, tenant and app never go on a span, and error messages never do either: drivers and plugins can quote the values they failed on. The layers are still TODO, so they have no spans yet. When one lands it starts a child span from the scan's context. The Forge extension now hands the app logger to the engine, and the engine hands its logger to the plugin registry. Until now both stayed on a no-op logger, so hook errors and failed scan writes were never logged anywhere. --- engine/engine.go | 38 +++- engine/telemetry_test.go | 208 ++++++++++++++++++ extension/extension.go | 13 +- go.mod | 6 +- internal/telemetry/telemetry.go | 50 +++++ internal/telemetry/telemetry_test.go | 27 +++ plugin/registry.go | 315 ++++++++++++--------------- 7 files changed, 468 insertions(+), 189 deletions(-) create mode 100644 engine/telemetry_test.go create mode 100644 internal/telemetry/telemetry.go create mode 100644 internal/telemetry/telemetry_test.go diff --git a/engine/engine.go b/engine/engine.go index 1150211..2558aff 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -19,10 +19,14 @@ import ( "errors" "time" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + log "github.com/xraph/go-utils/log" "github.com/xraph/shield" "github.com/xraph/shield/id" + "github.com/xraph/shield/internal/telemetry" "github.com/xraph/shield/plugin" "github.com/xraph/shield/scan" "github.com/xraph/shield/store" @@ -54,7 +58,8 @@ func WithConfig(cfg shield.Config) Option { return func(e *Engine) { e.config = cfg } } -// WithLogger sets a custom logger. +// WithLogger sets a custom logger. The plugin registry reports hook errors +// on it too. func WithLogger(l log.Logger) Option { return func(e *Engine) { e.logger = l } } @@ -69,6 +74,7 @@ func New(opts ...Option) (*Engine, error) { for _, opt := range opts { opt(e) } + e.registry.SetLogger(e.logger) return e, nil } @@ -85,10 +91,21 @@ func (e *Engine) ScanOutput(ctx context.Context, input *scan.Input) (*scan.Resul } // executeScan runs the layered safety evaluation. +// +// The scan runs under a "scan content" span. Its attributes are counts, +// enums and the scan ID; the scanned text, findings and tenant never go on +// it. Plugin hooks and the store write nest under it, and so will each +// layer once it exists. func (e *Engine) executeScan(ctx context.Context, input *scan.Input) (*scan.Result, error) { start := time.Now() scanID := id.NewScanID() + ctx, span := telemetry.Tracer().Start(ctx, "scan content", trace.WithAttributes( + attribute.String("shield.scan.id", scanID.String()), + attribute.String("shield.scan.direction", string(input.Direction)), + )) + defer span.End() + result := &scan.Result{ ID: scanID, Direction: input.Direction, @@ -100,6 +117,10 @@ func (e *Engine) executeScan(ctx context.Context, input *scan.Input) (*scan.Resu // Emit scan started e.registry.EmitScanStarted(ctx, scanID, string(input.Direction), input.Text) + // Each layer below, once implemented, gets a child span of ctx with a + // shield.scan.layer attribute naming it (instincts, awareness, + // boundaries, values, judgments, reflexes). + // Layer 1: INSTINCTS — fastest, pre-conscious threat detection // TODO: Execute registered instincts @@ -127,13 +148,22 @@ func (e *Engine) executeScan(ctx context.Context, input *scan.Input) (*scan.Resu e.registry.EmitScanBlocked(ctx, scanID, string(result.Decision)) } - // Persist result + // Persist result. A failed write doesn't fail the scan, but the span + // says so: the dashboard will never show this scan. if e.store != nil { - if err := e.store.CreateScan(ctx, result); err != nil { - e.logger.Warn("engine: failed to persist scan result", log.String("scan_id", scanID.String()), log.Error(err)) + persistErr := e.store.CreateScan(ctx, result) + if persistErr != nil { + e.logger.Warn("engine: failed to persist scan result", log.String("scan_id", scanID.String()), log.Error(persistErr)) + telemetry.Fail(span, "persist_failed", persistErr) } + span.SetAttributes(attribute.Bool("shield.scan.persisted", persistErr == nil)) } + span.SetAttributes( + attribute.String("shield.scan.decision", string(result.Decision)), + attribute.Bool("shield.scan.blocked", result.Blocked), + attribute.Int("shield.scan.findings", len(result.Findings)), + ) return result, nil } diff --git a/engine/telemetry_test.go b/engine/telemetry_test.go new file mode 100644 index 0000000..a41bec1 --- /dev/null +++ b/engine/telemetry_test.go @@ -0,0 +1,208 @@ +package engine + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" + + "github.com/xraph/shield" + "github.com/xraph/shield/id" + "github.com/xraph/shield/scan" + "github.com/xraph/shield/store" +) + +// secret stands in for anything that must never reach a span: scanned text, +// a tenant, an error message quoting a value. +const secret = "alice@example.com" + +func recordSpans(t *testing.T) *tracetest.SpanRecorder { + t.Helper() + prev := otel.GetTracerProvider() + rec := tracetest.NewSpanRecorder() + otel.SetTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(rec))) + t.Cleanup(func() { otel.SetTracerProvider(prev) }) + return rec +} + +func spansNamed(rec *tracetest.SpanRecorder, name string) []sdktrace.ReadOnlySpan { + var out []sdktrace.ReadOnlySpan + for _, s := range rec.Ended() { + if s.Name() == name { + out = append(out, s) + } + } + return out +} + +func attrs(s sdktrace.ReadOnlySpan) map[attribute.Key]attribute.Value { + m := map[attribute.Key]attribute.Value{} + for _, kv := range s.Attributes() { + m[kv.Key] = kv.Value + } + return m +} + +// assertNoSecret fails if secret appears in any attribute, event or status of +// any recorded span. +func assertNoSecret(t *testing.T, rec *tracetest.SpanRecorder) { + t.Helper() + for _, s := range rec.Ended() { + check := func(where string, kvs []attribute.KeyValue) { + for _, kv := range kvs { + if strings.Contains(kv.Value.String(), secret) { + t.Errorf("span %q leaks %s in %s %s", s.Name(), secret, where, kv.Key) + } + } + } + check("attribute", s.Attributes()) + for _, ev := range s.Events() { + check("event "+ev.Name, ev.Attributes) + } + if strings.Contains(s.Status().Description, secret) { + t.Errorf("span %q leaks %s in its status", s.Name(), secret) + } + } +} + +type hookPlugin struct { + startErr error +} + +func (hookPlugin) Name() string { return "test-hooks" } +func (p hookPlugin) OnScanStarted(context.Context, id.ScanID, string, string) error { + return p.startErr +} +func (hookPlugin) OnScanCompleted(context.Context, id.ScanID, string, int, time.Duration) error { + return nil +} + +type scanStore struct { + store.Store + err error +} + +func (s scanStore) CreateScan(context.Context, *scan.Result) error { return s.err } + +func TestScanRunsUnderOneSpanWithPluginHooksNested(t *testing.T) { + rec := recordSpans(t) + e, _ := New(WithPlugin(hookPlugin{startErr: errors.New("plugin rejected " + secret)}), WithStore(scanStore{})) + + ctx, parent := otel.Tracer("test").Start(context.Background(), "handle request") + ctx = shield.WithTenant(shield.WithApp(ctx, secret), secret) + res, err := e.ScanInput(ctx, &scan.Input{Text: "my email is " + secret}) + parent.End() + if err != nil { + t.Fatal(err) + } + + scans := spansNamed(rec, "scan content") + if len(scans) != 1 { + t.Fatalf("want one scan span, got %d", len(scans)) + } + sp := scans[0] + if sp.SpanKind() != trace.SpanKindInternal || sp.Parent().SpanID() != parent.SpanContext().SpanID() { + t.Fatalf("scan span kind %v, parent %v", sp.SpanKind(), sp.Parent().SpanID()) + } + if sp.InstrumentationScope().Name != "github.com/xraph/shield" { + t.Fatalf("scope = %q", sp.InstrumentationScope().Name) + } + a := attrs(sp) + want := map[attribute.Key]any{ + "shield.scan.id": res.ID.String(), + "shield.scan.direction": "input", + "shield.scan.decision": "allow", + "shield.scan.blocked": false, + "shield.scan.findings": int64(0), + "shield.scan.persisted": true, + } + for k, v := range want { + if got := a[k].AsInterface(); got != v { + t.Errorf("%s = %v, want %v", k, got, v) + } + } + if sp.Status().Code == codes.Error { + t.Fatalf("a failed plugin hook failed the scan span: %v", sp.Status()) + } + + hooks := spansNamed(rec, "run plugin hooks") + if len(hooks) != 2 { + t.Fatalf("want a hook span for scan_started and scan_completed, got %d", len(hooks)) + } + byHook := map[string]sdktrace.ReadOnlySpan{} + for _, h := range hooks { + if h.Parent().SpanID() != sp.SpanContext().SpanID() { + t.Errorf("hook span is not a child of the scan span") + } + ha := attrs(h) + if ha["shield.plugin.count"].AsInt64() != 1 { + t.Errorf("plugin count = %v", ha["shield.plugin.count"]) + } + byHook[ha["shield.plugin.hook"].AsString()] = h + } + started, completed := byHook["scan_started"], byHook["scan_completed"] + if started == nil || completed == nil { + t.Fatalf("hook spans = %v", byHook) + } + if started.Status().Code != codes.Error || started.Status().Description != "plugin_hook_failed" { + t.Errorf("failed hook status = %v", started.Status()) + } + if attrs(started)["shield.plugin.failures"].AsInt64() != 1 { + t.Errorf("failures = %v", attrs(started)["shield.plugin.failures"]) + } + if ev := started.Events(); len(ev) != 1 || ev[0].Name != "exception" || + len(ev[0].Attributes) != 1 || ev[0].Attributes[0].Value.AsString() != "*errors.errorString" { + t.Errorf("exception event = %+v", ev) + } + if completed.Status().Code == codes.Error { + t.Errorf("successful hook marked failed") + } + + assertNoSecret(t, rec) +} + +func TestScanSpanMarksAFailedStoreWrite(t *testing.T) { + rec := recordSpans(t) + e, _ := New(WithStore(scanStore{err: errors.New("insert failed for " + secret)})) + if _, err := e.ScanOutput(context.Background(), &scan.Input{Text: secret}); err != nil { + t.Fatal("a failed store write must not fail the scan:", err) + } + scans := spansNamed(rec, "scan content") + if len(scans) != 1 { + t.Fatalf("want one scan span, got %d", len(scans)) + } + sp := scans[0] + if sp.Parent().IsValid() { + t.Error("a scan with no caller span should be a root") + } + if sp.Status().Code != codes.Error || sp.Status().Description != "persist_failed" { + t.Errorf("status = %v", sp.Status()) + } + a := attrs(sp) + if a["shield.scan.persisted"].AsBool() || a["shield.scan.direction"].AsString() != "output" { + t.Errorf("attributes = %v", a) + } + if len(spansNamed(rec, "run plugin hooks")) != 0 { + t.Error("hook span started with no plugins registered") + } + assertNoSecret(t, rec) +} + +func TestScanWithoutStoreLeavesPersistedUnset(t *testing.T) { + rec := recordSpans(t) + e, _ := New() + if _, err := e.ScanInput(context.Background(), &scan.Input{Text: "hello"}); err != nil { + t.Fatal(err) + } + if _, ok := attrs(spansNamed(rec, "scan content")[0])["shield.scan.persisted"]; ok { + t.Error("persisted set with no store configured") + } +} diff --git a/extension/extension.go b/extension/extension.go index 867d3b7..a39dd03 100644 --- a/extension/extension.go +++ b/extension/extension.go @@ -105,10 +105,15 @@ func (e *Extension) Register(fapp forge.App) error { ) } - opts := append([]engine.Option{engine.WithConfig(shield.Config{ - DefaultProfile: e.config.DefaultProfile, ShutdownTimeout: e.config.ShutdownTimeout, - ScanConcurrency: e.config.ScanConcurrency, EnableShortCircuit: e.config.EnableShortCircuit, - })}, e.engineOpts...) + // The app's logger goes first, so an engine option you pass through + // WithEngineOption can still replace it. + opts := append([]engine.Option{ + engine.WithConfig(shield.Config{ + DefaultProfile: e.config.DefaultProfile, ShutdownTimeout: e.config.ShutdownTimeout, + ScanConcurrency: e.config.ScanConcurrency, EnableShortCircuit: e.config.EnableShortCircuit, + }), + engine.WithLogger(e.Logger()), + }, e.engineOpts...) eng, err := engine.New(opts...) if err != nil { return err diff --git a/go.mod b/go.mod index 1035e94..bbda158 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,9 @@ require ( github.com/xraph/vessel v1.0.4 go.jetify.com/typeid/v2 v2.0.0-alpha.3 go.mongodb.org/mongo-driver/v2 v2.5.0 + go.opentelemetry.io/otel v1.45.0 + go.opentelemetry.io/otel/sdk v1.45.0 + go.opentelemetry.io/otel/trace v1.45.0 ) require ( @@ -101,14 +104,11 @@ require ( github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect - go.opentelemetry.io/otel v1.45.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.45.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 // indirect go.opentelemetry.io/otel/metric v1.45.0 // indirect - go.opentelemetry.io/otel/sdk v1.45.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect - go.opentelemetry.io/otel/trace v1.45.0 // indirect go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go new file mode 100644 index 0000000..d939616 --- /dev/null +++ b/internal/telemetry/telemetry.go @@ -0,0 +1,50 @@ +// Package telemetry holds the tracing helpers the engine, the plugin +// registry and the admin service share. +// +// Spans go to the global OpenTelemetry tracer provider, which forge.New +// installs. +package telemetry + +import ( + "errors" + "fmt" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +// ScopeName is the instrumentation scope on every span shield starts. +const ScopeName = "github.com/xraph/shield" + +// Tracer returns shield's tracer from the global provider. It is looked up on +// each call, so a provider installed after shield was built still applies. +func Tracer() trace.Tracer { return otel.Tracer(ScopeName) } + +// Fail marks span as failed. class is a short, fixed description such as +// "audit_failed". The exception event carries only the error's Go type: error +// messages from drivers and host adapters can quote the values they failed +// on, so they never go on a span. +func Fail(span trace.Span, class string, err error) { + span.SetStatus(codes.Error, class) + Exception(span, err) +} + +// Exception adds an exception event that carries only the error's Go type, +// for a failure that doesn't fail the span as a whole. +func Exception(span trace.Span, err error) { + span.AddEvent("exception", trace.WithAttributes(attribute.String("exception.type", ErrorType(err)))) +} + +// ErrorType returns the Go type of the innermost error in err's chain, the +// value for the exception.type attribute. +func ErrorType(err error) string { + for { + next := errors.Unwrap(err) + if next == nil { + return fmt.Sprintf("%T", err) + } + err = next + } +} diff --git a/internal/telemetry/telemetry_test.go b/internal/telemetry/telemetry_test.go new file mode 100644 index 0000000..1a0b616 --- /dev/null +++ b/internal/telemetry/telemetry_test.go @@ -0,0 +1,27 @@ +package telemetry + +import ( + "errors" + "fmt" + "io/fs" + "testing" +) + +type codedError struct{} + +func (*codedError) Error() string { return "coded" } + +func TestErrorTypeNamesTheInnermostError(t *testing.T) { + for _, tc := range []struct { + err error + want string + }{ + {errors.New("x"), "*errors.errorString"}, + {fmt.Errorf("outer: %w", &codedError{}), "*telemetry.codedError"}, + {fmt.Errorf("a: %w", fmt.Errorf("b: %w", fs.ErrNotExist)), "*errors.errorString"}, + } { + if got := ErrorType(tc.err); got != tc.want { + t.Errorf("ErrorType(%v) = %s, want %s", tc.err, got, tc.want) + } + } +} diff --git a/plugin/registry.go b/plugin/registry.go index a6ac5df..d6db56d 100644 --- a/plugin/registry.go +++ b/plugin/registry.go @@ -4,86 +4,39 @@ import ( "context" "time" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + log "github.com/xraph/go-utils/log" "github.com/xraph/shield/id" + "github.com/xraph/shield/internal/telemetry" ) -// ── Entry types for type-cached slices ─────────────── - -type scanStartedEntry struct { - name string - hook ScanStarted -} - -type scanCompletedEntry struct { - name string - hook ScanCompleted -} - -type scanBlockedEntry struct { - name string - hook ScanBlocked -} - -type scanFailedEntry struct { - name string - hook ScanFailed -} - -type instinctTriggeredEntry struct { - name string - hook InstinctTriggered -} - -type awarenessDetectedEntry struct { - name string - hook AwarenessDetected -} - -type judgmentAssessedEntry struct { - name string - hook JudgmentAssessed -} - -type valueViolatedEntry struct { - name string - hook ValueViolated -} - -type reflexFiredEntry struct { - name string - hook ReflexFired -} - -type boundaryEnforcedEntry struct { - name string - hook BoundaryEnforced -} - -type piiDetectedEntry struct { - name string - hook PIIDetected -} - -type piiRedactedEntry struct { - name string - hook PIIRedacted -} - -type policyEvaluatedEntry struct { - name string - hook PolicyEvaluated -} - -type safetyProfileResolvedEntry struct { - name string - hook SafetyProfileResolved -} +// The shield.plugin.hook span attribute values. +const ( + hookScanStarted = "scan_started" + hookScanCompleted = "scan_completed" + hookScanBlocked = "scan_blocked" + hookScanFailed = "scan_failed" + hookInstinctTriggered = "instinct_triggered" + hookAwarenessDetected = "awareness_detected" + hookJudgmentAssessed = "judgment_assessed" + hookValueViolated = "value_violated" + hookReflexFired = "reflex_fired" + hookBoundaryEnforced = "boundary_enforced" + hookPIIDetected = "pii_detected" + hookPIIRedacted = "pii_redacted" + hookPolicyEvaluated = "policy_evaluated" + hookSafetyProfileResolved = "safety_profile_resolved" + hookShutdown = "shutdown" +) -type shutdownEntry struct { +// entry pairs a plugin's name with one hook it implements. +type entry[H any] struct { name string - hook Shutdown + hook H } // Registry holds registered plugins and dispatches lifecycle events. @@ -93,21 +46,21 @@ type Registry struct { plugins []Plugin logger log.Logger - scanStarted []scanStartedEntry - scanCompleted []scanCompletedEntry - scanBlocked []scanBlockedEntry - scanFailed []scanFailedEntry - instinctTriggered []instinctTriggeredEntry - awarenessDetected []awarenessDetectedEntry - judgmentAssessed []judgmentAssessedEntry - valueViolated []valueViolatedEntry - reflexFired []reflexFiredEntry - boundaryEnforced []boundaryEnforcedEntry - piiDetected []piiDetectedEntry - piiRedacted []piiRedactedEntry - policyEvaluated []policyEvaluatedEntry - safetyProfileResolved []safetyProfileResolvedEntry - shutdown []shutdownEntry + scanStarted []entry[ScanStarted] + scanCompleted []entry[ScanCompleted] + scanBlocked []entry[ScanBlocked] + scanFailed []entry[ScanFailed] + instinctTriggered []entry[InstinctTriggered] + awarenessDetected []entry[AwarenessDetected] + judgmentAssessed []entry[JudgmentAssessed] + valueViolated []entry[ValueViolated] + reflexFired []entry[ReflexFired] + boundaryEnforced []entry[BoundaryEnforced] + piiDetected []entry[PIIDetected] + piiRedacted []entry[PIIRedacted] + policyEvaluated []entry[PolicyEvaluated] + safetyProfileResolved []entry[SafetyProfileResolved] + shutdown []entry[Shutdown] } // NewRegistry creates a new plugin registry. @@ -118,55 +71,91 @@ func NewRegistry(logger log.Logger) *Registry { return &Registry{logger: logger} } +// SetLogger replaces the logger hook errors are reported on. A nil l is +// ignored. Call it before the first Emit. +func (r *Registry) SetLogger(l log.Logger) { + if l != nil { + r.logger = l + } +} + +// run calls one hook on every plugin that implements it. When there is at +// least one, the calls share a "run plugin hooks" span under ctx, so a slow +// or failing plugin shows up inside the scan that triggered it. Hook errors +// are logged, never returned: a plugin can't fail a scan. +func run[H any](ctx context.Context, r *Registry, hook string, entries []entry[H], call func(context.Context, H) error) { + if len(entries) == 0 { + return + } + ctx, span := telemetry.Tracer().Start(ctx, "run plugin hooks", trace.WithAttributes( + attribute.String("shield.plugin.hook", hook), + attribute.Int("shield.plugin.count", len(entries)), + )) + defer span.End() + + failures := 0 + for _, e := range entries { + if err := call(ctx, e.hook); err != nil { + failures++ + r.logger.Warn("plugin: hook error", log.String("hook", hook), log.String("plugin", e.name), log.Error(err)) + telemetry.Exception(span, err) + } + } + if failures > 0 { + span.SetAttributes(attribute.Int("shield.plugin.failures", failures)) + span.SetStatus(codes.Error, "plugin_hook_failed") + } +} + // Register adds a plugin and type-caches its hook implementations. func (r *Registry) Register(p Plugin) { r.plugins = append(r.plugins, p) name := p.Name() if h, ok := p.(ScanStarted); ok { - r.scanStarted = append(r.scanStarted, scanStartedEntry{name, h}) + r.scanStarted = append(r.scanStarted, entry[ScanStarted]{name, h}) } if h, ok := p.(ScanCompleted); ok { - r.scanCompleted = append(r.scanCompleted, scanCompletedEntry{name, h}) + r.scanCompleted = append(r.scanCompleted, entry[ScanCompleted]{name, h}) } if h, ok := p.(ScanBlocked); ok { - r.scanBlocked = append(r.scanBlocked, scanBlockedEntry{name, h}) + r.scanBlocked = append(r.scanBlocked, entry[ScanBlocked]{name, h}) } if h, ok := p.(ScanFailed); ok { - r.scanFailed = append(r.scanFailed, scanFailedEntry{name, h}) + r.scanFailed = append(r.scanFailed, entry[ScanFailed]{name, h}) } if h, ok := p.(InstinctTriggered); ok { - r.instinctTriggered = append(r.instinctTriggered, instinctTriggeredEntry{name, h}) + r.instinctTriggered = append(r.instinctTriggered, entry[InstinctTriggered]{name, h}) } if h, ok := p.(AwarenessDetected); ok { - r.awarenessDetected = append(r.awarenessDetected, awarenessDetectedEntry{name, h}) + r.awarenessDetected = append(r.awarenessDetected, entry[AwarenessDetected]{name, h}) } if h, ok := p.(JudgmentAssessed); ok { - r.judgmentAssessed = append(r.judgmentAssessed, judgmentAssessedEntry{name, h}) + r.judgmentAssessed = append(r.judgmentAssessed, entry[JudgmentAssessed]{name, h}) } if h, ok := p.(ValueViolated); ok { - r.valueViolated = append(r.valueViolated, valueViolatedEntry{name, h}) + r.valueViolated = append(r.valueViolated, entry[ValueViolated]{name, h}) } if h, ok := p.(ReflexFired); ok { - r.reflexFired = append(r.reflexFired, reflexFiredEntry{name, h}) + r.reflexFired = append(r.reflexFired, entry[ReflexFired]{name, h}) } if h, ok := p.(BoundaryEnforced); ok { - r.boundaryEnforced = append(r.boundaryEnforced, boundaryEnforcedEntry{name, h}) + r.boundaryEnforced = append(r.boundaryEnforced, entry[BoundaryEnforced]{name, h}) } if h, ok := p.(PIIDetected); ok { - r.piiDetected = append(r.piiDetected, piiDetectedEntry{name, h}) + r.piiDetected = append(r.piiDetected, entry[PIIDetected]{name, h}) } if h, ok := p.(PIIRedacted); ok { - r.piiRedacted = append(r.piiRedacted, piiRedactedEntry{name, h}) + r.piiRedacted = append(r.piiRedacted, entry[PIIRedacted]{name, h}) } if h, ok := p.(PolicyEvaluated); ok { - r.policyEvaluated = append(r.policyEvaluated, policyEvaluatedEntry{name, h}) + r.policyEvaluated = append(r.policyEvaluated, entry[PolicyEvaluated]{name, h}) } if h, ok := p.(SafetyProfileResolved); ok { - r.safetyProfileResolved = append(r.safetyProfileResolved, safetyProfileResolvedEntry{name, h}) + r.safetyProfileResolved = append(r.safetyProfileResolved, entry[SafetyProfileResolved]{name, h}) } if h, ok := p.(Shutdown); ok { - r.shutdown = append(r.shutdown, shutdownEntry{name, h}) + r.shutdown = append(r.shutdown, entry[Shutdown]{name, h}) } } @@ -174,135 +163,105 @@ func (r *Registry) Register(p Plugin) { // EmitScanStarted notifies all plugins that implement ScanStarted. func (r *Registry) EmitScanStarted(ctx context.Context, scanID id.ScanID, direction, text string) { - for _, e := range r.scanStarted { - if err := e.hook.OnScanStarted(ctx, scanID, direction, text); err != nil { - r.logger.Warn("plugin: hook error", log.String("hook", "ScanStarted"), log.String("plugin", e.name), log.Error(err)) - } - } + run(ctx, r, hookScanStarted, r.scanStarted, func(ctx context.Context, h ScanStarted) error { + return h.OnScanStarted(ctx, scanID, direction, text) + }) } // EmitScanCompleted notifies all plugins that implement ScanCompleted. func (r *Registry) EmitScanCompleted(ctx context.Context, scanID id.ScanID, decision string, findingCount int, elapsed time.Duration) { - for _, e := range r.scanCompleted { - if err := e.hook.OnScanCompleted(ctx, scanID, decision, findingCount, elapsed); err != nil { - r.logger.Warn("plugin: hook error", log.String("hook", "ScanCompleted"), log.String("plugin", e.name), log.Error(err)) - } - } + run(ctx, r, hookScanCompleted, r.scanCompleted, func(ctx context.Context, h ScanCompleted) error { + return h.OnScanCompleted(ctx, scanID, decision, findingCount, elapsed) + }) } // EmitScanBlocked notifies all plugins that implement ScanBlocked. func (r *Registry) EmitScanBlocked(ctx context.Context, scanID id.ScanID, reason string) { - for _, e := range r.scanBlocked { - if err := e.hook.OnScanBlocked(ctx, scanID, reason); err != nil { - r.logger.Warn("plugin: hook error", log.String("hook", "ScanBlocked"), log.String("plugin", e.name), log.Error(err)) - } - } + run(ctx, r, hookScanBlocked, r.scanBlocked, func(ctx context.Context, h ScanBlocked) error { + return h.OnScanBlocked(ctx, scanID, reason) + }) } // EmitScanFailed notifies all plugins that implement ScanFailed. func (r *Registry) EmitScanFailed(ctx context.Context, scanID id.ScanID, err error) { - for _, e := range r.scanFailed { - if hookErr := e.hook.OnScanFailed(ctx, scanID, err); hookErr != nil { - r.logger.Warn("plugin: hook error", log.String("hook", "ScanFailed"), log.String("plugin", e.name), log.Error(hookErr)) - } - } + run(ctx, r, hookScanFailed, r.scanFailed, func(ctx context.Context, h ScanFailed) error { + return h.OnScanFailed(ctx, scanID, err) + }) } // EmitInstinctTriggered notifies all plugins that implement InstinctTriggered. func (r *Registry) EmitInstinctTriggered(ctx context.Context, scanID id.ScanID, instinctName string, score float64) { - for _, e := range r.instinctTriggered { - if err := e.hook.OnInstinctTriggered(ctx, scanID, instinctName, score); err != nil { - r.logger.Warn("plugin: hook error", log.String("hook", "InstinctTriggered"), log.String("plugin", e.name), log.Error(err)) - } - } + run(ctx, r, hookInstinctTriggered, r.instinctTriggered, func(ctx context.Context, h InstinctTriggered) error { + return h.OnInstinctTriggered(ctx, scanID, instinctName, score) + }) } // EmitAwarenessDetected notifies all plugins that implement AwarenessDetected. func (r *Registry) EmitAwarenessDetected(ctx context.Context, scanID id.ScanID, detectorName string, findingCount int) { - for _, e := range r.awarenessDetected { - if err := e.hook.OnAwarenessDetected(ctx, scanID, detectorName, findingCount); err != nil { - r.logger.Warn("plugin: hook error", log.String("hook", "AwarenessDetected"), log.String("plugin", e.name), log.Error(err)) - } - } + run(ctx, r, hookAwarenessDetected, r.awarenessDetected, func(ctx context.Context, h AwarenessDetected) error { + return h.OnAwarenessDetected(ctx, scanID, detectorName, findingCount) + }) } // EmitJudgmentAssessed notifies all plugins that implement JudgmentAssessed. func (r *Registry) EmitJudgmentAssessed(ctx context.Context, scanID id.ScanID, assessorName, riskLevel string, confidence float64) { - for _, e := range r.judgmentAssessed { - if err := e.hook.OnJudgmentAssessed(ctx, scanID, assessorName, riskLevel, confidence); err != nil { - r.logger.Warn("plugin: hook error", log.String("hook", "JudgmentAssessed"), log.String("plugin", e.name), log.Error(err)) - } - } + run(ctx, r, hookJudgmentAssessed, r.judgmentAssessed, func(ctx context.Context, h JudgmentAssessed) error { + return h.OnJudgmentAssessed(ctx, scanID, assessorName, riskLevel, confidence) + }) } // EmitValueViolated notifies all plugins that implement ValueViolated. func (r *Registry) EmitValueViolated(ctx context.Context, scanID id.ScanID, valueName, severity string) { - for _, e := range r.valueViolated { - if err := e.hook.OnValueViolated(ctx, scanID, valueName, severity); err != nil { - r.logger.Warn("plugin: hook error", log.String("hook", "ValueViolated"), log.String("plugin", e.name), log.Error(err)) - } - } + run(ctx, r, hookValueViolated, r.valueViolated, func(ctx context.Context, h ValueViolated) error { + return h.OnValueViolated(ctx, scanID, valueName, severity) + }) } // EmitReflexFired notifies all plugins that implement ReflexFired. func (r *Registry) EmitReflexFired(ctx context.Context, scanID id.ScanID, reflexName, action string) { - for _, e := range r.reflexFired { - if err := e.hook.OnReflexFired(ctx, scanID, reflexName, action); err != nil { - r.logger.Warn("plugin: hook error", log.String("hook", "ReflexFired"), log.String("plugin", e.name), log.Error(err)) - } - } + run(ctx, r, hookReflexFired, r.reflexFired, func(ctx context.Context, h ReflexFired) error { + return h.OnReflexFired(ctx, scanID, reflexName, action) + }) } // EmitBoundaryEnforced notifies all plugins that implement BoundaryEnforced. func (r *Registry) EmitBoundaryEnforced(ctx context.Context, scanID id.ScanID, boundaryName string) { - for _, e := range r.boundaryEnforced { - if err := e.hook.OnBoundaryEnforced(ctx, scanID, boundaryName); err != nil { - r.logger.Warn("plugin: hook error", log.String("hook", "BoundaryEnforced"), log.String("plugin", e.name), log.Error(err)) - } - } + run(ctx, r, hookBoundaryEnforced, r.boundaryEnforced, func(ctx context.Context, h BoundaryEnforced) error { + return h.OnBoundaryEnforced(ctx, scanID, boundaryName) + }) } // EmitPIIDetected notifies all plugins that implement PIIDetected. func (r *Registry) EmitPIIDetected(ctx context.Context, scanID id.ScanID, piiType string, count int) { - for _, e := range r.piiDetected { - if err := e.hook.OnPIIDetected(ctx, scanID, piiType, count); err != nil { - r.logger.Warn("plugin: hook error", log.String("hook", "PIIDetected"), log.String("plugin", e.name), log.Error(err)) - } - } + run(ctx, r, hookPIIDetected, r.piiDetected, func(ctx context.Context, h PIIDetected) error { + return h.OnPIIDetected(ctx, scanID, piiType, count) + }) } // EmitPIIRedacted notifies all plugins that implement PIIRedacted. func (r *Registry) EmitPIIRedacted(ctx context.Context, scanID id.ScanID, piiType string, count int) { - for _, e := range r.piiRedacted { - if err := e.hook.OnPIIRedacted(ctx, scanID, piiType, count); err != nil { - r.logger.Warn("plugin: hook error", log.String("hook", "PIIRedacted"), log.String("plugin", e.name), log.Error(err)) - } - } + run(ctx, r, hookPIIRedacted, r.piiRedacted, func(ctx context.Context, h PIIRedacted) error { + return h.OnPIIRedacted(ctx, scanID, piiType, count) + }) } // EmitPolicyEvaluated notifies all plugins that implement PolicyEvaluated. func (r *Registry) EmitPolicyEvaluated(ctx context.Context, scanID id.ScanID, policyName, decision string) { - for _, e := range r.policyEvaluated { - if err := e.hook.OnPolicyEvaluated(ctx, scanID, policyName, decision); err != nil { - r.logger.Warn("plugin: hook error", log.String("hook", "PolicyEvaluated"), log.String("plugin", e.name), log.Error(err)) - } - } + run(ctx, r, hookPolicyEvaluated, r.policyEvaluated, func(ctx context.Context, h PolicyEvaluated) error { + return h.OnPolicyEvaluated(ctx, scanID, policyName, decision) + }) } // EmitSafetyProfileResolved notifies all plugins that implement SafetyProfileResolved. func (r *Registry) EmitSafetyProfileResolved(ctx context.Context, scanID id.ScanID, profileName string) { - for _, e := range r.safetyProfileResolved { - if err := e.hook.OnSafetyProfileResolved(ctx, scanID, profileName); err != nil { - r.logger.Warn("plugin: hook error", log.String("hook", "SafetyProfileResolved"), log.String("plugin", e.name), log.Error(err)) - } - } + run(ctx, r, hookSafetyProfileResolved, r.safetyProfileResolved, func(ctx context.Context, h SafetyProfileResolved) error { + return h.OnSafetyProfileResolved(ctx, scanID, profileName) + }) } // EmitShutdown notifies all plugins that implement Shutdown. func (r *Registry) EmitShutdown(ctx context.Context) { - for _, e := range r.shutdown { - if err := e.hook.OnShutdown(ctx); err != nil { - r.logger.Warn("plugin: hook error", log.String("hook", "Shutdown"), log.String("plugin", e.name), log.Error(err)) - } - } + run(ctx, r, hookShutdown, r.shutdown, func(ctx context.Context, h Shutdown) error { + return h.OnShutdown(ctx) + }) } From 00f94e7dfb88bbb7992a738c66a640275d4ec09c Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Fri, 16 Oct 2026 14:09:18 -0500 Subject: [PATCH 3/6] feat(admin): trace the pii retention purge RetentionExecute now runs the purge under a "purge pii tokens" span: the request audit, the token delete and the completion audit. It records how many tokens the preview selected and how many the delete removed. When a step fails, the status names it (audit_failed, delete_failed or completion_audit_failed), so you can tell "nothing was deleted" apart from "tokens are gone but the audit trail is missing" without reading logs. The permission and preview checks stay outside the span. Forge's dashboard dispatcher already traces and counts every intent with its error code, so a refused purge is visible there. The preview gets no span of its own for the same reason: it is one store read, which grove traces. The subject, scope and token IDs never go on the span, and neither do audit adapter error messages. --- admin/service.go | 15 ++++ admin/telemetry_test.go | 175 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+) create mode 100644 admin/telemetry_test.go diff --git a/admin/service.go b/admin/service.go index d78ccaf..406ede3 100644 --- a/admin/service.go +++ b/admin/service.go @@ -9,8 +9,12 @@ import ( "sync" "time" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "github.com/xraph/shield/engine" "github.com/xraph/shield/id" + "github.com/xraph/shield/internal/telemetry" "github.com/xraph/shield/store" ) @@ -423,17 +427,28 @@ func (s *Service) RetentionExecute(ctx context.Context, a Actor, key string) (in if !ok { return 0, fail("UNAVAILABLE", "PII retention is unavailable") } + // The purge itself (audit, delete, audit) runs under its own span inside + // forge's dispatch span, so you can see which step failed and how many + // tokens went. The checks above are the dispatch span's business. + ctx, span := telemetry.Tracer().Start(ctx, "purge pii tokens", trace.WithAttributes( + attribute.Int("shield.retention.tokens_selected", len(p.TokenIDs)), + )) + defer span.End() event := AuditEvent{Subject: a.Subject, Scope: a.Scope, Action: "pii.retention.requested", TokenIDs: p.TokenIDs, Cutoff: p.Cutoff} if err := s.audit(ctx, event); err != nil { + telemetry.Fail(span, "audit_failed", err) return 0, fail("UNAVAILABLE", "Could not record retention audit; nothing was deleted") } n, err := db.DashboardDeleteTokens(ctx, a.Scope, p.Cutoff, p.TokenIDs) if err != nil { + telemetry.Fail(span, "delete_failed", err) return 0, err } + span.SetAttributes(attribute.Int64("shield.retention.tokens_deleted", n)) event.Action = "pii.retention.completed" event.Affected = n if err = s.audit(ctx, event); err != nil { + telemetry.Fail(span, "completion_audit_failed", err) return n, fail("INTERNAL", "Retention completed but its completion audit failed") } return n, nil diff --git a/admin/telemetry_test.go b/admin/telemetry_test.go new file mode 100644 index 0000000..21a6e12 --- /dev/null +++ b/admin/telemetry_test.go @@ -0,0 +1,175 @@ +package admin + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + + "github.com/xraph/shield/id" + "github.com/xraph/shield/pii" + "github.com/xraph/shield/store" +) + +const secret = "alice@example.com" + +func purgeSpans(t *testing.T) *tracetest.SpanRecorder { + t.Helper() + prev := otel.GetTracerProvider() + rec := tracetest.NewSpanRecorder() + otel.SetTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(rec))) + t.Cleanup(func() { otel.SetTracerProvider(prev) }) + return rec +} + +func spanAttrs(s sdktrace.ReadOnlySpan) map[attribute.Key]attribute.Value { + m := map[attribute.Key]attribute.Value{} + for _, kv := range s.Attributes() { + m[kv.Key] = kv.Value + } + return m +} + +func seedExpiredToken(t *testing.T, s *Service, a Actor) { + t.Helper() + expired := time.Now().Add(-time.Hour) + tok := &pii.Token{ID: id.NewPIITokenID(), ScanID: id.NewScanID(), TenantID: a.Scope.TenantID, PIIType: "email", Placeholder: "[EMAIL]", EncryptedValue: []byte(secret), ExpiresAt: &expired} + if err := s.db.(store.Store).StorePIITokens(context.Background(), []*pii.Token{tok}); err != nil { + t.Fatal(err) + } +} + +func TestRetentionPurgeIsTraced(t *testing.T) { + s, a := serviceForTest(t) + a.Subject = secret + seedExpiredToken(t, s, a) + s.audit = func(context.Context, AuditEvent) error { return nil } + ctx := context.Background() + p, err := s.RetentionPreview(ctx, a) + if err != nil { + t.Fatal(err) + } + + rec := purgeSpans(t) + if _, err = s.RetentionExecute(ctx, a, "missing"); err == nil { + t.Fatal("unknown preview executed") + } + if len(rec.Ended()) != 0 { + t.Fatal("a refused purge started a span; the dispatch span covers it") + } + + n, err := s.RetentionExecute(ctx, a, p.ID) + if err != nil || n != 1 { + t.Fatal(n, err) + } + var purge sdktrace.ReadOnlySpan + for _, sp := range rec.Ended() { + if sp.Name() == "purge pii tokens" { + purge = sp + } + } + if purge == nil { + t.Fatal("no purge span") + } + at := spanAttrs(purge) + if at["shield.retention.tokens_selected"].AsInt64() != 1 || at["shield.retention.tokens_deleted"].AsInt64() != 1 { + t.Errorf("attributes = %v", at) + } + if purge.Status().Code == codes.Error { + t.Errorf("status = %v", purge.Status()) + } + // The token delete goes through grove, which nests its span here. + children := 0 + for _, sp := range rec.Ended() { + if sp == purge { + continue + } + if sp.Parent().SpanID() != purge.SpanContext().SpanID() { + t.Errorf("%q is not a child of the purge span", sp.Name()) + } + children++ + } + if children == 0 { + t.Error("the token delete left no span under the purge span") + } + assertClean(t, rec) +} + +func TestRetentionPurgeSpanNamesTheFailedStep(t *testing.T) { + for _, tc := range []struct { + name string + fail int // which audit call fails, 1 or 2 + class string + }{ + {"request audit", 1, "audit_failed"}, + {"completion audit", 2, "completion_audit_failed"}, + } { + t.Run(tc.name, func(t *testing.T) { + s, a := serviceForTest(t) + seedExpiredToken(t, s, a) + calls := 0 + s.audit = func(context.Context, AuditEvent) error { + calls++ + if calls == tc.fail { + return errors.New("audit sink refused " + secret) + } + return nil + } + ctx := context.Background() + p, err := s.RetentionPreview(ctx, a) + if err != nil { + t.Fatal(err) + } + rec := purgeSpans(t) + if _, err = s.RetentionExecute(ctx, a, p.ID); err == nil { + t.Fatal("audit failure hidden") + } + var purge sdktrace.ReadOnlySpan + for _, sp := range rec.Ended() { + if sp.Name() == "purge pii tokens" { + purge = sp + } + } + if purge == nil { + t.Fatal("no purge span") + } + if purge.Status().Code != codes.Error || purge.Status().Description != tc.class { + t.Errorf("status = %v, want %s", purge.Status(), tc.class) + } + ev := purge.Events() + if len(ev) != 1 || ev[0].Name != "exception" || ev[0].Attributes[0].Value.AsString() != "*errors.errorString" { + t.Errorf("events = %+v", ev) + } + _, deleted := spanAttrs(purge)["shield.retention.tokens_deleted"] + if deleted != (tc.fail == 2) { + t.Errorf("tokens_deleted present = %v", deleted) + } + assertClean(t, rec) + }) + } +} + +func assertClean(t *testing.T, rec *tracetest.SpanRecorder) { + t.Helper() + for _, sp := range rec.Ended() { + kvs := append([]attribute.KeyValue{}, sp.Attributes()...) + for _, ev := range sp.Events() { + kvs = append(kvs, ev.Attributes...) + } + for _, kv := range kvs { + if strings.Contains(kv.Value.String(), secret) { + t.Errorf("span %q leaks %s in %s", sp.Name(), secret, kv.Key) + } + } + if strings.Contains(sp.Status().Description, secret) { + t.Errorf("span %q leaks %s in its status", sp.Name(), secret) + } + } +} From a4e63e32fcf9f7534064488dd347aab898280046 Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Fri, 16 Oct 2026 14:09:36 -0500 Subject: [PATCH 4/6] feat(engine)!: record scan and plugin hook metrics The engine records three metrics when you give it a go-utils metrics sink with the new engine.WithMetrics option. The Forge extension passes app.Metrics() for you, so a Forge app gets them on /_/metrics with no setup: forge_shield_scans_completed_total{direction, decision} forge_shield_scan_duration_seconds{direction, decision} forge_shield_plugin_hooks_failed_total{hook} direction is input or output, decision is allow, block, flag or redact, and hook is one of the 15 plugin hooks in snake case (scan_started and so on). There are no tenant, app, scan or plugin labels. Pass engine.WithMetrics(nil) through WithEngineOption to turn them off. This replaces the observability plugin's 14 unlabelled counters. Those had dotted names (shield.scan.started and the rest). Built with NewMetricsExtension they went to a private collector and were never exported at all. Built from app.Metrics() they scraped as forge_shield_scan_started and friends, typed as gauges because the names lacked _total, and only the started and completed pair ever moved. BREAKING CHANGE: the forge_shield_scan_started style series are gone. If you scrape them, move to forge_shield_scans_completed_total. The observability package still compiles, but MetricsExtension is deprecated, implements no hooks and has lost its exported counter fields. Drop it from your plugin list. --- engine/engine.go | 37 ++++++ extension/extension.go | 5 +- extension/telemetry_test.go | 128 +++++++++++++++++++++ internal/telemetry/metrics.go | 92 +++++++++++++++ internal/telemetry/metrics_test.go | 13 +++ internal/telemetry/telemetry.go | 8 +- observability/extension.go | 177 ++++------------------------- plugin/registry.go | 24 +++- 8 files changed, 321 insertions(+), 163 deletions(-) create mode 100644 extension/telemetry_test.go create mode 100644 internal/telemetry/metrics.go create mode 100644 internal/telemetry/metrics_test.go diff --git a/engine/engine.go b/engine/engine.go index 2558aff..5bde0a8 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -23,6 +23,7 @@ import ( "go.opentelemetry.io/otel/trace" log "github.com/xraph/go-utils/log" + gometrics "github.com/xraph/go-utils/metrics" "github.com/xraph/shield" "github.com/xraph/shield/id" @@ -32,12 +33,30 @@ import ( "github.com/xraph/shield/store" ) +// The metrics the engine records when WithMetrics is set. A Forge app scrapes +// them with a forge_ prefix: forge_shield_scans_completed_total and +// forge_shield_scan_duration_seconds. Both carry direction (input, output) +// and decision (allow, block, flag, redact). +const ( + ScansCompletedMetric = "shield_scans_completed_total" + ScanDurationMetric = "shield_scan_duration_seconds" +) + +// ScanDurationBuckets are the histogram bounds for ScanDurationMetric, in +// seconds. They span the layer budgets, from a 5ms boundary check to a +// 500ms judgment, with headroom for storing the result. +var ScanDurationBuckets = []float64{0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5} + // Engine is the core safety execution engine. type Engine struct { store store.Store registry *plugin.Registry config shield.Config logger log.Logger + metrics gometrics.Metrics + + scansCompleted *telemetry.CounterVec + scanDuration *telemetry.HistogramVec } // Option configures the Engine. @@ -64,6 +83,14 @@ func WithLogger(l log.Logger) Option { return func(e *Engine) { e.logger = l } } +// WithMetrics records scan and plugin metrics on m: ScansCompletedMetric, +// ScanDurationMetric and plugin.HooksFailedMetric. In a Forge app pass +// app.Metrics(); the Shield Forge extension does this for you. A nil m, the +// default, records nothing. +func WithMetrics(m gometrics.Metrics) Option { + return func(e *Engine) { e.metrics = m } +} + // New creates a new safety engine with the given options. func New(opts ...Option) (*Engine, error) { e := &Engine{ @@ -75,6 +102,12 @@ func New(opts ...Option) (*Engine, error) { opt(e) } e.registry.SetLogger(e.logger) + e.registry.SetMetrics(e.metrics) + e.scansCompleted = telemetry.NewCounterVec(e.metrics, ScansCompletedMetric, + "Safety scans completed, by direction and decision.", "direction", "decision") + e.scanDuration = telemetry.NewHistogramVec(e.metrics, ScanDurationMetric, + "Wall time of one safety scan, plugin hooks and storing the result included.", + "s", ScanDurationBuckets, "direction", "decision") return e, nil } @@ -164,6 +197,10 @@ func (e *Engine) executeScan(ctx context.Context, input *scan.Input) (*scan.Resu attribute.Bool("shield.scan.blocked", result.Blocked), attribute.Int("shield.scan.findings", len(result.Findings)), ) + direction, decision := string(result.Direction), string(result.Decision) + e.scansCompleted.Inc(direction, decision) + e.scanDuration.Observe(time.Since(start).Seconds(), direction, decision) + return result, nil } diff --git a/extension/extension.go b/extension/extension.go index a39dd03..2d8d348 100644 --- a/extension/extension.go +++ b/extension/extension.go @@ -105,14 +105,15 @@ func (e *Extension) Register(fapp forge.App) error { ) } - // The app's logger goes first, so an engine option you pass through - // WithEngineOption can still replace it. + // The app's logger and metrics go first, so an engine option you pass + // through WithEngineOption can still replace them. opts := append([]engine.Option{ engine.WithConfig(shield.Config{ DefaultProfile: e.config.DefaultProfile, ShutdownTimeout: e.config.ShutdownTimeout, ScanConcurrency: e.config.ScanConcurrency, EnableShortCircuit: e.config.EnableShortCircuit, }), engine.WithLogger(e.Logger()), + engine.WithMetrics(e.Metrics()), }, e.engineOpts...) eng, err := engine.New(opts...) if err != nil { diff --git a/extension/telemetry_test.go b/extension/telemetry_test.go new file mode 100644 index 0000000..d8c3789 --- /dev/null +++ b/extension/telemetry_test.go @@ -0,0 +1,128 @@ +package extension + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + + "github.com/xraph/forge" + + "github.com/xraph/shield/engine" + "github.com/xraph/shield/id" + "github.com/xraph/shield/scan" +) + +type failingStart struct{} + +func (failingStart) Name() string { return "failing-start" } +func (failingStart) OnScanStarted(context.Context, id.ScanID, string, string) error { + return errors.New("refused") +} + +func scrape(t *testing.T, app forge.App) string { + t.Helper() + p, ok := app.Metrics().(interface{ PrometheusHandler() http.Handler }) + if !ok { + t.Fatalf("app metrics %T have no Prometheus handler", app.Metrics()) + } + rr := httptest.NewRecorder() + p.PrometheusHandler().ServeHTTP(rr, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/_/metrics", nil)) + body, err := io.ReadAll(rr.Body) + if err != nil { + t.Fatal(err) + } + return string(body) +} + +func series(body, prefix string) []string { + var out []string + for _, line := range strings.Split(body, "\n") { + if strings.HasPrefix(line, prefix) { + out = append(out, line) + } + } + return out +} + +func TestRegisterWiresAppMetricsAndTracing(t *testing.T) { + prevTP, prevProp := otel.GetTracerProvider(), otel.GetTextMapPropagator() + t.Cleanup(func() { + otel.SetTracerProvider(prevTP) + otel.SetTextMapPropagator(prevProp) + }) + + app := forge.New(forge.WithAppName("shield-telemetry-test"), forge.WithAppLogger(forge.NewNoopLogger()), + forge.WithEnableConfigAutoDiscovery(false)) + rec := tracetest.NewSpanRecorder() + if err := forge.RegisterSpanProcessor(app, rec); err != nil { + t.Fatal(err) + } + e := New(WithPlugin(failingStart{})) + if err := e.Register(app); err != nil { + t.Fatal(err) + } + + ctx := context.Background() + for range 2 { + if _, err := e.Engine().ScanInput(ctx, &scan.Input{Text: "hello"}); err != nil { + t.Fatal(err) + } + } + if _, err := e.Engine().ScanOutput(ctx, &scan.Input{Text: "hello"}); err != nil { + t.Fatal(err) + } + + body := scrape(t, app) + for _, want := range []string{ + "# TYPE forge_shield_scans_completed_total counter", + `forge_shield_scans_completed_total{decision="allow",direction="input"} 2`, + `forge_shield_scans_completed_total{decision="allow",direction="output"} 1`, + "# TYPE forge_shield_scan_duration_seconds histogram", + `forge_shield_scan_duration_seconds_count{decision="allow",direction="input"} 2`, + `forge_shield_scan_duration_seconds_count{decision="allow",direction="output"} 1`, + "# TYPE forge_shield_plugin_hooks_failed_total counter", + `forge_shield_plugin_hooks_failed_total{hook="scan_started"} 3`, + } { + if !strings.Contains(body, want) { + t.Errorf("scrape is missing %q", want) + } + } + if got, want := len(series(body, `forge_shield_scan_duration_seconds_bucket{`)), 2*(len(engine.ScanDurationBuckets)+1); got != want { + t.Errorf("bucket lines = %d, want %d", got, want) + } + if strings.Contains(body, "forge_shield_scan_started") || strings.Contains(body, "shield.scan") { + t.Error("an old dotted counter is still on the scrape") + } + + var scans int + for _, s := range rec.Ended() { + if s.Name() == "scan content" && s.InstrumentationScope().Name == "github.com/xraph/shield" { + scans++ + } + } + if scans != 3 { + t.Errorf("scan spans on forge's provider = %d, want 3", scans) + } +} + +func TestEngineOptionCanTurnMetricsOff(t *testing.T) { + app := forge.New(forge.WithAppName("shield-metrics-off"), forge.WithAppLogger(forge.NewNoopLogger()), + forge.WithEnableConfigAutoDiscovery(false)) + e := New(WithEngineOption(engine.WithMetrics(nil))) + if err := e.Register(app); err != nil { + t.Fatal(err) + } + if _, err := e.Engine().ScanInput(context.Background(), &scan.Input{Text: "hello"}); err != nil { + t.Fatal(err) + } + if strings.Contains(scrape(t, app), "forge_shield_") { + t.Error("shield recorded metrics after WithMetrics(nil)") + } +} diff --git a/internal/telemetry/metrics.go b/internal/telemetry/metrics.go new file mode 100644 index 0000000..1846aa2 --- /dev/null +++ b/internal/telemetry/metrics.go @@ -0,0 +1,92 @@ +package telemetry + +import ( + "strings" + "sync" + + gometrics "github.com/xraph/go-utils/metrics" +) + +// vec is a metric family with fixed label keys. A child is built on the first +// use of a label tuple and cached, so a repeat call takes no registry lock. +// Callers only pass values from fixed enums, which bounds the cache. +type vec[T any] struct { + keys []string + build func(labels map[string]string) T + cache sync.Map // joined label values -> T +} + +func (v *vec[T]) with(values []string) (T, bool) { + var zero T + if len(values) != len(v.keys) { + return zero, false + } + key := strings.Join(values, "\xff") + if c, ok := v.cache.Load(key); ok { + child, isT := c.(T) + return child, isT + } + labels := make(map[string]string, len(v.keys)) + for i, k := range v.keys { + labels[k] = values[i] + } + c, _ := v.cache.LoadOrStore(key, v.build(labels)) + child, isT := c.(T) + return child, isT +} + +// CounterVec is a counter with fixed label keys. A nil *CounterVec records +// nothing, which is what NewCounterVec returns for a nil sink. +type CounterVec struct{ v vec[gometrics.Counter] } + +// NewCounterVec declares a counter on m. A nil m returns nil. +func NewCounterVec(m gometrics.Metrics, name, help string, keys ...string) *CounterVec { + if m == nil { + return nil + } + return &CounterVec{v: vec[gometrics.Counter]{keys: keys, build: func(l map[string]string) gometrics.Counter { + return m.Counter(name, gometrics.WithDescription(help), gometrics.WithLabels(l)) + }}} +} + +// Inc adds one to the series for values, given in the declared key order. A +// call with the wrong number of values records nothing. +func (c *CounterVec) Inc(values ...string) { + if c == nil { + return + } + if child, ok := c.v.with(values); ok { + child.Inc() + } +} + +// HistogramVec is a histogram with fixed label keys. A nil *HistogramVec +// records nothing, which is what NewHistogramVec returns for a nil sink. +type HistogramVec struct{ v vec[gometrics.Histogram] } + +// NewHistogramVec declares a histogram on m with the given unit and buckets. +// A nil m returns nil. +func NewHistogramVec(m gometrics.Metrics, name, help, unit string, buckets []float64, keys ...string) *HistogramVec { + if m == nil { + return nil + } + return &HistogramVec{v: vec[gometrics.Histogram]{keys: keys, build: func(l map[string]string) gometrics.Histogram { + return m.Histogram(name, + gometrics.WithDescription(help), + gometrics.WithUnit(unit), + gometrics.WithBuckets(buckets...), + gometrics.WithLabels(l), + ) + }}} +} + +// Observe records v in the series for values, given in the declared key +// order. A call with the wrong number of values records nothing. +func (h *HistogramVec) Observe(v float64, values ...string) { + if h == nil { + return + } + if child, ok := h.v.with(values); ok { + child.Observe(v) + } +} diff --git a/internal/telemetry/metrics_test.go b/internal/telemetry/metrics_test.go new file mode 100644 index 0000000..36f12c3 --- /dev/null +++ b/internal/telemetry/metrics_test.go @@ -0,0 +1,13 @@ +package telemetry + +import "testing" + +func TestNilVecsRecordNothing(t *testing.T) { + if NewCounterVec(nil, "c_total", "", "k") != nil || NewHistogramVec(nil, "h_seconds", "", "s", nil, "k") != nil { + t.Fatal("a nil sink should give a nil vec") + } + var c *CounterVec + var h *HistogramVec + c.Inc("v") + h.Observe(1, "v") +} diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go index d939616..ac12ac7 100644 --- a/internal/telemetry/telemetry.go +++ b/internal/telemetry/telemetry.go @@ -1,8 +1,10 @@ -// Package telemetry holds the tracing helpers the engine, the plugin -// registry and the admin service share. +// Package telemetry holds the tracing and metric helpers the engine, the +// plugin registry and the admin service share. // // Spans go to the global OpenTelemetry tracer provider, which forge.New -// installs. +// installs. Metrics go to a go-utils metrics.Metrics, which in a Forge app is +// app.Metrics(); Forge scrapes a name like shield_scans_completed_total as +// forge_shield_scans_completed_total. package telemetry import ( diff --git a/observability/extension.go b/observability/extension.go index 3bb576b..77180a7 100644 --- a/observability/extension.go +++ b/observability/extension.go @@ -1,169 +1,38 @@ -// Package observability provides a metrics extension for Shield that records -// lifecycle event counts via go-utils MetricFactory. +// Package observability used to hold Shield's metrics plugin. +// +// Deprecated: the engine records its own metrics now, labelled by direction, +// decision and hook. Pass engine.WithMetrics, or let the Forge extension pass +// app.Metrics() for you, and drop this plugin. It implements no hooks and +// records nothing. The dotted counters it used to declare +// (shield.scan.started and the rest) are gone. package observability import ( - "context" - "time" - gu "github.com/xraph/go-utils/metrics" - "github.com/xraph/shield/id" "github.com/xraph/shield/plugin" ) -// Compile-time interface checks. -var ( - _ plugin.Plugin = (*MetricsExtension)(nil) - _ plugin.ScanStarted = (*MetricsExtension)(nil) - _ plugin.ScanCompleted = (*MetricsExtension)(nil) - _ plugin.ScanBlocked = (*MetricsExtension)(nil) - _ plugin.ScanFailed = (*MetricsExtension)(nil) - _ plugin.InstinctTriggered = (*MetricsExtension)(nil) - _ plugin.AwarenessDetected = (*MetricsExtension)(nil) - _ plugin.JudgmentAssessed = (*MetricsExtension)(nil) - _ plugin.ValueViolated = (*MetricsExtension)(nil) - _ plugin.ReflexFired = (*MetricsExtension)(nil) - _ plugin.BoundaryEnforced = (*MetricsExtension)(nil) - _ plugin.PIIDetected = (*MetricsExtension)(nil) - _ plugin.PIIRedacted = (*MetricsExtension)(nil) - _ plugin.PolicyEvaluated = (*MetricsExtension)(nil) - _ plugin.SafetyProfileResolved = (*MetricsExtension)(nil) -) +var _ plugin.Plugin = (*MetricsExtension)(nil) -// MetricsExtension records system-wide lifecycle metrics via go-utils MetricFactory. -// Register it as a Shield extension to automatically track scan rates, -// instinct triggers, boundary enforcements, and other safety events. -type MetricsExtension struct { - ScanStartedCount gu.Counter - ScanCompletedCount gu.Counter - ScanBlockedCount gu.Counter - ScanFailedCount gu.Counter - InstinctTriggeredCount gu.Counter - AwarenessDetectedCount gu.Counter - JudgmentAssessedCount gu.Counter - ValueViolatedCount gu.Counter - ReflexFiredCount gu.Counter - BoundaryEnforcedCount gu.Counter - PIIDetectedCount gu.Counter - PIIRedactedCount gu.Counter - PolicyEvaluatedCount gu.Counter - SafetyProfileResolvedCount gu.Counter -} +// MetricsExtension is a plugin that implements no hooks. +// +// Deprecated: use engine.WithMetrics. The Forge extension wires app.Metrics() +// into the engine without it. +type MetricsExtension struct{} -// NewMetricsExtension creates a MetricsExtension using a default metrics collector. -func NewMetricsExtension() *MetricsExtension { - return NewMetricsExtensionWithFactory(gu.NewMetricsCollector("shield/observability")) -} +// NewMetricsExtension returns a plugin that records nothing. +// +// Deprecated: use engine.WithMetrics. +func NewMetricsExtension() *MetricsExtension { return &MetricsExtension{} } -// NewMetricsExtensionWithFactory creates a MetricsExtension with the provided MetricFactory. -// Use fapp.Metrics() in forge extensions, or gu.NewMetricsCollector for testing. -func NewMetricsExtensionWithFactory(factory gu.MetricFactory) *MetricsExtension { - return &MetricsExtension{ - ScanStartedCount: factory.Counter("shield.scan.started"), - ScanCompletedCount: factory.Counter("shield.scan.completed"), - ScanBlockedCount: factory.Counter("shield.scan.blocked"), - ScanFailedCount: factory.Counter("shield.scan.failed"), - InstinctTriggeredCount: factory.Counter("shield.instinct.triggered"), - AwarenessDetectedCount: factory.Counter("shield.awareness.detected"), - JudgmentAssessedCount: factory.Counter("shield.judgment.assessed"), - ValueViolatedCount: factory.Counter("shield.value.violated"), - ReflexFiredCount: factory.Counter("shield.reflex.fired"), - BoundaryEnforcedCount: factory.Counter("shield.boundary.enforced"), - PIIDetectedCount: factory.Counter("shield.pii.detected"), - PIIRedactedCount: factory.Counter("shield.pii.redacted"), - PolicyEvaluatedCount: factory.Counter("shield.policy.evaluated"), - SafetyProfileResolvedCount: factory.Counter("shield.profile.resolved"), - } +// NewMetricsExtensionWithFactory returns a plugin that records nothing. The +// factory is ignored. +// +// Deprecated: use engine.WithMetrics with the same metrics. +func NewMetricsExtensionWithFactory(_ gu.MetricFactory) *MetricsExtension { + return &MetricsExtension{} } // Name implements plugin.Plugin. func (m *MetricsExtension) Name() string { return "observability-metrics" } - -// ── Scan lifecycle hooks ────────────────────────────── - -// OnScanStarted implements plugin.ScanStarted. -func (m *MetricsExtension) OnScanStarted(_ context.Context, _ id.ScanID, _, _ string) error { - m.ScanStartedCount.Inc() - return nil -} - -// OnScanCompleted implements plugin.ScanCompleted. -func (m *MetricsExtension) OnScanCompleted(_ context.Context, _ id.ScanID, _ string, _ int, _ time.Duration) error { - m.ScanCompletedCount.Inc() - return nil -} - -// OnScanBlocked implements plugin.ScanBlocked. -func (m *MetricsExtension) OnScanBlocked(_ context.Context, _ id.ScanID, _ string) error { - m.ScanBlockedCount.Inc() - return nil -} - -// OnScanFailed implements plugin.ScanFailed. -func (m *MetricsExtension) OnScanFailed(_ context.Context, _ id.ScanID, _ error) error { - m.ScanFailedCount.Inc() - return nil -} - -// ── Safety primitive lifecycle hooks ────────────────── - -// OnInstinctTriggered implements plugin.InstinctTriggered. -func (m *MetricsExtension) OnInstinctTriggered(_ context.Context, _ id.ScanID, _ string, _ float64) error { - m.InstinctTriggeredCount.Inc() - return nil -} - -// OnAwarenessDetected implements plugin.AwarenessDetected. -func (m *MetricsExtension) OnAwarenessDetected(_ context.Context, _ id.ScanID, _ string, _ int) error { - m.AwarenessDetectedCount.Inc() - return nil -} - -// OnJudgmentAssessed implements plugin.JudgmentAssessed. -func (m *MetricsExtension) OnJudgmentAssessed(_ context.Context, _ id.ScanID, _, _ string, _ float64) error { - m.JudgmentAssessedCount.Inc() - return nil -} - -// OnValueViolated implements plugin.ValueViolated. -func (m *MetricsExtension) OnValueViolated(_ context.Context, _ id.ScanID, _, _ string) error { - m.ValueViolatedCount.Inc() - return nil -} - -// OnReflexFired implements plugin.ReflexFired. -func (m *MetricsExtension) OnReflexFired(_ context.Context, _ id.ScanID, _, _ string) error { - m.ReflexFiredCount.Inc() - return nil -} - -// OnBoundaryEnforced implements plugin.BoundaryEnforced. -func (m *MetricsExtension) OnBoundaryEnforced(_ context.Context, _ id.ScanID, _ string) error { - m.BoundaryEnforcedCount.Inc() - return nil -} - -// OnPIIDetected implements plugin.PIIDetected. -func (m *MetricsExtension) OnPIIDetected(_ context.Context, _ id.ScanID, _ string, _ int) error { - m.PIIDetectedCount.Inc() - return nil -} - -// OnPIIRedacted implements plugin.PIIRedacted. -func (m *MetricsExtension) OnPIIRedacted(_ context.Context, _ id.ScanID, _ string, _ int) error { - m.PIIRedactedCount.Inc() - return nil -} - -// OnPolicyEvaluated implements plugin.PolicyEvaluated. -func (m *MetricsExtension) OnPolicyEvaluated(_ context.Context, _ id.ScanID, _, _ string) error { - m.PolicyEvaluatedCount.Inc() - return nil -} - -// OnSafetyProfileResolved implements plugin.SafetyProfileResolved. -func (m *MetricsExtension) OnSafetyProfileResolved(_ context.Context, _ id.ScanID, _ string) error { - m.SafetyProfileResolvedCount.Inc() - return nil -} diff --git a/plugin/registry.go b/plugin/registry.go index d6db56d..cab3810 100644 --- a/plugin/registry.go +++ b/plugin/registry.go @@ -9,12 +9,18 @@ import ( "go.opentelemetry.io/otel/trace" log "github.com/xraph/go-utils/log" + gometrics "github.com/xraph/go-utils/metrics" "github.com/xraph/shield/id" "github.com/xraph/shield/internal/telemetry" ) -// The shield.plugin.hook span attribute values. +// HooksFailedMetric counts hook calls that returned an error, labelled by +// hook. A Forge app scrapes it as forge_shield_plugin_hooks_failed_total. +const HooksFailedMetric = "shield_plugin_hooks_failed_total" + +// The hook label values on HooksFailedMetric and the shield.plugin.hook span +// attribute. const ( hookScanStarted = "scan_started" hookScanCompleted = "scan_completed" @@ -43,8 +49,9 @@ type entry[H any] struct { // It type-caches plugins at registration time so emit calls iterate // only over plugins that implement the relevant hook. type Registry struct { - plugins []Plugin - logger log.Logger + plugins []Plugin + logger log.Logger + hooksFailed *telemetry.CounterVec scanStarted []entry[ScanStarted] scanCompleted []entry[ScanCompleted] @@ -79,10 +86,18 @@ func (r *Registry) SetLogger(l log.Logger) { } } +// SetMetrics counts failed hook calls on m as HooksFailedMetric. In a Forge +// app pass app.Metrics(); the engine does this for you with engine.WithMetrics. +// A nil m records nothing. Call it before the first Emit. +func (r *Registry) SetMetrics(m gometrics.Metrics) { + r.hooksFailed = telemetry.NewCounterVec(m, HooksFailedMetric, + "Plugin hook calls that returned an error. Shield logs the error and carries on.", "hook") +} + // run calls one hook on every plugin that implements it. When there is at // least one, the calls share a "run plugin hooks" span under ctx, so a slow // or failing plugin shows up inside the scan that triggered it. Hook errors -// are logged, never returned: a plugin can't fail a scan. +// are logged and counted, never returned: a plugin can't fail a scan. func run[H any](ctx context.Context, r *Registry, hook string, entries []entry[H], call func(context.Context, H) error) { if len(entries) == 0 { return @@ -98,6 +113,7 @@ func run[H any](ctx context.Context, r *Registry, hook string, entries []entry[H if err := call(ctx, e.hook); err != nil { failures++ r.logger.Warn("plugin: hook error", log.String("hook", hook), log.String("plugin", e.name), log.Error(err)) + r.hooksFailed.Inc(hook) telemetry.Exception(span, err) } } From 0e07735a5e9b15cd4fef85d5776a1f78001f9a79 Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Fri, 16 Oct 2026 14:13:49 -0500 Subject: [PATCH 5/6] docs: document shield's spans and metrics Adds an observability guide that lists every span with its attributes and error classes, every metric with its scraped forge_ name and label values, what never goes on either, and a table that maps the old shield.scan.* counters to the new series. The footer's Observability link now points at it, and the README gets a short Telemetry section with the same tables. The getting started, Forge extension, full example, configuration, architecture and custom plugin pages stop telling you to register observability.NewMetricsExtension(). The Forge extension wires metrics itself, and outside Forge you pass engine.WithMetrics. They also stop saying hook errors are dropped silently, because the registry now logs and counts them. AGENTS.md describes internal/telemetry and the rules for adding a span or a metric. --- AGENTS.md | 8 +- README.md | 30 +++-- .../docs/api-reference/go-packages.mdx | 8 +- docs/content/docs/architecture.mdx | 14 ++- docs/content/docs/concepts/configuration.mdx | 7 +- docs/content/docs/getting-started.mdx | 12 +- docs/content/docs/guides/custom-plugin.mdx | 21 +--- docs/content/docs/guides/forge-extension.mdx | 22 +--- docs/content/docs/guides/full-example.mdx | 8 +- docs/content/docs/guides/meta.json | 8 +- docs/content/docs/guides/observability.mdx | 113 ++++++++++++++++++ docs/content/docs/index.mdx | 3 +- docs/src/components/landing/footer.tsx | 2 +- 13 files changed, 188 insertions(+), 68 deletions(-) create mode 100644 docs/content/docs/guides/observability.mdx diff --git a/AGENTS.md b/AGENTS.md index 44d62d0..12831d5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -87,7 +87,9 @@ For govulncheck, use CI's Go. CI always resolves the newest 1.26 patch, and an o | `store/storetest/` | The shared dashboard conformance suite | | `admin/` | `admin.Service`: validation, permissions, field schemas and audited retention between the contract and the scoped stores | | `plugin/` | Lifecycle hook interfaces and the registry | -| `audit_hook/` (package `audithook`), `observability/` | Plugins for an audit trail and for metrics | +| `audit_hook/` (package `audithook`) | Plugin for an audit trail | +| `observability/` | Deprecated metrics plugin, kept so old builds compile. It records nothing | +| `internal/telemetry/` | The shared tracer, the span error helper and the labelled metric vectors the engine, plugin registry and admin service use | | `extension/` | The Forge extension, its `Config` and options | | `extension/contract/` | The `shield` dashboard contributor: `manifest.yaml`, handlers, and the claim-based actor resolver in `scope.go` | | `docs/` | The Fumadocs site | @@ -107,7 +109,9 @@ IDs come from the `id` package (`id.NewInstinctID()` and friends). Don't build t `engine.New(opts...)` takes `type Option func(*Engine)` and returns `(*Engine, error)`. `extension.New(opts...)` takes `type Option func(*Extension)`. The extension reads YAML under `extensions.shield`, then `shield`, and merges it with the programmatic options. Its store comes from `WithStore`, a named grove DB (`WithGroveDatabase`), or the default `*grove.DB` in the container, chosen by driver name (`pg`, `sqlite`, `mongo`). With no store at all, `Health` reports that persistence is not configured. Shield mounts no HTTP routes of its own: `BasePath` is loaded and merged but nothing reads it, and `WithDisableRoutes` only stops the dashboard contract from registering. -Logging uses `github.com/xraph/go-utils/log`, with `log.NewNoopLogger()` as the engine default. +Logging uses `github.com/xraph/go-utils/log`, with `log.NewNoopLogger()` as the engine default. The Forge extension passes the app logger, and the engine passes its logger on to the plugin registry. + +Telemetry goes through `internal/telemetry`. Start spans from `telemetry.Tracer()` (scope `github.com/xraph/shield`), mark failures with `telemetry.Fail(span, "", err)`, and never call `span.RecordError` or put an error message, scanned text, a tenant, app, subject or user-chosen name on a span or a label. Metrics take a go-utils `metrics.Metrics` through `engine.WithMetrics` (nil means off), and the extension wires `e.Metrics()`. Name them `shield___total` or `shield__duration_seconds`; Forge scrapes them with a `forge_` prefix. `docs/content/docs/guides/observability.mdx` lists every span and metric, so update it when you add one. ## Dependencies diff --git a/README.md b/README.md index bf27283..675164a 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,8 @@ Safety rules for an AI product change often and differ per customer. One tenant | PII handling with retention | A PII token vault (placeholder, type, your encrypted value, expiry), per-tenant deletion and age-based purge | | Evidence for audits | Compliance report records for the EU AI Act, NIST AI RMF and SOC 2 | | An admin UI with real permissions | A Forge dashboard contract where scope comes only from the signed-in user's claims, with optimistic concurrency on edits | -| Hooks into your own systems | 15 lifecycle hooks, an audit bridge for [Chronicle](https://github.com/xraph/chronicle) and a metrics plugin | +| Hooks into your own systems | 15 lifecycle hooks and an audit bridge for [Chronicle](https://github.com/xraph/chronicle) | +| To see what it's doing | OpenTelemetry spans for every scan, plugin hook and PII purge, plus scan and hook metrics on Forge's `/_/metrics` | ## The safety model @@ -173,16 +174,13 @@ The `extension` package registers Shield as a Forge extension. It reads `extensi import ( "github.com/xraph/forge" shieldext "github.com/xraph/shield/extension" - "github.com/xraph/shield/observability" ) app := forge.NewApp(forge.AppConfig{ Name: "safety", HTTPAddress: ":8080", Extensions: []forge.Extension{ - shieldext.New( - shieldext.WithPlugin(observability.NewMetricsExtension()), - ), + shieldext.New(), }, }) ``` @@ -207,7 +205,25 @@ The dashboard takes tenant and app from the signed-in user's `tenant_id` and `ap ## Lifecycle hooks -Register a plugin with `engine.WithPlugin` (or `WithPlugin` on the extension). It can implement any of 15 hooks: scan started, completed, blocked and failed, one per layer (`OnInstinctTriggered`, `OnAwarenessDetected`, `OnBoundaryEnforced`, `OnValueViolated`, `OnJudgmentAssessed`, `OnReflexFired`), PII detected and redacted, policy evaluated, profile resolved, and shutdown. Today only the scan and shutdown hooks fire. `audithook.New(recorder)` (import path `github.com/xraph/shield/audit_hook`) writes them to an audit trail, and `observability.NewMetricsExtension()` counts them. +Register a plugin with `engine.WithPlugin` (or `WithPlugin` on the extension). It can implement any of 15 hooks: scan started, completed, blocked and failed, one per layer (`OnInstinctTriggered`, `OnAwarenessDetected`, `OnBoundaryEnforced`, `OnValueViolated`, `OnJudgmentAssessed`, `OnReflexFired`), PII detected and redacted, policy evaluated, profile resolved, and shutdown. Today only the scan and shutdown hooks fire. `audithook.New(recorder)` (import path `github.com/xraph/shield/audit_hook`) writes them to an audit trail. A hook that returns an error is logged and counted, and the scan carries on. + +## Telemetry + +You don't wire anything in a Forge app. The extension hands `app.Metrics()` and the app logger to the engine, and spans go to the tracer provider `forge.New` installs. Outside Forge, pass a go-utils `metrics.Metrics` to `engine.WithMetrics` and install a global OpenTelemetry tracer provider. + +| Span | Kind | Attributes | +| --- | --- | --- | +| `scan content` | Internal | `shield.scan.id`, `shield.scan.direction`, `shield.scan.decision`, `shield.scan.blocked`, `shield.scan.findings`, `shield.scan.persisted` (only with a store) | +| `run plugin hooks` | Internal, child of the scan | `shield.plugin.hook`, `shield.plugin.count`, `shield.plugin.failures` (on failure) | +| `purge pii tokens` | Internal | `shield.retention.tokens_selected`, `shield.retention.tokens_deleted` | + +| Metric (as scraped) | Type | Labels | +| --- | --- | --- | +| `forge_shield_scans_completed_total` | counter | `direction` (`input`, `output`), `decision` (`allow`, `block`, `flag`, `redact`) | +| `forge_shield_scan_duration_seconds` | histogram | `direction`, `decision` | +| `forge_shield_plugin_hooks_failed_total` | counter | `hook` (`scan_started`, `scan_completed` and the other 13 hooks in snake case) | + +Scanned text, findings, tenants, apps and error messages never go on a span or a label. The [observability guide](https://www.xraph.com/docs/shield/guides/observability) has the details, including what changed for anyone scraping the old `shield.scan.*` counters. ## Documentation @@ -216,7 +232,7 @@ Register a plugin with `engine.WithPlugin` (or `WithPlugin` on the extension). I | Overview | [Introduction](https://www.xraph.com/docs/shield), [architecture](https://www.xraph.com/docs/shield/architecture), [getting started](https://www.xraph.com/docs/shield/getting-started) | | Concepts | [Entities](https://www.xraph.com/docs/shield/concepts/entities), [multi-tenancy](https://www.xraph.com/docs/shield/concepts/multi-tenancy), [configuration](https://www.xraph.com/docs/shield/concepts/configuration), [errors](https://www.xraph.com/docs/shield/concepts/errors) | | Storage | [PostgreSQL](https://www.xraph.com/docs/shield/stores/postgres), [SQLite](https://www.xraph.com/docs/shield/stores/sqlite), [MongoDB](https://www.xraph.com/docs/shield/stores/mongo) | -| Forge and plugins | [Forge extension](https://www.xraph.com/docs/shield/guides/forge-extension), [custom plugins](https://www.xraph.com/docs/shield/guides/custom-plugin) | +| Forge and plugins | [Forge extension](https://www.xraph.com/docs/shield/guides/forge-extension), [custom plugins](https://www.xraph.com/docs/shield/guides/custom-plugin), [observability](https://www.xraph.com/docs/shield/guides/observability) | Some docs pages, including getting started, show scans that block and redact. They describe the design, not what the code does yet, so trust [Status](#status) over them. The Go API is on [pkg.go.dev](https://pkg.go.dev/github.com/xraph/shield). diff --git a/docs/content/docs/api-reference/go-packages.mdx b/docs/content/docs/api-reference/go-packages.mdx index 8e49ec3..7fa093a 100644 --- a/docs/content/docs/api-reference/go-packages.mdx +++ b/docs/content/docs/api-reference/go-packages.mdx @@ -41,7 +41,9 @@ The scan entry points. No evaluation layer runs yet. | `WithStore(s)` | Set composite store | | `WithPlugin(p)` | Register lifecycle plugin | | `WithConfig(cfg)` | Set engine configuration | -| `WithLogger(l)` | Set a go-utils `log.Logger` | +| `WithLogger(l)` | Set a go-utils `log.Logger`; plugin hook errors go to it too | +| `WithMetrics(m)` | Record `shield_scans_completed_total`, `shield_scan_duration_seconds` and `shield_plugin_hooks_failed_total` on a go-utils `metrics.Metrics` | +| `ScansCompletedMetric`, `ScanDurationMetric`, `ScanDurationBuckets` | The metric names and histogram bounds | ### `github.com/xraph/shield/scan` @@ -179,10 +181,12 @@ Compliance report records for the EU AI Act, NIST AI RMF and SOC 2. Shield store ### `github.com/xraph/shield/plugin` -The `Plugin` base interface, `Registry`, `NewRegistry` and all lifecycle hook interfaces: `ScanStarted`, `ScanCompleted`, `ScanBlocked`, `ScanFailed`, `InstinctTriggered`, `AwarenessDetected`, `JudgmentAssessed`, `ValueViolated`, `ReflexFired`, `BoundaryEnforced`, `PIIDetected`, `PIIRedacted`, `PolicyEvaluated`, `SafetyProfileResolved`, `Shutdown`. Only `ScanStarted`, `ScanCompleted` and `Shutdown` are emitted today. +The `Plugin` base interface, `Registry` (with `NewRegistry`, `SetLogger` and `SetMetrics`), `HooksFailedMetric` and all lifecycle hook interfaces: `ScanStarted`, `ScanCompleted`, `ScanBlocked`, `ScanFailed`, `InstinctTriggered`, `AwarenessDetected`, `JudgmentAssessed`, `ValueViolated`, `ReflexFired`, `BoundaryEnforced`, `PIIDetected`, `PIIRedacted`, `PolicyEvaluated`, `SafetyProfileResolved`, `Shutdown`. Only `ScanStarted`, `ScanCompleted` and `Shutdown` are emitted today. ### `github.com/xraph/shield/observability` +Deprecated. `MetricsExtension` implements no hooks and records nothing; use `engine.WithMetrics`. + ```go func NewMetricsExtension() *MetricsExtension func NewMetricsExtensionWithFactory(factory gu.MetricFactory) *MetricsExtension diff --git a/docs/content/docs/architecture.mdx b/docs/content/docs/architecture.mdx index 3516f33..4bc231d 100644 --- a/docs/content/docs/architecture.mdx +++ b/docs/content/docs/architecture.mdx @@ -50,8 +50,9 @@ eng, err := engine.New( EnableShortCircuit: true, ScanConcurrency: 10, }), - engine.WithPlugin(metricsPlugin), // optional: lifecycle hooks + engine.WithPlugin(auditPlugin), // optional: lifecycle hooks engine.WithLogger(logger), // optional: a go-utils log.Logger + engine.WithMetrics(metrics), // optional: a go-utils metrics.Metrics ) ``` @@ -68,6 +69,8 @@ What `ScanInput` and `ScanOutput` do today: 5. Emit `OnScanCompleted`, and `OnScanBlocked` if the result is blocked (it never is yet). 6. Persist the result with `CreateScan` when a store is set. A persistence error is logged as a warning and the scan still returns successfully. +All six steps run under one `scan content` span, so the plugin hooks and the store write show up as its children. When the scan returns, the engine counts it on `shield_scans_completed_total` and records its wall time on `shield_scan_duration_seconds`. See [Observability](/docs/guides/observability). + The layers, once implemented, are meant to run in this order: 1. **Instincts.** Pre-conscious, immediate threat detection: prompt injection, jailbreak attempts and data exfiltration. Fastest layer, and able to short-circuit the rest. @@ -109,14 +112,15 @@ type BoundaryEnforced interface { /* ... */ } // ... 15 hook interfaces in total ``` -`plugin.Registry` type-caches plugins at registration time, so emit calls iterate only over plugins that implement the relevant hook. Hook errors never reach the scan. The engine builds its registry with a no-op logger, so those errors are dropped silently. +`plugin.Registry` type-caches plugins at registration time, so emit calls iterate only over plugins that implement the relevant hook. Hook errors never reach the scan. The registry logs them on the engine's logger, counts them on `shield_plugin_hooks_failed_total` and marks the `run plugin hooks` span failed. Only `OnScanStarted`, `OnScanCompleted` and `OnShutdown` are emitted today. The other twelve hooks are defined and the built-in plugins implement them, but nothing calls them yet. -Built-in plugins: -- `observability.MetricsExtension`: 14 counters, one per hook except shutdown +Built-in plugin: - `audithook.Extension` (import path `audit_hook`): bridges lifecycle events to an audit trail backend +`observability.MetricsExtension` is deprecated and does nothing. The engine records its own metrics through `engine.WithMetrics`; see [Observability](/docs/guides/observability). + ## Package index | Package | Import path | Purpose | @@ -136,7 +140,7 @@ Built-in plugins: | `pii` | `.../pii` | PII token store (you supply the encrypted value) | | `compliance` | `.../compliance` | Compliance report records | | `plugin` | `.../plugin` | Plugin interfaces and Registry | -| `observability` | `.../observability` | Metrics plugin (14 counters) | +| `observability` | `.../observability` | Deprecated metrics plugin; records nothing | | `audithook` | `.../audit_hook` | Audit trail plugin | | `store` | `.../store` | Composite `Store`, plus the scoped `DashboardStore` contract | | `postgres`, `sqlite`, `mongo` | `.../store/postgres`, `.../store/sqlite`, `.../store/mongo` | Store backends on grove | diff --git a/docs/content/docs/concepts/configuration.mdx b/docs/content/docs/concepts/configuration.mdx index fd8074d..cc12458 100644 --- a/docs/content/docs/concepts/configuration.mdx +++ b/docs/content/docs/concepts/configuration.mdx @@ -51,9 +51,10 @@ eng, err := engine.New( | `engine.WithStore(s)` | Sets the composite `store.Store`. Scans are persisted only when a store is set | | `engine.WithConfig(cfg)` | Replaces the `shield.Config` (stored, not read) | | `engine.WithPlugin(p)` | Registers a lifecycle plugin | -| `engine.WithLogger(l)` | Sets the go-utils `log.Logger` the engine uses to warn when a scan can't be persisted | +| `engine.WithLogger(l)` | Sets the go-utils `log.Logger` the engine uses to warn when a scan can't be persisted. The plugin registry logs hook errors on it too | +| `engine.WithMetrics(m)` | Records scan and plugin-hook metrics on a go-utils `metrics.Metrics`. `nil`, the default, records nothing. See [Observability](/docs/guides/observability) | -The plugin registry always uses a no-op logger, so `WithLogger` doesn't make hook errors visible. A plugin that returns an error is skipped silently. +The Forge extension passes the app's logger and metrics for you. An option you pass with `WithEngineOption` comes later, so it wins. ## Top-level Shield options @@ -81,7 +82,7 @@ with `shield.WithConfig`, `shield.WithProfile`, `shield.WithInstincts`, `shield. ```go ext := shieldext.New( shieldext.WithStore(pgStore), - shieldext.WithPlugin(metricsPlugin), + shieldext.WithPlugin(auditPlugin), shieldext.WithDisableRoutes(), shieldext.WithDisableMigrate(), shieldext.WithGroveDatabase(""), diff --git a/docs/content/docs/getting-started.mdx b/docs/content/docs/getting-started.mdx index a50acff..fdd61c9 100644 --- a/docs/content/docs/getting-started.mdx +++ b/docs/content/docs/getting-started.mdx @@ -183,7 +183,7 @@ Delete `shield.db` before you run it again, since the instinct and profile names ## Step 7: Add a plugin -Plugins observe lifecycle events. Shield ships a metrics plugin and an audit plugin. Today only the scan started, scan completed and shutdown hooks fire. +Plugins observe lifecycle events. Shield ships an audit plugin. Today only the scan started, scan completed and shutdown hooks fire. ```go import ( @@ -192,10 +192,8 @@ import ( audithook "github.com/xraph/shield/audit_hook" "github.com/xraph/shield/engine" - "github.com/xraph/shield/observability" ) -metricsPlugin := observability.NewMetricsExtension() auditPlugin := audithook.New(audithook.RecorderFunc( func(ctx context.Context, ev *audithook.AuditEvent) error { log.Printf("audit: %s %s", ev.Action, ev.ResourceID) @@ -205,13 +203,14 @@ auditPlugin := audithook.New(audithook.RecorderFunc( eng, err := engine.New( engine.WithStore(st), - engine.WithPlugin(metricsPlugin), engine.WithPlugin(auditPlugin), ) ``` The import path is `github.com/xraph/shield/audit_hook`, but the package name is `audithook`. +Metrics aren't a plugin. Pass a go-utils `metrics.Metrics` with `engine.WithMetrics(m)` and the engine counts scans and failed hooks itself. In a Forge app the extension does that for you. See [Observability](/docs/guides/observability). + ## Step 8: Use it with Forge If you run a [Forge](https://github.com/xraph/forge) app, the `extension` package wires the engine into it and takes its store from the `*grove.DB` in the container: @@ -220,16 +219,13 @@ If you run a [Forge](https://github.com/xraph/forge) app, the `extension` packag import ( "github.com/xraph/forge" shieldext "github.com/xraph/shield/extension" - "github.com/xraph/shield/observability" ) app := forge.NewApp(forge.AppConfig{ Name: "safety", HTTPAddress: ":8080", Extensions: []forge.Extension{ - shieldext.New( - shieldext.WithPlugin(observability.NewMetricsExtension()), - ), + shieldext.New(), }, }) if err := app.Run(); err != nil { diff --git a/docs/content/docs/guides/custom-plugin.mdx b/docs/content/docs/guides/custom-plugin.mdx index 85674d7..1120326 100644 --- a/docs/content/docs/guides/custom-plugin.mdx +++ b/docs/content/docs/guides/custom-plugin.mdx @@ -1,6 +1,6 @@ --- title: Custom Plugins -description: Build plugins that react to Shield lifecycle events with metrics, audit trails, tracing, or custom logic. +description: Build plugins that react to Shield lifecycle events with audit trails, alerts or custom logic. --- Shield uses an opt-in plugin system. Implement the `plugin.Plugin` base interface and any combination of 15 lifecycle hook interfaces to receive only the events you care about. @@ -99,7 +99,6 @@ Register plugins when creating the engine: ```go eng, err := engine.New( engine.WithPlugin(&MyPlugin{}), - engine.WithPlugin(observability.NewMetricsExtension()), ) ``` @@ -127,7 +126,9 @@ If you forget to implement a method, the compiler will tell you. ## Error handling -Hook errors never reach the scan or `Stop`. The registry passes each error to its logger as a warning and moves on to the next plugin. The engine builds that registry with a no-op logger, and `engine.WithLogger` doesn't change it, so today a hook error is dropped silently. Log inside your hook if you need to see failures. +Hook errors never reach the scan or `Stop`. The registry logs each one as a warning, with the hook and your plugin's `Name()`, and moves on to the next plugin. It uses the engine's logger, which is whatever you pass to `engine.WithLogger` (the Forge extension passes the app logger). With a metrics sink it also counts the failure on `forge_shield_plugin_hooks_failed_total{hook="..."}`, and the `run plugin hooks` span for that call is marked failed. See [Observability](/docs/guides/observability). + +Each call to a hook runs inside that span, a child of the scan's span. If your plugin starts spans of its own from the `ctx` it's given, they nest there too. A panic in a hook isn't recovered. It propagates out of `ScanInput`, `ScanOutput` or `Stop`. @@ -169,19 +170,7 @@ func (p *SlackAlertPlugin) OnInstinctTriggered( ## Built-in plugins -Shield ships two built-in plugins. - -### `observability.MetricsExtension` - -Holds 14 `gu.Counter` values from a go-utils `metrics.MetricFactory`, one per hook except shutdown, named like `shield.scan.started`. Today only `ScanStartedCount` and `ScanCompletedCount` move. - -```go -import "github.com/xraph/shield/observability" - -metrics := observability.NewMetricsExtension() -// or with your own factory, such as a Forge app's metrics: -metrics = observability.NewMetricsExtensionWithFactory(app.Metrics()) -``` +Shield ships one built-in plugin. The `observability` package still exists, but its `MetricsExtension` is deprecated and does nothing: the engine records its own metrics now (see [Observability](/docs/guides/observability)). Drop it from your plugin list. ### `audithook.Extension` diff --git a/docs/content/docs/guides/forge-extension.mdx b/docs/content/docs/guides/forge-extension.mdx index 189d61b..83547d7 100644 --- a/docs/content/docs/guides/forge-extension.mdx +++ b/docs/content/docs/guides/forge-extension.mdx @@ -23,7 +23,6 @@ import ( "github.com/xraph/forge" shieldext "github.com/xraph/shield/extension" - "github.com/xraph/shield/observability" ) func main() { @@ -31,9 +30,7 @@ func main() { Name: "safety", HTTPAddress: ":8080", Extensions: []forge.Extension{ - shieldext.New( - shieldext.WithPlugin(observability.NewMetricsExtension()), - ), + shieldext.New(), }, }) @@ -158,20 +155,14 @@ The default resolver takes scope only from the signed-in user's claims, never fr PII retention also needs an audit sink from `WithDashboardAudit`. Without one, those commands return `UNAVAILABLE`. If your claims look different, pass your own resolver with `WithDashboardActorResolver`. It must authenticate the principal and authorize every scope and permission it returns. -## Adding metrics +## Metrics and tracing -`NewMetricsExtension()` uses its own collector. To report through your app's metrics, build the plugin from `app.Metrics()`: +There's nothing to add. `Register` hands the engine your app's metrics and logger, so scan and plugin-hook metrics show up on `/_/metrics` as `forge_shield_scans_completed_total`, `forge_shield_scan_duration_seconds` and `forge_shield_plugin_hooks_failed_total`. Scans, plugin hooks and PII purges are traced on the provider `forge.New` installs, under whatever span called them. The [observability guide](/docs/guides/observability) lists every span, attribute, metric and label. -```go -app := forge.NewApp(forge.AppConfig{Name: "safety"}) - -metricsPlugin := observability.NewMetricsExtensionWithFactory(app.Metrics()) +To turn the metrics off, pass a nil sink through to the engine: -if err := app.RegisterExtension(shieldext.New( - shieldext.WithPlugin(metricsPlugin), -)); err != nil { - log.Fatal(err) -} +```go +shieldext.New(shieldext.WithEngineOption(engine.WithMetrics(nil))) ``` ## Adding audit trails @@ -190,7 +181,6 @@ auditPlugin := audithook.New(myRecorder, ) ext := shieldext.New( - shieldext.WithPlugin(metricsPlugin), shieldext.WithPlugin(auditPlugin), ) ``` diff --git a/docs/content/docs/guides/full-example.mdx b/docs/content/docs/guides/full-example.mdx index ddb040b..50ad750 100644 --- a/docs/content/docs/guides/full-example.mdx +++ b/docs/content/docs/guides/full-example.mdx @@ -19,7 +19,6 @@ import ( "github.com/xraph/shield" "github.com/xraph/shield/engine" - "github.com/xraph/shield/observability" "github.com/xraph/shield/scan" sqlitestore "github.com/xraph/shield/store/sqlite" ) @@ -40,12 +39,10 @@ func main() { log.Fatal(err) } - // Build the engine with a store and metrics. - metricsPlugin := observability.NewMetricsExtension() - + // Build the engine with a store. Pass engine.WithMetrics to count + // scans; see the observability guide. eng, err := engine.New( engine.WithStore(st), - engine.WithPlugin(metricsPlugin), ) if err != nil { log.Fatal(err) @@ -171,7 +168,6 @@ auditPlugin := audithook.New(recorder) eng, err := engine.New( engine.WithStore(st), - engine.WithPlugin(metricsPlugin), engine.WithPlugin(auditPlugin), ) ``` diff --git a/docs/content/docs/guides/meta.json b/docs/content/docs/guides/meta.json index 8a0f269..db69d6f 100644 --- a/docs/content/docs/guides/meta.json +++ b/docs/content/docs/guides/meta.json @@ -1,4 +1,10 @@ { "title": "Guides", - "pages": ["full-example", "forge-extension", "custom-store", "custom-plugin"] + "pages": [ + "full-example", + "forge-extension", + "custom-store", + "custom-plugin", + "observability" + ] } diff --git a/docs/content/docs/guides/observability.mdx b/docs/content/docs/guides/observability.mdx new file mode 100644 index 0000000..2021093 --- /dev/null +++ b/docs/content/docs/guides/observability.mdx @@ -0,0 +1,113 @@ +--- +title: Observability +description: The spans and metrics Shield records, how to see them, and what changed from the old metrics plugin. +--- + +In a Forge app you don't set anything up. The extension hands the engine your app's metrics and logger, and Shield's spans go to the tracer provider `forge.New` installs. Run a scan, then look: + +```bash +curl -s localhost:8080/_/metrics | grep forge_shield_ +``` + +```text +forge_shield_scans_completed_total{decision="allow",direction="input"} 1 +forge_shield_scan_duration_seconds_count{decision="allow",direction="input"} 1 +``` + +Traces show up wherever your Forge app exports them. A scan made inside an HTTP handler sits under that request's server span. + +## Without Forge + +Give the engine a go-utils `metrics.Metrics` and install a global OpenTelemetry tracer provider: + +```go +import ( + "go.opentelemetry.io/otel" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + + "github.com/xraph/shield/engine" +) + +otel.SetTracerProvider(sdktrace.NewTracerProvider( /* your exporter */ )) + +eng, err := engine.New( + engine.WithStore(st), + engine.WithMetrics(m), // nil, the default, records nothing + engine.WithLogger(logger), +) +``` + +The sink has to keep one series per label set, the way a Forge app's `app.Metrics()` does. go-utils' plain `metrics.NewMetricsCollector` keys series by name alone, so every direction and decision would land on the same counter. + +To switch metrics off inside Forge, pass `shieldext.WithEngineOption(engine.WithMetrics(nil))`. Options you pass that way come after the extension's own, so yours win. + +## Spans + +Every span comes from the `github.com/xraph/shield` tracer and has kind Internal. + +| Span | Where | Attributes | +| --- | --- | --- | +| `scan content` | Every `ScanInput` and `ScanOutput` call | `shield.scan.id`, `shield.scan.direction` (`input`, `output`), `shield.scan.decision` (`allow`, `block`, `flag`, `redact`), `shield.scan.blocked`, `shield.scan.findings` (count), `shield.scan.persisted` (only when a store is set) | +| `run plugin hooks` | Child of the scan, once per hook that at least one plugin implements | `shield.plugin.hook` (the hook in snake case, such as `scan_started`), `shield.plugin.count`, `shield.plugin.failures` (only when a hook failed) | +| `purge pii tokens` | `pii.purge` and the other retention commands, after the permission and preview checks pass | `shield.retention.tokens_selected`, `shield.retention.tokens_deleted` | + +A span that failed has an error status with a short class and one `exception` event carrying `exception.type`, the error's Go type: + +| Span | Status | When | +| --- | --- | --- | +| `scan content` | `persist_failed` | The store write failed. The scan still returns its verdict, but the dashboard won't have it | +| `run plugin hooks` | `plugin_hook_failed` | At least one plugin returned an error. The scan carries on | +| `purge pii tokens` | `audit_failed` | The request audit failed, so nothing was deleted | +| `purge pii tokens` | `delete_failed` | The token delete failed | +| `purge pii tokens` | `completion_audit_failed` | The tokens are gone but the completion audit wasn't written | + +The store queries under these spans come from grove, which traces every call as a `db.client` span. The dashboard intents themselves (`dispatch:shield/pii.purge@1` and so on) are traced and counted by Forge's dispatcher, so Shield adds nothing there. + +The six layers don't run yet. When one lands, it gets its own child span of `scan content` with a `shield.scan.layer` attribute (`instincts`, `awareness`, `boundaries`, `values`, `judgments`, `reflexes`). + +## Metrics + +Shield records three. The names below are what a Forge app scrapes; the engine registers them without the `forge_` prefix (`engine.ScansCompletedMetric`, `engine.ScanDurationMetric`, `plugin.HooksFailedMetric`). + +| Metric | Type | Labels | +| --- | --- | --- | +| `forge_shield_scans_completed_total` | counter | `direction` (`input`, `output`), `decision` (`allow`, `block`, `flag`, `redact`) | +| `forge_shield_scan_duration_seconds` | histogram | `direction`, `decision` | +| `forge_shield_plugin_hooks_failed_total` | counter | `hook` | + +The duration is the wall time of one `ScanInput` or `ScanOutput` call, plugin hooks and the store write included. Its buckets run from 1ms to 5s (`engine.ScanDurationBuckets`), which covers the layer budgets from a 5ms boundary check to a 500ms judgment. + +`hook` is one of `scan_started`, `scan_completed`, `scan_blocked`, `scan_failed`, `instinct_triggered`, `awareness_detected`, `judgment_assessed`, `value_violated`, `reflex_fired`, `boundary_enforced`, `pii_detected`, `pii_redacted`, `policy_evaluated`, `safety_profile_resolved` or `shutdown`. + +A few queries to start from: + +```text +# Share of scans that blocked, over five minutes +sum(rate(forge_shield_scans_completed_total{decision="block"}[5m])) + / sum(rate(forge_shield_scans_completed_total[5m])) + +# p95 scan latency by direction +histogram_quantile(0.95, + sum by (le, direction) (rate(forge_shield_scan_duration_seconds_bucket[5m]))) + +# Any plugin failing +sum by (hook) (rate(forge_shield_plugin_hooks_failed_total[5m])) > 0 +``` + +## What never goes on a span or a label + +The scanned text, findings, matched patterns, PII values and types, tenant and app IDs, the signed-in subject, PII token IDs and error messages. Error messages are left off because drivers and plugins quote the values they failed on. Plugin, profile, policy and rule names aren't labels either, since you or your tenants choose them. The scan ID is on the scan span so you can find the stored record; it's an opaque ID and never a label. + +## Moving from the observability plugin + +Shield used to ship `observability.MetricsExtension`, a plugin with 14 unlabelled counters named like `shield.scan.started`. Built with `NewMetricsExtension()` they went to a private collector nobody could scrape. Built from `app.Metrics()` they showed up as `forge_shield_scan_started` and friends, typed as gauges, and only the started and completed pair ever moved. + +Those series are gone. If you scraped them, switch to the table above: + +| Old series | Now | +| --- | --- | +| `forge_shield_scan_started`, `forge_shield_scan_completed` | `forge_shield_scans_completed_total` (scans are counted once, when they finish) | +| `forge_shield_scan_blocked` | `forge_shield_scans_completed_total{decision="block"}` | +| `forge_shield_scan_failed` and the ten per-layer counters | Nothing yet. Those events never fired, and they'll get metrics when the layers land | + +The `observability` package still compiles so your build doesn't break, but `MetricsExtension` is deprecated, implements no hooks and has no counter fields. Take it out of your plugin list. diff --git a/docs/content/docs/index.mdx b/docs/content/docs/index.mdx index f8a439f..b2d61f7 100644 --- a/docs/content/docs/index.mdx +++ b/docs/content/docs/index.mdx @@ -33,7 +33,8 @@ Shield models AI safety the way you would describe human cognition. The speeds a - Tenant and app on the context through `shield.WithTenant` and `shield.WithApp`. The engine stamps them on scans; the subsystem stores filter by the IDs you pass them. - Policies with ordered rules, assigned at app or org level and to tenants. - Compliance report records for the EU AI Act, NIST AI RMF and SOC 2. Shield stores reports; it doesn't generate them. -- 15 lifecycle hook interfaces for plugins, a metrics plugin and an audit plugin. Only the scan started, scan completed and shutdown hooks fire today. +- 15 lifecycle hook interfaces for plugins, and an audit plugin. Only the scan started, scan completed and shutdown hooks fire today. +- OpenTelemetry spans for scans, plugin hooks and PII purges, and scan and hook metrics that a Forge app scrapes on `/_/metrics`. See [Observability](/docs/guides/observability). - A `forge.Extension` that provides `*engine.Engine` in the DI container, runs migrations and registers an authenticated dashboard contributor. ## Design philosophy diff --git a/docs/src/components/landing/footer.tsx b/docs/src/components/landing/footer.tsx index 6898b96..962682a 100644 --- a/docs/src/components/landing/footer.tsx +++ b/docs/src/components/landing/footer.tsx @@ -20,7 +20,7 @@ const footerLinks = { ], Extensions: [ { label: "Audit Hook", href: "/docs/extensions/audit-hook" }, - { label: "Observability", href: "/docs/extensions/observability" }, + { label: "Observability", href: "/docs/guides/observability" }, { label: "Custom Plugin", href: "/docs/guides/custom-plugin" }, { label: "Custom Store", href: "/docs/guides/custom-store" }, ], From b8b8e3b8c9202d5d84ab727323d31835ac262d1d Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Fri, 16 Oct 2026 15:03:24 -0500 Subject: [PATCH 6/6] docs: say plainly what the observability package no longer has MetricsExtension lost its exported counter fields and its On* hook methods in this release. Code that only builds and registers the plugin still compiles, but code that read a field or called a hook method directly doesn't, so AGENTS.md and the package reference now say so. --- AGENTS.md | 2 +- docs/content/docs/api-reference/go-packages.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 12831d5..d6829d3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -88,7 +88,7 @@ For govulncheck, use CI's Go. CI always resolves the newest 1.26 patch, and an o | `admin/` | `admin.Service`: validation, permissions, field schemas and audited retention between the contract and the scoped stores | | `plugin/` | Lifecycle hook interfaces and the registry | | `audit_hook/` (package `audithook`) | Plugin for an audit trail | -| `observability/` | Deprecated metrics plugin, kept so old builds compile. It records nothing | +| `observability/` | Deprecated metrics plugin. It records nothing. Code that only builds and registers it still compiles; its old counter fields and hook methods are gone | | `internal/telemetry/` | The shared tracer, the span error helper and the labelled metric vectors the engine, plugin registry and admin service use | | `extension/` | The Forge extension, its `Config` and options | | `extension/contract/` | The `shield` dashboard contributor: `manifest.yaml`, handlers, and the claim-based actor resolver in `scope.go` | diff --git a/docs/content/docs/api-reference/go-packages.mdx b/docs/content/docs/api-reference/go-packages.mdx index 7fa093a..ca31333 100644 --- a/docs/content/docs/api-reference/go-packages.mdx +++ b/docs/content/docs/api-reference/go-packages.mdx @@ -185,7 +185,7 @@ The `Plugin` base interface, `Registry` (with `NewRegistry`, `SetLogger` and `Se ### `github.com/xraph/shield/observability` -Deprecated. `MetricsExtension` implements no hooks and records nothing; use `engine.WithMetrics`. +Deprecated. `MetricsExtension` implements no hooks and records nothing; use `engine.WithMetrics`. Its exported counter fields (`ScanStartedCount` and the rest) and its `On*` hook methods were removed in v1.8.0, so code that read them no longer compiles. Code that only builds and registers the plugin still does. ```go func NewMetricsExtension() *MetricsExtension