Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions server/internal/app/route_sweep_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,12 @@ func TestEveryRouteAnswersWithoutAServerError(t *testing.T) {
for _, rt := range routes {
for _, at := range attempts {
status := env.sweep(rt.method, sweepURL(rt.pattern, env.org.Slug, at.uid), at.token, at.body)
// 503 is how an unconfigured integration (WebAuthn, bots) says so.
if status >= http.StatusInternalServerError && status != http.StatusServiceUnavailable {
// 503 is how an unconfigured integration (WebAuthn, bots) says so;
// 502 is how Freebox pairing says the box is unreachable (the
// default base URL is a public host, so the answer depends on
// the network the test runs on: 502 when refused, none on timeout).
if status >= http.StatusInternalServerError &&
status != http.StatusServiceUnavailable && status != http.StatusBadGateway {
serverErrors = append(serverErrors, rt.method+" "+rt.pattern)
}
}
Expand Down
15 changes: 10 additions & 5 deletions server/internal/app/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -2508,11 +2508,16 @@ func (s *Server) SetupRoutes(ctx context.Context) {
mgmt.GET("/version", s.getVersion)
mgmt.GET("/limits", s.getLimits)
// POST /report is anonymous (no auth, no org required) and outside
// limitedPrefix, so it gets its own dedicated, much stricter limiter
// rather than riding on the unlimited /api/mgmt traffic assumption (spec
// 2026-09-25-26) — see reportRateLimitConfig. RateLimitRoute (not
// RateLimit) because the route itself is the scope here, not a path
// prefix.
// limitedPrefix, so it gets its own dedicated, much stricter per-IP
// limiter rather than riding on the unlimited /api/mgmt traffic
// assumption (spec 2026-09-25-26) — see reportRateLimitConfig.
// RateLimitRoute (not RateLimit) because the route itself is the scope
// here, not a path prefix. What it can STORE is bounded in
// feedback.Service (spec 2026-10-29-01): anonymous screenshots spend an
// instance-wide hourly budget (app.feedback_anon_screenshots_per_hour)
// and need a free GitHub issue dispatch slot, every screenshot also fits
// the per-org quota (checked atomically with the insert), and the
// state-cleanup job purges report files past app.feedback_retention_days.
reportLimiter := middleware.NewRateLimiter(reportRateLimitConfig(s.config.Server.RateLimiting), ctx)
mgmtReport := mainGroup.NewGroup("/api/mgmt").Use(reportLimiter.RateLimitRoute)
mgmtReport.POST("/report", feedbackHandler.SubmitReport)
Expand Down
28 changes: 26 additions & 2 deletions server/internal/checkworker/repair_grant_agent_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ type agentGrantServer struct {
mu sync.Mutex
result *agents.ClientFrame
done chan struct{}
// stop ends every connection the fake still holds (the agent may redial after
// the first one closed); handlers tracks them so the test can wait them out.
stop chan struct{}
handlers sync.WaitGroup
}

func (s *agentGrantServer) serve(writer http.ResponseWriter, req *http.Request) {
Expand All @@ -39,7 +43,22 @@ func (s *agentGrantServer) serve(writer http.ResponseWriter, req *http.Request)
return
}

ctx := req.Context()
s.handlers.Add(1)
defer s.handlers.Done()

ctx, cancel := context.WithCancel(req.Context())
defer cancel()

go func() {
select {
case <-s.stop:
cancel()
case <-ctx.Done():
}
}()

defer func() { _ = conn.CloseNow() }()

hello := agents.ServerFrame{Type: agents.MsgTypeHello, AgentUID: "agent-1", WorkerUID: "worker-1", Region: "r"}

if req.Header.Get("Authorization") != "" {
Expand Down Expand Up @@ -141,14 +160,19 @@ func runAgentGrantJob(
region := "r"
fake := &agentGrantServer{
done: make(chan struct{}),
stop: make(chan struct{}),
job: agents.AgentJob{
UID: "job-1", CheckUID: "check-1", Type: string(checkerdef.CheckTypeJS),
Config: stored, ConfigSealed: &sealed, Period: "1m", ScheduledAt: &due, Region: &region,
},
}

srv := httptest.NewServer(http.HandlerFunc(fake.serve))
t.Cleanup(srv.Close)
t.Cleanup(func() {
close(fake.stop)
srv.Close()
fake.handlers.Wait()
})

wsb := backend.NewWSBackend(srv.URL, "spe_token", "agent", &backend.Identity{AgentKeys: *keys}, nil,
backend.WithPingInterval(400*time.Millisecond))
Expand Down
52 changes: 49 additions & 3 deletions server/internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -1237,6 +1237,17 @@ type AppConfig struct {
// (systemconfig.KeyFeedbackMaxStorageBytes), read at request time by
// feedback.Service so a DB-set value takes effect without a restart.
FeedbackMaxStorageBytes int64 `koanf:"feedback_max_storage_bytes"`
// FeedbackAnonScreenshotsPerHour caps how many screenshots ANONYMOUS bug
// reports may store per hour across the WHOLE instance (spec 2026-10-29-01).
// The per-org quota above is spent once per org, and the anonymous caller
// picks the org, so only an instance-wide budget bounds what they write.
// Counted through db.BumpGlobalCounter, so it holds across processes.
// Authenticated members are not counted. 0 or negative disables it.
FeedbackAnonScreenshotsPerHour int `koanf:"feedback_anon_screenshots_per_hour"`
// FeedbackRetentionDays is how long feedback-report screenshots are kept:
// the state-cleanup job purges (row and blob) every reports file older
// than this (spec 2026-10-29-01). 0 or negative keeps them forever.
FeedbackRetentionDays int `koanf:"feedback_retention_days"`
}

// AppGitHubConfig holds the GitHub credentials used for in-app feature integrations
Expand All @@ -1254,13 +1265,33 @@ const DefaultFeedbackMaxStorageBytes int64 = 100 * 1024 * 1024
// AppConfig.FeedbackMaxStorageBytes.
const EnvAppFeedbackMaxStorageBytes = "SP_APP_FEEDBACK_MAX_STORAGE_BYTES"

// applyFeedbackEnv binds SP_APP_FEEDBACK_MAX_STORAGE_BYTES.
// DefaultFeedbackAnonScreenshotsPerHour is the built-in instance-wide budget of
// anonymous bug-report screenshots per hour: the same as the GitHub issue
// dispatch cap, since a screenshot past that cap would never be filed.
const DefaultFeedbackAnonScreenshotsPerHour = 20

// EnvAppFeedbackAnonScreenshotsPerHour overrides
// AppConfig.FeedbackAnonScreenshotsPerHour.
const EnvAppFeedbackAnonScreenshotsPerHour = "SP_APP_FEEDBACK_ANON_SCREENSHOTS_PER_HOUR"

// DefaultFeedbackRetentionDays is the built-in retention of bug-report
// screenshots.
const DefaultFeedbackRetentionDays = 90

// EnvAppFeedbackRetentionDays overrides AppConfig.FeedbackRetentionDays.
const EnvAppFeedbackRetentionDays = "SP_APP_FEEDBACK_RETENTION_DAYS"

// applyFeedbackEnv binds SP_APP_FEEDBACK_MAX_STORAGE_BYTES,
// SP_APP_FEEDBACK_ANON_SCREENSHOTS_PER_HOUR and SP_APP_FEEDBACK_RETENTION_DAYS.
// feedback_max_storage_bytes is snake_case, so koanf's env loader can never
// reach it (SP_APP_FEEDBACK_MAX_STORAGE_BYTES would land on
// app.feedback.max.storage.bytes, not app.feedback_max_storage_bytes). An
// absent or unparseable variable leaves whatever config.yml /
// config.local.yml (or the built-in default) already set alone.
func applyFeedbackEnv(cfg *AppConfig) {
applyFeedbackIntEnv(EnvAppFeedbackAnonScreenshotsPerHour, &cfg.FeedbackAnonScreenshotsPerHour)
applyFeedbackIntEnv(EnvAppFeedbackRetentionDays, &cfg.FeedbackRetentionDays)

v := os.Getenv(EnvAppFeedbackMaxStorageBytes)
if v == "" {
return
Expand All @@ -1271,6 +1302,19 @@ func applyFeedbackEnv(cfg *AppConfig) {
}
}

// applyFeedbackIntEnv sets *dst from the integer environment variable name. An
// absent or unparseable variable leaves *dst alone.
func applyFeedbackIntEnv(name string, dst *int) {
v := os.Getenv(name)
if v == "" {
return
}

if n, err := strconv.Atoi(v); err == nil {
*dst = n
}
}

// AggregationConfig controls how aggressively raw/hour/day result data is compacted.
// Each value is the number of completed periods of that tier to retain before
// rolling up to the next tier. Minimum 1 (the previous behavior).
Expand Down Expand Up @@ -2267,8 +2311,10 @@ func Load() (*Config, error) {
App: AppConfig{
// No token and no repo: the in-app bug report is off unless the
// operator sets both SP_APP_GITHUB_ISSUES_TOKEN and SP_APP_GITHUB_REPO.
GitHub: AppGitHubConfig{},
FeedbackMaxStorageBytes: DefaultFeedbackMaxStorageBytes,
GitHub: AppGitHubConfig{},
FeedbackMaxStorageBytes: DefaultFeedbackMaxStorageBytes,
FeedbackAnonScreenshotsPerHour: DefaultFeedbackAnonScreenshotsPerHour,
FeedbackRetentionDays: DefaultFeedbackRetentionDays,
},
Google: GoogleOAuthConfig{Enabled: false},
GitHub: GitHubOAuthConfig{Enabled: false},
Expand Down
2 changes: 2 additions & 0 deletions server/internal/config/envvars.go
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,8 @@ func manualReaderServerEnvVars() []string {
// applyFeedbackEnv — feedback_max_storage_bytes is snake_case, so
// koanf's env loader cannot reach it.
EnvAppFeedbackMaxStorageBytes,
EnvAppFeedbackAnonScreenshotsPerHour,
EnvAppFeedbackRetentionDays,
// applyCheckersEnv — both keys have a snake_case segment, so koanf's
// env loader cannot reach them (see BrowserCheckerConfig).
"SP_CHECKERS_BROWSER_CDP_URL",
Expand Down
32 changes: 32 additions & 0 deletions server/internal/config/feedback_env_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package config

import (
"testing"

"github.com/stretchr/testify/require"
)

func TestApplyFeedbackEnv_IntVars(t *testing.T) {
cfg := &AppConfig{FeedbackAnonScreenshotsPerHour: 20, FeedbackRetentionDays: 90}

t.Setenv(EnvAppFeedbackAnonScreenshotsPerHour, "5")
t.Setenv(EnvAppFeedbackRetentionDays, "not-a-number")
applyFeedbackEnv(cfg)

require.Equal(t, 5, cfg.FeedbackAnonScreenshotsPerHour)
require.Equal(t, 90, cfg.FeedbackRetentionDays, "an unparseable value leaves the setting alone")

t.Setenv(EnvAppFeedbackRetentionDays, "0")
applyFeedbackEnv(cfg)
require.Zero(t, cfg.FeedbackRetentionDays)
}

func TestLoad_FeedbackDefaults(t *testing.T) {
t.Setenv(EnvAppFeedbackAnonScreenshotsPerHour, "")
t.Setenv(EnvAppFeedbackRetentionDays, "")

cfg, err := Load()
require.NoError(t, err)
require.Equal(t, DefaultFeedbackAnonScreenshotsPerHour, cfg.App.FeedbackAnonScreenshotsPerHour)
require.Equal(t, DefaultFeedbackRetentionDays, cfg.App.FeedbackRetentionDays)
}
31 changes: 31 additions & 0 deletions server/internal/db/postgres/postgres.go
Original file line number Diff line number Diff line change
Expand Up @@ -8027,6 +8027,37 @@ func (s *Service) SumFileSizeByGroup(ctx context.Context, orgUID, group string)
return total.Int64, nil
}

// ListFilesByGroup returns live files of every organization under the given
// storage group, created before `before`, oldest first, capped at limit. The
// group is matched on its "/<group>/" URI path segment, as in
// SumFileSizeByGroup.
func (s *Service) ListFilesByGroup(
ctx context.Context, group string, before time.Time, limit int,
) ([]*models.File, error) {
if group == "" {
return nil, nil
}

var files []*models.File

query := s.db.NewSelect().
Model(&files).
Where("deleted_at IS NULL").
Where("file_uri LIKE ? ESCAPE '\\'", "%/"+escapeLikePrefix(group)+"/%").
Where("created_at < ?", before).
Order("created_at ASC")

if limit > 0 {
query = query.Limit(limit)
}

if err := query.Scan(ctx); err != nil {
return nil, err
}

return files, nil
}

// ListAttachmentsByTopicPrefix returns live attachment rows across all orgs
// under a topic prefix, older than `before`, capped at limit. Cross-org because
// its only caller is the GC sweep.
Expand Down
5 changes: 5 additions & 0 deletions server/internal/db/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -1458,6 +1458,11 @@ type Service interface {
// the scheme/bucket portion varies by storage backend (local vs S3). Used
// by the feedback-report per-org storage quota (spec 2026-09-25-26).
SumFileSizeByGroup(ctx context.Context, orgUID, group string) (int64, error)
// ListFilesByGroup returns live files across ALL organizations whose
// storage URI belongs to the given filestorage.GroupType and which were
// created before `before`, oldest first, capped at limit. Cross-org by
// design: its only caller is the report-retention sweep (spec 2026-10-29-01).
ListFilesByGroup(ctx context.Context, group string, before time.Time, limit int) ([]*models.File, error)

// CheckDependency operations
CreateCheckDependency(ctx context.Context, dep *models.CheckDependency) error
Expand Down
31 changes: 31 additions & 0 deletions server/internal/db/sqlite/sqlite.go
Original file line number Diff line number Diff line change
Expand Up @@ -7991,6 +7991,37 @@ func (s *Service) SumFileSizeByGroup(ctx context.Context, orgUID, group string)
return total.Int64, nil
}

// ListFilesByGroup returns live files of every organization under the given
// storage group, created before `before`, oldest first, capped at limit. The
// group is matched on its "/<group>/" URI path segment, as in
// SumFileSizeByGroup.
func (s *Service) ListFilesByGroup(
ctx context.Context, group string, before time.Time, limit int,
) ([]*models.File, error) {
if group == "" {
return nil, nil
}

var files []*models.File

query := s.db.NewSelect().
Model(&files).
Where("deleted_at IS NULL").
Where("file_uri LIKE ? ESCAPE '\\'", "%/"+escapeLikePrefix(group)+"/%").
Where("created_at < ?", before).
Order("created_at ASC")

if limit > 0 {
query = query.Limit(limit)
}

if err := query.Scan(ctx); err != nil {
return nil, err
}

return files, nil
}

// ListAttachmentsByTopicPrefix returns live attachment rows across all orgs
// under a topic prefix, older than `before`, capped at limit. Cross-org because
// its only caller is the GC sweep.
Expand Down
Loading
Loading