fix(daemon): backoff + attempt cap on startup-timeout auto-retry #13468
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # CI - Build, test, and quality checks | |
| # | |
| # Test execution is flattened based on trigger: | |
| # | |
| # | Trigger | Tests Run | Rationale | | |
| # |--------------------------------------|--------------------------------------------------------------|-----------------------------------------| | |
| # | PR to dev | check + unit + mocked online + web | Fast feedback without real secrets | | |
| # | Push to dev | check + unit + mocked online + web | Merge safety without real secrets | | |
| # | workflow_dispatch (default) | mocked CI + no-LLM E2E | Manual mocked run on any branch | | |
| # | workflow_dispatch (run_e2e_only=true) | discover + build + no-LLM E2E only | Selective no-LLM E2E on any branch | | |
| # | |
| # NOTE: E2E tests are NOT run on any automatic trigger (push / PR). No-LLM E2E | |
| # remains manual-only here via workflow_dispatch on any branch. Real-API daemon | |
| # shards and LLM E2E are manual-only in real-api-tests.yml. | |
| # | |
| # Release is handled separately in release.yml (triggered by version tags). | |
| # Test coverage (unit, online, web) is uploaded to Coveralls. | |
| name: CI | |
| on: | |
| push: | |
| branches: [dev] | |
| pull_request: | |
| branches: [dev] | |
| workflow_dispatch: | |
| inputs: | |
| run_e2e_only: | |
| description: 'Skip all prerequisite jobs and run only E2E tests (any branch)' | |
| required: false | |
| default: 'false' | |
| type: boolean | |
| # Cancel in-progress runs when a new run is triggered on the same branch | |
| concurrency: | |
| group: ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| # ============================================ | |
| # CODE QUALITY CHECKS | |
| # Run on: PR to dev, push to dev | |
| # ============================================ | |
| check: | |
| name: Lint, Knip, Format & Type Check | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 3 | |
| if: github.event.inputs.run_e2e_only != 'true' | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Bun | |
| uses: ./.github/actions/setup-bun | |
| with: | |
| bun-version: 1.3.13 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: '24' | |
| - name: Cache node_modules | |
| uses: actions/cache@v5 | |
| with: | |
| path: | | |
| node_modules | |
| packages/*/node_modules | |
| key: bun-deps-${{ runner.os }}-${{ hashFiles('bun.lock', 'package.json') }} | |
| restore-keys: | | |
| bun-deps-${{ runner.os }}- | |
| - name: Install dependencies | |
| run: bun install --frozen-lockfile | |
| - name: Validate test matrix coverage (unit + web + online) | |
| run: bash scripts/validate-test-matrix.sh | |
| - name: Validate daemon shard configuration | |
| run: | | |
| bash scripts/lib/shard-split.sh --self-test | |
| ./scripts/test-daemon.sh --verify | |
| scripts/test-online.sh --verify | |
| - name: Validate space task RPC handler tests | |
| run: bun run check:space-task-handler-tests | |
| - name: Test space task RPC handler gate | |
| run: bun run test:space-task-handler-gate | |
| - name: Run linter | |
| id: lint | |
| run: bun run lint | |
| continue-on-error: true | |
| - name: Check formatting | |
| id: format | |
| run: bun run format:check | |
| continue-on-error: true | |
| - name: Run Knip | |
| id: knip | |
| run: bun run knip | |
| - name: Run type check | |
| id: typecheck | |
| run: bun run typecheck | |
| continue-on-error: true | |
| - name: Check DB schema parity | |
| id: db-schema-parity | |
| run: bun run check:db-schema-parity | |
| continue-on-error: true | |
| - name: Verify exact version pins | |
| id: exact-pins | |
| run: | | |
| if grep -rn '"[~^][0-9]' packages/*/package.json package.json; then | |
| echo "ERROR: Non-exact version pins found. Use exact versions only (no ^ or ~)." | |
| exit 1 | |
| fi | |
| continue-on-error: true | |
| - name: Check for failures | |
| if: steps.lint.outcome == 'failure' || steps.format.outcome == 'failure' || steps.knip.outcome == 'failure' || steps.typecheck.outcome == 'failure' || steps.db-schema-parity.outcome == 'failure' || steps.exact-pins.outcome == 'failure' | |
| run: | | |
| echo "One or more checks failed:" | |
| echo " Lint: ${{ steps.lint.outcome }}" | |
| echo " Format: ${{ steps.format.outcome }}" | |
| echo " Knip: ${{ steps.knip.outcome }}" | |
| echo " Typecheck: ${{ steps.typecheck.outcome }}" | |
| echo " DB schema parity: ${{ steps.db-schema-parity.outcome }}" | |
| echo " Exact pins: ${{ steps.exact-pins.outcome }}" | |
| exit 1 | |
| # ============================================ | |
| # DAEMON ONLINE TESTS (Mocked credentials only) | |
| # Run on: PR to dev, push to dev, workflow_dispatch | |
| # Real-key shards are manual-only in real-api-tests.yml. | |
| # ============================================ | |
| test-daemon-online: | |
| name: Daemon Online (${{ matrix.module }}) | |
| runs-on: ubuntu-latest | |
| if: github.ref_type != 'tag' && github.event.inputs.run_e2e_only != 'true' | |
| timeout-minutes: ${{ matrix.timeout || 15 }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| module: | |
| - agent-sdk | |
| - components | |
| - convo | |
| - coordinator | |
| # features/ is hash-split 3-way by scripts/test-online.sh (features-a/b/c). | |
| - features-a | |
| - features-b | |
| - features-c | |
| - git | |
| - lifecycle | |
| - mcp | |
| # Codex bridge tests disabled — requires OPENAI_API_KEY (real API, not mocked) | |
| # - providers-anthropic-to-codex-bridge | |
| # Copilot bridge tests disabled — COPILOT_GITHUB_TOKEN credential issue causing hard-fail | |
| # - providers-anthropic-copilot | |
| # rewind/ is hash-split 2-way by scripts/test-online.sh (rewind-a/b). | |
| - rewind-a | |
| - rewind-b | |
| # rpc/ is hash-split 6-way by scripts/test-online.sh (rpc-a…rpc-f). | |
| - rpc-a | |
| - rpc-b | |
| - rpc-c | |
| - rpc-d | |
| - rpc-e | |
| - rpc-f | |
| - sdk | |
| # space/ is hash-split 4-way by scripts/test-online.sh (space-a…space-d). | |
| - space-a | |
| - space-b | |
| - space-c | |
| - space-d | |
| - websocket | |
| include: | |
| # mock_sdk: true → uses Dev Proxy (HYPERNEO_USE_DEV_PROXY) for mocking instead of real API calls. | |
| # These modules test infrastructure/plumbing and don't need real AI responses. | |
| # To run with real API locally, just omit the env var. | |
| # | |
| # NO module carries a hand-listed test_path anymore: every module's | |
| # files are resolved at run time by scripts/test-online.sh (directory | |
| # globs; oversized directories hash-split via scripts/lib/shard-split.sh), | |
| # so a new online test file auto-routes to a shard with no YAML edit. | |
| # Rebalance a split by editing its split count in scripts/test-online.sh. | |
| # Run `bash scripts/validate-test-matrix.sh` + `scripts/test-online.sh --verify` | |
| # to verify all files are covered. | |
| - module: agent-sdk | |
| mock_sdk: true | |
| - module: components | |
| mock_sdk: true | |
| # GLM online tests disabled due to flakiness | |
| # - module: glm | |
| - module: convo | |
| mock_sdk: true | |
| - module: coordinator | |
| timeout: 10 | |
| mock_sdk: true | |
| - module: features-a | |
| mock_sdk: true | |
| - module: features-b | |
| mock_sdk: true | |
| - module: features-c | |
| mock_sdk: true | |
| - module: git | |
| timeout: 10 | |
| mock_sdk: true | |
| - module: lifecycle | |
| mock_sdk: true | |
| - module: mcp | |
| mock_sdk: true | |
| # - module: providers-anthropic-to-codex-bridge | |
| # - module: providers-anthropic-copilot | |
| # timeout: 15 | |
| - module: rewind-a | |
| mock_sdk: true | |
| - module: rewind-b | |
| mock_sdk: true | |
| - module: rpc-a | |
| mock_sdk: true | |
| - module: rpc-b | |
| mock_sdk: true | |
| - module: rpc-c | |
| mock_sdk: true | |
| - module: rpc-d | |
| mock_sdk: true | |
| - module: rpc-e | |
| mock_sdk: true | |
| - module: rpc-f | |
| mock_sdk: true | |
| - module: sdk | |
| mock_sdk: true | |
| - module: space-a | |
| mock_sdk: true | |
| - module: space-b | |
| mock_sdk: true | |
| - module: space-c | |
| mock_sdk: true | |
| - module: space-d | |
| mock_sdk: true | |
| - module: websocket | |
| mock_sdk: true | |
| # Real-key cross-provider shards are manual-only in real-api-tests.yml. | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Setup Bun | |
| uses: ./.github/actions/setup-bun | |
| with: | |
| bun-version: 1.3.13 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: '24' | |
| - name: Install sandbox dependencies (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo find /etc/apt/sources.list.d/ \( -name "*microsoft*" -o -name "*azure*" \) -delete 2>/dev/null || true | |
| sudo apt-get update | |
| sudo apt-get install -y bubblewrap socat ripgrep | |
| - name: Setup Dev Proxy | |
| if: runner.os == 'Linux' && matrix.mock_sdk == true | |
| uses: ./.github/actions/setup-devproxy | |
| - name: Cache node_modules | |
| uses: actions/cache@v5 | |
| with: | |
| path: | | |
| node_modules | |
| packages/*/node_modules | |
| key: bun-deps-${{ runner.os }}-${{ hashFiles('bun.lock', 'package.json') }} | |
| restore-keys: | | |
| bun-deps-${{ runner.os }}- | |
| - name: Install dependencies | |
| run: bun install --frozen-lockfile | |
| - name: Run daemon online tests (${{ matrix.module }}) | |
| run: >- | |
| bun run scripts/flaky-test-runner.ts | |
| --suite daemon-online | |
| --results-dir test-results/daemon-online-${{ matrix.module }} | |
| --report test-results/flaky-quarantine-daemon-online-${{ matrix.module }}.json | |
| -- | |
| bash -lc 'set -o pipefail; cd packages/daemon && paths=$(cd ../.. && scripts/test-online.sh ${{ matrix.module }}) && [ -n "$paths" ] || exit 1; node_modules/.bin/vitest run --config vitest.online.config.ts $paths | |
| --coverage | |
| --coverage.reporter=lcov | |
| --coverage.reportsDirectory=coverage-online-${{ matrix.module }} | |
| --reporter=junit | |
| --outputFile.junit=../../test-results/daemon-online-${{ matrix.module }}/junit.xml' | |
| env: | |
| # Use Dev Proxy for mock_sdk tests by setting ANTHROPIC_BASE_URL | |
| # This avoids proxy env var issues and TLS interception problems | |
| HYPERNEO_USE_DEV_PROXY: ${{ matrix.mock_sdk == true && '1' || '' }} | |
| HYPERNEO_TEST_DISABLE_SANDBOX: '1' | |
| # Persist daemon structured/lifecycle logs into the results dir so a | |
| # failing shard's artifact carries the delivery lifecycle trace. | |
| HYPERNEO_LOG_FILE: ${{ github.workspace }}/test-results/daemon-online-${{ matrix.module }}/daemon.jsonl | |
| ANTHROPIC_BASE_URL: ${{ matrix.mock_sdk == true && 'http://127.0.0.1:8000' || '' }} | |
| # For Dev Proxy tests, never pass real Anthropic credentials. | |
| ANTHROPIC_API_KEY: ${{ matrix.mock_sdk == true && 'sk-devproxy-test-key' || '' }} | |
| ANTHROPIC_AUTH_TOKEN: "" | |
| CLAUDE_CODE_OAUTH_TOKEN: "" | |
| OPENAI_API_KEY: "" | |
| COPILOT_GITHUB_TOKEN: "" | |
| GLM_API_KEY: "" | |
| MINIMAX_API_KEY: "" | |
| DEFAULT_PROVIDER: "" | |
| DEFAULT_MODEL: "" | |
| - name: Stop Dev Proxy | |
| if: always() && matrix.mock_sdk == true | |
| run: | | |
| if [ -n "$DEVPROXY_PID" ]; then | |
| kill $DEVPROXY_PID 2>/dev/null || true | |
| fi | |
| - name: Fix coverage paths and strip test utilities | |
| working-directory: packages/daemon | |
| run: | | |
| LCOV=coverage-online-${{ matrix.module }}/lcov.info | |
| # Fix paths: Bun generates relative to package dir, need repo root paths | |
| sed -i 's|^SF:src/|SF:packages/daemon/src/|g' $LCOV || true | |
| sed -i 's|^SF:\.\./shared/|SF:packages/shared/|g' $LCOV || true | |
| # Strip test utility coverage (helpers, fixtures, mocks) | |
| awk '/^SF:(packages\/daemon\/)?tests\//{skip=1} /^end_of_record/{if(skip){skip=0;next}} !skip' $LCOV > ${LCOV}.clean && mv ${LCOV}.clean $LCOV || true | |
| - name: Upload junit on failure (diagnostics) | |
| if: failure() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: daemon-online-junit-${{ matrix.module }}-${{ github.run_number }} | |
| path: test-results/daemon-online-${{ matrix.module }}/ | |
| retention-days: 7 | |
| - name: Upload to Coveralls (parallel) | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| file: packages/daemon/coverage-online-${{ matrix.module }}/lcov.info | |
| flag-name: daemon-online-${{ matrix.module }} | |
| parallel: true | |
| fail-on-error: false | |
| # ============================================ | |
| # DAEMON + SHARED UNIT TESTS (For coverage measurement) | |
| # Run on: PR to dev, push to dev | |
| # ============================================ | |
| test-daemon-shared-unit: | |
| name: Daemon Unit Tests (${{ matrix.shard }}) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 12 | |
| if: github.event.inputs.run_e2e_only != 'true' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: [shared, 0-shared-handlers-workflow, 1-core, 4-space-storage, 4-space-migrations-a, 4-space-migrations-b, 5-space-agent-other, 5-space-runtime-a, 5-space-runtime-b] | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Setup Bun | |
| uses: ./.github/actions/setup-bun | |
| with: | |
| bun-version: 1.3.13 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: '24' | |
| - name: Install sandbox dependencies (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo find /etc/apt/sources.list.d/ \( -name "*microsoft*" -o -name "*azure*" \) -delete 2>/dev/null || true | |
| sudo apt-get update | |
| sudo apt-get install -y bubblewrap socat ripgrep | |
| - name: Cache node_modules | |
| uses: actions/cache@v5 | |
| with: | |
| path: | | |
| node_modules | |
| packages/*/node_modules | |
| key: bun-deps-${{ runner.os }}-${{ hashFiles('bun.lock', 'package.json') }} | |
| restore-keys: | | |
| bun-deps-${{ runner.os }}- | |
| - name: Install dependencies | |
| run: bun install --frozen-lockfile | |
| - name: Run daemon + shared unit tests (${{ matrix.shard }}) | |
| run: >- | |
| bun run scripts/flaky-test-runner.ts | |
| --suite daemon-unit | |
| --results-dir test-results/daemon | |
| --report test-results/flaky-quarantine-daemon-${{ matrix.shard }}.json | |
| -- | |
| ./scripts/test-daemon.sh ${{ matrix.shard }} --coverage | |
| env: | |
| GLM_API_KEY: "" | |
| CLAUDE_CODE_OAUTH_TOKEN: "" | |
| - name: Fix coverage paths for monorepo | |
| if: always() | |
| working-directory: packages/${{ matrix.shard == 'shared' && 'shared' || 'daemon' }} | |
| run: | | |
| # Vitest writes SF: paths relative to the package dir (src/...); Coveralls | |
| # needs repo-root paths (packages/<pkg>/src/...). Mirror the online/web jobs. | |
| PKG="${{ matrix.shard == 'shared' && 'shared' || 'daemon' }}" | |
| LCOV=coverage/lcov.info | |
| [ -f "$LCOV" ] || exit 0 | |
| sed -i "s|^SF:src/|SF:packages/${PKG}/src/|g" "$LCOV" || true | |
| sed -i 's|^SF:\.\./shared/|SF:packages/shared/|g' "$LCOV" || true | |
| sed -i 's|^SF:\.\./daemon/|SF:packages/daemon/|g' "$LCOV" || true | |
| - name: Upload coverage (parallel) | |
| if: always() | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| # test-daemon.sh runs each shard from its package dir (packages/daemon | |
| # or packages/shared) with --coverage.reportsDirectory=coverage, so the | |
| # report lands at packages/<pkg>/coverage/lcov.info — not the repo root. | |
| file: packages/${{ matrix.shard == 'shared' && 'shared' || 'daemon' }}/coverage/lcov.info | |
| flag-name: daemon-${{ matrix.shard }} | |
| parallel: true | |
| fail-on-error: false | |
| # ============================================ | |
| # WEB TESTS | |
| # Run on: PR to dev, push to dev, workflow_dispatch | |
| # ============================================ | |
| test-web: | |
| name: Web Tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 9 | |
| if: (github.event_name == 'pull_request' && github.base_ref == 'dev' || github.event_name == 'push' && github.ref == 'refs/heads/dev' || github.event_name == 'workflow_dispatch') && github.event.inputs.run_e2e_only != 'true' | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Setup Bun | |
| uses: ./.github/actions/setup-bun | |
| with: | |
| bun-version: 1.3.13 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: '24' | |
| - name: Install sandbox dependencies (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo find /etc/apt/sources.list.d/ \( -name "*microsoft*" -o -name "*azure*" \) -delete 2>/dev/null || true | |
| sudo apt-get update | |
| sudo apt-get install -y bubblewrap socat ripgrep | |
| - name: Cache node_modules | |
| uses: actions/cache@v5 | |
| with: | |
| path: | | |
| node_modules | |
| packages/*/node_modules | |
| key: bun-deps-${{ runner.os }}-${{ hashFiles('bun.lock', 'package.json') }} | |
| restore-keys: | | |
| bun-deps-${{ runner.os }}- | |
| - name: Install dependencies | |
| run: bun install --frozen-lockfile | |
| - name: Run web tests | |
| run: >- | |
| bun run scripts/flaky-test-runner.ts | |
| --suite web | |
| --results-dir test-results/web | |
| --report test-results/flaky-quarantine-web.json | |
| -- | |
| bash -lc 'cd packages/web && bunx vitest run | |
| --reporter dot | |
| --reporter junit | |
| --outputFile.junit=../../test-results/web/junit.xml | |
| --coverage' | |
| - name: Fix coverage paths for monorepo | |
| run: | | |
| # Vitest generates paths relative to package dir (src/...) | |
| # Coveralls needs paths relative to repo root (packages/web/src/...) | |
| sed -i 's|^SF:src/|SF:packages/web/src/|g' packages/web/coverage/lcov.info | |
| # Fix cross-package paths (../shared/src/... -> packages/shared/src/...) | |
| sed -i 's|^SF:\.\./shared/|SF:packages/shared/|g' packages/web/coverage/lcov.info | |
| - name: Upload to Coveralls (parallel) | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| file: packages/web/coverage/lcov.info | |
| flag-name: web | |
| parallel: true | |
| fail-on-error: false | |
| # ============================================ | |
| # CLI TESTS | |
| # Run on: push to dev, workflow_dispatch (skip on PR to dev) | |
| # ============================================ | |
| test-cli: | |
| name: CLI Tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 2 | |
| if: (github.event_name == 'push' && github.ref == 'refs/heads/dev' || github.event_name == 'workflow_dispatch') && github.event.inputs.run_e2e_only != 'true' | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Setup Bun | |
| uses: ./.github/actions/setup-bun | |
| with: | |
| bun-version: 1.3.13 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: '24' | |
| - name: Install sandbox dependencies (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo find /etc/apt/sources.list.d/ \( -name "*microsoft*" -o -name "*azure*" \) -delete 2>/dev/null || true | |
| sudo apt-get update | |
| sudo apt-get install -y bubblewrap socat ripgrep | |
| - name: Cache node_modules | |
| uses: actions/cache@v5 | |
| with: | |
| path: | | |
| node_modules | |
| packages/*/node_modules | |
| key: bun-deps-${{ runner.os }}-${{ hashFiles('bun.lock', 'package.json') }} | |
| restore-keys: | | |
| bun-deps-${{ runner.os }}- | |
| - name: Install dependencies | |
| run: bun install --frozen-lockfile | |
| - name: Run CLI tests | |
| working-directory: packages/cli | |
| run: bunx vitest run | |
| # ============================================ | |
| # COVERALLS FINALIZATION | |
| # Merges all parallel coverage uploads | |
| # Run on: PR to dev, push to dev, workflow_dispatch | |
| # ============================================ | |
| coveralls-finalize: | |
| name: Finalize Coveralls | |
| runs-on: ubuntu-latest | |
| needs: [test-daemon-shared-unit, test-web, test-daemon-online] | |
| timeout-minutes: 1 | |
| if: github.event.inputs.run_e2e_only != 'true' | |
| steps: | |
| - name: Finalize Coveralls parallel build | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| parallel-finished: true | |
| fail-on-error: true | |
| # ============================================ | |
| # COVERAGE QUALITY GATE | |
| # Verifies minimum coverage and regression thresholds via Coveralls API. | |
| # Run on: PR to dev, push to dev, workflow_dispatch | |
| # ============================================ | |
| coverage-gate: | |
| name: Coverage Quality Gate | |
| runs-on: ubuntu-latest | |
| needs: [coveralls-finalize] | |
| timeout-minutes: 3 | |
| if: github.event.inputs.run_e2e_only != 'true' | |
| steps: | |
| - name: Check coverage thresholds via Coveralls API | |
| run: | | |
| echo "Querying Coveralls for commit ${{ github.sha }}..." | |
| # Retry logic — Coveralls may take time to process the finalized build | |
| MAX_RETRIES=6 | |
| RETRY_DELAY=15 | |
| RESPONSE="" | |
| for attempt in $(seq 1 $MAX_RETRIES); do | |
| HTTP_CODE=$(curl -s -o /tmp/coveralls.json -w "%{http_code}" \ | |
| "https://coveralls.io/builds/${{ github.sha }}.json" 2>/dev/null) || true | |
| if [ "$HTTP_CODE" = "200" ]; then | |
| RESPONSE=$(cat /tmp/coveralls.json) | |
| break | |
| fi | |
| echo "Attempt $attempt/$MAX_RETRIES: Coveralls API returned HTTP $HTTP_CODE, retrying in ${RETRY_DELAY}s..." | |
| sleep $RETRY_DELAY | |
| done | |
| if [ -z "$RESPONSE" ]; then | |
| echo "⚠️ Could not fetch coverage from Coveralls API after $MAX_RETRIES attempts" | |
| echo "Skipping coverage gate (API unavailable)" | |
| exit 0 | |
| fi | |
| COVERED_PERCENT=$(echo "$RESPONSE" | jq -r '.covered_percent // empty') | |
| COVERAGE_CHANGE=$(echo "$RESPONSE" | jq -r '.coverage_change // "0"') | |
| if [ -z "$COVERED_PERCENT" ]; then | |
| echo "⚠️ Coveralls response missing coverage data" | |
| echo "Skipping coverage gate" | |
| exit 0 | |
| fi | |
| echo "===============================" | |
| echo "📊 Project Coverage: ${COVERED_PERCENT}%" | |
| echo "📈 Coverage Change: ${COVERAGE_CHANGE}%" | |
| echo "===============================" | |
| FAILED=0 | |
| # --- Project gate: minimum overall coverage --- | |
| MIN_COVERAGE=30 | |
| if (( $(echo "$COVERED_PERCENT < $MIN_COVERAGE" | bc -l) )); then | |
| echo "❌ FAIL: Project coverage ${COVERED_PERCENT}% is below minimum ${MIN_COVERAGE}%" | |
| FAILED=1 | |
| else | |
| echo "✅ Project coverage meets minimum threshold (>= ${MIN_COVERAGE}%)" | |
| fi | |
| # --- Regression gate: coverage must not drop more than threshold --- | |
| MAX_REGRESSION=-2 | |
| if [ "$COVERAGE_CHANGE" != "null" ] && [ -n "$COVERAGE_CHANGE" ]; then | |
| if (( $(echo "$COVERAGE_CHANGE < $MAX_REGRESSION" | bc -l) )); then | |
| echo "❌ FAIL: Coverage regressed by ${COVERAGE_CHANGE}% (max allowed: ${MAX_REGRESSION}%)" | |
| FAILED=1 | |
| else | |
| echo "✅ Coverage regression within acceptable range (change: ${COVERAGE_CHANGE}%)" | |
| fi | |
| fi | |
| if [ "$FAILED" -eq 1 ]; then | |
| echo "" | |
| echo "Coverage quality gate FAILED" | |
| exit 1 | |
| fi | |
| echo "" | |
| echo "✅ Coverage quality gate passed!" | |
| # ============================================ | |
| # BUILD - Compile linux-x64 binary + smoke test | |
| # Run on: push to dev, workflow_dispatch | |
| # Waits for test jobs; proceeds if they passed or were skipped. | |
| # Discover may be skipped (E2E is workflow_dispatch-only); we tolerate that. | |
| # ============================================ | |
| build: | |
| name: Build Binary (linux-x64) | |
| runs-on: ubuntu-latest | |
| needs: [check, test-daemon-online, test-daemon-shared-unit, test-web, test-cli, discover] | |
| if: >- | |
| always() && | |
| (github.event_name == 'push' && github.ref == 'refs/heads/dev' || | |
| github.event_name == 'workflow_dispatch') && | |
| (github.event.inputs.run_e2e_only == 'true' || | |
| (contains(fromJSON('["success", "skipped"]'), needs.discover.result) && | |
| contains(fromJSON('["success", "skipped"]'), needs.check.result) && | |
| contains(fromJSON('["success", "skipped"]'), needs.test-daemon-online.result) && | |
| contains(fromJSON('["success", "skipped"]'), needs.test-daemon-shared-unit.result) && | |
| contains(fromJSON('["success", "skipped"]'), needs.test-web.result) && | |
| contains(fromJSON('["success", "skipped"]'), needs.test-cli.result))) | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Setup Bun | |
| uses: ./.github/actions/setup-bun | |
| with: | |
| bun-version: 1.3.13 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: '24' | |
| - name: Install sandbox dependencies (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo find /etc/apt/sources.list.d/ \( -name "*microsoft*" -o -name "*azure*" \) -delete 2>/dev/null || true | |
| sudo apt-get update | |
| sudo apt-get install -y bubblewrap socat ripgrep | |
| - name: Cache node_modules | |
| uses: actions/cache@v5 | |
| with: | |
| path: | | |
| node_modules | |
| packages/*/node_modules | |
| key: bun-deps-${{ runner.os }}-${{ hashFiles('bun.lock', 'package.json') }} | |
| restore-keys: | | |
| bun-deps-${{ runner.os }}- | |
| - name: Install dependencies | |
| run: bun install --frozen-lockfile | |
| - name: Build web frontend | |
| run: cd packages/web && bun run build | |
| - name: Generate embedded assets | |
| run: bun run scripts/generate-embedded-assets.ts | |
| - name: Compile binary | |
| run: bun build --compile --target=bun-linux-x64 --outfile=dist/bin/hyperneo-linux-x64 packages/cli/prod-entry.ts | |
| - name: Smoke test | |
| run: bun run scripts/smoke-test.ts ./dist/bin/hyperneo-linux-x64 | |
| env: | |
| GLM_API_KEY: smoke-test | |
| - name: Upload binary artifact | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: hyperneo-linux-x64-e2e | |
| path: dist/bin/hyperneo-linux-x64 | |
| # ============================================ | |
| # DISCOVER TESTS | |
| # Auto-discover No-LLM E2E test files | |
| # LLM tests are excluded here and run from real-api-tests.yml. | |
| # Run on: workflow_dispatch only (No-LLM E2E is manual-only) | |
| # ============================================ | |
| discover: | |
| name: Discover Tests | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'workflow_dispatch' | |
| outputs: | |
| tests_no_llm: ${{ steps.categorize.outputs.tests_no_llm }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Categorize E2E test files | |
| id: categorize | |
| run: | | |
| # Tests that require LLM API round-trips are excluded here and run in real-api-tests.yml. | |
| LLM_TESTS=(core/message-flow core/interrupt-button core/interrupt-error-bug core/context-features core/model-selection features/archive features/file-operations features/message-operations features/provider-model-switching features/reviewer-feedback-loop features/rewind-features features/session-operations features/slash-cmd features/space-approval-gate-rejection features/space-artifacts-panel features/space-canvas-mode features/space-happy-path-pipeline features/worktree-isolation responsive/mobile responsive/tablet settings/auto-title) | |
| # Tests to exclude entirely (disabled, pending stabilization) | |
| EXCLUDED_TESTS=( | |
| features/space-export-import | |
| features/space-workflow-rules | |
| ) | |
| # Build lookup sets | |
| declare -A LLM_SET EXCLUDED_SET | |
| for t in "${LLM_TESTS[@]}"; do LLM_SET[$t]=1; done | |
| for t in "${EXCLUDED_TESTS[@]}"; do EXCLUDED_SET[$t]=1; done | |
| # Find all test files recursively and extract subdir/name paths | |
| NO_LLM_JSON="[]" | |
| for file in $(find packages/e2e/tests -name "*.e2e.ts" -type f | sed 's|packages/e2e/tests/||; s|\.e2e\.ts$||' | sort); do | |
| if [[ -n "${EXCLUDED_SET[$file]}" ]]; then | |
| echo "EXCLUDED: $file" | |
| continue | |
| fi | |
| if [[ -n "${LLM_SET[$file]}" ]]; then | |
| echo "LLM (real-api-tests.yml): $file" | |
| continue | |
| fi | |
| # Create JSON object with path (for test runner) and name (for artifact/display) | |
| safe_name=$(echo "$file" | tr '/' '-') | |
| entry=$(jq -n -c --arg path "$file" --arg name "$safe_name" '{path: $path, name: $name}') | |
| NO_LLM_JSON=$(echo "$NO_LLM_JSON" | jq -c ". + [$entry]") | |
| done | |
| no_llm_count=$(echo "$NO_LLM_JSON" | jq length) | |
| echo "No-LLM tests ($no_llm_count): $NO_LLM_JSON" | |
| echo "tests_no_llm=$NO_LLM_JSON" >> $GITHUB_OUTPUT | |
| # ============================================ | |
| # E2E TESTS - No LLM (Matrix, fully parallel) | |
| # UI-only tests that don't require LLM API calls | |
| # Run on: workflow_dispatch only (E2E is manual-only on any branch) | |
| # ============================================ | |
| e2e-no-llm: | |
| name: E2E No-LLM (${{ matrix.test.name }}) | |
| runs-on: ubuntu-latest | |
| needs: [build, discover] | |
| # Manual-only: E2E must be explicitly triggered via workflow_dispatch. | |
| if: >- | |
| always() && | |
| github.event_name == 'workflow_dispatch' && | |
| needs.build.result == 'success' && | |
| needs.discover.result == 'success' | |
| timeout-minutes: 10 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test: ${{ fromJson(needs.discover.outputs.tests_no_llm) }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Setup Bun | |
| uses: ./.github/actions/setup-bun | |
| with: | |
| bun-version: 1.3.13 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: '24' | |
| - name: Install sandbox dependencies (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo find /etc/apt/sources.list.d/ \( -name "*microsoft*" -o -name "*azure*" \) -delete 2>/dev/null || true | |
| sudo apt-get update | |
| sudo apt-get install -y bubblewrap socat ripgrep | |
| - name: Cache node_modules | |
| uses: actions/cache@v5 | |
| with: | |
| path: | | |
| node_modules | |
| packages/*/node_modules | |
| key: bun-deps-${{ runner.os }}-${{ hashFiles('bun.lock', 'package.json') }} | |
| restore-keys: | | |
| bun-deps-${{ runner.os }}- | |
| - name: Install dependencies | |
| run: | | |
| echo "Starting bun install at $(date)" | |
| timeout 180 bun install --frozen-lockfile --verbose 2>&1 || { | |
| echo "bun install timed out or failed (lockfile may be stale — run 'bun install' locally and commit the updated bun.lock)" | |
| exit 1 | |
| } | |
| echo "Finished bun install at $(date)" | |
| - name: Cache Playwright browsers | |
| uses: actions/cache@v5 | |
| with: | |
| path: ~/.cache/ms-playwright | |
| key: playwright-chromium-${{ runner.os }}-${{ hashFiles('packages/e2e/package.json') }} | |
| restore-keys: | | |
| playwright-chromium-${{ runner.os }}- | |
| - name: Install Playwright browsers | |
| run: cd packages/e2e && bunx playwright install chromium | |
| - name: Download binary | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: hyperneo-linux-x64-e2e | |
| path: dist/bin/ | |
| - name: Run E2E test (${{ matrix.test.name }}) | |
| run: | | |
| chmod +x dist/bin/hyperneo-linux-x64 | |
| WORKSPACE=$(mktemp -d) | |
| # Get a free port via a short-lived Python socket with SO_REUSEADDR. | |
| # SO_REUSEADDR shrinks the reuse window on Linux so the binary can | |
| # re-bind quickly even if the port is briefly in TIME_WAIT state. | |
| PORT=$(python3 -c 'import socket; s=socket.socket(socket.AF_INET, socket.SOCK_STREAM); s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1); s.bind(("",0)); print(s.getsockname()[1])') | |
| echo "Reserved port $PORT for binary" | |
| HYPERNEO_WORKSPACE_ROOT="$WORKSPACE" ./dist/bin/hyperneo-linux-x64 --port "$PORT" & | |
| BINARY_PID=$! | |
| trap 'kill $BINARY_PID 2>/dev/null; rm -rf "$WORKSPACE"' EXIT | |
| for i in $(seq 1 60); do | |
| if curl -s "http://localhost:$PORT/" > /dev/null 2>&1; then | |
| echo "Server ready on port $PORT (PID $BINARY_PID)" | |
| break | |
| fi | |
| if ! kill -0 $BINARY_PID 2>/dev/null; then | |
| echo "Binary process died unexpectedly" | |
| exit 1 | |
| fi | |
| sleep 0.5 | |
| done | |
| if ! curl -s "http://localhost:$PORT/" > /dev/null 2>&1; then | |
| echo "Server failed to start within 30s" | |
| exit 1 | |
| fi | |
| export PLAYWRIGHT_BASE_URL="http://localhost:$PORT" | |
| cd packages/e2e | |
| xvfb-run --auto-servernum --server-args="-screen 0 1920x1080x24" \ | |
| bunx playwright test "tests/${{ matrix.test.path }}.e2e.ts" | |
| env: | |
| # Set a dummy API key so provider availability check passes. | |
| # No-LLM tests don't need real API calls; the key just satisfies isAvailable(). | |
| ANTHROPIC_API_KEY: "sk-devproxy-test-key" | |
| # Clear real auth tokens to avoid credential conflicts. | |
| ANTHROPIC_AUTH_TOKEN: "" | |
| CLAUDE_CODE_OAUTH_TOKEN: "" | |
| GLM_API_KEY: "" | |
| DEFAULT_MODEL: sonnet | |
| # Signal that the LLM will not respond in this environment. | |
| # rather than timing out waiting for a response from a dummy API key. | |
| CI: true | |
| - name: Upload test results | |
| uses: actions/upload-artifact@v7 | |
| if: failure() | |
| with: | |
| name: e2e-no-llm-results-${{ matrix.test.name }} | |
| path: | | |
| packages/e2e/test-results/ | |
| packages/e2e/playwright-report/ | |
| retention-days: 7 | |
| # ============================================ | |
| # GATE - All tests must pass (handles conditional job results) | |
| # ============================================ | |
| all-tests-pass: | |
| name: All Tests Pass | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 1 | |
| if: always() | |
| needs: | |
| - check | |
| - test-daemon-online | |
| - test-daemon-shared-unit | |
| - test-web | |
| - test-cli | |
| - coverage-gate | |
| - build | |
| - discover | |
| - e2e-no-llm | |
| steps: | |
| - name: Check all jobs passed | |
| run: | | |
| FAILED=0 | |
| # Check job: PR to dev, push to dev | |
| if [[ "${{ needs.check.result }}" == "failure" ]]; then | |
| echo "❌ check failed" | |
| FAILED=1 | |
| fi | |
| # test-daemon-online: PR to dev, push to dev | |
| if [[ "${{ needs.test-daemon-online.result }}" == "failure" ]]; then | |
| echo "❌ test-daemon-online failed" | |
| FAILED=1 | |
| fi | |
| # test-daemon-shared-unit: PR to dev, push to dev | |
| if [[ "${{ needs.test-daemon-shared-unit.result }}" == "failure" ]]; then | |
| echo "❌ test-daemon-shared-unit failed" | |
| FAILED=1 | |
| fi | |
| # test-web: PR to dev, push to dev | |
| if [[ "${{ needs.test-web.result }}" == "failure" ]]; then | |
| echo "❌ test-web failed" | |
| FAILED=1 | |
| fi | |
| # test-cli: push to dev, workflow_dispatch | |
| if [[ "${{ needs.test-cli.result }}" == "failure" ]]; then | |
| echo "❌ test-cli failed" | |
| FAILED=1 | |
| fi | |
| # coverage-gate: PR to dev, push to dev | |
| if [[ "${{ needs.coverage-gate.result }}" == "failure" ]]; then | |
| echo "❌ coverage-gate failed" | |
| FAILED=1 | |
| fi | |
| # build: push to dev, workflow_dispatch | |
| if [[ "${{ needs.build.result }}" == "failure" ]]; then | |
| echo "❌ build failed" | |
| FAILED=1 | |
| fi | |
| # discover: workflow_dispatch only (E2E is manual-only) | |
| if [[ "${{ needs.discover.result }}" == "failure" ]]; then | |
| echo "❌ discover failed" | |
| FAILED=1 | |
| fi | |
| # e2e-no-llm: workflow_dispatch only (E2E is manual-only) | |
| if [[ "${{ needs.e2e-no-llm.result }}" == "failure" ]]; then | |
| echo "❌ e2e-no-llm failed" | |
| FAILED=1 | |
| fi | |
| if [ $FAILED -eq 1 ]; then | |
| echo "" | |
| echo "One or more required jobs failed:" | |
| echo " check: ${{ needs.check.result }}" | |
| echo " test-daemon-online: ${{ needs.test-daemon-online.result }}" | |
| echo " test-daemon-shared-unit: ${{ needs.test-daemon-shared-unit.result }}" | |
| echo " test-web: ${{ needs.test-web.result }}" | |
| echo " test-cli: ${{ needs.test-cli.result }}" | |
| echo " coverage-gate: ${{ needs.coverage-gate.result }}" | |
| echo " build: ${{ needs.build.result }}" | |
| echo " discover: ${{ needs.discover.result }}" | |
| echo " e2e-no-llm: ${{ needs.e2e-no-llm.result }}" | |
| exit 1 | |
| fi | |
| echo "All tests passed!" |