diff --git a/.github/workflows/appstore-publish.yml b/.github/workflows/appstore-publish.yml index 32776cefa..2febd7c11 100644 --- a/.github/workflows/appstore-publish.yml +++ b/.github/workflows/appstore-publish.yml @@ -152,7 +152,7 @@ jobs: OUTPUT: dist/CHANGELOG_RELEASE.md - name: Upload workflow artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: appstore-publish-${{ steps.resolve_channel.outputs.channel }} path: dist/appstore-publish @@ -236,22 +236,21 @@ jobs: id: purge_urls run: | if [[ "${{ steps.resolve_channel.outputs.channel }}" == "dev" ]]; then - echo "urls=[\"https://artifact.websoft9.com/dev/websoft9/plugin/library/library-dev.zip\",\"https://artifact.websoft9.com/dev/websoft9/plugin/media/media-dev.zip\"]" >> "$GITHUB_OUTPUT" + echo "urls=https://artifact.websoft9.com/dev/websoft9/plugin/library/library-dev.zip https://artifact.websoft9.com/dev/websoft9/plugin/media/media-dev.zip" >> "$GITHUB_OUTPUT" else - echo "urls=[\"https://artifact.websoft9.com/release/websoft9/plugin/library/library-latest.zip\",\"https://artifact.websoft9.com/release/websoft9/plugin/media/media-latest.zip\"]" >> "$GITHUB_OUTPUT" + echo "urls=https://artifact.websoft9.com/release/websoft9/plugin/library/library-latest.zip https://artifact.websoft9.com/release/websoft9/plugin/media/media-latest.zip" >> "$GITHUB_OUTPUT" fi - name: Purge legacy Cloudflare Cache - uses: jakejarvis/cloudflare-purge-action@master - env: - CLOUDFLARE_ZONE: ${{ secrets.CLOUDFLARE_ZONE_ID }} - CLOUDFLARE_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + uses: NathanVaughn/actions-cloudflare-purge@v4.0.0 with: - PURGE_URLS: ${{ steps.purge_urls.outputs.urls }} + cf_zone: ${{ secrets.CLOUDFLARE_ZONE_ID }} + cf_auth: ${{ secrets.CLOUDFLARE_API_TOKEN }} + files: ${{ steps.purge_urls.outputs.urls }} - name: Create GitHub Release (release channel only) if: steps.resolve_channel.outputs.channel == 'release' - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v3 with: files: | dist/appstore-publish/legacy/release/library/library.json @@ -278,13 +277,13 @@ jobs: uses: actions/checkout@v7 - name: Setup Pages - uses: actions/configure-pages@v4 + uses: actions/configure-pages@v6 - name: Upload artifact - uses: actions/upload-pages-artifact@v3 + uses: actions/upload-pages-artifact@v5 with: path: '.' - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 \ No newline at end of file + uses: actions/deploy-pages@v5 \ No newline at end of file diff --git a/.github/workflows/i18n.yml b/.github/workflows/i18n.yml deleted file mode 100644 index a5fd9ad08..000000000 --- a/.github/workflows/i18n.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: i18n - -on: - workflow_dispatch: - push: - branches: [ main ] - paths: - - "apps/*/.env" - -permissions: - contents: write - pull-requests: write - -jobs: - create: - runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' - steps: - - uses: actions/checkout@v7 - name: Check out code - with: - ref: main - fetch-depth: 0 - - - name: Check automation token - env: - TOKEN: ${{ secrets.OPENCODE_GITHUB_TOKEN }} - run: | - if [ -z "$TOKEN" ]; then - echo "::error::Set OPENCODE_GITHUB_TOKEN so i18n PRs trigger the required PR checks." - exit 1 - fi - - - name: Update translation.json - run: | - echo "start to update translation.json" - cd i18n - pip3 install python-dotenv - python3 create.py - - - name: Create translation PR - uses: peter-evans/create-pull-request@v5 - with: - # A PAT is required so the generated PR triggers the required PR checks. - token: ${{ secrets.OPENCODE_GITHUB_TOKEN }} - branch: update-i18n - base: main - delete-branch: true - commit-message: "chore: update i18n translation keys" - title: "chore: update i18n translation keys" - body: "Automated update from the i18n workflow." - - pr: - runs-on: ubuntu-latest - if: github.event_name == 'workflow_dispatch' - strategy: - matrix: - repository: ['Websoft9/plugin-appstore', 'Websoft9/plugin-myapps'] - steps: - - name: Checkout - uses: actions/checkout@v7 - with: - repository: ${{ matrix.repository }} - ref: main - - - name: Update languages files - run: | - cd public - wget https://raw.githubusercontent.com/Websoft9/docker-library/main/i18n/translation.json - wget https://raw.githubusercontent.com/Websoft9/docker-library/main/i18n/pr.py - python3 pr.py translation.json po.en_US.js po.zh_CN.js - rm -rf pr.py translation.json - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v5 - with: - token: ${{ secrets.ZJ_ADMIN_TOKEN }} - branch: update-${{ matrix.repository }}-version-${{ github.run_number }} - delete-branch: true - commit-message: "${{ matrix.repository }} combine translation.json" - title: "${{ matrix.repository }} combine translation.json" - body: "${{ matrix.repository }} combine translation.json" \ No newline at end of file diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index ab02d2248..3405b6968 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -18,14 +18,13 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - name: Validate PR title + - name: Check PR title format env: TITLE: ${{ github.event.pull_request.title }} PATTERN: '^(feat|fix|docs|ci|chore|refactor|perf|test|build|revert)(\([^)]+\))?!?:[[:space:]]+.+' run: | if ! [[ "$TITLE" =~ $PATTERN ]]; then - echo "::error::PR title must look like 'type: summary', e.g. 'fix: handle empty config'" - exit 1 + echo "::warning::PR title should look like 'type: summary', e.g. 'fix: handle empty config'" fi - name: Checkout repository @@ -52,6 +51,12 @@ jobs: - name: Run build tests run: python -m pytest tests/build -q + - name: Validate i18n keys for changed apps + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: make i18n-check ARGS="--base $BASE_SHA --head $HEAD_SHA --github-annotations --warn-only" + - name: Validate changed catalog files env: BASE_SHA: ${{ github.event.pull_request.base.sha }} diff --git a/AGENTS.md b/AGENTS.md index 93175b268..b3a41b57c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,7 +39,7 @@ Read in this order: - Translatable env keys are `W9_*_SET` and `W9_LOGIN*`. - When a task adds or changes such keys, ensure `i18n/translation.json` gets the key. - New keys may stay `["", ""]`; translations are filled by humans, not AI. -- Do not edit downstream repos; the `i18n.yml` workflow syncs them by PR. +- Do not edit downstream repos. i18n checks cover only this repo's `translation.json`; downstream sync workflows were removed. ## Delivery Model diff --git a/Makefile b/Makefile index 1e9f0c4ac..348a18ce0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help opencode opencode-clear install libs cli remote connector github-token install-git-hooks dns-bind dns-delete websoft9-upgrade create-aliyun-ecs build-ai-commands test test-cli test-build test-skills +.PHONY: help opencode opencode-clear install libs cli remote connector github-token install-git-hooks dns-bind dns-delete websoft9-upgrade create-aliyun-ecs build-ai-commands i18n-check i18n-update test test-cli test-build test-skills ifeq ($(OS),Windows_NT) PYTHON ?= py @@ -29,6 +29,8 @@ help: @printf ' \033[2mmake \033[0m\033[1;32m%-18s\033[0m %s\n' 'test-build' 'run build pipeline smoke tests' @printf ' \033[2mmake \033[0m\033[1;32m%-18s\033[0m %s\n' 'test-skills' 'run skills asset and workflow tests' @printf ' \033[2mmake \033[0m\033[1;32m%-18s\033[0m %s\n' 'build-ai-commands' 'generate opencode/copilot/codex command adapters from manifest' + @printf ' \033[2mmake \033[0m\033[1;32m%-18s\033[0m %s\n' 'i18n-check' 'check translation.json for missing keys (ARGS: --strict, --app )' + @printf ' \033[2mmake \033[0m\033[1;32m%-18s\033[0m %s\n' 'i18n-update' 'add missing translation keys as ["", ""]' @printf '\n \033[1;35m▸ Config & Setup\033[0m\n' @printf ' \033[2mmake \033[0m\033[1;32m%-18s\033[0m %s\n' 'help' 'show this help' @printf ' \033[2mmake \033[0m\033[1;32m%-18s\033[0m %s\n' 'install' 'create .venv and install the libs CLI' @@ -239,6 +241,12 @@ connector: chmod 600 "$$file"; \ echo "wrote $$file"' +i18n-check: + $(PYTHON) i18n/manage.py --check $(ARGS) + +i18n-update: + $(PYTHON) i18n/manage.py --update $(ARGS) + test-cli: $(VENV_BIN)/python -m pytest cli/tests -q diff --git a/apps/grafana/CHANGELOG.md b/apps/grafana/CHANGELOG.md index 4418587f0..52f249791 100644 --- a/apps/grafana/CHANGELOG.md +++ b/apps/grafana/CHANGELOG.md @@ -1,5 +1,10 @@ # CHANGELOG +## 2026-10-23 + +- Added a startup log regression check for unsupported Grafana plugins. +- Version scans now identify patch releases already covered by the current floating minor tag. + ## 2026-10-22 - Kept `W9_VERSION=13.2`; the upstream stable `13.2` tag currently resolves to Grafana `13.2.4`. diff --git a/apps/grafana/tests/cases.yml b/apps/grafana/tests/cases.yml index 49c6f9d57..7cfc1eeba 100644 --- a/apps/grafana/tests/cases.yml +++ b/apps/grafana/tests/cases.yml @@ -3,3 +3,6 @@ optional: type: web-access path: /api/health expect_status: 200 + - id: grafana-plugin-validation + type: script + script: check-plugin-validation.sh diff --git a/apps/grafana/tests/check-plugin-validation.sh b/apps/grafana/tests/check-plugin-validation.sh new file mode 100755 index 000000000..7c87cf6cd --- /dev/null +++ b/apps/grafana/tests/check-plugin-validation.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +container="${W9_ID:-grafana}" +stale_plugin="/var/lib/grafana/plugins/grafana-simple-json-datasource" + +if docker exec "$container" test -d "$stale_plugin"; then + echo "Unsupported stale plugin remains in the Grafana plugins volume: $stale_plugin" >&2 + echo "Remove that directory once before validating an existing instance." >&2 + exit 1 +fi + +started_at="$(docker inspect -f '{{.State.StartedAt}}' "$container")" +logs="$(docker logs --since "$started_at" "$container" 2>&1 || true)" + +if grep -Eiq 'Plugin validation failed|angular plugins are not supported|Refusing to initialize plugin' <<<"$logs"; then + echo "Grafana reported an unsupported plugin during this container startup." >&2 + grep -Ei 'Plugin validation failed|angular plugins are not supported|Refusing to initialize plugin' <<<"$logs" >&2 + exit 1 +fi + +echo "No unsupported plugin validation errors during the current startup." diff --git a/apps/metabase/.env b/apps/metabase/.env index b14374897..1b87f8352 100644 --- a/apps/metabase/.env +++ b/apps/metabase/.env @@ -1,6 +1,6 @@ W9_REPO=metabase/metabase W9_DIST=community -W9_VERSION=v0.64.x +W9_VERSION=v0.64.1.x # Optional password seed: enable only when the package actually controls a DB or built-in login. # See docs/w9-env-spec.md for when W9_POWER_PASSWORD is appropriate. diff --git a/apps/metabase/CHANGELOG.md b/apps/metabase/CHANGELOG.md index 4f55e6cfc..76806e913 100644 --- a/apps/metabase/CHANGELOG.md +++ b/apps/metabase/CHANGELOG.md @@ -1,5 +1,10 @@ # CHANGELOG +## 2026-10-23 + +- Updated Metabase to the official `v0.64.1.x` Docker image tag from the invalid `v0.64.x` placeholder. +- Updated version metadata so upstream scans can parse and compare the current Metabase image correctly. + ## 2026-10-08 - Updated Metabase from `v0.57.5.3` to `v0.64.x`. diff --git a/apps/metabase/README.md b/apps/metabase/README.md index 48bfef10a..2e51f48f5 100644 --- a/apps/metabase/README.md +++ b/apps/metabase/README.md @@ -6,6 +6,7 @@ 1. In the [Websoft9](https://www.websoft9.com) console, open **My Apps** and select **Metabase**. 2. In the **Access** tab, get the login URL and credentials. + 3. Open the login URL in a browser and sign in to confirm the app works. @@ -33,7 +34,7 @@ Apps run as containers; rebuild after any configuration change. ### Version Support -Supported versions: v0.64.x, latest. +Supported versions: v0.64.1.x, latest. The `latest` tag is not guaranteed to remain valid; pin a specific version for production. diff --git a/apps/metabase/variables.json b/apps/metabase/variables.json index fc8245a1d..c558e9b3a 100644 --- a/apps/metabase/variables.json +++ b/apps/metabase/variables.json @@ -14,7 +14,7 @@ { "dist": "community", "version": [ - "v0.64.x", + "v0.64.1.x", "latest" ] } diff --git a/cli/libs/__pycache__/__init__.cpython-312.pyc b/cli/libs/__pycache__/__init__.cpython-312.pyc index fce0f4529..6c1c4fce2 100644 Binary files a/cli/libs/__pycache__/__init__.cpython-312.pyc and b/cli/libs/__pycache__/__init__.cpython-312.pyc differ diff --git a/cli/libs/__pycache__/app_deploy.cpython-312.pyc b/cli/libs/__pycache__/app_deploy.cpython-312.pyc index 0f249a9e9..5c62e1de0 100644 Binary files a/cli/libs/__pycache__/app_deploy.cpython-312.pyc and b/cli/libs/__pycache__/app_deploy.cpython-312.pyc differ diff --git a/cli/libs/__pycache__/http.cpython-312.pyc b/cli/libs/__pycache__/http.cpython-312.pyc index 1e0c9788a..d8c618fb6 100644 Binary files a/cli/libs/__pycache__/http.cpython-312.pyc and b/cli/libs/__pycache__/http.cpython-312.pyc differ diff --git a/cli/libs/__pycache__/main.cpython-312.pyc b/cli/libs/__pycache__/main.cpython-312.pyc index 62d403f7b..5d317d91b 100644 Binary files a/cli/libs/__pycache__/main.cpython-312.pyc and b/cli/libs/__pycache__/main.cpython-312.pyc differ diff --git a/cli/libs/__pycache__/remote.cpython-312.pyc b/cli/libs/__pycache__/remote.cpython-312.pyc index e15c8d8fb..e3d6dba17 100644 Binary files a/cli/libs/__pycache__/remote.cpython-312.pyc and b/cli/libs/__pycache__/remote.cpython-312.pyc differ diff --git a/cli/libs/__pycache__/repo.cpython-312.pyc b/cli/libs/__pycache__/repo.cpython-312.pyc index 7a3c6e372..0d56e0194 100644 Binary files a/cli/libs/__pycache__/repo.cpython-312.pyc and b/cli/libs/__pycache__/repo.cpython-312.pyc differ diff --git a/cli/libs/__pycache__/sources.cpython-312.pyc b/cli/libs/__pycache__/sources.cpython-312.pyc index 17f657d6e..f6e4f5f6a 100644 Binary files a/cli/libs/__pycache__/sources.cpython-312.pyc and b/cli/libs/__pycache__/sources.cpython-312.pyc differ diff --git a/cli/libs/__pycache__/versioning.cpython-312.pyc b/cli/libs/__pycache__/versioning.cpython-312.pyc index 5b0ee983a..03331d454 100644 Binary files a/cli/libs/__pycache__/versioning.cpython-312.pyc and b/cli/libs/__pycache__/versioning.cpython-312.pyc differ diff --git a/cli/libs/__pycache__/versions.cpython-312.pyc b/cli/libs/__pycache__/versions.cpython-312.pyc index 25ac4209c..b65a2cfc5 100644 Binary files a/cli/libs/__pycache__/versions.cpython-312.pyc and b/cli/libs/__pycache__/versions.cpython-312.pyc differ diff --git a/cli/libs/app_deploy.py b/cli/libs/app_deploy.py index 57036bb79..88cf03961 100644 --- a/cli/libs/app_deploy.py +++ b/cli/libs/app_deploy.py @@ -25,6 +25,7 @@ SS_PORT_RE = re.compile(r":(\d+)\s+\S+\s*$") FREE_PORT_LOW = 9000 FREE_PORT_HIGH = 9999 +PULL_POLICIES = {"always", "missing", "never"} def _run(command: list[str], *, progress: ProgressWriter | None = None, verbose: bool = False) -> subprocess.CompletedProcess: @@ -87,6 +88,10 @@ def _remote_compose_script(target: str, action: str) -> str: return f"docker compose -f {target}/docker-compose.yml --env-file {target}/.env {action}" +def _compose_up_action(pull_policy: str) -> str: + return "up -d --pull never" if pull_policy == "never" else "up -d" + + def _compose_services(target: Path) -> dict: compose_path = target / "docker-compose.yml" if not compose_path.exists(): @@ -388,12 +393,15 @@ def deploy( version: str | None = None, down: bool = False, auto_port: bool = True, + pull_policy: str = "always", progress: ProgressWriter | None = None, verbose: bool = False, ) -> dict: source = app_dir(app_name) if not source: raise FileNotFoundError(app_name) + if pull_policy not in PULL_POLICIES: + raise ValueError(f"invalid pull policy: {pull_policy}; expected one of {sorted(PULL_POLICIES)}") mode = _target_mode(target, ssh_host) action = "down -v" if down else "up -d" @@ -425,9 +433,9 @@ def deploy( raise RuntimeError(config.stderr.strip() or config.stdout.strip() or "compose config failed") if not down: - _announce(progress, step + 1, total, "pulling images") - if pull_services: - pull = _run(_local_compose_args(target_path, env_file) + ["pull", *pull_services], progress=progress, verbose=verbose) + _announce(progress, step + 1, total, "pulling images" if pull_policy != "never" else "using existing images") + if pull_services and pull_policy != "never": + pull = _run(_local_compose_args(target_path, env_file) + ["pull", "--policy", pull_policy, *pull_services], progress=progress, verbose=verbose) if pull.returncode != 0: raise RuntimeError(pull.stderr.strip() or pull.stdout.strip() or "compose pull failed") if needs_build: @@ -439,11 +447,11 @@ def deploy( progress=progress, ) _announce(progress, step + 3, total, "starting application") - result = _run(_local_compose_args(target_path, env_file) + ["up", "-d"], progress=progress, verbose=verbose) + result = _run(_local_compose_args(target_path, env_file) + _compose_up_action(pull_policy).split(), progress=progress, verbose=verbose) ps_step = step + 4 else: _announce(progress, step + 2, total, "starting application") - result = _run(_local_compose_args(target_path, env_file) + ["up", "-d"], progress=progress, verbose=verbose) + result = _run(_local_compose_args(target_path, env_file) + _compose_up_action(pull_policy).split(), progress=progress, verbose=verbose) ps_step = step + 3 else: _announce(progress, step + 1, total, "stopping application") @@ -461,6 +469,7 @@ def deploy( "deploy_root": str(repo_path("apps")), "app_target": str(target_path), "action": action, + "pull_policy": pull_policy, "port_overrides": {key: str(value) for key, value in overrides.items()}, "base_url": base_url, "ps": ps.stdout.strip(), @@ -505,9 +514,9 @@ def deploy( raise RuntimeError(config.stderr.strip() or config.stdout.strip() or "remote compose config failed") if not down: - _announce(progress, step + 1, total, "pulling images") - if pull_services: - pull = _run_remote(host, user, secret_path, _remote_compose_script(app_target, f"pull {' '.join(pull_services)}"), progress=progress, verbose=verbose) + _announce(progress, step + 1, total, "pulling images" if pull_policy != "never" else "using existing images") + if pull_services and pull_policy != "never": + pull = _run_remote(host, user, secret_path, _remote_compose_script(app_target, f"pull --policy {pull_policy} {' '.join(pull_services)}"), progress=progress, verbose=verbose) if pull.returncode != 0: raise RuntimeError(pull.stderr.strip() or pull.stdout.strip() or "remote compose pull failed") if needs_build: @@ -522,7 +531,7 @@ def deploy( progress=progress, ) _announce(progress, step + 3, total, "starting application") - result = _run_remote(host, user, secret_path, _remote_compose_script(app_target, "up -d"), progress=progress, verbose=verbose) + result = _run_remote(host, user, secret_path, _remote_compose_script(app_target, _compose_up_action(pull_policy)), progress=progress, verbose=verbose) ps_step = step + 4 else: _announce(progress, step + 2, total, "starting application") @@ -547,6 +556,7 @@ def deploy( "app_target": app_target, "version": version, "action": action, + "pull_policy": pull_policy, "port_overrides": {key: str(value) for key, value in overrides.items()}, "base_url": base_url, "ps": ps.stdout.strip(), diff --git a/cli/libs/http.py b/cli/libs/http.py index 7c6765034..98a4ae446 100644 --- a/cli/libs/http.py +++ b/cli/libs/http.py @@ -14,6 +14,68 @@ PROXY_KEYS = ("https_proxy", "HTTPS_PROXY", "http_proxy", "HTTP_PROXY", "all_proxy", "ALL_PROXY") REPO_CONFIG_PATH = Path(__file__).resolve().parent.parent / "proxy.conf" _REMOTE_ONLY = False +_LOCAL_UNAVAILABLE = False +_REMOTE_UNAVAILABLE = False +_NETWORK_DEADLINE: float | None = None +_NETWORK_LAST_ERROR = "" +_NETWORK_STATS = { + "local_probe_attempts": 0, + "local_requests": 0, + "remote_attempts": 0, + "remote_retries": 0, + "local_unavailable": False, + "remote_unavailable": False, +} + + +def reset_network_state(deadline_seconds: float | None = None) -> None: + """Reset per-command network state and optionally set a hard deadline.""" + global _LOCAL_UNAVAILABLE, _REMOTE_ONLY, _REMOTE_UNAVAILABLE, _NETWORK_DEADLINE, _NETWORK_STATS, _NETWORK_LAST_ERROR + _REMOTE_ONLY = False + _LOCAL_UNAVAILABLE = False + _REMOTE_UNAVAILABLE = False + _NETWORK_DEADLINE = time.monotonic() + deadline_seconds if deadline_seconds is not None else None + _NETWORK_LAST_ERROR = "" + _NETWORK_STATS = { + "local_probe_attempts": 0, + "local_requests": 0, + "remote_attempts": 0, + "remote_retries": 0, + "local_unavailable": False, + "remote_unavailable": False, + } + + +def network_snapshot() -> dict: + remaining = None if _NETWORK_DEADLINE is None else max(0, int(_NETWORK_DEADLINE - time.monotonic())) + if _REMOTE_ONLY: + path = "remote" + elif _REMOTE_UNAVAILABLE: + path = "unavailable" + elif _LOCAL_UNAVAILABLE: + path = "local-failed" + else: + path = "local" + return {**_NETWORK_STATS, "path": path, "deadline_remaining_seconds": remaining} + + +def _remaining_budget() -> float | None: + if _NETWORK_DEADLINE is None: + return None + return max(0.0, _NETWORK_DEADLINE - time.monotonic()) + + +def _bounded_timeout(timeout: tuple[float, float] | None) -> tuple[float, float] | None: + remaining = _remaining_budget() + if remaining is None: + return timeout + if remaining <= 0: + return (0.1, 0.1) + base = timeout or default_timeout() + # Version discovery uses a separate bounded remote fallback. Keep one + # stalled local request from consuming the entire command budget. + base = (min(base[0], 2.0), min(base[1], 2.0)) + return (min(base[0], remaining), min(base[1], remaining)) def default_timeout() -> tuple[float, float]: @@ -141,28 +203,61 @@ def remote_get_json(url: str, params: dict | None, headers: dict | None) -> tupl # Import lazily so http.py stays usable without a remote profile. from libs import remote + global _REMOTE_UNAVAILABLE, _NETWORK_STATS, _NETWORK_LAST_ERROR + if _REMOTE_UNAVAILABLE: + return None, None, _NETWORK_LAST_ERROR or "remote egress unavailable after retry budget was exhausted" + + remaining = _remaining_budget() + if remaining is not None and remaining <= 0: + _REMOTE_UNAVAILABLE = True + _NETWORK_STATS["remote_unavailable"] = True + return None, None, "network scan deadline exceeded" + host = remote.ssh_host() if not host: + _REMOTE_UNAVAILABLE = True + _NETWORK_STATS["remote_unavailable"] = True return None, None, "local network unreachable and no remote SSH host configured" secret_path = remote.resolve_secret_path() if not secret_path.exists(): + _REMOTE_UNAVAILABLE = True + _NETWORK_STATS["remote_unavailable"] = True return None, None, f"local network unreachable and SSH secret not found: {secret_path}" target = url if params: target = f"{target}{'&' if '?' in target else '?'}{urlencode(params)}" - parts = ["curl", "-fsSL", "--connect-timeout", "5", "--max-time", "20"] - for key, value in (headers or {}).items(): - parts.extend(["-H", f"{key}: {value}"]) - parts.append(target) - script = " ".join(shlex.quote(part) for part in parts) - result = remote.run_ssh(host, remote.ssh_user(), secret_path, script, connect_timeout=5) - if result.returncode != 0: - return None, None, (result.stderr or result.stdout or "remote fetch failed").strip() - try: - return json.loads(result.stdout), {}, None - except json.JSONDecodeError as error: - return None, None, f"remote response was not JSON: {error}" + # Keep the remote fallback bounded as well; a blocked local path should not + # turn version discovery into a long retry cycle on the remote host. + attempts = 2 + last_error = "remote fetch failed" + for attempt in range(attempts): + remaining = _remaining_budget() + if remaining is not None and remaining <= 0: + break + curl_timeout = min(4.0, remaining) if remaining is not None else 4.0 + parts = ["curl", "-fsSL", "--connect-timeout", "2", "--max-time", str(max(1, int(curl_timeout)))] + for key, value in (headers or {}).items(): + parts.extend(["-H", f"{key}: {value}"]) + parts.append(target) + script = " ".join(shlex.quote(part) for part in parts) + _NETWORK_STATS["remote_attempts"] += 1 + connect_timeout = min(3, max(1, int(remaining))) if remaining is not None else 3 + result = remote.run_ssh(host, remote.ssh_user(), secret_path, script, connect_timeout=connect_timeout) + if result.returncode == 0: + try: + return json.loads(result.stdout), {}, None + except json.JSONDecodeError as error: + last_error = f"remote response was not JSON: {error}" + else: + last_error = (result.stderr or result.stdout or "remote fetch failed").strip() + if attempt == 0: + _NETWORK_STATS["remote_retries"] += 1 + + _REMOTE_UNAVAILABLE = True + _NETWORK_STATS["remote_unavailable"] = True + _NETWORK_LAST_ERROR = last_error + return None, None, last_error def get_json_with_fallback( @@ -180,22 +275,53 @@ def get_json_with_fallback( After one successful remote fallback, later calls in the same process skip the local probe and go straight to the remote path. """ - global _REMOTE_ONLY - - if not _REMOTE_ONLY: - probe = probe_url(url) + global _LOCAL_UNAVAILABLE, _REMOTE_ONLY, _REMOTE_UNAVAILABLE, _NETWORK_STATS, _NETWORK_LAST_ERROR + + remaining = _remaining_budget() + if remaining is not None and remaining <= 0: + _LOCAL_UNAVAILABLE = True + _REMOTE_UNAVAILABLE = True + _NETWORK_STATS["local_unavailable"] = True + _NETWORK_STATS["remote_unavailable"] = True + return None, None, "network scan deadline exceeded" + + if not _REMOTE_ONLY and not _LOCAL_UNAVAILABLE and not _REMOTE_UNAVAILABLE: + probe_timeout = min(2.0, remaining) if remaining is not None else 2.0 + probe_retries = 0 if _NETWORK_DEADLINE is not None else 2 + probe = probe_url(url, timeout=max(0.1, probe_timeout), retries=probe_retries) + _NETWORK_STATS["local_probe_attempts"] += probe.attempts if probe.ok: try: - response = get(url, params, timeout=timeout, headers=headers) + _NETWORK_STATS["local_requests"] += 1 + response = get(url, params, timeout=_bounded_timeout(timeout), headers=headers) if response.status_code == 429: - time.sleep(1) - response = get(url, params, timeout=timeout, headers=headers) + remaining = _remaining_budget() + if remaining is not None: + time.sleep(min(1, remaining)) + else: + time.sleep(1) + response = get(url, params, timeout=_bounded_timeout(timeout), headers=headers) response.raise_for_status() return response.json(), dict(response.headers), None except Exception as error: - return None, None, str(error) + _LOCAL_UNAVAILABLE = True + _NETWORK_STATS["local_unavailable"] = True + local_error = str(error) + else: + _LOCAL_UNAVAILABLE = True + _NETWORK_STATS["local_unavailable"] = True + local_error = probe.error or "local network probe failed" + else: + local_error = "local network path unavailable" + + if _REMOTE_UNAVAILABLE: + return None, None, _NETWORK_LAST_ERROR or "remote egress unavailable after retry budget was exhausted" data, response_headers, error = remote_get_json(url, params, headers) if not error: _REMOTE_ONLY = True + else: + _REMOTE_UNAVAILABLE = True + _NETWORK_STATS["remote_unavailable"] = True + _NETWORK_LAST_ERROR = error return data, response_headers, error diff --git a/cli/libs/main.py b/cli/libs/main.py index 42792f01a..7de2f902f 100644 --- a/cli/libs/main.py +++ b/cli/libs/main.py @@ -168,6 +168,7 @@ def scan_command( page_size: int = typer.Option(100, help="Number of tags per Docker Hub page"), major_ahead: int = typer.Option(3, help="How many future majors to probe"), app_name: str | None = typer.Option(None, "--app", help="Scan only this app"), + network_timeout: float = typer.Option(15.0, "--network-timeout", min=1.0, max=120.0, help="Total seconds for upstream network checks"), as_json: bool = typer.Option(False, "--json", help="Machine-readable output"), ) -> None: """Scan upstream versions for selected apps.""" @@ -178,6 +179,7 @@ def scan_command( page_size=page_size, major_ahead=major_ahead, app_filter=app_name, + network_timeout=network_timeout, ) print_output(payload, as_json) @@ -599,6 +601,7 @@ def app_deploy_command( version: str | None = typer.Option(None, "--version", help="Deploy a specific image tag by overriding W9_VERSION; repo .env is not modified"), down: bool = typer.Option(False, "--down", help="Tear the app down with docker compose down -v instead of up -d"), auto_port: bool = typer.Option(True, "--auto-port/--no-auto-port", help="Automatically use a free 9000-9999 host port when a published port is occupied"), + pull_policy: str = typer.Option("always", "--pull-policy", help="always | missing | never"), progress: bool = typer.Option(False, "--progress", help="Show step progress on stderr"), verbose: bool = typer.Option(False, "--verbose", help="Show step progress and raw subprocess output on stderr"), as_json: bool = typer.Option(False, "--json", help="Machine-readable output"), @@ -616,6 +619,7 @@ def app_deploy_command( version=version, down=down, auto_port=auto_port, + pull_policy=pull_policy, progress=progress_writer, verbose=verbose, ) diff --git a/cli/libs/sources.py b/cli/libs/sources.py index 15b8f91d0..c5b1c6298 100644 --- a/cli/libs/sources.py +++ b/cli/libs/sources.py @@ -283,25 +283,28 @@ def version_key(name): def _dockerhub_verify(api_url: str, candidate: str, page_size: int) -> dict | None: candidate_ver = version.parse(normalize_tag(candidate)) - params = {"page_size": page_size, "name": candidate.lstrip("v")} - data, error = _get_json(api_url, params) - if error or not data: - return None + queries = [candidate] + stripped = candidate.lstrip("v") + if stripped != candidate: + queries.append(stripped) - best = None - for tag in (data or {}).get("results") or []: - tag_name = tag["name"] - try: - if version.parse(normalize_tag(tag_name)) == candidate_ver: - best = { - "version": candidate, - "last_updated": tag["last_updated"], - } - break - except version.InvalidVersion: + for query in queries: + params = {"page_size": page_size, "name": query} + data, error = _get_json(api_url, params) + if error or not data: continue - return best + for tag in (data or {}).get("results") or []: + tag_name = tag["name"] + try: + if version.parse(normalize_tag(tag_name)) == candidate_ver: + return { + "version": tag_name, + "last_updated": tag["last_updated"], + } + except version.InvalidVersion: + continue + return None def fetch_verified_candidates( @@ -331,7 +334,7 @@ def fallback() -> tuple[dict | None, str | None]: candidates = _stable_candidates(index_tags, current_version) minors = sorted( - {version.parse(name).release[:2] for name in candidates}, + {version.parse(normalize_tag(name)).release[:2] for name in candidates}, reverse=True, ) wanted = [".".join(str(p) for p in minor) for minor in minors] + candidates diff --git a/cli/libs/versioning.py b/cli/libs/versioning.py index 65801ff87..41b4ef9fa 100644 --- a/cli/libs/versioning.py +++ b/cli/libs/versioning.py @@ -21,6 +21,9 @@ def normalize_tag(name: str) -> str: so prerelease filtering still works. """ normalized = re.sub(r"-ls\d+$", "", name) + # Metabase publishes release trains with an ``.x`` patch placeholder, + # such as ``v0.64.1.x``. Treat it as the numeric release for comparison. + normalized = re.sub(r"\.x$", "", normalized, flags=re.IGNORECASE) if normalized.endswith("-cli"): normalized = normalized[:-4] try: @@ -117,3 +120,22 @@ def find_latest_version(tags: list[dict], current_version: str) -> dict | None: "version": best[1], "last_updated": best[2], } + + +def update_status(current_version: str, candidate: dict | None) -> str: + """Classify a candidate while respecting floating minor image tags. + + A current ``x.x`` tag already receives upstream ``x.x.y`` patch updates, + so a patch candidate in the same minor is informational rather than a + package update. + """ + if not candidate: + return "up-to-date" + + current = version.parse(normalize_tag(current_version)) + latest = version.parse(normalize_tag(str(candidate.get("version", "")))) + if latest <= current: + return "up-to-date" + if len(current.release) == 2 and latest.release[:2] == current.release[:2]: + return "covered-by-current-minor-tag" + return "update-available" diff --git a/cli/libs/versions.py b/cli/libs/versions.py index aa793cffd..2bf3ba524 100644 --- a/cli/libs/versions.py +++ b/cli/libs/versions.py @@ -1,14 +1,16 @@ from __future__ import annotations import json +import time from datetime import date, datetime import typer +from libs import http from libs.metadata import active_app_dirs, resolve_app_metadata from libs.output import print_output from libs.sources import detect_source_type, fetch_candidates, fetch_verified_candidates -from libs.versioning import get_current_versions +from libs.versioning import get_current_versions, update_status app = typer.Typer(help="Scan upstream versions", context_settings={"help_option_names": ["-h", "--help"]}) @@ -56,7 +58,10 @@ def scan_apps( page_size: int = 100, major_ahead: int = 3, app_filter: str | None = None, + network_timeout: float = 15.0, ) -> list[dict]: + started = time.monotonic() + http.reset_network_state(network_timeout) target = _parse_target_date(target_date) output = [] @@ -97,12 +102,16 @@ def scan_apps( if not variables.get("release", False) and not app_filter: payload["error"] = "App release=false" payload["latest_version"] = None + payload["network"] = http.network_snapshot() + payload["scan_elapsed_ms"] = int((time.monotonic() - started) * 1000) output.append(payload) continue if not current_versions or ("latest" in current_version_strs and len(current_version_strs) == 1): payload["error"] = "No valid current versions found" payload["latest_version"] = None + payload["network"] = http.network_snapshot() + payload["scan_elapsed_ms"] = int((time.monotonic() - started) * 1000) output.append(payload) continue @@ -129,9 +138,13 @@ def scan_apps( if error: payload["error"] = f"Failed to fetch tags: {error}" payload["latest_version"] = None + payload["version_status"] = "unavailable" else: payload["latest_version"] = latest_version + payload["version_status"] = update_status(str(highest_version), latest_version) + payload["network"] = http.network_snapshot() + payload["scan_elapsed_ms"] = int((time.monotonic() - started) * 1000) output.append(payload) return output @@ -144,8 +157,9 @@ def scan( plan_only: bool = typer.Option(False, help="Only list selected apps without remote version checks"), max_pages: int = typer.Option(1, help="Deprecated; ignored"), page_size: int = typer.Option(100, help="Number of tags per Docker Hub page"), + network_timeout: float = typer.Option(15.0, "--network-timeout", min=1.0, max=120.0, help="Total seconds for upstream network checks"), as_json: bool = typer.Option(False, "--json", help="Machine-readable output"), ) -> None: _ = max_pages - output = scan_apps(selection=selection, target_date=target_date, plan_only=plan_only, page_size=page_size) + output = scan_apps(selection=selection, target_date=target_date, plan_only=plan_only, page_size=page_size, network_timeout=network_timeout) print_output(output, as_json) diff --git a/cli/tests/__pycache__/conftest.cpython-312-pytest-9.1.1.pyc b/cli/tests/__pycache__/conftest.cpython-312-pytest-9.1.1.pyc index a438ecddd..07040f5f1 100644 Binary files a/cli/tests/__pycache__/conftest.cpython-312-pytest-9.1.1.pyc and b/cli/tests/__pycache__/conftest.cpython-312-pytest-9.1.1.pyc differ diff --git a/cli/tests/__pycache__/test_app_deploy.cpython-312-pytest-9.1.1.pyc b/cli/tests/__pycache__/test_app_deploy.cpython-312-pytest-9.1.1.pyc index 5e267969b..15f8304f2 100644 Binary files a/cli/tests/__pycache__/test_app_deploy.cpython-312-pytest-9.1.1.pyc and b/cli/tests/__pycache__/test_app_deploy.cpython-312-pytest-9.1.1.pyc differ diff --git a/cli/tests/__pycache__/test_http.cpython-312-pytest-9.1.1.pyc b/cli/tests/__pycache__/test_http.cpython-312-pytest-9.1.1.pyc index 381ab7107..4d52bee04 100644 Binary files a/cli/tests/__pycache__/test_http.cpython-312-pytest-9.1.1.pyc and b/cli/tests/__pycache__/test_http.cpython-312-pytest-9.1.1.pyc differ diff --git a/cli/tests/__pycache__/test_sources_fetch.cpython-312-pytest-9.1.1.pyc b/cli/tests/__pycache__/test_sources_fetch.cpython-312-pytest-9.1.1.pyc index ca0b3a399..34ac25e0b 100644 Binary files a/cli/tests/__pycache__/test_sources_fetch.cpython-312-pytest-9.1.1.pyc and b/cli/tests/__pycache__/test_sources_fetch.cpython-312-pytest-9.1.1.pyc differ diff --git a/cli/tests/__pycache__/test_versioning.cpython-312-pytest-9.1.1.pyc b/cli/tests/__pycache__/test_versioning.cpython-312-pytest-9.1.1.pyc index 7d7f039dc..e74ae7cb1 100644 Binary files a/cli/tests/__pycache__/test_versioning.cpython-312-pytest-9.1.1.pyc and b/cli/tests/__pycache__/test_versioning.cpython-312-pytest-9.1.1.pyc differ diff --git a/cli/tests/__pycache__/test_versions.cpython-312-pytest-9.1.1.pyc b/cli/tests/__pycache__/test_versions.cpython-312-pytest-9.1.1.pyc index 3b0e45a41..dddb09b0a 100644 Binary files a/cli/tests/__pycache__/test_versions.cpython-312-pytest-9.1.1.pyc and b/cli/tests/__pycache__/test_versions.cpython-312-pytest-9.1.1.pyc differ diff --git a/cli/tests/test_app_deploy.py b/cli/tests/test_app_deploy.py index 6f5e16f90..0400fe736 100644 --- a/cli/tests/test_app_deploy.py +++ b/cli/tests/test_app_deploy.py @@ -171,7 +171,7 @@ def test_app_deploy_remote_version_patches_remote_env(monkeypatch, repo_fixture, assert payload["version"] == "7.0" assert any("W9_VERSION='7.0'" in call for call in calls) - assert any("pull db" in call for call in calls) + assert any("pull --policy always db" in call for call in calls) assert not any("pull jobs" in call for call in calls) assert build_calls == [("demo", "remote", {"ssh_host": "1.2.3.4", "ssh_user": "root", "ssh_secret_path": str(repo_fixture / ".secrets" / "ssh" / "default.pem"), "deploy_root": "/websoft9/library/apps", "progress": None})] @@ -200,6 +200,23 @@ def fake_run(command, *, progress=None, verbose=False): assert not any(command[:4] == ["docker", "compose", "--progress", "plain"] for command in calls) +def test_app_deploy_never_pull_uses_existing_images(monkeypatch, repo_fixture, app_factory): + app_factory("demo", compose="services:\n demo:\n image: nginx:latest\n") + calls = [] + + def fake_run(command, *, progress=None, verbose=False): + calls.append(command) + return types.SimpleNamespace(returncode=0, stdout="ok", stderr="") + + monkeypatch.setattr(app_deploy, "_run", fake_run) + + payload = app_deploy.deploy("demo", target="local", pull_policy="never") + + assert payload["pull_policy"] == "never" + assert not any("pull" in command[-3:] and "--pull" not in command[-3:] for command in calls) + assert any(command[-4:] == ["up", "-d", "--pull", "never"] for command in calls) + + def test_compose_published_ports_resolves_env_reference(repo_fixture, app_factory): app = app_factory( "demo", diff --git a/cli/tests/test_http.py b/cli/tests/test_http.py index 58abf2399..eb1e2119f 100644 --- a/cli/tests/test_http.py +++ b/cli/tests/test_http.py @@ -1,5 +1,6 @@ from __future__ import annotations +import subprocess from pathlib import Path import pytest @@ -9,9 +10,9 @@ @pytest.fixture(autouse=True) def _reset_remote_only(): - http._REMOTE_ONLY = False + http.reset_network_state() yield - http._REMOTE_ONLY = False + http.reset_network_state() def test_default_timeout_uses_env_and_falls_back_on_invalid(monkeypatch): @@ -103,3 +104,69 @@ def test_get_json_with_fallback_uses_remote_after_probe_failure(monkeypatch): assert data == {"ok": True} assert headers == {} assert error is None + + +def test_get_json_with_fallback_reuses_remote_failure_without_retrying(monkeypatch): + calls = {"remote": 0} + + monkeypatch.setattr(http, "probe_url", lambda url, **kwargs: http.Reachability(False, url, 3, 5, error="timeout")) + + def fail_remote(*args, **kwargs): + calls["remote"] += 1 + return None, None, "ssh timeout" + + monkeypatch.setattr(http, "remote_get_json", fail_remote) + + first = http.get_json_with_fallback("https://hub.docker.com/tags") + second = http.get_json_with_fallback("https://hub.docker.com/tags") + + assert first[2] == "ssh timeout" + assert second[2] == "ssh timeout" + assert calls["remote"] == 1 + assert http.network_snapshot()["path"] == "unavailable" + + +def test_remote_json_fallback_uses_short_curl_timeouts(monkeypatch): + from libs import remote + + captured = {} + + def fake_run_ssh(host, user, secret_path, script, connect_timeout=15): + captured["script"] = script + return subprocess.CompletedProcess(["ssh"], 0, '{"ok": true}', "") + + monkeypatch.setattr(remote, "ssh_host", lambda: "remote.example") + monkeypatch.setattr(remote, "ssh_user", lambda: "root") + monkeypatch.setattr(remote, "resolve_secret_path", lambda: Path("/tmp/key")) + monkeypatch.setattr(Path, "exists", lambda self: True) + monkeypatch.setattr(remote, "run_ssh", fake_run_ssh) + + data, _, error = http.remote_get_json("https://example.com/api", {"page": 1}, None) + + assert data == {"ok": True} + assert error is None + assert "--connect-timeout 2" in captured["script"] + assert "--max-time 4" in captured["script"] + + +def test_remote_json_fallback_retries_ssh_once(monkeypatch): + from libs import remote + + calls = [] + + def fake_run_ssh(host, user, secret_path, script, connect_timeout=15): + calls.append(connect_timeout) + return subprocess.CompletedProcess(["ssh"], 1, "", "connection timeout") + + monkeypatch.setattr(remote, "ssh_host", lambda: "remote.example") + monkeypatch.setattr(remote, "ssh_user", lambda: "root") + monkeypatch.setattr(remote, "resolve_secret_path", lambda: Path("/tmp/key")) + monkeypatch.setattr(Path, "exists", lambda self: True) + monkeypatch.setattr(remote, "run_ssh", fake_run_ssh) + + data, _, error = http.remote_get_json("https://example.com/api", None, None) + + assert data is None + assert error == "connection timeout" + assert calls == [3, 3] + assert http.network_snapshot()["remote_retries"] == 1 diff --git a/cli/tests/test_sources_fetch.py b/cli/tests/test_sources_fetch.py index f5561873d..b7943f08f 100644 --- a/cli/tests/test_sources_fetch.py +++ b/cli/tests/test_sources_fetch.py @@ -86,3 +86,55 @@ def test_fetch_verified_candidates_prefers_verified_dockerhub_candidate(monkeypa assert error is None assert latest == {"version": "1.2", "last_updated": "2026-02-01"} + + +def test_fetch_verified_candidates_accepts_x_placeholder_release_tags(monkeypatch): + monkeypatch.setattr( + sources, + "_github_tags_raw", + lambda url, page_size: ([{"name": "v1.52.5.x", "last_updated": "2026-02-01"}], None), + ) + monkeypatch.setattr( + sources, + "_dockerhub_api_url", + lambda url: "https://hub.docker.com/v2/repositories/example/app/tags", + ) + monkeypatch.setattr( + sources, + "_dockerhub_verify", + lambda api_url, candidate, page_size: { + "version": "v1.52.5.x", + "last_updated": "2026-02-01", + } if candidate == "1.52" or candidate == "1.52.5" else None, + ) + + latest, error = sources.fetch_verified_candidates( + verify_type="dockerhub-tags", + verify_url="https://hub.docker.com/r/example/app", + index_type="github-tags", + index_url="https://github.com/example/app", + current_version="v1.52.4.x", + ) + + assert error is None + assert latest == {"version": "v1.52.5.x", "last_updated": "2026-02-01"} + + +def test_dockerhub_verify_preserves_v_prefixed_tag_query(monkeypatch): + calls = [] + + def fake_get_json(url, params=None, headers=None): + calls.append(params["name"]) + results = [{"name": "v0.64.1.x", "last_updated": "2026-10-07"}] if params["name"] == "v0.64.1.x" else [] + return {"results": results}, None + + monkeypatch.setattr(sources, "_get_json", fake_get_json) + + result = sources._dockerhub_verify( + "https://hub.docker.com/v2/repositories/metabase/metabase/tags", + "v0.64.1.x", + 100, + ) + + assert result == {"version": "v0.64.1.x", "last_updated": "2026-10-07"} + assert calls == ["v0.64.1.x"] diff --git a/cli/tests/test_versioning.py b/cli/tests/test_versioning.py index a1cec5584..9c399079f 100644 --- a/cli/tests/test_versioning.py +++ b/cli/tests/test_versioning.py @@ -2,7 +2,7 @@ from packaging import version -from libs.versioning import find_latest_version, get_current_versions, normalize_tag +from libs.versioning import find_latest_version, get_current_versions, normalize_tag, update_status def test_normalize_tag_strips_ls_build_and_cli_suffixes(): @@ -12,6 +12,11 @@ def test_normalize_tag_strips_ls_build_and_cli_suffixes(): assert normalize_tag("version-9.0-cli") == "version-9.0" +def test_normalize_tag_strips_metabase_patch_placeholder(): + assert normalize_tag("v0.64.1.x") == "v0.64.1" + assert normalize_tag("v0.64.x") == "v0.64" + + def test_get_current_versions_accepts_ls_suffixed_tags(): valid, all_versions = get_current_versions( [{"dist": "community", "version": ["9.0-cli-ls78", "latest"]}] @@ -82,6 +87,12 @@ def test_find_latest_version_uses_patch_candidates_when_no_minor_candidate_exist } +def test_update_status_treats_same_minor_patch_as_covered_by_floating_tag(): + assert update_status("13.2", {"version": "13.2.4"}) == "covered-by-current-minor-tag" + assert update_status("13.2", {"version": "13.3"}) == "update-available" + assert update_status("13.2", None) == "up-to-date" + + def test_normalize_tag_strips_image_variants_but_preserves_prereleases(): from packaging import version as packaging_version diff --git a/cli/tests/test_versions.py b/cli/tests/test_versions.py index 3aaf2fa1a..7f2f8cf68 100644 --- a/cli/tests/test_versions.py +++ b/cli/tests/test_versions.py @@ -90,3 +90,22 @@ def test_scan_apps_uses_fetch_candidates_when_release_enabled(repo_fixture, app_ result = versions.scan_apps(app_filter="demo") assert result[0]["latest_version"] == {"version": "1.1", "last_updated": "2026-01-01"} + assert result[0]["version_status"] == "update-available" + + +def test_scan_apps_reports_same_minor_patch_as_covered(repo_fixture, app_factory, monkeypatch): + app_factory("grafana", variables={ + "name": "grafana", + "release": True, + "upstream": {"image": "https://hub.docker.com/r/grafana/grafana"}, + "edition": [{"dist": "community", "version": ["13.2", "latest"]}], + }) + monkeypatch.setattr( + versions, + "fetch_candidates", + lambda *args, **kwargs: ({"version": "13.2.4", "last_updated": "2026-10-13"}, None), + ) + + result = versions.scan_apps(app_filter="grafana") + + assert result[0]["version_status"] == "covered-by-current-minor-tag" diff --git a/i18n/README.md b/i18n/README.md index 89aee755c..1fb2abb40 100644 --- a/i18n/README.md +++ b/i18n/README.md @@ -1,13 +1,31 @@ # i18n -`translation.json` holds translatable env labels for downstream repos. +`translation.json` holds translatable env labels. Translatable keys are +`W9_*_SET` and `W9_LOGIN*`. Values are `["English label", "中文标签"]`. -Mechanism: -- Translatable keys: `W9_*_SET` and `W9_LOGIN*` -- `create.py` scans `apps/*/.env` and registers missing keys as `["", ""]` -- Values are filled by humans: `["English label", "中文标签"]` -- `.github/workflows/i18n.yml` pushes updates to the plugin repos by PR +## Commands -Rules for AI: -- register new translatable keys -- do not author final translations +```bash +make i18n-check # check all apps; fails only on missing keys +make i18n-check ARGS="--strict" # also fail when a used key has an empty translation +make i18n-check ARGS="--app drupal" +make i18n-check ARGS="--warn-only" # report missing/untranslated keys, exit 0 +make i18n-update # add missing keys as ["", ""] +make i18n-update ARGS="--app drupal" +``` + +The PR check uses the same entrypoint, restricted to keys added by the PR. It +is warning-only and never blocks a PR: + +```bash +make i18n-check ARGS="--base --head --github-annotations --warn-only" +``` + +`i18n/manage.py` is the single implementation. `create.py` is kept as a thin +`--update` wrapper for older local scripts. + +## Rules for AI + +- Register new translatable keys in the same PR that adds them. +- New keys may stay `["", ""]`; do not author final translations. +- Use `--strict` when you want to audit translation completeness. diff --git a/i18n/create.py b/i18n/create.py index dfb96ca48..35f65073e 100644 --- a/i18n/create.py +++ b/i18n/create.py @@ -1,23 +1,5 @@ -import glob -import json -from dotenv import dotenv_values +#!/usr/bin/env python3 +from manage import main -# 读取已有的 translation.json 文件 -with open('translation.json', 'r') as f: - translation = json.load(f) - -# 遍历所有 .env 文件 -for filename in glob.glob('../apps/*/.env'): - # 解析 .env 文件 - env_dict = dotenv_values(filename) - # 遍历所有环境变量 - for key in env_dict.keys(): - # 如果 key 以 'W9_' 开头并以 '_SET' 结尾,或者以 'W9_LOGIN' 开头 - if (key.startswith('W9_') and key.endswith('_SET')) or key.startswith('W9_LOGIN'): - # 如果 key 不在 translation.json 中,将其添加到 translation.json 中 - if key not in translation: - translation[key] = ["", ""] - -# 将修改后的 translation.json 写回到文件中 -with open('translation.json', 'w') as f: - json.dump(translation, f, indent=4, ensure_ascii=False) \ No newline at end of file +if __name__ == "__main__": + raise SystemExit(main(["--update"])) diff --git a/i18n/manage.py b/i18n/manage.py new file mode 100755 index 000000000..1f7dc0aa2 --- /dev/null +++ b/i18n/manage.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_TRANSLATION = ROOT / "i18n" / "translation.json" +TRANSLATABLE_KEY_RE = re.compile(r"^(W9_.*_SET|W9_LOGIN.*)$") +ENV_LINE_RE = re.compile(r"^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=") + + +def is_translatable(key: str) -> bool: + return bool(TRANSLATABLE_KEY_RE.match(key)) + + +def parse_env_keys(text: str) -> set[str]: + keys: set[str] = set() + for raw_line in text.splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + match = ENV_LINE_RE.match(line) + if match and is_translatable(match.group(1)): + keys.add(match.group(1)) + return keys + + +def added_env_keys(diff_text: str) -> set[str]: + keys: set[str] = set() + for line in diff_text.splitlines(): + if not line.startswith("+") or line.startswith("+++"): + continue + content = line[1:].strip() + if not content or content.startswith("#"): + continue + match = ENV_LINE_RE.match(content) + if match and is_translatable(match.group(1)): + keys.add(match.group(1)) + return keys + + +def missing_keys(keys: set[str], translation: dict) -> list[str]: + return sorted(key for key in keys if key not in translation) + + +def untranslated_keys(keys: set[str], translation: dict) -> list[str]: + """Return keys whose English or Chinese translation is still empty.""" + result = [] + for key in sorted(keys): + values = translation.get(key) + if not isinstance(values, list) or len(values) < 2: + continue + if not str(values[0]).strip() or not str(values[1]).strip(): + result.append(key) + return result + + +def app_from_path(path: Path) -> str: + parts = path.parts + if "apps" in parts: + index = parts.index("apps") + if index + 1 < len(parts): + return parts[index + 1] + return "" + + +def app_env_files(app: str | None = None) -> list[Path]: + files = sorted((ROOT / "apps").glob("*/.env")) + if app: + return [path for path in files if app_from_path(path) == app] + return files + + +def changed_env_files(base: str, head: str, app: str | None = None) -> list[Path]: + result = subprocess.run( + ["git", "diff", "--name-only", base, head, "--", "apps/*/.env"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + files = [ROOT / line for line in result.stdout.splitlines() if line.strip()] + if app: + files = [path for path in files if app_from_path(path) == app] + return files + + +def file_added_keys(base: str, head: str, path: Path) -> set[str]: + result = subprocess.run( + ["git", "diff", "--unified=0", base, head, "--", str(path.relative_to(ROOT))], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + return added_env_keys(result.stdout) + + +def collect_keys( + *, + base: str | None = None, + head: str | None = None, + app: str | None = None, +) -> dict[str, set[str]]: + if bool(base) != bool(head): + raise ValueError("--base and --head must be provided together") + if base and head: + files = changed_env_files(base, head, app) + return {app_from_path(path): file_added_keys(base, head, path) for path in files} + return {app_from_path(path): parse_env_keys(path.read_text(encoding="utf-8")) for path in app_env_files(app)} + + +def write_translation(path: Path, translation: dict) -> None: + path.write_text(json.dumps(translation, indent=4, ensure_ascii=False), encoding="utf-8") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Check or update i18n/translation.json for W9_*_SET and W9_LOGIN* env keys." + ) + parser.add_argument("--check", action="store_true", help="Check only; exit 1 when keys are missing") + parser.add_argument("--update", action="store_true", help='Add missing keys as ["", ""]') + parser.add_argument("--base", help="Base commit SHA; only inspect keys added between base and head") + parser.add_argument("--head", help="Head commit SHA; only inspect keys added between base and head") + parser.add_argument("--app", help="Limit to one app") + parser.add_argument("--translation", type=Path, default=DEFAULT_TRANSLATION, help="translation.json path") + parser.add_argument("--strict", action="store_true", help="Fail when a used key has an empty translation") + parser.add_argument("--github-annotations", action="store_true", help="Emit GitHub error/warning annotations") + parser.add_argument("--warn-only", action="store_true", help="Never fail on missing or untranslated keys") + parser.add_argument("--json", action="store_true", help="Machine-readable output") + args = parser.parse_args(argv) + + if not args.check and not args.update: + parser.error("pass --check or --update") + + translation_path = args.translation if args.translation.is_absolute() else ROOT / args.translation + if not translation_path.exists(): + print(f"missing translation file: {translation_path}", file=sys.stderr) + return 1 + translation = json.loads(translation_path.read_text(encoding="utf-8")) + + try: + keys_by_app = collect_keys(base=args.base, head=args.head, app=args.app) + except (ValueError, subprocess.CalledProcessError) as error: + print(str(error), file=sys.stderr) + return 2 + + missing_by_app = {} + untranslated_by_app = {} + for app, keys in sorted(keys_by_app.items()): + missing = missing_keys(keys, translation) + if missing: + missing_by_app[app] = missing + untranslated = untranslated_keys(keys, translation) + if untranslated: + untranslated_by_app[app] = untranslated + + if args.update: + added = [] + for app, keys in sorted(missing_by_app.items()): + for key in keys: + translation[key] = ["", ""] + added.append({"app": app, "key": key}) + if added: + write_translation(translation_path, translation) + if args.json: + print(json.dumps({ + "updated": added, + "untranslated": untranslated_by_app, + "translation": str(translation_path.relative_to(ROOT)), + }, indent=2, ensure_ascii=False)) + else: + for item in added: + print(f"added {item['key']} ({item['app']})") + if not added: + print("no missing i18n keys") + for app, keys in sorted(untranslated_by_app.items()): + print(f"needs translation ({app}): {', '.join(keys)}") + return 0 + + ok = not missing_by_app + strict_ok = ok and not untranslated_by_app + if args.json: + print(json.dumps({ + "missing": missing_by_app, + "untranslated": untranslated_by_app, + "ok": ok, + "strict_ok": strict_ok, + }, indent=2, ensure_ascii=False)) + elif args.github_annotations: + missing_level = "warning" if args.warn_only else "error" + for app, keys in sorted(missing_by_app.items()): + joined = ", ".join(keys) + print(f"::{missing_level} file=apps/{app}/.env,title=Missing i18n keys::{joined}") + for app, keys in sorted(untranslated_by_app.items()): + joined = ", ".join(keys) + print(f"::warning file=apps/{app}/.env,title=Untranslated i18n keys::{joined}") + if missing_by_app: + print("add missing keys to i18n/translation.json (empty values are allowed)", file=sys.stderr) + elif untranslated_by_app and not args.strict: + print("some keys still have empty translations; use --strict to fail on them", file=sys.stderr) + else: + for app, keys in sorted(missing_by_app.items()): + print(f"missing ({app}): {', '.join(keys)}") + for app, keys in sorted(untranslated_by_app.items()): + print(f"needs translation ({app}): {', '.join(keys)}") + if missing_by_app: + print("add missing keys to i18n/translation.json (empty values are allowed)", file=sys.stderr) + elif untranslated_by_app and not args.strict: + print("some keys still have empty translations; use --strict to fail on them", file=sys.stderr) + if ok and not untranslated_by_app: + print("i18n keys OK") + elif ok: + print("i18n keys registered; translations pending") + if args.warn_only: + return 0 + return 0 if (strict_ok if args.strict else ok) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/i18n/pr.py b/i18n/pr.py deleted file mode 100644 index 2a04daf0e..000000000 --- a/i18n/pr.py +++ /dev/null @@ -1,75 +0,0 @@ -import json -import argparse - -# 设置命令行参数 -parser = argparse.ArgumentParser(description='Update JS translation files from a JSON file.') -parser.add_argument('json_file', help='The path to the translation.json file') -parser.add_argument('en_js_file', help='The path to the po.en_US.js file') -parser.add_argument('zh_js_file', help='The path to the po.zh_CN.js file') - -# 解析命令行参数 -args = parser.parse_args() - -# 读取translation.json文件 -with open(args.json_file, "r") as f: - translation = json.load(f) - -# 读取po.en_US.js文件 -with open(args.en_js_file, "r") as f: - en_lines = f.readlines() - -# 读取po.zh_CN.js文件 -with open(args.zh_js_file, "r") as f: - zh_lines = f.readlines() - -# 定义一个函数,用于在js文件中查找或替换key和value -def update_js(lines, key, value): - # 如果值为空字符串,则不更新 - if value == "": - return False - - # 遍历每一行 - for i, line in enumerate(lines): - # 如果找到了key - if f'"{key}": [' in line: - # 替换value, 确保不在最后一行添加逗号 - lines[i+1] = f" null,\n" - lines[i+2] = f" \"{value}\"\n" - # 检查下一行是否是数组结束,如果不是则添加逗号 - if not lines[i+3].strip().startswith(']'): - lines[i+2] = lines[i+2].rstrip() + ",\n" - # 返回True表示已经更新 - return True - # 如果没有找到key,就在最后一行前插入新的key和value - # 确保在倒数第二行添加逗号 - if not lines[-2].strip().endswith(','): - lines[-2] = lines[-2].rstrip() + ",\n" - lines.insert(-1, f" \"{key}\": [\n") - lines.insert(-1, f" null,\n") - lines.insert(-1, f" \"{value}\"\n") - lines.insert(-1, f" ]\n") - # 返回False表示已经插入 - return False - -# 遍历translation.json的每一个键值对 -for key, values in translation.items(): - # 获取英文和中文的value - en_value = values[0] - zh_value = values[1] - # 如果英文值不为空,则在po.en_US.js文件中更新或插入key和value - if en_value: - update_js(en_lines, key, en_value) - # 如果中文值不为空,则在po.zh_CN.js文件中更新或插入key和value - if zh_value: - update_js(zh_lines, key, zh_value) - -# 写入po.en_US.js文件 -with open(args.en_js_file, "w") as f: - f.writelines(en_lines) - -# 写入po.zh_CN.js文件 -with open(args.zh_js_file, "w") as f: - f.writelines(zh_lines) - -# 打印完成信息 -print("同步完成!") diff --git a/i18n/translation.json b/i18n/translation.json index eb0e803b8..60c136a29 100644 --- a/i18n/translation.json +++ b/i18n/translation.json @@ -8,16 +8,16 @@ "应用 HTTP 端口" ], "W9_GRPC_PORT_SET": [ - "", - "" + "GRPC PORT", + "应用 GRPC 端口" ], "W9_AGENT_PORT_SET": [ "App Agent Port", "应用代理端口" ], "W9_ENROLLMENT_PORT_SET": [ - "", - "" + "Enrollment Port", + "注册端口" ], "W9_SSH_PORT_SET": [ "App SSH Port", @@ -132,12 +132,12 @@ "管理员密码" ], "W9_LOGIN_MYSQL_CONNECTION_STRING": [ - "", - "" + "MySQL Connection String", + "MySQL 连接字符串" ], "W9_LOGIN_MYSQL_CONNECTION_STRING_PASSWORD": [ - "", - "" + "MySQL Connection String Password", + "MySQL 连接字符串密码" ], "W9_LOGIN_WORKSPACE": [ "Login Workspace", diff --git a/tests/build/__pycache__/test_fetch_catalog.cpython-312-pytest-9.1.1.pyc b/tests/build/__pycache__/test_fetch_catalog.cpython-312-pytest-9.1.1.pyc index 7b02dd8bd..0fa5bbb3b 100644 Binary files a/tests/build/__pycache__/test_fetch_catalog.cpython-312-pytest-9.1.1.pyc and b/tests/build/__pycache__/test_fetch_catalog.cpython-312-pytest-9.1.1.pyc differ diff --git a/tests/build/__pycache__/test_library_publish.cpython-312-pytest-9.1.1.pyc b/tests/build/__pycache__/test_library_publish.cpython-312-pytest-9.1.1.pyc index d9ae37dd6..adf28cd99 100644 Binary files a/tests/build/__pycache__/test_library_publish.cpython-312-pytest-9.1.1.pyc and b/tests/build/__pycache__/test_library_publish.cpython-312-pytest-9.1.1.pyc differ diff --git a/tests/build/test_i18n_manage.py b/tests/build/test_i18n_manage.py new file mode 100644 index 000000000..1bbb1bd8f --- /dev/null +++ b/tests/build/test_i18n_manage.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "i18n" / "manage.py" + + +def load_manage(): + spec = importlib.util.spec_from_file_location("i18n_manage", SCRIPT) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_parse_env_keys_only_returns_translatable_keys(): + manage = load_manage() + + keys = manage.parse_env_keys( + "W9_HTTP_PORT_SET=9001\n" + "W9_LOGIN_USER=admin\n" + "APP_SECRET=value\n" + "# W9_COMMENTED_SET=9003\n" + ) + + assert keys == {"W9_HTTP_PORT_SET", "W9_LOGIN_USER"} + + +def test_added_env_keys_ignores_context_and_removed_lines(): + manage = load_manage() + diff = """diff --git a/apps/demo/.env b/apps/demo/.env +@@ -1 +1,3 @@ +-W9_OLD_SET=1 ++W9_NEW_SET=2 ++APP_SECRET=value +""" + + assert manage.added_env_keys(diff) == {"W9_NEW_SET"} + + +def test_untranslated_keys_reports_empty_values(): + manage = load_manage() + + assert manage.untranslated_keys( + {"W9_HTTP_PORT_SET", "W9_LOGIN_USER"}, + {"W9_HTTP_PORT_SET": ["HTTP Port", "HTTP 端口"], "W9_LOGIN_USER": ["", "用户"]}, + ) == ["W9_LOGIN_USER"] + + +def test_check_and_update_use_translation_file(tmp_path, monkeypatch): + manage = load_manage() + monkeypatch.setattr(manage, "ROOT", tmp_path) + app = tmp_path / "apps" / "demo" + app.mkdir(parents=True) + (app / ".env").write_text("W9_HTTP_PORT_SET=9001\nW9_SSH_PORT_SET=9022\n", encoding="utf-8") + translation = tmp_path / "translation.json" + translation.write_text(json.dumps({"W9_HTTP_PORT_SET": ["", ""]}), encoding="utf-8") + + assert manage.main(["--check", "--translation", str(translation)]) == 1 + assert manage.main(["--check", "--app", "demo", "--json", "--translation", str(translation)]) == 1 + + assert manage.main(["--update", "--translation", str(translation)]) == 0 + updated = json.loads(translation.read_text(encoding="utf-8")) + assert updated["W9_SSH_PORT_SET"] == ["", ""] + assert manage.main(["--check", "--translation", str(translation)]) == 0 + assert manage.main(["--check", "--strict", "--translation", str(translation)]) == 1 + + +def test_check_emits_github_annotations(tmp_path, monkeypatch, capsys): + manage = load_manage() + monkeypatch.setattr(manage, "ROOT", tmp_path) + app = tmp_path / "apps" / "demo" + app.mkdir(parents=True) + (app / ".env").write_text("W9_HTTP_PORT_SET=9001\nW9_SSH_PORT_SET=9022\n", encoding="utf-8") + translation = tmp_path / "translation.json" + translation.write_text(json.dumps({"W9_HTTP_PORT_SET": ["", ""]}), encoding="utf-8") + + assert manage.main(["--check", "--github-annotations", "--translation", str(translation)]) == 1 + + output = capsys.readouterr().out + assert "::error file=apps/demo/.env,title=Missing i18n keys::W9_SSH_PORT_SET" in output + assert "::warning file=apps/demo/.env,title=Untranslated i18n keys::W9_HTTP_PORT_SET" in output + + +def test_warn_only_emits_warnings_and_exits_zero(tmp_path, monkeypatch, capsys): + manage = load_manage() + monkeypatch.setattr(manage, "ROOT", tmp_path) + app = tmp_path / "apps" / "demo" + app.mkdir(parents=True) + (app / ".env").write_text("W9_HTTP_PORT_SET=9001\nW9_SSH_PORT_SET=9022\n", encoding="utf-8") + translation = tmp_path / "translation.json" + translation.write_text(json.dumps({"W9_HTTP_PORT_SET": ["", ""]}), encoding="utf-8") + + assert manage.main([ + "--check", "--github-annotations", "--warn-only", "--translation", str(translation) + ]) == 0 + + output = capsys.readouterr().out + assert "::warning file=apps/demo/.env,title=Missing i18n keys::W9_SSH_PORT_SET" in output + assert "::warning file=apps/demo/.env,title=Untranslated i18n keys::W9_HTTP_PORT_SET" in output