From 0cdfbad38326cf3d22978c0a551e840d999c14ef Mon Sep 17 00:00:00 2001 From: cocofhu Date: Fri, 20 Nov 2026 17:06:39 +0800 Subject: [PATCH 1/3] feat: pull the faster of ghcr.io and the NJU mirror Time both registries during install and pull so a domestic link does not have to set MIRROR by hand. Co-authored-by: Cursor --- .env.example | 2 +- CHANGELOG.md | 2 + README.md | 6 +- README.zh-CN.md | 6 +- docs/deployment.md | 22 +-- scripts/ghcr-mirror.sh | 311 +++++++++++++++++++++++++++++++++++++--- scripts/test-install.py | 162 ++++++++++++++++++++- start.sh | 13 +- 8 files changed, 477 insertions(+), 47 deletions(-) diff --git a/.env.example b/.env.example index a03f66b3d..8c44bebcb 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,6 @@ # ./start.sh (or bash scripts/configure.sh) generates .env with unique local database/session secrets. # Platform image: empty uses the GHCR release pinned in compose.yaml. Prefer an @sha256: digest when overriding. -# Slow ghcr.io links: MIRROR=ghcr.nju.edu.cn ./start.sh replaces the ghcr.io host and tags the image back. See README.zh-CN.md. +# ./start.sh times ghcr.io and ghcr.nju.edu.cn. MIRROR=ghcr.io or MIRROR=ghcr.nju.edu.cn forces one host. See README.zh-CN.md. LOOPFORGE_IMAGE= # Source build of this checkout (./start.sh dev writes this line; ./start.sh release removes it): # COMPOSE_FILE=compose.yaml:compose.dev.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index d3664e718..9ecfa50e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ Platform releases use `vX.Y.Z` tags and publish `ghcr.io/tencent/loopforge`. Loo Repositories saved earlier with a token in `cloneUrl` keep cloning until they are saved again. Saving one from the studio stores the URL without the token, so first put the Git token in 个人配置, then save the repository. Until then the token also lands in each workspace's `.git/config`, where the agent can read it, so re-save such repositories soon after upgrading. +- `./start.sh` and `./start.sh pull` time `ghcr.io` and `ghcr.nju.edu.cn`, then pull the faster registry. A faster Nanjing image is tagged back to the name pinned in `compose.yaml`. `MIRROR=ghcr.io` forces the upstream registry; `MIRROR=` still forces that host. Accepting Nanjing, or setting `MIRROR`, trusts that host. Different manifest digests stay on `ghcr.io`. The choice is not saved in `.env`. `registry-mirrors` such as `https://docker.nju.edu.cn` only accelerates Docker Hub. + ## 0.3.8 — header sign-in hardening and a GHCR mirror option - With `AGENTFLOW_TRUST_IDENTITY_HEADERS=true`, identity headers such as `Staffname` are ignored on requests that carry `X-Forwarded-For`, `Forwarded` or `X-Real-IP`, unless the gateway also sends `X-AgentFlow-Gateway-Secret` matching the new `AGENTFLOW_IDENTITY_HEADER_SECRET`. A client behind a pass-through reverse proxy could otherwise sign in as any account. Deployments with the option off are unaffected. diff --git a/README.md b/README.md index bfb8f3533..0167e9640 100644 --- a/README.md +++ b/README.md @@ -77,10 +77,12 @@ cd LoopForge `./start.sh` pulls the GHCR release image pinned in `compose.yaml` without building locally (`bash scripts/install.sh` does the same). It creates the MySQL and platform services, generates configuration and session secrets, and installs the built-in sandboxes, starter SOPs and welcome project. Host Go and Node.js installations are not required. Configure model API keys and GitHub authorization after startup. Use `./start.sh logs`, `stop`, `restart` or `pull` to manage it, and `./start.sh dev` to build the checkout from source. -If downloads from `ghcr.io` stall, set `MIRROR` to a registry host that can stand in for `ghcr.io`. `./start.sh` pulls from that host and tags the image back to the name pinned in `compose.yaml`. Setting `MIRROR` trusts that host, because the download is tagged with the official name. The GitHub Release records the digest; `docker image inspect --format '{{index .RepoDigests 0}}' ghcr.io/tencent/loopforge:` should show the same digest after `@`. Without `MIRROR`, `./start.sh` still pulls `ghcr.io` directly, and `compose.yaml` stays unchanged. Upgrade with the same variable and `./start.sh pull`. [ghcr.nju.edu.cn](https://doc.nju.edu.cn/books/e1654/page/ghcr) is one such host. A Docker `registry-mirrors` entry such as `https://docker.nju.edu.cn` only accelerates Docker Hub and is not a value for `MIRROR`. +If downloads from `ghcr.io` stall, `./start.sh` and `./start.sh pull` time `ghcr.io` against the Nanjing University mirror [ghcr.nju.edu.cn](https://doc.nju.edu.cn/books/e1654/page/ghcr) and pull the faster one. A faster Nanjing pull is tagged back to the name pinned in `compose.yaml`, which trusts that host because the result is tagged with the official name. Different manifest digests stay on `ghcr.io`. `MIRROR=ghcr.io` forces the upstream registry and `MIRROR=ghcr.nju.edu.cn` forces Nanjing; the choice is not written to `.env`. The GitHub Release records the digest; `docker image inspect --format '{{index .RepoDigests 0}}' ghcr.io/tencent/loopforge:` should show the same digest after `@`. `compose.yaml` stays unchanged. A Docker `registry-mirrors` entry such as `https://docker.nju.edu.cn` only accelerates Docker Hub and is not a value for `MIRROR`. ```bash -MIRROR=ghcr.nju.edu.cn ./start.sh +./start.sh +MIRROR=ghcr.io ./start.sh +MIRROR=ghcr.nju.edu.cn ./start.sh pull ``` Open the address printed by the installer and create your account; the first one becomes the administrator. The service listens on `0.0.0.0:8090` by default. On a cloud host, the installer reads the public IP from the provider's metadata service and prints it; allow the port in your security group, or put an HTTPS reverse proxy in front. Restrict access to your own IP until the administrator account exists. The home page's quick-access guide walks you through adding a model key, connecting a repository and choosing an SOP. diff --git a/README.zh-CN.md b/README.zh-CN.md index 3f55f4513..d236cf42b 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -77,10 +77,12 @@ cd LoopForge `./start.sh` 直接拉取 `compose.yaml` 中 pin 的 GHCR 发布镜像,不在本地构建(`bash scripts/install.sh` 效果相同)。安装流程会创建 MySQL 和平台服务,生成配置与会话密钥,并导入内置沙箱、通用 SOP 和体验项目;无需在宿主机安装 Go 或 Node.js。模型 API Key 和 GitHub 授权在启动后由用户配置。用 `./start.sh logs`、`stop`、`restart`、`pull` 管理服务;需要从源码构建时用 `./start.sh dev`。 -国内拉取 `ghcr.io` 经常中断。把 `MIRROR` 设成可以替换 `ghcr.io` 的镜像站主机名,`./start.sh` 会用它拉取,再改回 `compose.yaml` 里的原名。设置 `MIRROR` 表示信任该主机,因为下载结果会被标成官方镜像名。发布说明里有 digest,可用 `docker image inspect --format '{{index .RepoDigests 0}}' ghcr.io/tencent/loopforge:` 核对,`@` 后面的摘要应一致。不设置时仍直接从 `ghcr.io` 拉取。`compose.yaml` 不用改。升级时带上同一个变量执行 `./start.sh pull`。南京大学的 GHCR 镜像站是 [ghcr.nju.edu.cn](https://doc.nju.edu.cn/books/e1654/page/ghcr)。Docker 引擎里的 `registry-mirrors`(包括 `https://docker.nju.edu.cn`)只加速 Docker Hub,不能填在这里。 +国内拉取 `ghcr.io` 经常中断。`./start.sh` 和 `./start.sh pull` 会并行探测 `ghcr.io` 与南京大学镜像 [ghcr.nju.edu.cn](https://doc.nju.edu.cn/books/e1654/page/ghcr),对同一个 tag 抽样比较下载速度,走更快的一边。选中南京时,脚本从该主机拉取,再改回 `compose.yaml` 里的原名;这表示信任该主机,因为下载结果会被标成官方镜像名。两边 manifest 的 digest 不一致时仍走 `ghcr.io`。`MIRROR=ghcr.io` 固定走官方源,`MIRROR=ghcr.nju.edu.cn` 固定走南京,选择不会写入 `.env`。发布说明里有 digest,可用 `docker image inspect --format '{{index .RepoDigests 0}}' ghcr.io/tencent/loopforge:` 核对,`@` 后面的摘要应一致。`compose.yaml` 不用改。Docker 引擎里的 `registry-mirrors`(包括 `https://docker.nju.edu.cn`)只加速 Docker Hub,不能填在 `MIRROR` 里。 ```bash -MIRROR=ghcr.nju.edu.cn ./start.sh +./start.sh +MIRROR=ghcr.io ./start.sh +MIRROR=ghcr.nju.edu.cn ./start.sh pull ``` 打开安装脚本输出的地址创建账号,第一个账号自动成为管理员。服务默认监听 `0.0.0.0:8090`;在云主机上,安装脚本会从云厂商的元数据服务读取公网 IP 并直接打印出来,放通安全组端口即可访问,或配置 HTTPS 反向代理;管理员账号创建前,建议只允许你的 IP 访问。首页的「快速接入」会带你配置模型凭据、绑定仓库并选好默认 SOP。 diff --git a/docs/deployment.md b/docs/deployment.md index b2ec4f3b3..77766a804 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -22,14 +22,15 @@ Use `AGENTFLOW_MYSQL_DSN` for your own MySQL deployment. A fresh account command ## Container installation ```bash -./start.sh # install or start in the background (same as bash scripts/install.sh) -./start.sh logs # follow platform logs -./start.sh stop # stop; ./start.sh down removes containers but keeps data volumes -./start.sh restart # restart the platform container -./start.sh pull # refresh images and recreate changed containers -MIRROR=ghcr.nju.edu.cn ./start.sh # same install; replace the ghcr.io host with this registry -./start.sh dev # build this checkout instead (compose.dev.yaml) -./start.sh release # switch a source build back to the published image +./start.sh # install or start in the background (same as bash scripts/install.sh) +./start.sh logs # follow platform logs +./start.sh stop # stop; ./start.sh down removes containers but keeps data volumes +./start.sh restart # restart the platform container +./start.sh pull # refresh images; times ghcr.io against ghcr.nju.edu.cn +MIRROR=ghcr.io ./start.sh # install from ghcr.io without the speed test +MIRROR=ghcr.nju.edu.cn ./start.sh # force that host instead of the speed test +./start.sh dev # build this checkout instead (compose.dev.yaml) +./start.sh release # switch a source build back to the published image ``` The installer checks Docker, Compose, Python 3 and curl. It offers to install missing Python/curl packages using apt, dnf or yum on Linux (Homebrew on macOS), Docker using its official installer, and the missing Linux x86_64/arm64 Compose plugin using the pinned official release with SHA-256 verification. A plugin is installed for the active Docker user, or system-wide when Docker requires sudo. macOS Compose comes from Docker Desktop. Set `LOOPFORGE_YES=1` to accept installation prompts; internet access and package-manager/administrator permissions are still required. The installer generates `.env` with unique secrets, pulls and starts the containers, installs the observability columns and applies `migrations/seed/*.sql`. It is safe to re-run. Set `AGENTFLOW_HTTP_PORT` or `LOOPFORGE_MYSQL_PORT` in the environment before the first run (or edit `.env` later) when the default ports are taken. @@ -38,10 +39,11 @@ Run it as the installation owner, including an ordinary Linux user. If that user `compose.yaml` is the published-image stack and never builds: `./start.sh`, `scripts/install.sh` and a plain `docker compose --profile platform up` all pull the GHCR release pinned there (currently `ghcr.io/tencent/loopforge:0.3.8`). The pin matches the checkout's migrations and seeds, so check out a release tag when you install. Set `LOOPFORGE_IMAGE` (for example to an `@sha256:` digest) only to override it; a value saved in `.env` takes precedence over later pin changes. -Where `ghcr.io` downloads stall, set `MIRROR` to the registry host that replaces `ghcr.io`. `./start.sh` and `./start.sh pull` pull that reference and tag it back to the name in `compose.yaml`. Setting `MIRROR` trusts that host, because the result is tagged with the official name. The GitHub Release records the image digest; `docker image inspect --format '{{index .RepoDigests 0}}' ghcr.io/tencent/loopforge:` should show the same digest after `@`. The pull policy stays `missing` on install, so a local image is kept; `./start.sh pull` refreshes it from the same host. Leaving `MIRROR` unset preserves the direct `ghcr.io` pull. [ghcr.nju.edu.cn](https://doc.nju.edu.cn/books/e1654/page/ghcr) is one compatible host. Docker's `registry-mirrors` setting, including `https://docker.nju.edu.cn`, only applies to Docker Hub and cannot be used as `MIRROR`. A `LOOPFORGE_IMAGE` value that is not a `ghcr.io` name:tag is left unchanged. +Where `ghcr.io` downloads stall, `./start.sh` and `./start.sh pull` time `ghcr.io` against [ghcr.nju.edu.cn](https://doc.nju.edu.cn/books/e1654/page/ghcr) and use the faster registry. Nanjing is tagged back to the name in `compose.yaml`, which trusts that host. If the two manifest digests differ, the pull stays on `ghcr.io`. `MIRROR=ghcr.io` forces the upstream registry; any other `MIRROR` host forces that registry and skips the speed test. The choice is not written to `.env`. The pull policy stays `missing` on install, so a local image is kept and is not timed again; `./start.sh pull` times the registries and refreshes from the winner. The GitHub Release records the image digest; `docker image inspect --format '{{index .RepoDigests 0}}' ghcr.io/tencent/loopforge:` should show the same digest after `@`. Docker's `registry-mirrors` setting, including `https://docker.nju.edu.cn`, only applies to Docker Hub and cannot be used as `MIRROR`. A `LOOPFORGE_IMAGE` value that is not a `ghcr.io` name:tag is left unchanged. ```bash -MIRROR=ghcr.nju.edu.cn ./start.sh +./start.sh +MIRROR=ghcr.io ./start.sh pull MIRROR=ghcr.nju.edu.cn ./start.sh pull ``` diff --git a/scripts/ghcr-mirror.sh b/scripts/ghcr-mirror.sh index 287090dc3..d3768f947 100644 --- a/scripts/ghcr-mirror.sh +++ b/scripts/ghcr-mirror.sh @@ -1,30 +1,38 @@ #!/usr/bin/env bash -# Optional pull path for a slow ghcr.io link. Sourced by start.sh and install.sh. -# MIRROR is unset: callers keep their existing pull. MIRROR= replaces the -# ghcr.io host, for example MIRROR=ghcr.nju.edu.cn, then tags the image back -# so compose.yaml and the default pull policy stay unchanged. +# Pull path for the published ghcr.io image. Sourced by start.sh and install.sh. +# MIRROR is unset: time ghcr.io and ghcr.nju.edu.cn, then pull the faster one. +# MIRROR= replaces the ghcr.io host, for example MIRROR=ghcr.nju.edu.cn, +# and tags the image back so compose.yaml stays unchanged. MIRROR=ghcr.io keeps +# the direct pull. The choice is not written to .env. mirror_published_ghcr() { local policy="${1:-missing}" - [[ -n "${MIRROR:-}" ]] || return 1 - local mirror_host - mirror_host=$(printf '%s' "$MIRROR" | tr '[:upper:]' '[:lower:]') - mirror_host="${mirror_host#"${mirror_host%%[![:space:]]*}"}" - mirror_host="${mirror_host%"${mirror_host##*[![:space:]]}"}" - mirror_host=${mirror_host%/} - # The upstream host is the normal pull. Any other registry host replaces ghcr.io. - if [[ "$mirror_host" == ghcr.io || "$mirror_host" == ghcr.io:443 ]]; then - return 1 - fi - if [[ ! "$mirror_host" =~ ^(localhost|[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+)(:[0-9]{1,5})?$ ]]; then - printf '\n✗ MIRROR must be the registry host that replaces ghcr.io, for example ghcr.nju.edu.cn.\n' >&2 - exit 1 + local mirror_host="" + local auto=0 + + if [[ -n "${MIRROR:-}" ]]; then + mirror_host=$(printf '%s' "$MIRROR" | tr '[:upper:]' '[:lower:]') + mirror_host="${mirror_host#"${mirror_host%%[![:space:]]*}"}" + mirror_host="${mirror_host%"${mirror_host##*[![:space:]]}"}" + mirror_host=${mirror_host%/} + # The upstream host is the normal pull. Any other registry host replaces ghcr.io. + if [[ "$mirror_host" == ghcr.io || "$mirror_host" == ghcr.io:443 ]]; then + return 1 + fi + if [[ ! "$mirror_host" =~ ^(localhost|[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+)(:[0-9]{1,5})?$ ]]; then + printf '\n✗ MIRROR must be the registry host that replaces ghcr.io, for example ghcr.nju.edu.cn.\n' >&2 + exit 1 + fi + else + auto=1 fi local decision - if ! decision=$(python3 - <<'PY' + if ! decision=$(python3 - "$auto" <<'PY' import os, re, pathlib, sys +auto = sys.argv[1] == '1' + def quoted(value): value = value.strip() if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"": @@ -42,26 +50,34 @@ def stored(name): return quoted(matches[-1]) return '' -if stored('COMPOSE_FILE') == 'compose.yaml:compose.dev.yaml': +def skip(): print('skip') raise SystemExit(0) +if stored('COMPOSE_FILE') == 'compose.yaml:compose.dev.yaml': + skip() + image = stored('LOOPFORGE_IMAGE') if not image: compose = pathlib.Path('compose.yaml') if not compose.is_file(): + if auto: + skip() sys.stderr.write('MIRROR needs compose.yaml to find the pinned image.\n') raise SystemExit(1) match = re.search(r'(?m)^[ \t]*image:[ \t]*\$\{LOOPFORGE_IMAGE:-([^}]+)\}[ \t]*$', compose.read_text()) if not match: + if auto: + skip() sys.stderr.write('MIRROR could not find the pinned image in compose.yaml.\n') raise SystemExit(1) image = match.group(1).strip() if not image.startswith('ghcr.io/') or '@' in image: - print('skip') - raise SystemExit(0) + skip() if not re.fullmatch(r'ghcr\.io/[A-Za-z0-9._/-]+:[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}', image): + if auto: + skip() sys.stderr.write('MIRROR cannot rewrite this image reference: %s\n' % image) raise SystemExit(1) print('mirror ' + image) @@ -71,16 +87,267 @@ PY fi [[ "$decision" == skip ]] && return 1 [[ "$decision" == mirror\ * ]] || { + if [[ "$auto" == 1 ]]; then + return 1 + fi printf '\n✗ Could not resolve the image for MIRROR=%s.\n' "$mirror_host" >&2 exit 1 } local image="${decision#mirror }" - local mirror_image="${mirror_host}/${image#ghcr.io/}" + # A re-run that already has the pinned image does not need a speed test. + # An explicit mirror still reports that it kept the local image. if [[ "$policy" != always ]] && docker_access image inspect "$image" >/dev/null 2>&1; then + if [[ "$auto" == 1 ]]; then + return 1 + fi printf '\n▸ Using local image %s\n' "$image" return 0 fi + + if [[ "$auto" == 1 ]]; then + local probe_line="" + if ! probe_line=$(python3 - "$image" <<'PY' +import json, re, shutil, subprocess, sys, tempfile, threading, time +from pathlib import Path +from urllib.parse import urlencode + +image = sys.argv[1] +ref = image[len('ghcr.io/'):] +repo, tag = ref.rsplit(':', 1) +hosts = ('ghcr.io', 'ghcr.nju.edu.cn') +accept = ', '.join([ + 'application/vnd.oci.image.index.v1+json', + 'application/vnd.docker.distribution.manifest.list.v2+json', + 'application/vnd.oci.image.manifest.v1+json', + 'application/vnd.docker.distribution.manifest.v2+json', +]) +min_layer = 262144 +min_sample = 65536 +deadline = 6.0 + +class Result: + def __init__(self): + self.digest = '' + self.speed_ok = False + self.bps = 0.0 + self.timed_out = False + +def curl(url, headers, timeout, body, header_file): + cmd = [ + 'curl', '-sS', '--connect-timeout', '2', '--max-time', f'{max(timeout, 0.1):.1f}', + '-o', str(body), '-D', str(header_file), + '-w', '%{http_code} %{size_download} %{time_total}', + ] + for header in headers: + cmd.extend(['-H', header]) + cmd.append(url) + proc = subprocess.run(cmd, capture_output=True, text=True) + parts = proc.stdout.split() + code = size = elapsed = None + if len(parts) >= 3: + try: + code = int(float(parts[0])) + size = int(float(parts[1])) + elapsed = float(parts[2]) + except ValueError: + code = size = elapsed = None + return proc.returncode, code, size, elapsed, header_file.read_text(errors='replace') if header_file.is_file() else '' + +def bearer(headers): + match = re.search(r'(?im)^www-authenticate:\s*(.+)$', headers) + if not match or not match.group(1).lower().startswith('bearer'): + return {} + return dict(re.findall(r'(\w+)="([^"]*)"', match.group(1))) + +def header_value(headers, name): + match = re.search(r'(?im)^%s:\s*(\S+)' % re.escape(name), headers) + return match.group(1).strip() if match else '' + +def probe(host): + result = Result() + started = time.monotonic() + work = Path(tempfile.mkdtemp(prefix='loopforge-mirror-')) + token = '' + try: + return probe_host(host, result, started, work, token) + finally: + shutil.rmtree(work, ignore_errors=True) + +def probe_host(host, result, started, work, token): + + def remaining(): + return deadline - (time.monotonic() - started) + + def request(url, headers): + if remaining() < 0.3: + result.timed_out = True + return None + body = work / 'body' + headers_path = work / 'headers' + status, code, size, elapsed, text = curl(url, headers, remaining(), body, headers_path) + if status == 28: + result.timed_out = True + return None + if status != 0 or code is None: + return None + payload = body.read_bytes() if body.is_file() else b'' + return code, size, elapsed, text, payload + + challenge = request(f'https://{host}/v2/', []) + if challenge is None: + return result + code, _, _, text, _ = challenge + auth = [] + if code == 401: + params = bearer(text) + realm = params.get('realm', '') + service = params.get('service', '') + if not realm: + return result + query = urlencode({'service': service, 'scope': f'repository:{repo}:pull'}) + join = '&' if '?' in realm else '?' + token_response = request(realm + join + query, ['Accept: application/json']) + if token_response is None or token_response[0] != 200: + return result + try: + payload = json.loads(token_response[4].decode()) + except (ValueError, UnicodeError): + return result + token = payload.get('token') or payload.get('access_token') or '' + if not token: + return result + auth = [f'Authorization: Bearer {token}'] + elif code != 200: + return result + + def fetch_manifest(reference): + response = request( + f'https://{host}/v2/{repo}/manifests/{reference}', + auth + [f'Accept: {accept}'], + ) + if response is None or response[0] != 200: + return None, None + digest = header_value(response[3], 'Docker-Content-Digest') + try: + document = json.loads(response[4].decode()) + except (ValueError, UnicodeError): + return digest, None + return digest, document + + digest, document = fetch_manifest(tag) + if digest: + result.digest = digest + if not isinstance(document, dict): + return result + if 'manifests' in document and 'layers' not in document: + chosen = '' + fallback = '' + for item in document.get('manifests') or []: + if not isinstance(item, dict) or not item.get('digest'): + continue + platform = item.get('platform') or {} + if platform.get('os') == 'unknown' or platform.get('architecture') == 'unknown': + continue + if platform.get('os') == 'linux' and platform.get('architecture') in ('amd64', 'arm64'): + chosen = item['digest'] + break + fallback = fallback or item['digest'] + child = chosen or fallback + if not child: + return result + _, document = fetch_manifest(child) + if not isinstance(document, dict): + return result + + layer = '' + for item in document.get('layers') or []: + if not isinstance(item, dict): + continue + size = item.get('size') or 0 + layer_digest = item.get('digest') or '' + if isinstance(size, int) and size >= min_layer and layer_digest.startswith('sha256:'): + layer = layer_digest + break + if not layer: + return result + blob = request( + f'https://{host}/v2/{repo}/blobs/{layer}', + auth + ['Range: bytes=0-262143'], + ) + if blob is None or blob[0] not in (200, 206): + return result + _, size, elapsed, _, _ = blob + if size is None or size < min_sample or elapsed is None or elapsed <= 0: + return result + result.speed_ok = True + result.bps = size / elapsed + return result + +results = {} + +def run(host): + try: + results[host] = probe(host) + except Exception: + results[host] = Result() + +threads = [threading.Thread(target=run, args=(host,)) for host in hosts] +for thread in threads: + thread.start() +for thread in threads: + thread.join(deadline + 1) +for host in hosts: + results.setdefault(host, Result()) + +ghcr = results['ghcr.io'] +nju = results['ghcr.nju.edu.cn'] + +def rate(bps): + if bps >= 1024 * 1024: + return f'{bps / (1024 * 1024):.1f} MB/s' + if bps >= 1024: + return f'{bps / 1024:.0f} KB/s' + return f'{bps:.0f} B/s' + +def side(result): + if result.speed_ok: + return f'was {rate(result.bps)}' + if result.timed_out: + return 'timed out' + return 'failed' + +if ghcr.digest and nju.digest and ghcr.digest != nju.digest: + print('ghcr.io\tRegistry digests differ; using ghcr.io') + raise SystemExit(0) +if ghcr.speed_ok and nju.speed_ok: + if nju.bps >= ghcr.bps * 1.5: + print(f'ghcr.nju.edu.cn\tFastest registry is ghcr.nju.edu.cn ({rate(nju.bps)}); ghcr.io {side(ghcr)}') + else: + print(f'ghcr.io\tFastest registry is ghcr.io ({rate(ghcr.bps)}); ghcr.nju.edu.cn {side(nju)}') + raise SystemExit(0) +if nju.speed_ok: + print(f'ghcr.nju.edu.cn\tFastest registry is ghcr.nju.edu.cn ({rate(nju.bps)}); ghcr.io {side(ghcr)}') + raise SystemExit(0) +if ghcr.speed_ok: + print(f'ghcr.io\tFastest registry is ghcr.io ({rate(ghcr.bps)}); ghcr.nju.edu.cn {side(nju)}') + raise SystemExit(0) +print('ghcr.io\tCould not time either registry; using ghcr.io') +PY + ); then + probe_line=$'ghcr.io\tCould not time either registry; using ghcr.io' + fi + if [[ "$probe_line" != *$'\t'* ]]; then + probe_line=$'ghcr.io\tCould not time either registry; using ghcr.io' + fi + printf '\n▸ %s\n' "${probe_line#*$'\t'}" + mirror_host=${probe_line%%$'\t'*} + if [[ "$mirror_host" != ghcr.nju.edu.cn ]]; then + return 1 + fi + fi + + local mirror_image="${mirror_host}/${image#ghcr.io/}" printf '\n▸ Pulling %s and tagging it %s\n' "$mirror_image" "$image" # Called from `if`, so a failing docker command does not exit on its own. # Exit rather than return: a return would make the caller pull ghcr.io directly. diff --git a/scripts/test-install.py b/scripts/test-install.py index 131fddf56..34ec3c4b9 100644 --- a/scripts/test-install.py +++ b/scripts/test-install.py @@ -65,7 +65,7 @@ def setUp(self): exec "$@" ''') self.environ = dict(os.environ, PATH=str(self.root / 'mock') + ':' + os.environ['PATH'], MOCK_LOG=str(self.root / 'docker.log'), AGENTFLOW_HTTP_PORT='18765', LOOPFORGE_PUBLIC_URL='https://team.example.test', LOOPFORGE_BIND='0.0.0.0') - for name in ('LOOPFORGE_DOCKER_USE_SUDO', 'LOOPFORGE_DATA_DIR', 'LOOPFORGE_IMAGE', 'COMPOSE_FILE', 'LOOPFORGE_BUILD_FROM_SOURCE', 'LOOPFORGE_PULL_POLICY', 'MIRROR', 'MOCK_USER', 'MOCK_DIRECT', 'MOCK_SUDO_DENIED', 'MOCK_IMAGE_PRESENT', 'MOCK_PULL_FAIL', 'MOCK_TAG_FAIL'): + for name in ('LOOPFORGE_DOCKER_USE_SUDO', 'LOOPFORGE_DATA_DIR', 'LOOPFORGE_IMAGE', 'COMPOSE_FILE', 'LOOPFORGE_BUILD_FROM_SOURCE', 'LOOPFORGE_PULL_POLICY', 'MIRROR', 'MOCK_USER', 'MOCK_DIRECT', 'MOCK_SUDO_DENIED', 'MOCK_IMAGE_PRESENT', 'MOCK_PULL_FAIL', 'MOCK_TAG_FAIL', 'MOCK_MIRROR_GHCR', 'MOCK_MIRROR_NJU', 'MOCK_CURL_LOG'): self.environ.pop(name, None) def executable(self, name, contents): @@ -169,6 +169,160 @@ def test_upstream_mirror_host_keeps_the_direct_pull(self): self.assertIn('pull --policy missing platform mysql', log) self.assertNotIn(' tag ', log) + def mirror_probe_curl(self): + self.executable('curl', r'''#!/usr/bin/env bash +printf '%s\n' "$*" >> "${MOCK_CURL_LOG:?}" +body="" +header_file="" +prev="" +url="" +for arg in "$@"; do + case "$prev" in + -o) body=$arg ;; + -D) header_file=$arg ;; + esac + prev=$arg + url=$arg +done +case "$url" in + https://ghcr.nju.edu.cn/*) spec="${MOCK_MIRROR_NJU:?}"; host=ghcr.nju.edu.cn ;; + https://ghcr.io/*) spec="${MOCK_MIRROR_GHCR:?}"; host=ghcr.io ;; + *) printf 'unexpected %s\n' "$url" >&2; exit 1 ;; +esac +case "$spec" in + timeout) exit 28 ;; + fail) exit 22 ;; +esac +digest=${spec#ok:} +seconds=${digest##*:} +digest=${digest%:*} +case "$url" in + */token\?*) + printf '%s' '{"token":"t"}' > "$body" + printf 'HTTP/1.1 200 OK\r\n\r\n' > "$header_file" + printf '200 %s 0.01\n' "$(wc -c < "$body")" + ;; + */blobs/*) + printf 'x' > "$body" + printf 'HTTP/1.1 206 Partial Content\r\n\r\n' > "$header_file" + printf '206 262144 %s\n' "$seconds" + ;; + */manifests/*) + printf '%s' '{"schemaVersion":2,"layers":[{"digest":"sha256:layer","size":1048576}]}' > "$body" + printf 'HTTP/1.1 200 OK\r\nDocker-Content-Digest: %s\r\n\r\n' "$digest" > "$header_file" + printf '200 %s 0.01\n' "$(wc -c < "$body")" + ;; + */v2/) + : > "$body" + printf 'HTTP/1.1 401 Unauthorized\r\nWWW-Authenticate: Bearer realm="https://%s/token",service="%s"\r\n\r\n' "$host" "$host" > "$header_file" + printf '401 0 0.01\n' + ;; + *) + printf 'unhandled %s\n' "$url" >&2 + exit 1 + ;; +esac +''') + + def probe_env(self, ghcr, nju): + return { + 'MOCK_CURL_LOG': str(self.root / 'curl.log'), + 'MOCK_MIRROR_GHCR': ghcr, + 'MOCK_MIRROR_NJU': nju, + } + + def test_explicit_mirror_does_not_probe(self): + self.use_compose_file() + self.executable('curl', '#!/usr/bin/env bash\nprintf \'%s\\n\' "$*" >> "$MOCK_CURL_LOG"\nexit 99\n') + curl_log = self.root / 'curl.log' + result = self.install(MIRROR='ghcr.nju.edu.cn', MOCK_CURL_LOG=str(curl_log)) + self.assert_success(result) + self.assertFalse(curl_log.exists()) + self.assertIn('pull ghcr.nju.edu.cn/tencent/loopforge:', (self.root / 'docker.log').read_text()) + self.assertNotIn('Fastest registry', result.stdout) + + def test_faster_nju_mirror_is_pulled_and_tagged(self): + self.use_compose_file() + self.mirror_probe_curl() + result = self.install(**self.probe_env('ok:sha256:same:0.40', 'ok:sha256:same:0.10')) + self.assert_success(result) + image = self.pinned_image() + mirror = 'ghcr.nju.edu.cn/' + image[len('ghcr.io/'):] + log = (self.root / 'docker.log').read_text() + self.assertLess(log.index(f'pull {mirror}'), log.index(f'tag {mirror} {image}')) + self.assertIn('Fastest registry is ghcr.nju.edu.cn', result.stdout) + self.assertIn('ghcr.io was', result.stdout) + self.assertNotIn('MIRROR=', (self.root / '.env').read_text()) + + def test_faster_ghcr_keeps_the_direct_pull(self): + self.use_compose_file() + self.mirror_probe_curl() + result = self.install(**self.probe_env('ok:sha256:same:0.10', 'ok:sha256:same:0.40')) + self.assert_success(result) + log = (self.root / 'docker.log').read_text() + self.assertIn('pull --policy missing platform mysql', log) + self.assertNotIn('ghcr.nju.edu.cn', log) + self.assertIn('Fastest registry is ghcr.io', result.stdout) + + def test_digest_mismatch_keeps_the_direct_pull(self): + self.use_compose_file() + self.mirror_probe_curl() + result = self.install(**self.probe_env('ok:sha256:upstream:0.40', 'ok:sha256:cached:0.05')) + self.assert_success(result) + log = (self.root / 'docker.log').read_text() + self.assertIn('pull --policy missing platform mysql', log) + self.assertNotIn('ghcr.nju.edu.cn', log) + self.assertIn('Registry digests differ; using ghcr.io', result.stdout) + + def test_both_mirrors_unreachable_keeps_the_direct_pull(self): + self.use_compose_file() + self.mirror_probe_curl() + result = self.install(**self.probe_env('timeout', 'fail')) + self.assert_success(result) + log = (self.root / 'docker.log').read_text() + self.assertIn('pull --policy missing platform mysql', log) + self.assertNotIn('ghcr.nju.edu.cn', log) + self.assertIn('Could not time either registry; using ghcr.io', result.stdout) + + def test_nju_wins_when_ghcr_times_out(self): + self.use_compose_file() + self.mirror_probe_curl() + result = self.install(**self.probe_env('timeout', 'ok:sha256:same:0.20')) + self.assert_success(result) + self.assertIn('pull ghcr.nju.edu.cn/tencent/loopforge:', (self.root / 'docker.log').read_text()) + self.assertIn('ghcr.io timed out', result.stdout) + + def test_local_image_skips_the_speed_test(self): + self.use_compose_file() + self.executable('curl', '#!/usr/bin/env bash\nprintf \'%s\\n\' "$*" >> "$MOCK_CURL_LOG"\nexit 99\n') + curl_log = self.root / 'curl.log' + result = self.install(MOCK_IMAGE_PRESENT='1', MOCK_CURL_LOG=str(curl_log)) + self.assert_success(result) + self.assertFalse(curl_log.exists()) + log = (self.root / 'docker.log').read_text() + self.assertIn('pull --policy missing platform mysql', log) + self.assertNotIn('ghcr.nju.edu.cn', log) + self.assertNotIn('Fastest registry', result.stdout) + + def test_start_pull_uses_the_faster_nju_mirror(self): + self.assert_success(self.install()) + self.use_compose_file() + self.mirror_probe_curl() + (self.root / 'docker.log').unlink() + result = subprocess.run( + ['bash', 'start.sh', 'pull'], cwd=self.root, + env=dict(self.environ, **self.probe_env('timeout', 'ok:sha256:same:0.20')), + text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + image = self.pinned_image() + mirror = 'ghcr.nju.edu.cn/' + image[len('ghcr.io/'):] + log = (self.root / 'docker.log').read_text() + self.assertIn(f'pull {mirror}', log) + self.assertIn(f'tag {mirror} {image}', log) + self.assertIn('pull --policy always mysql', log) + self.assertNotIn('pull --ignore-buildable --policy always', log) + self.assertIn('ghcr.io timed out', result.stdout) + def test_invalid_mirror_fails_before_pull(self): self.use_compose_file() for mirror in ('NJU', 'https://ghcr.nju.edu.cn', 'ghcr.nju.edu.cn/tencent'): @@ -190,8 +344,8 @@ def test_start_pull_without_mirror_is_unchanged(self): self.assertNotIn('ghcr.nju.edu.cn', log) def test_start_pull_with_nju_mirror_does_not_contact_ghcr_for_platform(self): - self.use_compose_file() self.assert_success(self.install()) + self.use_compose_file() (self.root / 'docker.log').unlink() result = subprocess.run(['bash', 'start.sh', 'pull'], cwd=self.root, env=dict(self.environ, MIRROR='ghcr.nju.edu.cn'), text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) @@ -204,8 +358,8 @@ def test_start_pull_with_nju_mirror_does_not_contact_ghcr_for_platform(self): self.assertNotIn('pull --ignore-buildable --policy always', log) def test_mirror_uses_the_last_image_assignment_in_env(self): - self.use_compose_file() self.assert_success(self.install()) + self.use_compose_file() env = self.root / '.env' env.write_text(env.read_text() + 'LOOPFORGE_IMAGE=ghcr.io/example/ignored:1\nLOOPFORGE_IMAGE=ghcr.io/example/chosen:9\n') (self.root / 'docker.log').unlink() @@ -228,8 +382,8 @@ def test_failed_mirror_pull_does_not_retag_or_start(self): self.assertNotIn('up -d --no-build', log) def test_failed_mirror_refresh_does_not_retag_a_cached_image(self): - self.use_compose_file() self.assert_success(self.install()) + self.use_compose_file() (self.root / 'docker.log').unlink() result = subprocess.run( ['bash', 'start.sh', 'pull'], cwd=self.root, diff --git a/start.sh b/start.sh index 27e37eff3..4a7eab9c5 100755 --- a/start.sh +++ b/start.sh @@ -12,6 +12,9 @@ # ./start.sh pull refresh images and recreate changed containers # ./start.sh dev build this checkout (compose.dev.yaml) and start it # ./start.sh release switch a source-built installation back to GHCR +# ./start.sh time ghcr.io and ghcr.nju.edu.cn, then pull the faster one +# MIRROR=ghcr.io ./start.sh +# pull ghcr.io directly # MIRROR=ghcr.nju.edu.cn ./start.sh # pull ghcr.io images from that host and tag them back set -euo pipefail @@ -39,12 +42,10 @@ PY COMPOSE_FILE=compose.yaml bash scripts/install.sh ;; pull) - # An empty MIRROR keeps the original refresh. A registry host pulls the pinned - # ghcr.io tag from that host and tags it back before MySQL is refreshed. - if [[ -n "${MIRROR:-}" ]]; then - [[ -f .env ]] || docker_access_fail 'Not installed yet. Run ./start.sh first.' - select_docker_access || docker_access_fail 'Docker is not running, or this user cannot access it.' - fi + # Docker is selected before the speed test: a faster Nanjing mirror pulls + # immediately, and the direct ghcr.io path needs it too. + [[ -f .env ]] || docker_access_fail 'Not installed yet. Run ./start.sh first.' + select_docker_access || docker_access_fail 'Docker is not running, or this user cannot access it.' if mirror_published_ghcr always; then compose pull --policy always mysql else From c3f98c31d1cf474528656194f7fa3b067b6a445f Mon Sep 17 00:00:00 2001 From: cocofhu Date: Fri, 20 Nov 2026 17:35:49 +0800 Subject: [PATCH 2/3] fix: pull the NJU mirror only by a verified ghcr.io digest Automatic selection now follows layer redirects without forwarding the token, and a failed mirror pull falls back to ghcr.io instead of stopping the install. Co-authored-by: Cursor --- CHANGELOG.md | 2 +- README.md | 2 +- README.zh-CN.md | 2 +- docs/deployment.md | 2 +- scripts/ghcr-mirror.sh | 81 ++++++++++++++++++++++++++++++++--------- scripts/test-install.py | 78 +++++++++++++++++++++++++++++++++------ 6 files changed, 133 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 648b77125..95fbf01c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Platform releases use `vX.Y.Z` tags and publish `ghcr.io/tencent/loopforge`. Loo ## Unreleased -- `./start.sh` and `./start.sh pull` time `ghcr.io` and `ghcr.nju.edu.cn`, then pull the faster registry. A faster Nanjing image is tagged back to the name pinned in `compose.yaml`. `MIRROR=ghcr.io` forces the upstream registry; `MIRROR=` still forces that host. Accepting Nanjing, or setting `MIRROR`, trusts that host. Different manifest digests stay on `ghcr.io`. The choice is not saved in `.env`. `registry-mirrors` such as `https://docker.nju.edu.cn` only accelerates Docker Hub. +- `./start.sh` and `./start.sh pull` time `ghcr.io` and `ghcr.nju.edu.cn`. Nanjing is used only when the ghcr.io manifest digest was verified and the mirror returned the same digest; the pull is that digest, then tagged back to the name pinned in `compose.yaml`. A missing official digest, a digest mismatch, or a failed mirror pull stays on `ghcr.io`. `MIRROR=ghcr.io` forces the upstream registry; `MIRROR=` still forces that host and does not fall back. Accepting Nanjing, or setting `MIRROR`, trusts that host. The choice is not saved in `.env`. `registry-mirrors` such as `https://docker.nju.edu.cn` only accelerates Docker Hub. ## 0.4.0 — scheduled task history, an MR node in delivery SOPs and agent CLIs in conversation workspaces diff --git a/README.md b/README.md index 01190fd74..a1183806f 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ cd LoopForge `./start.sh` pulls the GHCR release image pinned in `compose.yaml` without building locally (`bash scripts/install.sh` does the same). It creates the MySQL and platform services, generates configuration and session secrets, and installs the built-in sandboxes, starter SOPs and welcome project. Host Go and Node.js installations are not required. Configure model API keys and GitHub authorization after startup. Use `./start.sh logs`, `stop`, `restart` or `pull` to manage it, and `./start.sh dev` to build the checkout from source. -If downloads from `ghcr.io` stall, `./start.sh` and `./start.sh pull` time `ghcr.io` against the Nanjing University mirror [ghcr.nju.edu.cn](https://doc.nju.edu.cn/books/e1654/page/ghcr) and pull the faster one. A faster Nanjing pull is tagged back to the name pinned in `compose.yaml`, which trusts that host because the result is tagged with the official name. Different manifest digests stay on `ghcr.io`. `MIRROR=ghcr.io` forces the upstream registry and `MIRROR=ghcr.nju.edu.cn` forces Nanjing; the choice is not written to `.env`. The GitHub Release records the digest; `docker image inspect --format '{{index .RepoDigests 0}}' ghcr.io/tencent/loopforge:` should show the same digest after `@`. `compose.yaml` stays unchanged. A Docker `registry-mirrors` entry such as `https://docker.nju.edu.cn` only accelerates Docker Hub and is not a value for `MIRROR`. +If downloads from `ghcr.io` stall, `./start.sh` and `./start.sh pull` time `ghcr.io` against the Nanjing University mirror [ghcr.nju.edu.cn](https://doc.nju.edu.cn/books/e1654/page/ghcr). Nanjing is used only when its manifest digest matches the digest just read from ghcr.io; the script pulls that digest and tags it back to the name pinned in `compose.yaml`, which trusts that host because the result is tagged with the official name. A missing official digest, a mismatch, or a failed mirror pull stays on `ghcr.io`. `MIRROR=ghcr.io` forces the upstream registry and `MIRROR=ghcr.nju.edu.cn` forces Nanjing; the choice is not written to `.env`. The GitHub Release records the digest; `docker image inspect --format '{{index .RepoDigests 0}}' ghcr.io/tencent/loopforge:` should show the same digest after `@`. `compose.yaml` stays unchanged. A Docker `registry-mirrors` entry such as `https://docker.nju.edu.cn` only accelerates Docker Hub and is not a value for `MIRROR`. ```bash ./start.sh diff --git a/README.zh-CN.md b/README.zh-CN.md index 18e85dc65..78d179ed8 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -80,7 +80,7 @@ cd LoopForge `./start.sh` 直接拉取 `compose.yaml` 中 pin 的 GHCR 发布镜像,不在本地构建(`bash scripts/install.sh` 效果相同)。安装流程会创建 MySQL 和平台服务,生成配置与会话密钥,并导入内置沙箱、通用 SOP 和体验项目;无需在宿主机安装 Go 或 Node.js。模型 API Key 和 GitHub 授权在启动后由用户配置。用 `./start.sh logs`、`stop`、`restart`、`pull` 管理服务;需要从源码构建时用 `./start.sh dev`。 -国内拉取 `ghcr.io` 经常中断。`./start.sh` 和 `./start.sh pull` 会并行探测 `ghcr.io` 与南京大学镜像 [ghcr.nju.edu.cn](https://doc.nju.edu.cn/books/e1654/page/ghcr),对同一个 tag 抽样比较下载速度,走更快的一边。选中南京时,脚本从该主机拉取,再改回 `compose.yaml` 里的原名;这表示信任该主机,因为下载结果会被标成官方镜像名。两边 manifest 的 digest 不一致时仍走 `ghcr.io`。`MIRROR=ghcr.io` 固定走官方源,`MIRROR=ghcr.nju.edu.cn` 固定走南京,选择不会写入 `.env`。发布说明里有 digest,可用 `docker image inspect --format '{{index .RepoDigests 0}}' ghcr.io/tencent/loopforge:` 核对,`@` 后面的摘要应一致。`compose.yaml` 不用改。Docker 引擎里的 `registry-mirrors`(包括 `https://docker.nju.edu.cn`)只加速 Docker Hub,不能填在 `MIRROR` 里。 +国内拉取 `ghcr.io` 经常中断。`./start.sh` 和 `./start.sh pull` 会并行探测 `ghcr.io` 与南京大学镜像 [ghcr.nju.edu.cn](https://doc.nju.edu.cn/books/e1654/page/ghcr)。只有先从 ghcr.io 拿到 manifest 摘要,且南京返回同一摘要时,才按这个摘要拉取并改回 `compose.yaml` 里的原名;这表示信任该主机,因为下载结果会被标成官方镜像名。拿不到官方摘要、摘要不一致,或镜像拉取失败时,仍走 `ghcr.io`。`MIRROR=ghcr.io` 固定走官方源,`MIRROR=ghcr.nju.edu.cn` 固定走南京,选择不会写入 `.env`。发布说明里有 digest,可用 `docker image inspect --format '{{index .RepoDigests 0}}' ghcr.io/tencent/loopforge:` 核对,`@` 后面的摘要应一致。`compose.yaml` 不用改。Docker 引擎里的 `registry-mirrors`(包括 `https://docker.nju.edu.cn`)只加速 Docker Hub,不能填在 `MIRROR` 里。 ```bash ./start.sh diff --git a/docs/deployment.md b/docs/deployment.md index 2a73019c2..115148319 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -50,7 +50,7 @@ git fetch --tags && git checkout vX.Y.Z The new `compose.yaml` brings the image pin and any new platform settings together, and `./start.sh pull` recreates the platform container when either changes. A source build from `./start.sh dev` is not rebuilt by `./start.sh pull`; run `./start.sh dev` again after updating the checkout. Running `./start.sh pull` alone keeps the pinned version. With `LOOPFORGE_IMAGE` saved in `.env`, change it to the checked-out release as well; a newer image with an older `compose.yaml` can miss settings that release needs. -Where `ghcr.io` downloads stall, `./start.sh` and `./start.sh pull` time `ghcr.io` against [ghcr.nju.edu.cn](https://doc.nju.edu.cn/books/e1654/page/ghcr) and use the faster registry. Nanjing is tagged back to the name in `compose.yaml`, which trusts that host. If the two manifest digests differ, the pull stays on `ghcr.io`. `MIRROR=ghcr.io` forces the upstream registry; any other `MIRROR` host forces that registry and skips the speed test. The choice is not written to `.env`. The pull policy stays `missing` on install, so a local image is kept and is not timed again; `./start.sh pull` times the registries and refreshes from the winner. The GitHub Release records the image digest; `docker image inspect --format '{{index .RepoDigests 0}}' ghcr.io/tencent/loopforge:` should show the same digest after `@`. Docker's `registry-mirrors` setting, including `https://docker.nju.edu.cn`, only applies to Docker Hub and cannot be used as `MIRROR`. A `LOOPFORGE_IMAGE` value that is not a `ghcr.io` name:tag is left unchanged. +Where `ghcr.io` downloads stall, `./start.sh` and `./start.sh pull` time `ghcr.io` against [ghcr.nju.edu.cn](https://doc.nju.edu.cn/books/e1654/page/ghcr). Nanjing is used only when its manifest digest matches the digest just read from ghcr.io; that digest is what gets pulled, then tagged back to the name in `compose.yaml`, which trusts that host. A missing official digest, a mismatch, or a failed mirror pull stays on `ghcr.io`. `MIRROR=ghcr.io` forces the upstream registry; any other `MIRROR` host forces that registry, skips the speed test, and does not fall back. The choice is not written to `.env`. The pull policy stays `missing` on install, so a local image is kept and is not timed again; `./start.sh pull` times the registries and refreshes from the winner. The GitHub Release records the image digest; `docker image inspect --format '{{index .RepoDigests 0}}' ghcr.io/tencent/loopforge:` should show the same digest after `@`. Docker's `registry-mirrors` setting, including `https://docker.nju.edu.cn`, only applies to Docker Hub and cannot be used as `MIRROR`. A `LOOPFORGE_IMAGE` value that is not a `ghcr.io` name:tag is left unchanged. ```bash ./start.sh diff --git a/scripts/ghcr-mirror.sh b/scripts/ghcr-mirror.sh index d3768f947..1c3fbdb03 100644 --- a/scripts/ghcr-mirror.sh +++ b/scripts/ghcr-mirror.sh @@ -105,6 +105,7 @@ PY return 0 fi + local mirror_digest="" if [[ "$auto" == 1 ]]; then local probe_line="" if ! probe_line=$(python3 - "$image" <<'PY' @@ -142,7 +143,8 @@ def curl(url, headers, timeout, body, header_file): for header in headers: cmd.extend(['-H', header]) cmd.append(url) - proc = subprocess.run(cmd, capture_output=True, text=True) + # stdout/stderr/universal_newlines are the names available on Python 3.6. + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) parts = proc.stdout.split() code = size = elapsed = None if len(parts) >= 3: @@ -164,6 +166,13 @@ def header_value(headers, name): match = re.search(r'(?im)^%s:\s*(\S+)' % re.escape(name), headers) return match.group(1).strip() if match else '' +def host_of(url): + match = re.match(r'https://([^/?#]+)', url or '') + return match.group(1).lower() if match else '' + +def trusted_digest(value): + return re.fullmatch(r'sha256:[a-f0-9]{64}', value or '') is not None + def probe(host): result = Result() started = time.monotonic() @@ -179,7 +188,7 @@ def probe_host(host, result, started, work, token): def remaining(): return deadline - (time.monotonic() - started) - def request(url, headers): + def request(url, headers, redirects=0): if remaining() < 0.3: result.timed_out = True return None @@ -191,6 +200,18 @@ def probe_host(host, result, started, work, token): return None if status != 0 or code is None: return None + # GHCR answers a layer range with 307 and an empty body, then the CDN + # returns 206. Follow only HTTPS, and do not send the registry token to + # another host. The sample is the final response. + if code in (301, 302, 303, 307, 308): + location = header_value(text, 'Location') + if redirects >= 3 or not location.startswith('https://'): + return None + next_headers = [ + header for header in headers + if host_of(location) == host_of(url) or not header.lower().startswith('authorization:') + ] + return request(location, next_headers, redirects + 1) payload = body.read_bytes() if body.is_file() else b'' return code, size, elapsed, text, payload @@ -317,41 +338,65 @@ def side(result): return 'timed out' return 'failed' -if ghcr.digest and nju.digest and ghcr.digest != nju.digest: - print('ghcr.io\tRegistry digests differ; using ghcr.io') +# The official manifest digest is the only trusted identity. Without it, or +# when the mirror disagrees, keep the ghcr.io pull. A faster mirror is pulled +# by that digest, not by the mutable tag. +official = ghcr.digest if trusted_digest(ghcr.digest) else '' + +def emit(host, digest, message): + print('%s\t%s\t%s' % (host, digest, message)) + +if not official: + emit('ghcr.io', '-', 'Could not verify the ghcr.io digest; using ghcr.io') raise SystemExit(0) -if ghcr.speed_ok and nju.speed_ok: - if nju.bps >= ghcr.bps * 1.5: - print(f'ghcr.nju.edu.cn\tFastest registry is ghcr.nju.edu.cn ({rate(nju.bps)}); ghcr.io {side(ghcr)}') - else: - print(f'ghcr.io\tFastest registry is ghcr.io ({rate(ghcr.bps)}); ghcr.nju.edu.cn {side(nju)}') +if trusted_digest(nju.digest) and nju.digest != official: + emit('ghcr.io', '-', 'Registry digests differ; using ghcr.io') raise SystemExit(0) -if nju.speed_ok: - print(f'ghcr.nju.edu.cn\tFastest registry is ghcr.nju.edu.cn ({rate(nju.bps)}); ghcr.io {side(ghcr)}') +nju_verified = nju.speed_ok and nju.digest == official +if nju_verified and (not ghcr.speed_ok or nju.bps >= ghcr.bps * 1.5): + emit('ghcr.nju.edu.cn', official, 'Fastest registry is ghcr.nju.edu.cn (%s); ghcr.io %s' % (rate(nju.bps), side(ghcr))) raise SystemExit(0) if ghcr.speed_ok: - print(f'ghcr.io\tFastest registry is ghcr.io ({rate(ghcr.bps)}); ghcr.nju.edu.cn {side(nju)}') + emit('ghcr.io', '-', 'Fastest registry is ghcr.io (%s); ghcr.nju.edu.cn %s' % (rate(ghcr.bps), side(nju))) + raise SystemExit(0) +if nju.speed_ok: + emit('ghcr.io', '-', 'Could not verify the mirror digest; using ghcr.io') raise SystemExit(0) -print('ghcr.io\tCould not time either registry; using ghcr.io') +emit('ghcr.io', '-', 'Could not time either registry; using ghcr.io') PY ); then - probe_line=$'ghcr.io\tCould not time either registry; using ghcr.io' + probe_line=$'ghcr.io\t-\tCould not time either registry; using ghcr.io' fi - if [[ "$probe_line" != *$'\t'* ]]; then - probe_line=$'ghcr.io\tCould not time either registry; using ghcr.io' + if [[ "$probe_line" != *$'\t'*$'\t'* ]]; then + probe_line=$'ghcr.io\t-\tCould not time either registry; using ghcr.io' fi - printf '\n▸ %s\n' "${probe_line#*$'\t'}" + printf '\n▸ %s\n' "${probe_line##*$'\t'}" mirror_host=${probe_line%%$'\t'*} + mirror_digest=${probe_line#*$'\t'} + mirror_digest=${mirror_digest%%$'\t'*} if [[ "$mirror_host" != ghcr.nju.edu.cn ]]; then return 1 fi + if [[ ! "$mirror_digest" =~ ^sha256:[a-f0-9]{64}$ ]]; then + printf '\n▸ Mirror digest is not the ghcr.io digest; using ghcr.io\n' + return 1 + fi fi local mirror_image="${mirror_host}/${image#ghcr.io/}" + if [[ -n "$mirror_digest" ]]; then + local repo="${image#ghcr.io/}" + repo="${repo%:*}" + mirror_image="${mirror_host}/${repo}@${mirror_digest}" + fi printf '\n▸ Pulling %s and tagging it %s\n' "$mirror_image" "$image" # Called from `if`, so a failing docker command does not exit on its own. - # Exit rather than return: a return would make the caller pull ghcr.io directly. + # An explicit mirror stops here. Automatic selection falls back to ghcr.io. if ! docker_access pull "$mirror_image"; then + if [[ "$auto" == 1 ]]; then + printf '\n▸ Could not pull %s; using ghcr.io\n' "$mirror_image" + return 1 + fi printf '\n✗ Could not pull %s. Stopped without retagging the previous image.\n' "$mirror_image" >&2 exit 1 fi diff --git a/scripts/test-install.py b/scripts/test-install.py index dd1d52c3a..94b3734b5 100644 --- a/scripts/test-install.py +++ b/scripts/test-install.py @@ -187,7 +187,9 @@ def mirror_probe_curl(self): prev=$arg url=$arg done +mode=ok case "$url" in + https://cdn.example.test/*) spec="${MOCK_MIRROR_GHCR:?}"; host=cdn.example.test ;; https://ghcr.nju.edu.cn/*) spec="${MOCK_MIRROR_NJU:?}"; host=ghcr.nju.edu.cn ;; https://ghcr.io/*) spec="${MOCK_MIRROR_GHCR:?}"; host=ghcr.io ;; *) printf 'unexpected %s\n' "$url" >&2; exit 1 ;; @@ -195,6 +197,8 @@ def mirror_probe_curl(self): case "$spec" in timeout) exit 28 ;; fail) exit 22 ;; + redirect:*) mode=redirect; spec=${spec#redirect:} ;; + blobtimeout:*) mode=blobtimeout; spec=${spec#blobtimeout:} ;; esac digest=${spec#ok:} seconds=${digest##*:} @@ -205,7 +209,14 @@ def mirror_probe_curl(self): printf 'HTTP/1.1 200 OK\r\n\r\n' > "$header_file" printf '200 %s 0.01\n' "$(wc -c < "$body")" ;; - */blobs/*) + */blobs/*|https://cdn.example.test/*) + if [[ "$mode" == blobtimeout && "$url" != https://cdn.example.test/* ]]; then exit 28; fi + if [[ "$mode" == redirect && "$url" != https://cdn.example.test/* ]]; then + : > "$body" + printf 'HTTP/1.1 307 Temporary Redirect\r\nLocation: https://cdn.example.test/v2/blob\r\n\r\n' > "$header_file" + printf '307 0 0.01\n' + exit 0 + fi printf 'x' > "$body" printf 'HTTP/1.1 206 Partial Content\r\n\r\n' > "$header_file" printf '206 262144 %s\n' "$seconds" @@ -227,6 +238,9 @@ def mirror_probe_curl(self): esac ''') + DIGEST = 'sha256:' + ('ab' * 32) + OTHER = 'sha256:' + ('cd' * 32) + def probe_env(self, ghcr, nju): return { 'MOCK_CURL_LOG': str(self.root / 'curl.log'), @@ -234,6 +248,9 @@ def probe_env(self, ghcr, nju): 'MOCK_MIRROR_NJU': nju, } + def timed(self, digest, seconds): + return f'ok:{digest}:{seconds}' + def test_explicit_mirror_does_not_probe(self): self.use_compose_file() self.executable('curl', '#!/usr/bin/env bash\nprintf \'%s\\n\' "$*" >> "$MOCK_CURL_LOG"\nexit 99\n') @@ -247,12 +264,13 @@ def test_explicit_mirror_does_not_probe(self): def test_faster_nju_mirror_is_pulled_and_tagged(self): self.use_compose_file() self.mirror_probe_curl() - result = self.install(**self.probe_env('ok:sha256:same:0.40', 'ok:sha256:same:0.10')) + result = self.install(**self.probe_env(self.timed(self.DIGEST, '0.40'), self.timed(self.DIGEST, '0.10'))) self.assert_success(result) image = self.pinned_image() - mirror = 'ghcr.nju.edu.cn/' + image[len('ghcr.io/'):] + mirror = 'ghcr.nju.edu.cn/tencent/loopforge@' + self.DIGEST log = (self.root / 'docker.log').read_text() self.assertLess(log.index(f'pull {mirror}'), log.index(f'tag {mirror} {image}')) + self.assertNotIn('pull ghcr.nju.edu.cn/tencent/loopforge:', log) self.assertIn('Fastest registry is ghcr.nju.edu.cn', result.stdout) self.assertIn('ghcr.io was', result.stdout) self.assertNotIn('MIRROR=', (self.root / '.env').read_text()) @@ -260,7 +278,7 @@ def test_faster_nju_mirror_is_pulled_and_tagged(self): def test_faster_ghcr_keeps_the_direct_pull(self): self.use_compose_file() self.mirror_probe_curl() - result = self.install(**self.probe_env('ok:sha256:same:0.10', 'ok:sha256:same:0.40')) + result = self.install(**self.probe_env(self.timed(self.DIGEST, '0.10'), self.timed(self.DIGEST, '0.40'))) self.assert_success(result) log = (self.root / 'docker.log').read_text() self.assertIn('pull --policy missing platform mysql', log) @@ -270,7 +288,7 @@ def test_faster_ghcr_keeps_the_direct_pull(self): def test_digest_mismatch_keeps_the_direct_pull(self): self.use_compose_file() self.mirror_probe_curl() - result = self.install(**self.probe_env('ok:sha256:upstream:0.40', 'ok:sha256:cached:0.05')) + result = self.install(**self.probe_env(self.timed(self.DIGEST, '0.40'), self.timed(self.OTHER, '0.05'))) self.assert_success(result) log = (self.root / 'docker.log').read_text() self.assertIn('pull --policy missing platform mysql', log) @@ -285,16 +303,52 @@ def test_both_mirrors_unreachable_keeps_the_direct_pull(self): log = (self.root / 'docker.log').read_text() self.assertIn('pull --policy missing platform mysql', log) self.assertNotIn('ghcr.nju.edu.cn', log) - self.assertIn('Could not time either registry; using ghcr.io', result.stdout) + self.assertIn('Could not verify the ghcr.io digest; using ghcr.io', result.stdout) - def test_nju_wins_when_ghcr_times_out(self): + def test_missing_official_digest_keeps_the_direct_pull(self): self.use_compose_file() self.mirror_probe_curl() - result = self.install(**self.probe_env('timeout', 'ok:sha256:same:0.20')) + result = self.install(**self.probe_env('timeout', self.timed(self.DIGEST, '0.20'))) self.assert_success(result) - self.assertIn('pull ghcr.nju.edu.cn/tencent/loopforge:', (self.root / 'docker.log').read_text()) + log = (self.root / 'docker.log').read_text() + self.assertIn('pull --policy missing platform mysql', log) + self.assertNotIn('ghcr.nju.edu.cn', log) + self.assertIn('Could not verify the ghcr.io digest', result.stdout) + + def test_verified_mirror_wins_when_the_ghcr_sample_times_out(self): + self.use_compose_file() + self.mirror_probe_curl() + result = self.install(**self.probe_env('blobtimeout:' + self.timed(self.DIGEST, '0.40'), self.timed(self.DIGEST, '0.10'))) + self.assert_success(result) + self.assertIn('pull ghcr.nju.edu.cn/tencent/loopforge@' + self.DIGEST, (self.root / 'docker.log').read_text()) self.assertIn('ghcr.io timed out', result.stdout) + def test_ghcr_blob_redirect_is_timed_without_sending_the_token(self): + self.use_compose_file() + self.mirror_probe_curl() + result = self.install(**self.probe_env('redirect:' + self.timed(self.DIGEST, '0.10'), self.timed(self.DIGEST, '0.40'))) + self.assert_success(result) + log = (self.root / 'docker.log').read_text() + curl_log = (self.root / 'curl.log').read_text() + self.assertIn('pull --policy missing platform mysql', log) + self.assertNotIn('ghcr.nju.edu.cn', log) + self.assertIn('Fastest registry is ghcr.io', result.stdout) + cdn = [line for line in curl_log.splitlines() if 'https://cdn.example.test/' in line] + self.assertTrue(cdn, curl_log) + self.assertNotIn('Authorization:', cdn[0]) + self.assertTrue(any('Authorization:' in line and '/blobs/' in line for line in curl_log.splitlines())) + + def test_auto_mirror_pull_failure_uses_ghcr(self): + self.use_compose_file() + self.mirror_probe_curl() + result = self.install(**self.probe_env(self.timed(self.DIGEST, '0.40'), self.timed(self.DIGEST, '0.10')), MOCK_PULL_FAIL='1') + self.assert_success(result) + log = (self.root / 'docker.log').read_text() + self.assertIn('pull ghcr.nju.edu.cn/tencent/loopforge@' + self.DIGEST, log) + self.assertNotIn(' tag ', log) + self.assertIn('pull --policy missing platform mysql', log) + self.assertIn('Could not pull', result.stdout) + def test_local_image_skips_the_speed_test(self): self.use_compose_file() self.executable('curl', '#!/usr/bin/env bash\nprintf \'%s\\n\' "$*" >> "$MOCK_CURL_LOG"\nexit 99\n') @@ -314,17 +368,17 @@ def test_start_pull_uses_the_faster_nju_mirror(self): (self.root / 'docker.log').unlink() result = subprocess.run( ['bash', 'start.sh', 'pull'], cwd=self.root, - env=dict(self.environ, **self.probe_env('timeout', 'ok:sha256:same:0.20')), + env=dict(self.environ, **self.probe_env(self.timed(self.DIGEST, '0.40'), self.timed(self.DIGEST, '0.10'))), text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) image = self.pinned_image() - mirror = 'ghcr.nju.edu.cn/' + image[len('ghcr.io/'):] + mirror = 'ghcr.nju.edu.cn/tencent/loopforge@' + self.DIGEST log = (self.root / 'docker.log').read_text() self.assertIn(f'pull {mirror}', log) self.assertIn(f'tag {mirror} {image}', log) self.assertIn('pull --policy always mysql', log) self.assertNotIn('pull --ignore-buildable --policy always', log) - self.assertIn('ghcr.io timed out', result.stdout) + self.assertIn('Fastest registry is ghcr.nju.edu.cn', result.stdout) def test_invalid_mirror_fails_before_pull(self): self.use_compose_file() From 627cb8fde0cd42acb26e8f34a7fcfe5508ceb660 Mon Sep 17 00:00:00 2001 From: cocofhu Date: Sat, 21 Nov 2026 10:50:08 +0800 Subject: [PATCH 3/3] fix: return Docker failures to callers on Bash 3.2 macOS /bin/bash aborts set -e when a command fails inside docker_access, so a cold install and a failed automatic mirror pull never reach the official pull. Co-authored-by: Cursor --- CHANGELOG.md | 1 + scripts/docker-access.sh | 8 +++-- scripts/ghcr-mirror.sh | 2 +- scripts/test-install.py | 71 +++++++++++++++++++++++++++++++++------- 4 files changed, 68 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95fbf01c9..11dfae5f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Platform releases use `vX.Y.Z` tags and publish `ghcr.io/tencent/loopforge`. Loo ## Unreleased +- On macOS `/bin/bash` 3.2, a missing local image or a failed automatic mirror pull no longer aborts `./start.sh` before the official image pull. - `./start.sh` and `./start.sh pull` time `ghcr.io` and `ghcr.nju.edu.cn`. Nanjing is used only when the ghcr.io manifest digest was verified and the mirror returned the same digest; the pull is that digest, then tagged back to the name pinned in `compose.yaml`. A missing official digest, a digest mismatch, or a failed mirror pull stays on `ghcr.io`. `MIRROR=ghcr.io` forces the upstream registry; `MIRROR=` still forces that host and does not fall back. Accepting Nanjing, or setting `MIRROR`, trusts that host. The choice is not saved in `.env`. `registry-mirrors` such as `https://docker.nju.edu.cn` only accelerates Docker Hub. ## 0.4.0 — scheduled task history, an MR node in delivery SOPs and agent CLIs in conversation workspaces diff --git a/scripts/docker-access.sh b/scripts/docker-access.sh index 34863ab49..bf03ba48f 100644 --- a/scripts/docker-access.sh +++ b/scripts/docker-access.sh @@ -1,11 +1,15 @@ #!/usr/bin/env bash docker_access() { + # macOS /bin/bash 3.2 does not inherit an if/&& caller into a function. + # A failing command there aborts set -e before the caller can see the status. + # A subshell returns Docker's status instead, including to a cold install's + # image inspect and to the automatic mirror pull fallback. if [[ "${LOOPFORGE_DOCKER_USE_SUDO:-0}" == 1 ]]; then require_sudo - command sudo -n docker "$@" + ( command sudo -n docker "$@" ) else - command docker "$@" + ( command docker "$@" ) fi } diff --git a/scripts/ghcr-mirror.sh b/scripts/ghcr-mirror.sh index 1c3fbdb03..60c1d078e 100644 --- a/scripts/ghcr-mirror.sh +++ b/scripts/ghcr-mirror.sh @@ -390,7 +390,7 @@ PY mirror_image="${mirror_host}/${repo}@${mirror_digest}" fi printf '\n▸ Pulling %s and tagging it %s\n' "$mirror_image" "$image" - # Called from `if`, so a failing docker command does not exit on its own. + # docker_access returns the failure on Bash 3.2 and later. # An explicit mirror stops here. Automatic selection falls back to ghcr.io. if ! docker_access pull "$mirror_image"; then if [[ "$auto" == 1 ]]; then diff --git a/scripts/test-install.py b/scripts/test-install.py index 94b3734b5..4307a8404 100644 --- a/scripts/test-install.py +++ b/scripts/test-install.py @@ -76,8 +76,29 @@ def executable(self, name, contents): target.write_text(contents) target.chmod(0o755) - def install(self, **values): - return subprocess.run(['bash', 'scripts/install.sh'], cwd=self.root, env=dict(self.environ, **values), text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) + def shell(self): + return os.environ.get('LOOPFORGE_TEST_BASH', 'bash') + + def bash32(self): + candidates = [] + for name in ('LOOPFORGE_TEST_BASH32', 'LOOPFORGE_TEST_BASH'): + value = os.environ.get(name) + if value: + candidates.append(value) + candidates.append('/bin/bash') + seen = set() + for candidate in candidates: + path = candidate if os.path.isabs(candidate) else shutil.which(candidate) + if not path or path in seen: + continue + seen.add(path) + version = subprocess.run([path, '-c', 'printf %s "${BASH_VERSINFO[0]}.${BASH_VERSINFO[1]}"'], capture_output=True, text=True) + if version.returncode == 0 and version.stdout == '3.2': + return path + self.skipTest('Bash 3.2 is not available') + + def install(self, shell=None, **values): + return subprocess.run([shell or self.shell(), 'scripts/install.sh'], cwd=self.root, env=dict(self.environ, **values), text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) def assert_success(self, result): self.assertEqual(result.returncode, 0, result.stdout + result.stderr) @@ -349,6 +370,34 @@ def test_auto_mirror_pull_failure_uses_ghcr(self): self.assertIn('pull --policy missing platform mysql', log) self.assertIn('Could not pull', result.stdout) + def test_bash32_cold_install_continues_when_the_image_is_missing(self): + shell = self.bash32() + self.use_compose_file() + self.mirror_probe_curl() + result = self.install(shell=shell, **self.probe_env('timeout', 'timeout')) + self.assert_success(result) + log = (self.root / 'docker.log').read_text() + self.assertIn('image inspect ', log) + self.assertIn('pull --policy missing platform mysql', log) + self.assertNotIn('ghcr.nju.edu.cn', log) + + def test_bash32_auto_pull_failure_falls_back(self): + shell = self.bash32() + self.assert_success(self.install(shell=shell)) + self.use_compose_file() + self.mirror_probe_curl() + (self.root / 'docker.log').unlink() + result = subprocess.run( + [shell, 'start.sh', 'pull'], cwd=self.root, + env=dict(self.environ, **self.probe_env(self.timed(self.DIGEST, '0.40'), self.timed(self.DIGEST, '0.10')), MOCK_PULL_FAIL='1'), + text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + log = (self.root / 'docker.log').read_text() + self.assertIn('pull ghcr.nju.edu.cn/tencent/loopforge@' + self.DIGEST, log) + self.assertNotIn(' tag ', log) + self.assertIn('pull --ignore-buildable --policy always', log) + self.assertIn('Could not pull', result.stdout) + def test_local_image_skips_the_speed_test(self): self.use_compose_file() self.executable('curl', '#!/usr/bin/env bash\nprintf \'%s\\n\' "$*" >> "$MOCK_CURL_LOG"\nexit 99\n') @@ -367,7 +416,7 @@ def test_start_pull_uses_the_faster_nju_mirror(self): self.mirror_probe_curl() (self.root / 'docker.log').unlink() result = subprocess.run( - ['bash', 'start.sh', 'pull'], cwd=self.root, + [self.shell(), 'start.sh', 'pull'], cwd=self.root, env=dict(self.environ, **self.probe_env(self.timed(self.DIGEST, '0.40'), self.timed(self.DIGEST, '0.10'))), text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) @@ -394,7 +443,7 @@ def test_invalid_mirror_fails_before_pull(self): def test_start_pull_without_mirror_is_unchanged(self): self.assert_success(self.install()) (self.root / 'docker.log').unlink() - result = subprocess.run(['bash', 'start.sh', 'pull'], cwd=self.root, env=self.environ, text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) + result = subprocess.run([self.shell(), 'start.sh', 'pull'], cwd=self.root, env=self.environ, text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) log = (self.root / 'docker.log').read_text() self.assertIn('pull --ignore-buildable --policy always', log) @@ -403,7 +452,7 @@ def test_start_pull_without_mirror_is_unchanged(self): def test_start_pull_reapplies_schema_extensions_after_containers_are_up(self): self.assert_success(self.install()) (self.root / 'docker.log').unlink() - result = subprocess.run(['bash', 'start.sh', 'pull'], cwd=self.root, env=dict(self.environ, MOCK_COLUMN_COUNT='0'), text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) + result = subprocess.run([self.shell(), 'start.sh', 'pull'], cwd=self.root, env=dict(self.environ, MOCK_COLUMN_COUNT='0'), text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) log = (self.root / 'docker.log').read_text() sql = (self.root / 'mysql.log').read_text() @@ -417,7 +466,7 @@ def test_start_pull_reapplies_schema_extensions_after_containers_are_up(self): def test_start_pull_names_the_migration_when_columns_cannot_be_checked(self): self.assert_success(self.install()) - result = subprocess.run(['bash', 'start.sh', 'pull'], cwd=self.root, env=dict(self.environ, MOCK_COLUMN_COUNT='?'), text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) + result = subprocess.run([self.shell(), 'start.sh', 'pull'], cwd=self.root, env=dict(self.environ, MOCK_COLUMN_COUNT='?'), text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) self.assertNotEqual(result.returncode, 0) self.assertIn('Could not inspect run_telemetry.adapter', result.stderr) self.assertIn('migrations/001_run_obs_generated_columns.sql', result.stderr) @@ -426,7 +475,7 @@ def test_start_pull_with_nju_mirror_does_not_contact_ghcr_for_platform(self): self.assert_success(self.install()) self.use_compose_file() (self.root / 'docker.log').unlink() - result = subprocess.run(['bash', 'start.sh', 'pull'], cwd=self.root, env=dict(self.environ, MIRROR='ghcr.nju.edu.cn'), text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) + result = subprocess.run([self.shell(), 'start.sh', 'pull'], cwd=self.root, env=dict(self.environ, MIRROR='ghcr.nju.edu.cn'), text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) image = self.pinned_image() mirror = 'ghcr.nju.edu.cn/' + image[len('ghcr.io/'):] @@ -442,7 +491,7 @@ def test_mirror_uses_the_last_image_assignment_in_env(self): env = self.root / '.env' env.write_text(env.read_text() + 'LOOPFORGE_IMAGE=ghcr.io/example/ignored:1\nLOOPFORGE_IMAGE=ghcr.io/example/chosen:9\n') (self.root / 'docker.log').unlink() - result = subprocess.run(['bash', 'start.sh', 'pull'], cwd=self.root, env=dict(self.environ, MIRROR='ghcr.nju.edu.cn'), text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) + result = subprocess.run([self.shell(), 'start.sh', 'pull'], cwd=self.root, env=dict(self.environ, MIRROR='ghcr.nju.edu.cn'), text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) log = (self.root / 'docker.log').read_text() self.assertIn('pull ghcr.nju.edu.cn/example/chosen:9', log) @@ -465,7 +514,7 @@ def test_failed_mirror_refresh_does_not_retag_a_cached_image(self): self.use_compose_file() (self.root / 'docker.log').unlink() result = subprocess.run( - ['bash', 'start.sh', 'pull'], cwd=self.root, + [self.shell(), 'start.sh', 'pull'], cwd=self.root, env=dict(self.environ, MIRROR='ghcr.nju.edu.cn', MOCK_PULL_FAIL='1', MOCK_IMAGE_PRESENT='1'), text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) @@ -505,7 +554,7 @@ def test_source_build_persists_dev_compose_file(self): def test_start_switches_source_build_back_to_release(self): self.assert_success(self.install(LOOPFORGE_BUILD_FROM_SOURCE='1')) (self.root / 'docker.log').unlink() - result = subprocess.run(['bash', 'start.sh', 'release'], cwd=self.root, env=self.environ, text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) + result = subprocess.run([self.shell(), 'start.sh', 'release'], cwd=self.root, env=self.environ, text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) self.assert_success(result) self.assertNotIn('COMPOSE_FILE=', (self.root / '.env').read_text()) log = (self.root / 'docker.log').read_text() @@ -513,7 +562,7 @@ def test_start_switches_source_build_back_to_release(self): self.assertNotIn('build platform', log) def test_start_refuses_local_build(self): - result = subprocess.run(['bash', 'start.sh', 'build'], cwd=self.root, env=self.environ, text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) + result = subprocess.run([self.shell(), 'start.sh', 'build'], cwd=self.root, env=self.environ, text=True, capture_output=True, stdin=subprocess.DEVNULL, timeout=15) self.assertNotEqual(result.returncode, 0) self.assertIn('./start.sh dev', result.stderr)