From 15e53d6f377fda7d6977e8966326616413e5ce57 Mon Sep 17 00:00:00 2001 From: Volker Christian Date: Wed, 14 Oct 2026 12:46:08 +0200 Subject: [PATCH] ci: support manual fresh package builds for existing tags --- ci/dispatch.py | 8 +++- docs/maintainers.md | 4 +- tests/check_manual_package_workflows.py | 56 +++++++++++++++++++++++++ tests/test_handoffs.py | 26 ++++++++++++ tests/test_packaging.py | 3 +- 5 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 tests/check_manual_package_workflows.py diff --git a/ci/dispatch.py b/ci/dispatch.py index ebad97c..b8492dd 100644 --- a/ci/dispatch.py +++ b/ci/dispatch.py @@ -41,7 +41,7 @@ def receive(payload): if not all(value.isdecimal() and int(value) > 0 for value in (capture, build_run, attempt)): raise ValueError('Invalid run identity') origin = api(f'repos/SNodeC/{project}/actions/runs/{build_run}') - if (origin['event'] not in {'push', 'repository_dispatch'} or origin['path'] != '.github/workflows/build-distribution-packages.yml' + if (origin['event'] not in {'push', 'repository_dispatch', 'workflow_dispatch'} or origin['path'] != '.github/workflows/build-distribution-packages.yml' or origin['head_repository']['full_name'] != f'SNodeC/{project}' or origin['run_attempt'] < int(attempt)): raise ValueError('Artifact must come from the current upstream package build attempt') @@ -50,6 +50,10 @@ def receive(payload): if origin['event'] == 'push': if capture != build_run or not re.fullmatch(r'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)', origin['head_branch']): raise ValueError('Capture must be the originating version-tag run') + elif origin['event'] == 'workflow_dispatch': + if (capture != build_run or origin['display_title'] != f'Packages · {capture} · -' + or origin['head_branch'] != api(f'repos/SNodeC/{project}')['default_branch']): + raise ValueError('Manual capture must be its own default-branch package run') elif project != 'mqttsuite' or origin['display_title'] != f'Packages · {capture} · {row["id"]}': raise ValueError('Only a published SNode.C target dispatches a dependent build') job_url = origin['html_url'] @@ -65,7 +69,7 @@ def receive(payload): name = f'packages-{project}-{row["id"]}-{attempt}' return dict(project=project, target=json.dumps(row), capture=capture, run=build_run, attempt=attempt, artifact=name, run_url=job_url, - capture_repository='SNodeC/' + (project if origin['event'] == 'push' else 'snode.c')) + capture_repository='SNodeC/' + (project if origin['event'] != 'repository_dispatch' else 'snode.c')) def baseline(bundle, target, ref): diff --git a/docs/maintainers.md b/docs/maintainers.md index 16a1c5b..0e917f0 100644 --- a/docs/maintainers.md +++ b/docs/maintainers.md @@ -63,7 +63,9 @@ Superseded files remain for 30 days after leaving the active index. Cleanup runs The manual-only **Render package status** workflow in DistributionPackages runs the default-branch renderer against the current Packages state, with the existing publication concurrency group and metadata retry transaction. It does not build, sign or publish packages, and is not triggered automatically after merge. Run it only after explicit approval. Locally: `python3 -m ci.publish.publication render /path/to/Packages`. The local renderer preserves authored documentation, feed records and `status.json`. The approved manual workflow also hands off the current `site/` README, documentation and keys guide through the same metadata retry transaction; it removes obsolete documentation without changing key bytes or installer behavior. -There are no workflows in Packages. Shared platform scripts have one implementation in DistributionPackages; the source repositories own and execute the build jobs. Signing and publication tools remain checked out from DistributionPackages’ own Git history at the captured packaging commit, never supplied by an uploaded artifact. The presentation-only exception is an isolated, read-only navigation worker from `SNodeC/.github@v1.1`, with PyYAML; it receives no feed paths, CI credentials or signing environment. Signing secrets exist only in DistributionPackages. OpenWrt index signing and RPM package/index signing happen there after upload; APT metadata is also signed there. Builds need no private signing key. +SNode.C and MQTTSuite package workflows also support a manual fresh start: choose **Run workflow** on the source repository’s default branch and supply an existing `vMAJOR.MINOR.PATCH` tag as `release_tag`. This allocates a new package revision and captures that tag with current DistributionPackages tooling; no tag is created, moved or deleted. Publication authenticates the manual run’s repository, workflow, default branch, capture identity, attempt and target job. Tag/Version validation remains in the existing capture path. A job retry retains the original capture and requires its artifact; a new manual start is the recovery route when a full rerun loses that artifact. + +There are no workflows in Packages. Shared platform scripts have one implementation in DistributionPackages; the source repositories own and execute the build jobs. Signing and publication tools remain checked out from DistributionPackages’ own Git history at the captured packaging commit, never supplied by an uploaded artifact. The presentation-only exception is an isolated, read-only navigation worker from `SNodeC/.github@main`, with PyYAML; it receives no feed paths, CI credentials or signing environment. Signing secrets exist only in DistributionPackages. OpenWrt index signing and RPM package/index signing happen there after upload; APT metadata is also signed there. Builds need no private signing key. OpenWrt signer executables (`usign` and `apk`) are stored in this repository’s GitHub Actions cache, not in Git or Packages. The exact key includes runner OS/architecture, runner-image version and the SHA-256 content hash of the capture-pinned `ci/publish/tools.sh`; pinned source revisions and build options are inside that script. There are no fallback keys. A miss, eviction or missing executable rebuilds both tools from the audited pinned sources; a hit installs the cached executables without cloning or compiling them. Only executables are cached, never private keys, tokens or package artifacts. Existing captures keep their original script; new captures after this change use the cache. Publication still verifies signatures before committing feeds. diff --git a/tests/check_manual_package_workflows.py b/tests/check_manual_package_workflows.py new file mode 100644 index 0000000..9c923c7 --- /dev/null +++ b/tests/check_manual_package_workflows.py @@ -0,0 +1,56 @@ +"""Check source package-workflow boundaries locally without dispatches or tags.""" +import os +from pathlib import Path +import subprocess +import sys +import tempfile + +import yaml + + +def check(path): + workflow = yaml.load(path.read_text(), Loader=yaml.BaseLoader) + inputs = workflow['on']['workflow_dispatch']['inputs'] + assert inputs['release_tag']['required'] == 'true' + assert workflow['on']['push']['tags'] == ['v[0-9]*.[0-9]*.[0-9]*'] + assert workflow['on']['repository_dispatch']['types'] == ['build-dependent-packages'] + prepare = workflow['jobs']['prepare'] + assert "github.event_name == 'workflow_dispatch' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)" in prepare['if'] + steps = prepare['steps'] + download = next(s for s in steps if s.get('uses', '').startswith('actions/download-artifact@')) + assert download['if'] == "github.event_name == 'repository_dispatch' || github.run_attempt > 1" + initialize = next(s for s in steps if s.get('id') == 'initialize') + assert initialize['if'] == "(github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.run_attempt == 1" + assert initialize['env']['RELEASE_TAG'] == '${{ inputs.release_tag || github.ref_name }}' + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + status = root / 'feed/ci/publish/status.sh' + status.parent.mkdir(parents=True) + status.write_text('#!/bin/bash\nprintf "%s" "$RELEASE_TAG" > reserved\n') + binary = root / 'bin' + binary.mkdir() + python = binary / 'python3' + python.write_text('#!/bin/bash\n[[ "$*" == "-m ci.repository prepare published bundle" ]]\nprintf "%s" "$RELEASE_TAG" > captured\n') + python.chmod(0o755) + for tag in ('v2.0.0', 'v1.0.2', 'v12.34.56', 'master', 'Book-1.0', 'v01.0.0', 'v1.0', 'v1.0.0;touch injected'): + for name in ('reserved', 'captured', 'injected'): + (root / name).unlink(missing_ok=True) + result = subprocess.run(['bash', '-e', '-o', 'pipefail', '-c', initialize['run']], + cwd=root, capture_output=True, text=True, + env=os.environ | {'PATH': str(binary) + ':' + os.defpath, + 'RELEASE_TAG': tag, 'GITHUB_REF': 'refs/heads/master'}) + valid = tag in ('v2.0.0', 'v1.0.2', 'v12.34.56') + assert (result.returncode == 0) == valid, (path, tag, result.stderr) + assert (root / 'captured').exists() == valid + assert (root / 'reserved').exists() == valid + assert not (root / 'injected').exists() + if valid: + assert (root / 'captured').read_text() == tag + print(f'{path}: manual/tag/dependent routing and 8 tag-validation cases passed') + + +if __name__ == '__main__': + if len(sys.argv) < 2: + raise SystemExit('Provide source build-distribution-packages.yml paths') + for argument in sys.argv[1:]: + check(Path(argument)) diff --git a/tests/test_handoffs.py b/tests/test_handoffs.py index 39c4807..1ea4c2b 100644 --- a/tests/test_handoffs.py +++ b/tests/test_handoffs.py @@ -33,6 +33,32 @@ def test_receipt_requires_upstream_workflow_capture_and_attempt(self): with self.assertRaises(ValueError): dispatch.receive(payload) + def test_manual_publication_requires_own_capture_and_default_branch(self): + target = publication.targets()[0]['id'] + for project in ('snode.c', 'mqttsuite'): + payload = dict(project=project, target=target, capture='123', run='123', attempt='1') + origin = dict(event='workflow_dispatch', head_branch='trunk', + path='.github/workflows/build-distribution-packages.yml', + head_repository={'full_name': f'SNodeC/{project}'}, run_attempt=1, + display_title='Packages · 123 · -', html_url='https://example.invalid/123') + def response(path): + return {'default_branch': 'trunk'} if path == f'repos/SNodeC/{project}' else origin + with self.subTest(project=project), patch.object(dispatch, 'api', side_effect=response): + result = dispatch.receive(payload) + self.assertEqual(result['capture_repository'], f'SNodeC/{project}') + self.assertEqual(result['artifact'], f'packages-{project}-{target}-1') + with self.assertRaises(ValueError): + dispatch.receive(payload | {'capture': '456'}) + for changes in (dict(head_branch='feature/manual'), dict(run_attempt=0), + dict(path='.github/workflows/other.yml'), + dict(head_repository={'full_name': 'other/fork'}), + dict(display_title=f'Packages · 123 · {target}')): + invalid = origin | changes + with self.subTest(project=project, changes=changes), patch.object(dispatch, 'api', + side_effect=lambda path: {'default_branch': 'trunk'} if path == f'repos/SNodeC/{project}' else invalid): + with self.assertRaises(ValueError): + dispatch.receive(payload) + def test_receipt_locates_capture_for_application_tag_and_dependency_build(self): target = publication.targets()[0]['id'] payload = dict(project='mqttsuite', target=target, capture='123', run='123', attempt='1') diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 9846e58..e9577c8 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -127,7 +127,8 @@ def test_status_render_is_immutable_and_does_not_copy_documentation(self): (root / 'status/old-target.svg').write_text('obsolete') publication.render(root, {'targets': {}}) before = {p.name: p.read_bytes() for p in (root / 'status/badges').iterdir()} - self.assertEqual(set(before), {label + ".svg" for label in ["current", "outdated", "pending", "running", "publishing", "published", "failed", "cancelled", "skipped", "superseded", "not-built"]}) + self.assertEqual(set(before), {label + ".svg" for label in ["current", "outdated", "pending", "running", "publishing", "failed", "cancelled", "skipped", "superseded", "not-built"]}) + self.assertNotIn("published.svg", before) for name, svg in before.items(): self.assertIn(b'width="14" height="14"', svg) self.assertIn(b'