From 9c319cdba8db65843803683ddf269e97a2d5f1b6 Mon Sep 17 00:00:00 2001 From: Wu Sheng Date: Sat, 17 Oct 2026 18:28:20 +0800 Subject: [PATCH 1/2] Make release packaging work on macOS and Linux, and harden the release scripts The 0.15.0 release, packed on macOS, put 543 AppleDouble `._*` files into the src tarball and 116 into the bin tarball, plus com.apple.provenance xattr headers on every entry. macOS bsdtar archives extended attributes by default, and every file in a fresh clone carries com.apple.provenance. Makefile: - Pack with `COPYFILE_DISABLE=1 tar --no-xattrs`, which works with both bsdtar and GNU tar, and exclude coverage.txt from the src tarball. - Stop ignoring packing errors. The src tarball is packed into a temporary directory and moved in, because GNU tar exits 1 ("file changed as we read it") when it writes the archive inside the directory being packed. - New `release-check`: fail if a package is missing, unreadable, or contains `._*`, `.DS_Store` or `__MACOSX` entries. On bsdtar it lists with `--options !mac-ext`, because plain `tar -t` hides AppleDouble entries. `release` runs it before signing, as a recipe step with `-f`, so it works with `make -j` and with `make -f` against an older checkout. - Add the missing `license` target. `verify` depends on it: Linux failed with "No rule to make target 'license'", and on macOS the LICENSE file satisfied it, so no license check ever ran. scripts/pre-release.sh: - Ask (y/N) before removing an existing ./skywalking-cli or ./skywalking, and remove only the paths it listed. - Push only the release tag, and only after `make release` succeeds. - Put every step that must succeed on its own line, since `set -e` does not stop on a failing non-final command of an `&&` chain. - Remove the svn working copy after a successful upload. - Stop printing each sha512 filename twice in the vote email. scripts/release.sh: drop a pointless `rm -rf skywalking` and the `|| true` that hid a failed `svn mv`, and give `read` a variable so it works under dash. docs/How-to-release.md: document the above, build with the CI Go version through GOTOOLCHAIN, fix the tag name and the KEYS link in the email templates, and add a vote check for macOS metadata. Tested on macOS (bsdtar 3.5.3; sh, bash) and Linux (GNU tar 1.35; dash, bash). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EkgWrRt7QJvPTNx2N5wZr9 --- Makefile | 55 ++++++++++++++++++++++++++++++++++-------- docs/How-to-release.md | 37 ++++++++++++++++++++++------ scripts/pre-release.sh | 47 +++++++++++++++++++++++++++--------- scripts/release.sh | 10 +++----- 4 files changed, 113 insertions(+), 36 deletions(-) diff --git a/Makefile b/Makefile index d321b1a3..52b23131 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,10 @@ HUB ?= docker.io/apache RELEASE_BIN = skywalking-cli-$(VERSION)-bin RELEASE_SRC = skywalking-cli-$(VERSION)-src +# COPYFILE_DISABLE stops macOS bsdtar from adding AppleDouble `._*` files, and --no-xattrs (bsdtar and GNU tar) +# keeps extended attributes such as com.apple.provenance out of the tar headers. +TAR = COPYFILE_DISABLE=1 tar --no-xattrs + GO = go GO_PATH = $$($(GO) env GOPATH) GO_BUILD = $(GO) build @@ -70,6 +74,11 @@ lint: $(GO_LINT) fix-lint: $(GO_LINT) $(GO_LINT) run -v --fix ./... +# `all` and `verify` depend on `license`. Without this target, case-insensitive macOS filesystems let the LICENSE file +# satisfy it (so no license check ran), and Linux failed with "No rule to make target 'license'". +.PHONY: license +license: license-header dependency-license + .PHONY: license-header license-header: clean $(LICENSE_EYE) @$(LICENSE_EYE) header check @@ -118,26 +127,52 @@ clean: -rm -rf *.sha512 @go mod tidy &> /dev/null +# The source tarball is packed into a temporary directory and moved here: GNU tar exits 1 ("file changed as we read it") +# when the archive it writes is inside the directory being packed. Packing errors must fail the release. release-src: clean - -tar -zcvf $(RELEASE_SRC).tgz \ + tmp=$$(mktemp -d) && trap 'rm -rf "$$tmp"' EXIT && \ + $(TAR) -zcvf "$$tmp/$(RELEASE_SRC).tgz" \ --exclude bin \ --exclude .git \ --exclude .idea \ --exclude .DS_Store \ --exclude dist \ + --exclude coverage.txt \ --exclude $(RELEASE_SRC).tgz \ - . + --exclude $(RELEASE_BIN) \ + --exclude $(RELEASE_BIN).tgz \ + . && \ + mv "$$tmp/$(RELEASE_SRC).tgz" . release-bin: build - -mkdir $(RELEASE_BIN) - -cp -R bin $(RELEASE_BIN) - -cp -R dist/* $(RELEASE_BIN) - -cp -R CHANGES.md $(RELEASE_BIN) - -cp -R README.md $(RELEASE_BIN) - -tar -zcvf $(RELEASE_BIN).tgz $(RELEASE_BIN) - -rm -rf $(RELEASE_BIN) - + rm -rf $(RELEASE_BIN) + mkdir $(RELEASE_BIN) + cp -R bin $(RELEASE_BIN) + cp -R dist/* $(RELEASE_BIN) + cp -R CHANGES.md $(RELEASE_BIN) + cp -R README.md $(RELEASE_BIN) + $(TAR) -zcvf $(RELEASE_BIN).tgz $(RELEASE_BIN) + rm -rf $(RELEASE_BIN) + +# macOS bsdtar folds AppleDouble `._*` entries into the next entry when listing, so `-t` alone would hide them. +# `--options !mac-ext` lists every entry as it is. GNU tar has no such option and lists them anyway. +.PHONY: release-check +release-check: + @list_opts=""; \ + if tar --version 2>/dev/null | grep -q bsdtar; then list_opts="--options !mac-ext"; fi; \ + for pkg in $(RELEASE_SRC).tgz $(RELEASE_BIN).tgz; do \ + if [ ! -f "$$pkg" ]; then echo "$$pkg was not created"; exit 1; fi; \ + entries=$$(tar $$list_opts -tzf "$$pkg") || { echo "$$pkg is not a readable tar.gz"; exit 1; }; \ + if echo "$$entries" | grep -E '(^|/)(\._|(\.DS_Store|__MACOSX)(/|$$))'; then \ + echo "$$pkg contains the macOS metadata files listed above"; \ + exit 1; \ + fi; \ + done + +# release-check runs as a recipe step, not a prerequisite, so it always runs after both packages exist, even with -j. +# `-f` keeps the sub-make on this Makefile when it was given with `make -f`, since a sub-make does not inherit `-f`. release: verify release-src release-bin + @$(MAKE) -f $(firstword $(MAKEFILE_LIST)) --no-print-directory release-check gpg --batch --yes --armor --detach-sig $(RELEASE_SRC).tgz shasum -a 512 $(RELEASE_SRC).tgz > $(RELEASE_SRC).tgz.sha512 gpg --batch --yes --armor --detach-sig $(RELEASE_BIN).tgz diff --git a/docs/How-to-release.md b/docs/How-to-release.md index acdb878f..45be9d8e 100644 --- a/docs/How-to-release.md +++ b/docs/How-to-release.md @@ -22,27 +22,44 @@ check the release for vote. ## Build and sign the source code package +Run the release from an empty directory. Build with the Go version CI uses (`go-version` in +`.github/workflows/CI.yaml`), not a newer local Go: `make release` runs the linter, and the release binaries embed the +Go standard library. `GOTOOLCHAIN` selects that version without installing it separately. + ```shell export VERSION= +export GOTOOLCHAIN=go1.26. +mkdir release-cli && cd release-cli git clone --recurse-submodules git@github.com:apache/skywalking-cli && cd skywalking-cli git tag -a "$VERSION" -m "Release Apache SkyWalking-CLI $VERSION" -git push --tags -make clean && make release +make clean && make release && git push origin "$VERSION" ``` +Push the tag only after `make release` succeeds, which the `&&` chain above enforces. `scripts/pre-release.sh` runs +these steps and the svn upload below. + **In total, six files should be automatically generated in the directory**: `skywalking-cli-${VERSION}-bin.tgz` , `skywalking-cli-${VERSION}-src.tgz`, and their corresponding `asc`, `sha512` files. +`make release` packs both tarballs with `COPYFILE_DISABLE=1 tar --no-xattrs`, so a release built on macOS carries no +AppleDouble `._*` files or extended attributes. It fails if either package still contains `._*`, `.DS_Store` or +`__MACOSX` entries. + ## Upload to Apache svn +Run this from the release directory, the parent of `skywalking-cli`. The svn working copy is removed only after the +commit succeeds. + ```bash +cd .. svn co https://dist.apache.org/repos/dist/dev/skywalking/ mkdir -p skywalking/cli/"$VERSION" cp skywalking-cli/skywalking*.tgz skywalking/cli/"$VERSION" cp skywalking-cli/skywalking*.tgz.asc skywalking/cli/"$VERSION" cp skywalking-cli/skywalking-cli*.tgz.sha512 skywalking/cli/"$VERSION" -cd skywalking/cli && svn add "$VERSION" && svn commit -m "Draft Apache SkyWalking-CLI release $VERSION" +cd skywalking/cli && svn add "$VERSION" && svn commit -m "Draft Apache SkyWalking-CLI release $VERSION" && \ + cd ../.. && rm -rf skywalking ``` ## Make the internal announcement @@ -72,7 +89,7 @@ Release Candidate: Release Tag : - * (Git Tag) v$VERSION + * (Git Tag) $VERSION Release Commit Hash : @@ -80,7 +97,7 @@ Release Commit Hash : Keys to verify the Release Candidate : - * http://pgp.mit.edu:11371/pks/lookup?op=get&search=0x8BD99F552D9F33D7 corresponding to kezhenxu94@apache.org + * https://dist.apache.org/repos/dist/release/skywalking/KEYS Guide to build the release from source : @@ -120,7 +137,7 @@ Release Candidate: Release Tag : - * (Git Tag) v$VERSION + * (Git Tag) $VERSION Release Commit Hash : @@ -150,10 +167,14 @@ Thanks. All PMC members and committers should check these before voting +1: 1. Features test. -1. All artifacts in staging repository are published with `.asc`, `.md5`, and `sha` files. +1. All artifacts in staging repository are published with `.asc` and `.sha512` files. 1. Source codes and distribution packages (`skywalking-cli-$VERSION-{src,bin}.tgz`) are in `https://dist.apache.org/repos/dist/dev/skywalking/cli/$VERSION` with `.asc`, `.sha512`. 1. `LICENSE` and `NOTICE` are in source codes and distribution package. +1. The packages contain no macOS metadata files: + `tar -tzf skywalking-cli-$VERSION-src.tgz | grep -E '(^|/)(\._|(\.DS_Store|__MACOSX)(/|$))'` prints nothing, and the + same for the bin package. On macOS, use `tar --options '!mac-ext' -tzf ...` instead: the system bsdtar hides + AppleDouble `._*` entries when listing an archive. 1. Check `shasum -c skywalking-cli-$VERSION-{src,bin}.tgz.sha512`. 1. Check `gpg --verify skywalking-cli-$VERSION-{src,bin}.tgz.asc skywalking-cli-$VERSION-{src,bin}.tgz`. 1. Build distribution from source code package by following @@ -204,7 +225,7 @@ have [to enable BuildKit in the Docker daemon](https://docs.docker.com/develop/d make docker.push ``` -1. Refer to the previous [PR](https://github.com/apache/skywalking-website/pull/212), update the event and download +1. Refer to the previous [PR](https://github.com/apache/skywalking-website/pull/751), update the event and download links on the website. 1. Update [Github release page](https://github.com/apache/skywalking-cli/releases), follow the previous convention. diff --git a/scripts/pre-release.sh b/scripts/pre-release.sh index 7fee61fb..341cec85 100755 --- a/scripts/pre-release.sh +++ b/scripts/pre-release.sh @@ -19,26 +19,44 @@ # 1. update change log # 2. clear milestone issues, and create a new one if needed # 3. export VERSION= +# 4. build with the Go version CI uses, e.g. export GOTOOLCHAIN=go1.26. +# 5. run this script from an empty directory, it clones skywalking-cli and checks out the svn dev area there set -ex [ -z "$VERSION" ] && echo "VERSION is not set" && exit 1 -if ls skywalking-cli > /dev/null 2>&1; then - rm -rf skywalking-cli +# Both directories are recreated below, so ask before deleting anything that already exists here. +# Under `set -e`, a failing command that is not the last one of an `a && b` chain does not stop the script, so every +# step below that must succeed sits on its own line. +existing="" +for dir in skywalking-cli skywalking; do + if [ -e "$dir" ] || [ -L "$dir" ]; then + existing="$existing $dir" + fi +done +if [ -n "$existing" ]; then + printf 'Already in %s:%s. Remove and continue? [y/N] ' "$(pwd)" "$existing" + read -r answer + # $existing is unquoted on purpose: it only holds the fixed names listed in the prompt. + # shellcheck disable=SC2086 + case "$answer" in + [yY] | [yY][eE][sS]) rm -rf $existing ;; + *) echo "Aborted, nothing was removed" && exit 1 ;; + esac fi -git clone --recurse-submodules git@github.com:apache/skywalking-cli && cd skywalking-cli +git clone --recurse-submodules git@github.com:apache/skywalking-cli +cd skywalking-cli git tag -a "$VERSION" -m "Release Apache SkyWalking-CLI $VERSION" -git push --tags -make clean && make release +make clean +make release -cd .. +# Push the tag only after the packages are built and signed, so a failed build leaves no published tag. +git push origin "$VERSION" -if ls skywalking > /dev/null 2>&1; then - rm -rf skywalking -fi +cd .. svn co https://dist.apache.org/repos/dist/dev/skywalking/ mkdir -p skywalking/cli/"$VERSION" @@ -46,9 +64,14 @@ cp skywalking-cli/skywalking*.tgz skywalking/cli/"$VERSION" cp skywalking-cli/skywalking*.tgz.asc skywalking/cli/"$VERSION" cp skywalking-cli/skywalking-cli*.tgz.sha512 skywalking/cli/"$VERSION" -cd skywalking/cli && svn add "$VERSION" && svn commit -m "Draft Apache SkyWalking-CLI release $VERSION" +cd skywalking/cli +svn add "$VERSION" +svn commit -m "Draft Apache SkyWalking-CLI release $VERSION" cd ../.. +# The svn working copy is only needed for the upload above, the packages stay in ./skywalking-cli. +rm -rf skywalking + cd skywalking-cli cat << EOF @@ -70,8 +93,8 @@ Release Candidate: * https://dist.apache.org/repos/dist/dev/skywalking/cli/$VERSION/ * sha512 checksums - - $(cat skywalking-cli*bin*tgz.sha512) skywalking-cli-$VERSION-bin.tgz - - $(cat skywalking-cli*src*tgz.sha512) skywalking-cli-$VERSION-src.tgz + - $(cat skywalking-cli*bin*tgz.sha512) + - $(cat skywalking-cli*src*tgz.sha512) Release Tag : diff --git a/scripts/release.sh b/scripts/release.sh index 8298510c..9ca3c661 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -23,13 +23,11 @@ set -ex [ -z "$VERSION" ] && echo "VERSION is not set" && exit 1 - if ls skywalking > /dev/null 2>&1; then - rm -rf skywalking - fi + # No `|| true`: if the packages cannot be promoted, stop before printing the announcement. + svn mv https://dist.apache.org/repos/dist/dev/skywalking/cli/"$VERSION" https://dist.apache.org/repos/dist/release/skywalking/cli/"$VERSION" -m"Release Apache SkyWalking-CLI $VERSION" - svn mv https://dist.apache.org/repos/dist/dev/skywalking/cli/"$VERSION" https://dist.apache.org/repos/dist/release/skywalking/cli/"$VERSION" -m"Release Apache SkyWalking-CLI $VERSION" || true - - echo "Make sure you have released the GitHub release, updated the events, download links and menus?" && read -r + # POSIX `read` needs a variable name, dash fails without one. + echo "Make sure you have released the GitHub release, updated the events, download links and menus?" && read -r _ cat << EOF ========================================================================= From a5e071b7f02f226baa83855899d49de2212c2074 Mon Sep 17 00:00:00 2001 From: Wu Sheng Date: Sat, 17 Oct 2026 18:39:21 +0800 Subject: [PATCH 2/2] Run release steps serially, and check packages without a sub-make Follow-ups from a second review round: - `make -j` (or -j in MAKEFLAGS) could run `verify`, `release-src` and `release-bin` in parallel. `release-bin` would then copy dist/LICENSE while `dependency-license` rewrites it, or `clean` would delete bin/ while `build` writes it. `.NOTPARALLEL` is now set only when a release goal is requested, so CI's `make build -j3` stays parallel. A plain `.NOTPARALLEL` would serialize every target on GNU make before 4.4. - The package check is now a `define` macro used by both `release-check` and `release`, instead of a `$(MAKE) -f $(firstword $(MAKEFILE_LIST))` sub-make, which broke when the Makefile path contained spaces. - docs: the svn block's `cd ..` assumes you continue from inside skywalking-cli, and the text now says so. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EkgWrRt7QJvPTNx2N5wZr9 --- Makefile | 42 ++++++++++++++++++++++++++---------------- docs/How-to-release.md | 4 ++-- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 52b23131..c2900e29 100644 --- a/Makefile +++ b/Makefile @@ -154,25 +154,35 @@ release-bin: build $(TAR) -zcvf $(RELEASE_BIN).tgz $(RELEASE_BIN) rm -rf $(RELEASE_BIN) -# macOS bsdtar folds AppleDouble `._*` entries into the next entry when listing, so `-t` alone would hide them. -# `--options !mac-ext` lists every entry as it is. GNU tar has no such option and lists them anyway. +# Fails if a package is missing, unreadable, or contains macOS metadata. macOS bsdtar folds AppleDouble `._*` entries +# into the next entry when listing, so `-t` alone would hide them. `--options !mac-ext` lists every entry as it is. +# GNU tar has no such option and lists them anyway. Shared as a macro so `release` runs it without a sub-make. +define check_release_packages +@list_opts=""; \ +if tar --version 2>/dev/null | grep -q bsdtar; then list_opts="--options !mac-ext"; fi; \ +for pkg in $(RELEASE_SRC).tgz $(RELEASE_BIN).tgz; do \ + if [ ! -f "$$pkg" ]; then echo "$$pkg was not created"; exit 1; fi; \ + entries=$$(tar $$list_opts -tzf "$$pkg") || { echo "$$pkg is not a readable tar.gz"; exit 1; }; \ + if echo "$$entries" | grep -E '(^|/)(\._|(\.DS_Store|__MACOSX)(/|$$))'; then \ + echo "$$pkg contains the macOS metadata files listed above"; \ + exit 1; \ + fi; \ +done +endef + .PHONY: release-check release-check: - @list_opts=""; \ - if tar --version 2>/dev/null | grep -q bsdtar; then list_opts="--options !mac-ext"; fi; \ - for pkg in $(RELEASE_SRC).tgz $(RELEASE_BIN).tgz; do \ - if [ ! -f "$$pkg" ]; then echo "$$pkg was not created"; exit 1; fi; \ - entries=$$(tar $$list_opts -tzf "$$pkg") || { echo "$$pkg is not a readable tar.gz"; exit 1; }; \ - if echo "$$entries" | grep -E '(^|/)(\._|(\.DS_Store|__MACOSX)(/|$$))'; then \ - echo "$$pkg contains the macOS metadata files listed above"; \ - exit 1; \ - fi; \ - done - -# release-check runs as a recipe step, not a prerequisite, so it always runs after both packages exist, even with -j. -# `-f` keeps the sub-make on this Makefile when it was given with `make -f`, since a sub-make does not inherit `-f`. + $(check_release_packages) + +# Packaging copies dist/LICENSE, which `verify` rewrites, and bin/, which `clean` deletes, so the release steps must +# not run in parallel, even with -j in MAKEFLAGS. Scoped to the release goals to keep `make build -j3` (CI) parallel. +ifneq ($(filter release release-src release-bin,$(MAKECMDGOALS)),) +.NOTPARALLEL: +endif + +# The package check is a recipe step, not a prerequisite, so it always runs after both packages exist. release: verify release-src release-bin - @$(MAKE) -f $(firstword $(MAKEFILE_LIST)) --no-print-directory release-check + $(check_release_packages) gpg --batch --yes --armor --detach-sig $(RELEASE_SRC).tgz shasum -a 512 $(RELEASE_SRC).tgz > $(RELEASE_SRC).tgz.sha512 gpg --batch --yes --armor --detach-sig $(RELEASE_BIN).tgz diff --git a/docs/How-to-release.md b/docs/How-to-release.md index 45be9d8e..ee6d89e9 100644 --- a/docs/How-to-release.md +++ b/docs/How-to-release.md @@ -47,8 +47,8 @@ AppleDouble `._*` files or extended attributes. It fails if either package still ## Upload to Apache svn -Run this from the release directory, the parent of `skywalking-cli`. The svn working copy is removed only after the -commit succeeds. +Continue from inside `skywalking-cli`, where the build step above leaves you: the block first moves up to the release +directory. The svn working copy is removed only after the commit succeeds. ```bash cd ..