From e206f1a745beb7fea82b5e7f705671a302adfb01 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 06:26:40 +0000 Subject: [PATCH 1/7] Package release archives as flat shared libraries. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The release tar step was publishing the whole CMake target directory, so Windows artifacts included project files, object files, and import libraries. Stage only runtime shared libraries, and add a vcpkg DLL only when a staged binary imports it. Co-authored-by: Michał Komor --- .github/scripts/package_shared_libs.py | 386 ++++++++++++++++++++ .github/scripts/test_package_shared_libs.py | 330 +++++++++++++++++ .github/workflows/release.yml | 21 +- pubsub/BUILD_NOTES.md | 1 + pubsub/Readme.md | 4 +- samples/pubsub/README.md | 2 +- 6 files changed, 739 insertions(+), 5 deletions(-) create mode 100644 .github/scripts/package_shared_libs.py create mode 100644 .github/scripts/test_package_shared_libs.py diff --git a/.github/scripts/package_shared_libs.py b/.github/scripts/package_shared_libs.py new file mode 100644 index 0000000..afaffe2 --- /dev/null +++ b/.github/scripts/package_shared_libs.py @@ -0,0 +1,386 @@ +#!/usr/bin/env python3 +"""Build a flat release archive that contains only runtime shared libraries. + +The CMake target directory on Windows is a multi-config build tree. Publishing +that tree with ``tar -C .`` ships ``CMakeFiles/``, ``.vcxproj``, +import libraries (``.lib`` / ``.exp``), object files, and other build junk. +Consumers need the plugin and the shared libraries it actually loads, with no +extra nesting. + +Windows DLLs that the staged binaries import, but that vcpkg applocal left only +under ``build/vcpkg_installed//bin``, are copied in as well. DLLs that +are merely installed and never imported (for example ``libssl`` when curl is +built with Schannel) are left out. +""" + +from __future__ import annotations + +import argparse +import os +import shutil +import struct +import sys +import tarfile +from pathlib import Path, PureWindowsPath + + +# Prefer the Release output when the same basename exists in more than one +# MSVC config directory. Single-config generators (Ninja) have no config +# directory and sit between Release and Debug. +_CONFIG_RANK = { + "release": 30, + "relwithdebinfo": 20, + "minsizerel": 10, + "debug": 0, +} +_UNCONFIGURED_RANK = 15 + + +def is_shared_library_name(name: str) -> bool: + """True for runtime shared libraries, not import libs or static archives.""" + lower = name.lower() + if lower.endswith(".dll") or lower.endswith(".dylib"): + return True + stem, sep, rest = lower.rpartition(".so") + if not sep or not stem: + return False + if rest == "": + return True + if not rest.startswith("."): + return False + version = rest[1:] + return bool(version) and all(part.isdigit() for part in version.split(".")) + + +def _collision_key(name: str) -> str: + lower = name.lower() + if lower.endswith(".dll") or lower.endswith(".dylib"): + return lower + return name + + +def _path_rank(path: Path) -> tuple[int, float]: + rank = _UNCONFIGURED_RANK + for part in path.parts: + configured = _CONFIG_RANK.get(part.lower()) + if configured is not None: + rank = configured + break + try: + mtime = path.stat().st_mtime + except OSError: + mtime = 0.0 + return (rank, mtime) + + +def iter_shared_libraries(root: Path) -> list[Path]: + found: list[Path] = [] + for dirpath, _dirnames, filenames in os.walk(root): + for name in filenames: + if not is_shared_library_name(name): + continue + path = Path(dirpath) / name + if path.is_file(): + found.append(path) + return found + + +def choose_libraries(paths: list[Path]) -> dict[str, Path]: + """One file per archive basename. Release wins over Debug and loose copies.""" + chosen: dict[str, Path] = {} + ranks: dict[str, tuple[int, float]] = {} + for path in paths: + key = _collision_key(path.name) + rank = _path_rank(path) + if key not in chosen or rank > ranks[key]: + chosen[key] = path + ranks[key] = rank + return chosen + + +def vcpkg_release_bin_dirs(artifact_path: Path) -> list[Path]: + installed = artifact_path.parent / "vcpkg_installed" + if not installed.is_dir(): + return [] + bins: list[Path] = [] + for child in sorted(installed.iterdir()): + bin_dir = child / "bin" + if bin_dir.is_dir(): + bins.append(bin_dir) + return bins + + +def index_vcpkg_dlls(bin_dirs: list[Path]) -> dict[str, Path]: + found: dict[str, Path] = {} + for bin_dir in bin_dirs: + for pattern in ("*.dll", "*.DLL"): + for path in sorted(bin_dir.glob(pattern)): + if path.is_file(): + found.setdefault(path.name.lower(), path) + return found + + +def _rva_to_offset( + sections: list[tuple[int, int, int, int]], + rva: int, +) -> int | None: + for virtual_address, virtual_size, raw_ptr, raw_size in sections: + span = max(virtual_size, raw_size) + if virtual_address <= rva < virtual_address + max(span, 1): + delta = rva - virtual_address + if delta < raw_size: + return raw_ptr + delta + return None + + +def _cstring(data: bytes, offset: int | None) -> str | None: + if offset is None or offset < 0 or offset >= len(data): + return None + end = data.find(b"\x00", offset) + if end < 0: + return None + raw = data[offset:end] + try: + text = raw.decode("ascii") + except UnicodeDecodeError: + return None + text = text.strip() + if not text: + return None + return PureWindowsPath(text).name + + +def _read_sections( + data: bytes, + section_off: int, + count: int, +) -> list[tuple[int, int, int, int]]: + sections: list[tuple[int, int, int, int]] = [] + for index in range(count): + off = section_off + index * 40 + if off + 40 > len(data): + break + _name, virtual_size, virtual_address, raw_size, raw_ptr = struct.unpack_from( + "<8sIIII", + data, + off, + ) + sections.append((virtual_address, virtual_size, raw_ptr, raw_size)) + return sections + + +def imported_dll_names(path: Path) -> list[str]: + """DLL names from a PE import table and delay-load directory. + + Non-PE files (``.so`` / ``.dylib``) and truncated images yield an empty + list so packaging can still archive the plugin itself. + """ + try: + data = path.read_bytes() + except OSError: + return [] + if len(data) < 64 or data[:2] != b"MZ": + return [] + e_lfanew = struct.unpack_from(" len(data): + return [] + if data[e_lfanew : e_lfanew + 4] != b"PE\0\0": + return [] + coff = e_lfanew + 4 + _machine, nsections, _timestamp, _symptr, _nsyms, opt_size, _chars = ( + struct.unpack_from(" len(data): + return [] + magic = struct.unpack_from(" opt + opt_size or dir_off < 4: + return [] + dir_count = struct.unpack_from(" tuple[int, int] | None: + if index >= dir_count: + return None + off = dir_off + index * 8 + if off + 8 > len(data): + return None + return struct.unpack_from(" None: + if not name: + return + key = name.lower() + if key in seen: + return + seen.add(key) + names.append(name) + + import_dir = directory(1) + if import_dir and import_dir[0]: + cursor = _rva_to_offset(sections, import_dir[0]) + # IMAGE_IMPORT_DESCRIPTOR is 20 bytes; Name RVA is at +12. + for _ in range(4096): + if cursor is None or cursor + 20 > len(data): + break + descriptor = data[cursor : cursor + 20] + if descriptor == b"\x00" * 20: + break + name_rva = struct.unpack_from(" len(data): + break + descriptor = data[cursor : cursor + 32] + if descriptor == b"\x00" * 32: + break + attributes = struct.unpack_from(" dict[str, Path]: + staged = dict(selected) + pending = [path for path in staged.values() if path.name.lower().endswith(".dll")] + seen: set[str] = set() + while pending: + current = pending.pop() + try: + marker = str(current.resolve()) + except OSError: + marker = str(current) + if marker in seen: + continue + seen.add(marker) + for dep in imported_dll_names(current): + key = dep.lower() + if key in staged: + continue + source = vcpkg_dlls.get(key) + if source is None: + continue + staged[key] = source + pending.append(source) + return staged + + +def plugin_library_present(names: list[str], stem: str) -> bool: + lowered = {name.lower() for name in names} + exact = { + f"{stem}.dll".lower(), + f"{stem}.so".lower(), + f"lib{stem}.so".lower(), + f"{stem}.dylib".lower(), + f"lib{stem}.dylib".lower(), + } + if lowered & exact: + return True + prefixes = (f"lib{stem}.so.".lower(), f"{stem}.so.".lower()) + return any(name.startswith(prefixes) for name in lowered) + + +def stage_and_archive( + artifact_path: Path, + plugin_stem: str, + staging: Path, + archive: Path, +) -> list[str]: + if not artifact_path.is_dir(): + raise SystemExit(f"artifact path does not exist: {artifact_path}") + if not plugin_stem: + raise SystemExit("plugin stem is empty") + + selected = choose_libraries(iter_shared_libraries(artifact_path)) + vcpkg_dlls = index_vcpkg_dlls(vcpkg_release_bin_dirs(artifact_path)) + selected = add_transitive_windows_dlls(selected, vcpkg_dlls) + + if staging.exists(): + shutil.rmtree(staging) + staging.mkdir(parents=True) + + archived_names: list[str] = [] + for path in selected.values(): + destination = staging / path.name + shutil.copy2(path, destination) + archived_names.append(destination.name) + archived_names.sort(key=str.lower) + + if not archived_names: + raise SystemExit( + f"no shared libraries found under {artifact_path}" + ) + if not plugin_library_present(archived_names, plugin_stem): + raise SystemExit( + "plugin shared library " + f"{plugin_stem} (.dll / .so / .dylib) was not found under {artifact_path}; " + f"staged: {', '.join(archived_names)}" + ) + + archive.parent.mkdir(parents=True, exist_ok=True) + if archive.exists(): + archive.unlink() + with tarfile.open(archive, "w:gz") as tar: + for name in archived_names: + tar.add(staging / name, arcname=name, recursive=False) + return archived_names + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--artifact-path", required=True, type=Path) + parser.add_argument("--plugin-stem", required=True) + parser.add_argument("--staging", required=True, type=Path) + parser.add_argument("--archive", required=True, type=Path) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(sys.argv[1:] if argv is None else argv) + names = stage_and_archive( + args.artifact_path, + args.plugin_stem, + args.staging, + args.archive, + ) + print(f"Packaged {len(names)} shared libraries into {args.archive}:") + for name in names: + print(f" {name}") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except SystemExit: + raise + except Exception as exc: # noqa: BLE001 — surface a single CI error line + print(f"failed to package shared libraries: {exc}", file=sys.stderr) + raise SystemExit(1) from exc diff --git a/.github/scripts/test_package_shared_libs.py b/.github/scripts/test_package_shared_libs.py new file mode 100644 index 0000000..0ea32c1 --- /dev/null +++ b/.github/scripts/test_package_shared_libs.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +"""Fixture checks for the shared-library release packager.""" + +from __future__ import annotations + +import struct +import sys +import tarfile +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import package_shared_libs as packager + + +def _pe_headers( + *, + magic: int, + import_rva: int, + import_size: int, + delay_rva: int = 0, + delay_size: int = 0, +) -> tuple[bytes, int]: + """Return (header bytes padded to 0x200, SizeOfOptionalHeader).""" + dos = bytearray(64) + dos[0:2] = b"MZ" + struct.pack_into(" 0x200: + raise AssertionError(f"headers are {len(headers)} bytes") + return headers.ljust(0x200, b"\x00"), opt_size + + +def make_pe_dll( + imports: list[str], + delay_imports: list[str] | None = None, + *, + pe32_plus: bool = False, +) -> bytes: + """Minimal PE image whose import directory lists ``imports``.""" + delay_imports = delay_imports or [] + blob = bytearray() + descriptors = bytearray() + for _name in imports: + descriptors += b"\x00" * 12 + b"\x00\x00\x00\x00" + b"\x00" * 4 + descriptors += b"\x00" * 20 + delay = bytearray() + for _name in delay_imports: + delay += b"\x01\x00\x00\x00" + b"\x00" * 28 + if delay_imports: + delay += b"\x00" * 32 + + strings_at = len(descriptors) + len(delay) + names = bytearray() + import_name_rvas: list[int] = [] + for name in imports: + import_name_rvas.append(0x1000 + strings_at + len(names)) + names += name.encode("ascii") + b"\x00" + delay_name_rvas: list[int] = [] + for name in delay_imports: + delay_name_rvas.append(0x1000 + strings_at + len(names)) + names += name.encode("ascii") + b"\x00" + + for index, rva in enumerate(import_name_rvas): + struct.pack_into(" None: + self.assertTrue(packager.is_shared_library_name("PubSubPlugin.dll")) + self.assertTrue(packager.is_shared_library_name("libPubSubPlugin.so")) + self.assertTrue(packager.is_shared_library_name("libssl.so.3")) + self.assertTrue(packager.is_shared_library_name("libcrypto.so.3.0.0")) + self.assertTrue(packager.is_shared_library_name("libPubSubPlugin.dylib")) + self.assertFalse(packager.is_shared_library_name("PubSubPlugin.lib")) + self.assertFalse(packager.is_shared_library_name("PubSubPlugin.exp")) + self.assertFalse(packager.is_shared_library_name("PubSubPlugin.pdb")) + self.assertFalse(packager.is_shared_library_name("foo.obj")) + self.assertFalse(packager.is_shared_library_name("libPubSubPlugin.so.bak")) + self.assertFalse(packager.is_shared_library_name("CMakeCXXCompilerId.exe")) + + +class PeImportTests(unittest.TestCase): + def test_pe32_and_pe32_plus_imports(self) -> None: + for pe32_plus in (False, True): + blob = make_pe_dll( + [r".\libcurl.dll", "Z.DLL"], + ["libssl-3-x64.dll"], + pe32_plus=pe32_plus, + ) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "plugin.dll" + path.write_bytes(blob) + self.assertEqual( + [name.lower() for name in packager.imported_dll_names(path)], + ["libcurl.dll", "z.dll", "libssl-3-x64.dll"], + ) + + def test_non_pe_is_ignored(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "libPubSubPlugin.so" + path.write_bytes(b"\x7fELF not a real object") + self.assertEqual(packager.imported_dll_names(path), []) + + +class PackageTests(unittest.TestCase): + def test_windows_tree_is_flat_runtime_dlls_only(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + artifact = root / "pubsub" / "build" / "PubSubPlugin" + release = artifact / "Release" + debug = artifact / "Debug" + release.mkdir(parents=True) + debug.mkdir() + (artifact / "CMakeFiles").mkdir() + (artifact / "CMakeFiles" / "cmake.check_cache").write_text("x") + (artifact / "PubSubPlugin.dir" / "Release").mkdir(parents=True) + (artifact / "PubSubPlugin.dir" / "Release" / "PubSubUtil.obj").write_bytes(b"obj") + (artifact / "INSTALL.dir").mkdir() + (artifact / "RUN_TESTS.dir").mkdir() + (artifact / "PubSubPlugin.vcxproj").write_text("") + (artifact / "PubSubPlugin.vcxproj.filters").write_text("") + (artifact / "cmake_install.cmake").write_text("install()") + (artifact / "CTestTestfile.cmake").write_text("add_test()") + (artifact / "PubSubPlugin.tlog").write_text("log") + + plugin = make_pe_dll([r"libcurl.dll", "extra.dll"]) + (release / "PubSubPlugin.dll").write_bytes(plugin) + (release / "PubSubPlugin.exp").write_bytes(b"exp") + (release / "PubSubPlugin.lib").write_bytes(b"lib") + (release / "PubSubPlugin.pdb").write_bytes(b"pdb") + (release / "libcurl.dll").write_bytes(b"MZ not parsed further") + (release / "libcrypto-3-x64.dll").write_bytes(b"crypto") + (release / "z.dll").write_bytes(b"zlib") + # Newer Debug copy of the same basename must not win. + (debug / "PubSubPlugin.dll").write_bytes(b"debug-plugin") + + vcpkg_bin = ( + root / "pubsub" / "build" / "vcpkg_installed" / "x64-windows" / "bin" + ) + vcpkg_bin.mkdir(parents=True) + (vcpkg_bin / "extra.dll").write_bytes(b"transitive") + (vcpkg_bin / "libssl-3-x64.dll").write_bytes(b"unused-tls") + debug_bin = ( + root + / "pubsub" + / "build" + / "vcpkg_installed" + / "x64-windows" + / "debug" + / "bin" + ) + debug_bin.mkdir(parents=True) + (debug_bin / "extra.dll").write_bytes(b"debug-extra") + + staging = root / "package-staging" + archive = root / "release-payloads" / "pubsub-windows-x86_64.tar.gz" + names = packager.stage_and_archive( + artifact, + "PubSubPlugin", + staging, + archive, + ) + self.assertEqual( + names, + [ + "extra.dll", + "libcrypto-3-x64.dll", + "libcurl.dll", + "PubSubPlugin.dll", + "z.dll", + ], + ) + self.assertEqual((staging / "PubSubPlugin.dll").read_bytes(), plugin) + self.assertEqual((staging / "extra.dll").read_bytes(), b"transitive") + with tarfile.open(archive, "r:gz") as tar: + self.assertEqual(tar.getnames(), names) + for member in tar.getmembers(): + self.assertFalse(member.isdir()) + self.assertFalse("/" in member.name or "\\" in member.name) + + def test_imported_libssl_is_taken_from_vcpkg_bin(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + artifact = root / "build" / "PubSubPlugin" / "Release" + artifact.mkdir(parents=True) + (artifact / "PubSubPlugin.dll").write_bytes( + make_pe_dll(["libssl-3-x64.dll"]) + ) + vcpkg_bin = root / "build" / "vcpkg_installed" / "x64-windows" / "bin" + vcpkg_bin.mkdir(parents=True) + (vcpkg_bin / "libssl-3-x64.dll").write_bytes(b"ssl") + names = packager.stage_and_archive( + artifact.parent, + "PubSubPlugin", + root / "staging", + root / "out.tar.gz", + ) + self.assertEqual(names, ["libssl-3-x64.dll", "PubSubPlugin.dll"]) + + def test_linux_tree_keeps_plugin_and_shipped_so_deps_only(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + artifact = root / "pubsub" / "build" / "PubSubPlugin" + artifact.mkdir(parents=True) + (artifact / "CMakeFiles").mkdir() + (artifact / "cmake_install.cmake").write_text("install()") + plugin = artifact / "libPubSubPlugin.so" + plugin.write_bytes(b"plugin-so") + plugin.chmod(0o755) + real_dep = artifact / "libcrypto.so.3" + real_dep.write_bytes(b"crypto-so") + link = artifact / "libcrypto.so" + link.symlink_to(real_dep.name) + + staging = root / "package-staging" + archive = root / "out.tar.gz" + names = packager.stage_and_archive( + artifact, + "PubSubPlugin", + staging, + archive, + ) + self.assertEqual(names, ["libcrypto.so", "libcrypto.so.3", "libPubSubPlugin.so"]) + self.assertEqual((staging / "libcrypto.so").read_bytes(), b"crypto-so") + self.assertTrue((staging / "libPubSubPlugin.so").stat().st_mode & 0o111) + with tarfile.open(archive, "r:gz") as tar: + self.assertEqual(tar.getnames(), names) + + def test_missing_plugin_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + artifact = root / "build" / "PubSubPlugin" / "Release" + artifact.mkdir(parents=True) + (artifact / "libcurl.dll").write_bytes(b"curl") + with self.assertRaises(SystemExit) as raised: + packager.stage_and_archive( + artifact.parent, + "PubSubPlugin", + root / "staging", + root / "out.tar.gz", + ) + self.assertIn("plugin shared library", str(raised.exception)) + + def test_empty_tree_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + artifact = root / "build" / "PubSubPlugin" + artifact.mkdir(parents=True) + (artifact / "CMakeFiles").mkdir() + (artifact / "cmake_install.cmake").write_text("x") + with self.assertRaises(SystemExit) as raised: + packager.stage_and_archive( + artifact, + "PubSubPlugin", + root / "staging", + root / "out.tar.gz", + ) + self.assertIn("no shared libraries", str(raised.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a56b38e..8b0b718 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -173,10 +173,27 @@ jobs: run: | ctest --test-dir build -C Release --output-on-failure --timeout 180 + # Publish a flat archive of runtime shared libraries only. The MSVC + # target directory also holds CMake/project files and import libraries. - name: Package (${{ matrix.plugin }}) + shell: bash run: | - mkdir -p release-payloads - tar -czf "release-payloads/${{ env.PLUGIN_NAME }}-${{ env.PACKAGE_SUFFIX }}.tar.gz" -C "${{ env.ARTIFACT_PATH }}" . + set -euo pipefail + if command -v python3 >/dev/null 2>&1; then + PY=(python3) + elif command -v python >/dev/null 2>&1; then + PY=(python) + elif command -v py >/dev/null 2>&1; then + PY=(py -3) + else + echo "python3 is required to package shared libraries" >&2 + exit 1 + fi + "${PY[@]}" .github/scripts/package_shared_libs.py \ + --artifact-path "${ARTIFACT_PATH}" \ + --plugin-stem "$(basename "${ARTIFACT_PATH}")" \ + --staging package-staging \ + --archive "release-payloads/${PLUGIN_NAME}-${PACKAGE_SUFFIX}.tar.gz" - name: Upload artifact (${{ matrix.plugin }}) uses: actions/upload-artifact@v6 diff --git a/pubsub/BUILD_NOTES.md b/pubsub/BUILD_NOTES.md index a3d1252..0bc7901 100644 --- a/pubsub/BUILD_NOTES.md +++ b/pubsub/BUILD_NOTES.md @@ -4,6 +4,7 @@ Dependency choice: vcpkg `curl`, `openssl`, and `nlohmann-json` (same toolchain `google-cloud-cpp` (FetchContent or the vcpkg `pubsub` feature) pulls gRPC, protobuf, and Abseil. That superbuild is too heavy for this repo's six-platform CI matrix: multi-gigabyte RAM spikes and long compiles make the jobs unreliable. The REST client covers the emulator (`PUBSUB_EMULATOR_HOST`) and production (service-account JWT or a bearer token) without that stack. +- Windows curl uses Schannel (vcpkg's default `ssl` feature). The plugin links `OpenSSL::Crypto` for RS256 JWTs, which loads `libcrypto` but not `libssl`. Production HTTPS does not need `libssl-3-x64.dll`. Switching curl to OpenSSL would require a CA bundle or `CURLSSLOPT_NATIVE_CA`, because Schannel is what verifies Google's certificates today. - Hermetic `ctest` does not need an emulator. Invalid config, base64, JWT signing, and factory tests always run. - `PubSubLive.RpcRoundTrip` runs only when `PUBSUB_EMULATOR_HOST` or `PUBSUB_LIVE` is set. - First configure with the vcpkg toolchain needs network access so vcpkg can fetch ports. The manifest baseline matches `sqs/vcpkg.json`. diff --git a/pubsub/Readme.md b/pubsub/Readme.md index 8bb6347..e9d4bf9 100644 --- a/pubsub/Readme.md +++ b/pubsub/Readme.md @@ -52,7 +52,7 @@ Output: - Windows: `pubsub/build/PubSubPlugin/PubSubPlugin.dll` — config `"name": "PubSubPlugin"` - Linux/macOS: `pubsub/build/PubSubPlugin/libPubSubPlugin.so` or `.dylib` — config `"name": "libPubSubPlugin"` -On Windows, copy `PubSubPlugin.dll` and the vcpkg runtime DLLs (libcurl, libssl, libcrypto, zlib) from `build/vcpkg_installed//bin` next to `gg.exe`. The loader does not search the build tree. +On Windows, copy `PubSubPlugin.dll` and the vcpkg runtime DLLs next to `gg.exe`: `libcurl.dll`, the libcrypto DLL (for example `libcrypto-3-x64.dll`), and the zlib DLL (`z.dll` or `zlib1.dll`). vcpkg builds curl against Schannel on Windows, so HTTPS uses the Windows certificate store and `libssl` is not a runtime dependency. `libcrypto` signs the service-account JWT. The loader does not search the build tree. Hermetic tests always run. The round-trip test runs only when `PUBSUB_EMULATOR_HOST` or `PUBSUB_LIVE` is set. @@ -81,7 +81,7 @@ This layout is the same shape as the SQS sample. Put these files in one folder: - `gg.exe` - the hosted module, for example `netapp.dll` - `PubSubPlugin.dll` -- the libcurl and OpenSSL DLLs from the vcpkg `bin` directory +- the libcurl, libcrypto, and zlib DLLs from the vcpkg `bin` directory (`libssl` is not required on Windows; curl uses Schannel) `pubsubplugin.json` (same contents as [`samples/pubsub/pluginConfig.gateway.json`](../samples/pubsub/pluginConfig.gateway.json)): diff --git a/samples/pubsub/README.md b/samples/pubsub/README.md index d5656c7..afb9470 100644 --- a/samples/pubsub/README.md +++ b/samples/pubsub/README.md @@ -29,7 +29,7 @@ Colocate these next to each other: - `gg.exe` - `netapp.dll` (the hosted module) - `PubSubPlugin.dll` -- the libcurl and OpenSSL runtime DLLs from `pubsub/build/vcpkg_installed//bin` (typically `libcurl.dll`, `libssl-3-x64.dll`, `libcrypto-3-x64.dll`, and `zlib1.dll`) +- the libcurl, libcrypto, and zlib runtime DLLs from `pubsub/build/vcpkg_installed//bin` (typically `libcurl.dll`, `libcrypto-3-x64.dll`, and `z.dll`). Windows curl is built with Schannel, so `libssl-3-x64.dll` is not loaded. Copy `pluginConfig.gateway.json` to that folder as `pubsubplugin.json` and run: From 4338d11936f2469f816d11371f179119237adc64 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 06:39:44 +0000 Subject: [PATCH 2/7] Install release artifacts with CMake instead of Python. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage each plugin with install(RUNTIME_DEPENDENCY_SET) and tar the plugin-runtime component so the archive is a flat set of shared libraries without CMake or import-library files. Co-authored-by: Michał Komor --- .github/scripts/package_shared_libs.py | 386 -------------------- .github/scripts/test_package_shared_libs.py | 330 ----------------- .github/workflows/release.yml | 64 +++- cmake/InstallPluginRuntime.cmake | 80 ++++ kafka/CMakeLists.txt | 3 + pubsub/CMakeLists.txt | 3 + rabbitmq/CMakeLists.txt | 3 + servicebus/CMakeLists.txt | 3 + sqs/CMakeLists.txt | 3 + 9 files changed, 144 insertions(+), 731 deletions(-) delete mode 100644 .github/scripts/package_shared_libs.py delete mode 100644 .github/scripts/test_package_shared_libs.py create mode 100644 cmake/InstallPluginRuntime.cmake diff --git a/.github/scripts/package_shared_libs.py b/.github/scripts/package_shared_libs.py deleted file mode 100644 index afaffe2..0000000 --- a/.github/scripts/package_shared_libs.py +++ /dev/null @@ -1,386 +0,0 @@ -#!/usr/bin/env python3 -"""Build a flat release archive that contains only runtime shared libraries. - -The CMake target directory on Windows is a multi-config build tree. Publishing -that tree with ``tar -C .`` ships ``CMakeFiles/``, ``.vcxproj``, -import libraries (``.lib`` / ``.exp``), object files, and other build junk. -Consumers need the plugin and the shared libraries it actually loads, with no -extra nesting. - -Windows DLLs that the staged binaries import, but that vcpkg applocal left only -under ``build/vcpkg_installed//bin``, are copied in as well. DLLs that -are merely installed and never imported (for example ``libssl`` when curl is -built with Schannel) are left out. -""" - -from __future__ import annotations - -import argparse -import os -import shutil -import struct -import sys -import tarfile -from pathlib import Path, PureWindowsPath - - -# Prefer the Release output when the same basename exists in more than one -# MSVC config directory. Single-config generators (Ninja) have no config -# directory and sit between Release and Debug. -_CONFIG_RANK = { - "release": 30, - "relwithdebinfo": 20, - "minsizerel": 10, - "debug": 0, -} -_UNCONFIGURED_RANK = 15 - - -def is_shared_library_name(name: str) -> bool: - """True for runtime shared libraries, not import libs or static archives.""" - lower = name.lower() - if lower.endswith(".dll") or lower.endswith(".dylib"): - return True - stem, sep, rest = lower.rpartition(".so") - if not sep or not stem: - return False - if rest == "": - return True - if not rest.startswith("."): - return False - version = rest[1:] - return bool(version) and all(part.isdigit() for part in version.split(".")) - - -def _collision_key(name: str) -> str: - lower = name.lower() - if lower.endswith(".dll") or lower.endswith(".dylib"): - return lower - return name - - -def _path_rank(path: Path) -> tuple[int, float]: - rank = _UNCONFIGURED_RANK - for part in path.parts: - configured = _CONFIG_RANK.get(part.lower()) - if configured is not None: - rank = configured - break - try: - mtime = path.stat().st_mtime - except OSError: - mtime = 0.0 - return (rank, mtime) - - -def iter_shared_libraries(root: Path) -> list[Path]: - found: list[Path] = [] - for dirpath, _dirnames, filenames in os.walk(root): - for name in filenames: - if not is_shared_library_name(name): - continue - path = Path(dirpath) / name - if path.is_file(): - found.append(path) - return found - - -def choose_libraries(paths: list[Path]) -> dict[str, Path]: - """One file per archive basename. Release wins over Debug and loose copies.""" - chosen: dict[str, Path] = {} - ranks: dict[str, tuple[int, float]] = {} - for path in paths: - key = _collision_key(path.name) - rank = _path_rank(path) - if key not in chosen or rank > ranks[key]: - chosen[key] = path - ranks[key] = rank - return chosen - - -def vcpkg_release_bin_dirs(artifact_path: Path) -> list[Path]: - installed = artifact_path.parent / "vcpkg_installed" - if not installed.is_dir(): - return [] - bins: list[Path] = [] - for child in sorted(installed.iterdir()): - bin_dir = child / "bin" - if bin_dir.is_dir(): - bins.append(bin_dir) - return bins - - -def index_vcpkg_dlls(bin_dirs: list[Path]) -> dict[str, Path]: - found: dict[str, Path] = {} - for bin_dir in bin_dirs: - for pattern in ("*.dll", "*.DLL"): - for path in sorted(bin_dir.glob(pattern)): - if path.is_file(): - found.setdefault(path.name.lower(), path) - return found - - -def _rva_to_offset( - sections: list[tuple[int, int, int, int]], - rva: int, -) -> int | None: - for virtual_address, virtual_size, raw_ptr, raw_size in sections: - span = max(virtual_size, raw_size) - if virtual_address <= rva < virtual_address + max(span, 1): - delta = rva - virtual_address - if delta < raw_size: - return raw_ptr + delta - return None - - -def _cstring(data: bytes, offset: int | None) -> str | None: - if offset is None or offset < 0 or offset >= len(data): - return None - end = data.find(b"\x00", offset) - if end < 0: - return None - raw = data[offset:end] - try: - text = raw.decode("ascii") - except UnicodeDecodeError: - return None - text = text.strip() - if not text: - return None - return PureWindowsPath(text).name - - -def _read_sections( - data: bytes, - section_off: int, - count: int, -) -> list[tuple[int, int, int, int]]: - sections: list[tuple[int, int, int, int]] = [] - for index in range(count): - off = section_off + index * 40 - if off + 40 > len(data): - break - _name, virtual_size, virtual_address, raw_size, raw_ptr = struct.unpack_from( - "<8sIIII", - data, - off, - ) - sections.append((virtual_address, virtual_size, raw_ptr, raw_size)) - return sections - - -def imported_dll_names(path: Path) -> list[str]: - """DLL names from a PE import table and delay-load directory. - - Non-PE files (``.so`` / ``.dylib``) and truncated images yield an empty - list so packaging can still archive the plugin itself. - """ - try: - data = path.read_bytes() - except OSError: - return [] - if len(data) < 64 or data[:2] != b"MZ": - return [] - e_lfanew = struct.unpack_from(" len(data): - return [] - if data[e_lfanew : e_lfanew + 4] != b"PE\0\0": - return [] - coff = e_lfanew + 4 - _machine, nsections, _timestamp, _symptr, _nsyms, opt_size, _chars = ( - struct.unpack_from(" len(data): - return [] - magic = struct.unpack_from(" opt + opt_size or dir_off < 4: - return [] - dir_count = struct.unpack_from(" tuple[int, int] | None: - if index >= dir_count: - return None - off = dir_off + index * 8 - if off + 8 > len(data): - return None - return struct.unpack_from(" None: - if not name: - return - key = name.lower() - if key in seen: - return - seen.add(key) - names.append(name) - - import_dir = directory(1) - if import_dir and import_dir[0]: - cursor = _rva_to_offset(sections, import_dir[0]) - # IMAGE_IMPORT_DESCRIPTOR is 20 bytes; Name RVA is at +12. - for _ in range(4096): - if cursor is None or cursor + 20 > len(data): - break - descriptor = data[cursor : cursor + 20] - if descriptor == b"\x00" * 20: - break - name_rva = struct.unpack_from(" len(data): - break - descriptor = data[cursor : cursor + 32] - if descriptor == b"\x00" * 32: - break - attributes = struct.unpack_from(" dict[str, Path]: - staged = dict(selected) - pending = [path for path in staged.values() if path.name.lower().endswith(".dll")] - seen: set[str] = set() - while pending: - current = pending.pop() - try: - marker = str(current.resolve()) - except OSError: - marker = str(current) - if marker in seen: - continue - seen.add(marker) - for dep in imported_dll_names(current): - key = dep.lower() - if key in staged: - continue - source = vcpkg_dlls.get(key) - if source is None: - continue - staged[key] = source - pending.append(source) - return staged - - -def plugin_library_present(names: list[str], stem: str) -> bool: - lowered = {name.lower() for name in names} - exact = { - f"{stem}.dll".lower(), - f"{stem}.so".lower(), - f"lib{stem}.so".lower(), - f"{stem}.dylib".lower(), - f"lib{stem}.dylib".lower(), - } - if lowered & exact: - return True - prefixes = (f"lib{stem}.so.".lower(), f"{stem}.so.".lower()) - return any(name.startswith(prefixes) for name in lowered) - - -def stage_and_archive( - artifact_path: Path, - plugin_stem: str, - staging: Path, - archive: Path, -) -> list[str]: - if not artifact_path.is_dir(): - raise SystemExit(f"artifact path does not exist: {artifact_path}") - if not plugin_stem: - raise SystemExit("plugin stem is empty") - - selected = choose_libraries(iter_shared_libraries(artifact_path)) - vcpkg_dlls = index_vcpkg_dlls(vcpkg_release_bin_dirs(artifact_path)) - selected = add_transitive_windows_dlls(selected, vcpkg_dlls) - - if staging.exists(): - shutil.rmtree(staging) - staging.mkdir(parents=True) - - archived_names: list[str] = [] - for path in selected.values(): - destination = staging / path.name - shutil.copy2(path, destination) - archived_names.append(destination.name) - archived_names.sort(key=str.lower) - - if not archived_names: - raise SystemExit( - f"no shared libraries found under {artifact_path}" - ) - if not plugin_library_present(archived_names, plugin_stem): - raise SystemExit( - "plugin shared library " - f"{plugin_stem} (.dll / .so / .dylib) was not found under {artifact_path}; " - f"staged: {', '.join(archived_names)}" - ) - - archive.parent.mkdir(parents=True, exist_ok=True) - if archive.exists(): - archive.unlink() - with tarfile.open(archive, "w:gz") as tar: - for name in archived_names: - tar.add(staging / name, arcname=name, recursive=False) - return archived_names - - -def parse_args(argv: list[str]) -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--artifact-path", required=True, type=Path) - parser.add_argument("--plugin-stem", required=True) - parser.add_argument("--staging", required=True, type=Path) - parser.add_argument("--archive", required=True, type=Path) - return parser.parse_args(argv) - - -def main(argv: list[str] | None = None) -> int: - args = parse_args(sys.argv[1:] if argv is None else argv) - names = stage_and_archive( - args.artifact_path, - args.plugin_stem, - args.staging, - args.archive, - ) - print(f"Packaged {len(names)} shared libraries into {args.archive}:") - for name in names: - print(f" {name}") - return 0 - - -if __name__ == "__main__": - try: - raise SystemExit(main()) - except SystemExit: - raise - except Exception as exc: # noqa: BLE001 — surface a single CI error line - print(f"failed to package shared libraries: {exc}", file=sys.stderr) - raise SystemExit(1) from exc diff --git a/.github/scripts/test_package_shared_libs.py b/.github/scripts/test_package_shared_libs.py deleted file mode 100644 index 0ea32c1..0000000 --- a/.github/scripts/test_package_shared_libs.py +++ /dev/null @@ -1,330 +0,0 @@ -#!/usr/bin/env python3 -"""Fixture checks for the shared-library release packager.""" - -from __future__ import annotations - -import struct -import sys -import tarfile -import tempfile -import unittest -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent)) - -import package_shared_libs as packager - - -def _pe_headers( - *, - magic: int, - import_rva: int, - import_size: int, - delay_rva: int = 0, - delay_size: int = 0, -) -> tuple[bytes, int]: - """Return (header bytes padded to 0x200, SizeOfOptionalHeader).""" - dos = bytearray(64) - dos[0:2] = b"MZ" - struct.pack_into(" 0x200: - raise AssertionError(f"headers are {len(headers)} bytes") - return headers.ljust(0x200, b"\x00"), opt_size - - -def make_pe_dll( - imports: list[str], - delay_imports: list[str] | None = None, - *, - pe32_plus: bool = False, -) -> bytes: - """Minimal PE image whose import directory lists ``imports``.""" - delay_imports = delay_imports or [] - blob = bytearray() - descriptors = bytearray() - for _name in imports: - descriptors += b"\x00" * 12 + b"\x00\x00\x00\x00" + b"\x00" * 4 - descriptors += b"\x00" * 20 - delay = bytearray() - for _name in delay_imports: - delay += b"\x01\x00\x00\x00" + b"\x00" * 28 - if delay_imports: - delay += b"\x00" * 32 - - strings_at = len(descriptors) + len(delay) - names = bytearray() - import_name_rvas: list[int] = [] - for name in imports: - import_name_rvas.append(0x1000 + strings_at + len(names)) - names += name.encode("ascii") + b"\x00" - delay_name_rvas: list[int] = [] - for name in delay_imports: - delay_name_rvas.append(0x1000 + strings_at + len(names)) - names += name.encode("ascii") + b"\x00" - - for index, rva in enumerate(import_name_rvas): - struct.pack_into(" None: - self.assertTrue(packager.is_shared_library_name("PubSubPlugin.dll")) - self.assertTrue(packager.is_shared_library_name("libPubSubPlugin.so")) - self.assertTrue(packager.is_shared_library_name("libssl.so.3")) - self.assertTrue(packager.is_shared_library_name("libcrypto.so.3.0.0")) - self.assertTrue(packager.is_shared_library_name("libPubSubPlugin.dylib")) - self.assertFalse(packager.is_shared_library_name("PubSubPlugin.lib")) - self.assertFalse(packager.is_shared_library_name("PubSubPlugin.exp")) - self.assertFalse(packager.is_shared_library_name("PubSubPlugin.pdb")) - self.assertFalse(packager.is_shared_library_name("foo.obj")) - self.assertFalse(packager.is_shared_library_name("libPubSubPlugin.so.bak")) - self.assertFalse(packager.is_shared_library_name("CMakeCXXCompilerId.exe")) - - -class PeImportTests(unittest.TestCase): - def test_pe32_and_pe32_plus_imports(self) -> None: - for pe32_plus in (False, True): - blob = make_pe_dll( - [r".\libcurl.dll", "Z.DLL"], - ["libssl-3-x64.dll"], - pe32_plus=pe32_plus, - ) - with tempfile.TemporaryDirectory() as tmp: - path = Path(tmp) / "plugin.dll" - path.write_bytes(blob) - self.assertEqual( - [name.lower() for name in packager.imported_dll_names(path)], - ["libcurl.dll", "z.dll", "libssl-3-x64.dll"], - ) - - def test_non_pe_is_ignored(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - path = Path(tmp) / "libPubSubPlugin.so" - path.write_bytes(b"\x7fELF not a real object") - self.assertEqual(packager.imported_dll_names(path), []) - - -class PackageTests(unittest.TestCase): - def test_windows_tree_is_flat_runtime_dlls_only(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - artifact = root / "pubsub" / "build" / "PubSubPlugin" - release = artifact / "Release" - debug = artifact / "Debug" - release.mkdir(parents=True) - debug.mkdir() - (artifact / "CMakeFiles").mkdir() - (artifact / "CMakeFiles" / "cmake.check_cache").write_text("x") - (artifact / "PubSubPlugin.dir" / "Release").mkdir(parents=True) - (artifact / "PubSubPlugin.dir" / "Release" / "PubSubUtil.obj").write_bytes(b"obj") - (artifact / "INSTALL.dir").mkdir() - (artifact / "RUN_TESTS.dir").mkdir() - (artifact / "PubSubPlugin.vcxproj").write_text("") - (artifact / "PubSubPlugin.vcxproj.filters").write_text("") - (artifact / "cmake_install.cmake").write_text("install()") - (artifact / "CTestTestfile.cmake").write_text("add_test()") - (artifact / "PubSubPlugin.tlog").write_text("log") - - plugin = make_pe_dll([r"libcurl.dll", "extra.dll"]) - (release / "PubSubPlugin.dll").write_bytes(plugin) - (release / "PubSubPlugin.exp").write_bytes(b"exp") - (release / "PubSubPlugin.lib").write_bytes(b"lib") - (release / "PubSubPlugin.pdb").write_bytes(b"pdb") - (release / "libcurl.dll").write_bytes(b"MZ not parsed further") - (release / "libcrypto-3-x64.dll").write_bytes(b"crypto") - (release / "z.dll").write_bytes(b"zlib") - # Newer Debug copy of the same basename must not win. - (debug / "PubSubPlugin.dll").write_bytes(b"debug-plugin") - - vcpkg_bin = ( - root / "pubsub" / "build" / "vcpkg_installed" / "x64-windows" / "bin" - ) - vcpkg_bin.mkdir(parents=True) - (vcpkg_bin / "extra.dll").write_bytes(b"transitive") - (vcpkg_bin / "libssl-3-x64.dll").write_bytes(b"unused-tls") - debug_bin = ( - root - / "pubsub" - / "build" - / "vcpkg_installed" - / "x64-windows" - / "debug" - / "bin" - ) - debug_bin.mkdir(parents=True) - (debug_bin / "extra.dll").write_bytes(b"debug-extra") - - staging = root / "package-staging" - archive = root / "release-payloads" / "pubsub-windows-x86_64.tar.gz" - names = packager.stage_and_archive( - artifact, - "PubSubPlugin", - staging, - archive, - ) - self.assertEqual( - names, - [ - "extra.dll", - "libcrypto-3-x64.dll", - "libcurl.dll", - "PubSubPlugin.dll", - "z.dll", - ], - ) - self.assertEqual((staging / "PubSubPlugin.dll").read_bytes(), plugin) - self.assertEqual((staging / "extra.dll").read_bytes(), b"transitive") - with tarfile.open(archive, "r:gz") as tar: - self.assertEqual(tar.getnames(), names) - for member in tar.getmembers(): - self.assertFalse(member.isdir()) - self.assertFalse("/" in member.name or "\\" in member.name) - - def test_imported_libssl_is_taken_from_vcpkg_bin(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - artifact = root / "build" / "PubSubPlugin" / "Release" - artifact.mkdir(parents=True) - (artifact / "PubSubPlugin.dll").write_bytes( - make_pe_dll(["libssl-3-x64.dll"]) - ) - vcpkg_bin = root / "build" / "vcpkg_installed" / "x64-windows" / "bin" - vcpkg_bin.mkdir(parents=True) - (vcpkg_bin / "libssl-3-x64.dll").write_bytes(b"ssl") - names = packager.stage_and_archive( - artifact.parent, - "PubSubPlugin", - root / "staging", - root / "out.tar.gz", - ) - self.assertEqual(names, ["libssl-3-x64.dll", "PubSubPlugin.dll"]) - - def test_linux_tree_keeps_plugin_and_shipped_so_deps_only(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - artifact = root / "pubsub" / "build" / "PubSubPlugin" - artifact.mkdir(parents=True) - (artifact / "CMakeFiles").mkdir() - (artifact / "cmake_install.cmake").write_text("install()") - plugin = artifact / "libPubSubPlugin.so" - plugin.write_bytes(b"plugin-so") - plugin.chmod(0o755) - real_dep = artifact / "libcrypto.so.3" - real_dep.write_bytes(b"crypto-so") - link = artifact / "libcrypto.so" - link.symlink_to(real_dep.name) - - staging = root / "package-staging" - archive = root / "out.tar.gz" - names = packager.stage_and_archive( - artifact, - "PubSubPlugin", - staging, - archive, - ) - self.assertEqual(names, ["libcrypto.so", "libcrypto.so.3", "libPubSubPlugin.so"]) - self.assertEqual((staging / "libcrypto.so").read_bytes(), b"crypto-so") - self.assertTrue((staging / "libPubSubPlugin.so").stat().st_mode & 0o111) - with tarfile.open(archive, "r:gz") as tar: - self.assertEqual(tar.getnames(), names) - - def test_missing_plugin_fails(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - artifact = root / "build" / "PubSubPlugin" / "Release" - artifact.mkdir(parents=True) - (artifact / "libcurl.dll").write_bytes(b"curl") - with self.assertRaises(SystemExit) as raised: - packager.stage_and_archive( - artifact.parent, - "PubSubPlugin", - root / "staging", - root / "out.tar.gz", - ) - self.assertIn("plugin shared library", str(raised.exception)) - - def test_empty_tree_fails(self) -> None: - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - artifact = root / "build" / "PubSubPlugin" - artifact.mkdir(parents=True) - (artifact / "CMakeFiles").mkdir() - (artifact / "cmake_install.cmake").write_text("x") - with self.assertRaises(SystemExit) as raised: - packager.stage_and_archive( - artifact, - "PubSubPlugin", - root / "staging", - root / "out.tar.gz", - ) - self.assertIn("no shared libraries", str(raised.exception)) - - -if __name__ == "__main__": - unittest.main() diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8b0b718..83ea2d9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -173,27 +173,61 @@ jobs: run: | ctest --test-dir build -C Release --output-on-failure --timeout 180 - # Publish a flat archive of runtime shared libraries only. The MSVC - # target directory also holds CMake/project files and import libraries. + # cmake --install --component plugin-runtime copies the plugin and the + # shared libraries it actually loads. FetchContent install rules and + # MSVC import libraries stay out of that component. - name: Package (${{ matrix.plugin }}) shell: bash run: | set -euo pipefail - if command -v python3 >/dev/null 2>&1; then - PY=(python3) - elif command -v python >/dev/null 2>&1; then - PY=(python) - elif command -v py >/dev/null 2>&1; then - PY=(py -3) - else - echo "python3 is required to package shared libraries" >&2 + staging="${GITHUB_WORKSPACE}/package-staging" + flat="${GITHUB_WORKSPACE}/package-flat" + rm -rf "${staging}" "${flat}" + mkdir -p "${staging}" "${flat}" release-payloads + cmake --install "${DIRECTORY}/build" --config Release --prefix "${staging}" --component plugin-runtime + + find "${staging}" \( -type f -o -type l \) \( \ + -name '*.dll' -o -name '*.DLL' -o \ + -name '*.dylib' -o \ + -name '*.so' -o -name '*.so.*' \ + \) -print0 | + while IFS= read -r -d '' shared; do + cp -a "${shared}" "${flat}/$(basename "${shared}")" + done + + count="$(find "${flat}" -maxdepth 1 \( -type f -o -type l \) | wc -l)" + if [[ "${count}" -eq 0 ]]; then + echo "error: no shared libraries installed from ${DIRECTORY}/build" >&2 + find "${staging}" -print >&2 || true exit 1 fi - "${PY[@]}" .github/scripts/package_shared_libs.py \ - --artifact-path "${ARTIFACT_PATH}" \ - --plugin-stem "$(basename "${ARTIFACT_PATH}")" \ - --staging package-staging \ - --archive "release-payloads/${PLUGIN_NAME}-${PACKAGE_SUFFIX}.tar.gz" + + stem="$(basename "${ARTIFACT_PATH}")" + plugin_found=0 + for candidate in \ + "${flat}/${stem}.dll" \ + "${flat}/${stem}.DLL" \ + "${flat}/lib${stem}.so" \ + "${flat}/${stem}.so" \ + "${flat}/lib${stem}.dylib" \ + "${flat}/${stem}.dylib" + do + if [[ -e "${candidate}" ]]; then + plugin_found=1 + fi + done + if compgen -G "${flat}/lib${stem}.so.*" >/dev/null || compgen -G "${flat}/${stem}.so.*" >/dev/null; then + plugin_found=1 + fi + if [[ "${plugin_found}" -ne 1 ]]; then + echo "error: plugin shared library ${stem} was not installed" >&2 + find "${flat}" -maxdepth 1 -print >&2 + exit 1 + fi + + echo "Packaged shared libraries:" + find "${flat}" -maxdepth 1 \( -type f -o -type l \) -exec basename {} \; | sort + tar -czf "release-payloads/${PLUGIN_NAME}-${PACKAGE_SUFFIX}.tar.gz" -C "${flat}" . - name: Upload artifact (${{ matrix.plugin }}) uses: actions/upload-artifact@v6 diff --git a/cmake/InstallPluginRuntime.cmake b/cmake/InstallPluginRuntime.cmake new file mode 100644 index 0000000..de04a43 --- /dev/null +++ b/cmake/InstallPluginRuntime.cmake @@ -0,0 +1,80 @@ +# Install one plugin as a flat set of runtime shared libraries. +# +# Archive/import libraries (.lib, .exp, static .a) are not installed. System +# libraries are left on the host. Imported shared libraries (vcpkg DLLs on +# Windows) are installed next to the plugin because install(RUNTIME_DEPENDENCY_SET) +# scans the built binary. A DLL is included only when that scan finds it, so +# Windows Pub/Sub does not pick up libssl while curl is built with Schannel. +# +# Multi-config generators (Visual Studio) honor `cmake --install --config Release`. +# Single-config generators (Ninja, Makefiles) use CMAKE_BUILD_TYPE, which CI sets +# to Release. The install component is `plugin-runtime` so other install() +# rules in the same build (FetchContent headers, static libraries) are not +# packaged. + +function(graftcode_install_plugin target) + if(NOT TARGET "${target}") + message(FATAL_ERROR "graftcode_install_plugin: target '${target}' does not exist") + endif() + + set(dep_dirs "$") + if(DEFINED VCPKG_INSTALLED_DIR AND NOT "${VCPKG_INSTALLED_DIR}" STREQUAL "") + # Manifest mode points VCPKG_INSTALLED_DIR at either the triplet root + # (.../vcpkg_installed/x64-windows) or the parent (.../vcpkg_installed). + # Search both layouts. Debug bins are used only for the Debug config. + set(vcpkg_roots "${VCPKG_INSTALLED_DIR}") + if(DEFINED VCPKG_TARGET_TRIPLET AND NOT "${VCPKG_TARGET_TRIPLET}" STREQUAL "") + list(APPEND vcpkg_roots "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}") + endif() + foreach(root IN LISTS vcpkg_roots) + list(APPEND dep_dirs + "$,${root}/debug/bin,${root}/bin>" + "$,${root}/debug/lib,${root}/lib>" + ) + endforeach() + endif() + + # Flat install: consumers extract the archive next to the host executable. + if(APPLE) + set_property(TARGET "${target}" PROPERTY INSTALL_RPATH "@loader_path") + elseif(UNIX) + set_property(TARGET "${target}" PROPERTY INSTALL_RPATH "$ORIGIN") + endif() + + set(runtime_set "graftcode_${target}_runtime") + install(TARGETS "${target}" + RUNTIME_DEPENDENCY_SET "${runtime_set}" + RUNTIME DESTINATION . + COMPONENT plugin-runtime + LIBRARY DESTINATION . + COMPONENT plugin-runtime + ) + # Regex and DIRECTORIES arguments must precede the RUNTIME/LIBRARY + # groups. CMake otherwise parses them as per-artifact options. + install(RUNTIME_DEPENDENCY_SET "${runtime_set}" + DIRECTORIES ${dep_dirs} + PRE_EXCLUDE_REGEXES + [[api-ms-win-.*]] + [[ext-ms-.*]] + POST_EXCLUDE_REGEXES + # Windows system and toolchain directories. vcpkg copies are elsewhere. + [[.*[Ss]ystem32[/\\].*]] + [[.*[Ss]ys[Ww][Oo][Ww]64[/\\].*]] + [[.*[Ww]in[Ss]x[Ss][/\\].*]] + [[^[A-Za-z]:[/\\][Ww]indows[/\\].*]] + [[.*[Ww]indows [Kk]its[/\\].*]] + [[.*Microsoft Visual Studio[/\\].*]] + # Linux and macOS system loaders. Homebrew is the kafka macOS OpenSSL. + [[^/lib/.*]] + [[^/lib64/.*]] + [[^/usr/lib/.*]] + [[^/usr/lib64/.*]] + [[^/usr/local/.*]] + [[^/System/.*]] + [[^/opt/homebrew/.*]] + RUNTIME DESTINATION . + COMPONENT plugin-runtime + LIBRARY DESTINATION . + COMPONENT plugin-runtime + ) +endfunction() diff --git a/kafka/CMakeLists.txt b/kafka/CMakeLists.txt index a7f50a4..6761891 100644 --- a/kafka/CMakeLists.txt +++ b/kafka/CMakeLists.txt @@ -20,3 +20,6 @@ endif() add_subdirectory("KafkaPlugin") add_subdirectory("KafkaPluginTest") + +include("${CMAKE_CURRENT_SOURCE_DIR}/../cmake/InstallPluginRuntime.cmake") +graftcode_install_plugin(KafkaPlugin) diff --git a/pubsub/CMakeLists.txt b/pubsub/CMakeLists.txt index e3b429f..cef2f9c 100644 --- a/pubsub/CMakeLists.txt +++ b/pubsub/CMakeLists.txt @@ -20,3 +20,6 @@ endif() add_subdirectory("PubSubPlugin") add_subdirectory("PubSubPluginTest") + +include("${CMAKE_CURRENT_SOURCE_DIR}/../cmake/InstallPluginRuntime.cmake") +graftcode_install_plugin(PubSubPlugin) diff --git a/rabbitmq/CMakeLists.txt b/rabbitmq/CMakeLists.txt index 3ec0172..cefa250 100644 --- a/rabbitmq/CMakeLists.txt +++ b/rabbitmq/CMakeLists.txt @@ -28,3 +28,6 @@ endif() add_subdirectory ("RabbitmqPlugin") add_subdirectory ("RabbitmqPluginTest") +include("${CMAKE_CURRENT_SOURCE_DIR}/../cmake/InstallPluginRuntime.cmake") +graftcode_install_plugin(RabbitmqPlugin) + diff --git a/servicebus/CMakeLists.txt b/servicebus/CMakeLists.txt index b07e32e..1f3ea8b 100644 --- a/servicebus/CMakeLists.txt +++ b/servicebus/CMakeLists.txt @@ -22,3 +22,6 @@ endif() add_subdirectory ("ServiceBusPlugin") add_subdirectory ("ServiceBusPluginTest") + +include("${CMAKE_CURRENT_SOURCE_DIR}/../cmake/InstallPluginRuntime.cmake") +graftcode_install_plugin(ServiceBusPlugin) diff --git a/sqs/CMakeLists.txt b/sqs/CMakeLists.txt index a6fbf37..33ee2d2 100644 --- a/sqs/CMakeLists.txt +++ b/sqs/CMakeLists.txt @@ -19,3 +19,6 @@ endif() add_subdirectory("SqsPlugin") add_subdirectory("SqsPluginTest") + +include("${CMAKE_CURRENT_SOURCE_DIR}/../cmake/InstallPluginRuntime.cmake") +graftcode_install_plugin(SqsPlugin) From a15deecff43d9c981ee3f13b61c02d16f5db89d5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 06:55:27 +0000 Subject: [PATCH 3/7] Include non-system runtime libraries in plugin packages. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Search OpenSSL and vcpkg for shared libraries the plugin imports, and stop dropping Homebrew and /usr copies of those libraries. Keep the C/C++ runtime on the host, and point bundled ELF and Mach-O dependencies at the flat directory. Co-authored-by: Michał Komor --- .github/workflows/release.yml | 12 +- cmake/InstallPluginRuntime.cmake | 191 ++++++++++++++++++++++++++++--- 2 files changed, 185 insertions(+), 18 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 83ea2d9..4a5fc97 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,6 +11,7 @@ on: - "kafka/**" - "sqs/**" - "pubsub/**" + - "cmake/**" - ".github/workflows/release.yml" pull_request: branches: [ "main" ] @@ -20,6 +21,7 @@ on: - "kafka/**" - "sqs/**" - "pubsub/**" + - "cmake/**" - ".github/workflows/release.yml" workflow_dispatch: inputs: @@ -88,7 +90,7 @@ jobs: echo "ARTIFACT_PATH=servicebus/build/ServiceBusPlugin" >> "$GITHUB_ENV" echo "CONFIGURE_ARGS=-DCMAKE_TOOLCHAIN_FILE=${VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" >> "$GITHUB_ENV" if [ "${{ runner.os }}" = "Linux" ]; then - echo "INSTALL_PACKAGES=build-essential cmake git ninja-build pkg-config" >> "$GITHUB_ENV" + echo "INSTALL_PACKAGES=build-essential cmake git ninja-build pkg-config patchelf" >> "$GITHUB_ENV" elif [ "${{ runner.os }}" = "macOS" ]; then echo "INSTALL_PACKAGES=cmake ninja pkg-config" >> "$GITHUB_ENV" else @@ -100,7 +102,7 @@ jobs: echo "ARTIFACT_PATH=sqs/build/SqsPlugin" >> "$GITHUB_ENV" echo "CONFIGURE_ARGS=-DCMAKE_TOOLCHAIN_FILE=${VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" >> "$GITHUB_ENV" if [ "${{ runner.os }}" = "Linux" ]; then - echo "INSTALL_PACKAGES=build-essential cmake git ninja-build pkg-config" >> "$GITHUB_ENV" + echo "INSTALL_PACKAGES=build-essential cmake git ninja-build pkg-config patchelf" >> "$GITHUB_ENV" elif [ "${{ runner.os }}" = "macOS" ]; then echo "INSTALL_PACKAGES=cmake ninja pkg-config" >> "$GITHUB_ENV" else @@ -112,7 +114,7 @@ jobs: echo "ARTIFACT_PATH=pubsub/build/PubSubPlugin" >> "$GITHUB_ENV" echo "CONFIGURE_ARGS=-DCMAKE_TOOLCHAIN_FILE=${VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" >> "$GITHUB_ENV" if [ "${{ runner.os }}" = "Linux" ]; then - echo "INSTALL_PACKAGES=build-essential cmake git ninja-build pkg-config" >> "$GITHUB_ENV" + echo "INSTALL_PACKAGES=build-essential cmake git ninja-build pkg-config patchelf" >> "$GITHUB_ENV" elif [ "${{ runner.os }}" = "macOS" ]; then echo "INSTALL_PACKAGES=cmake ninja pkg-config" >> "$GITHUB_ENV" else @@ -124,7 +126,7 @@ jobs: echo "ARTIFACT_PATH=kafka/build/KafkaPlugin" >> "$GITHUB_ENV" echo "CONFIGURE_ARGS=" >> "$GITHUB_ENV" if [ "${{ runner.os }}" = "Linux" ]; then - echo "INSTALL_PACKAGES=build-essential cmake git libssl-dev libsasl2-dev zlib1g-dev" >> "$GITHUB_ENV" + echo "INSTALL_PACKAGES=build-essential cmake git libssl-dev libsasl2-dev zlib1g-dev patchelf" >> "$GITHUB_ENV" elif [ "${{ runner.os }}" = "macOS" ]; then echo "INSTALL_PACKAGES=cmake openssl" >> "$GITHUB_ENV" else @@ -136,7 +138,7 @@ jobs: echo "ARTIFACT_PATH=rabbitmq/build/RabbitmqPlugin" >> "$GITHUB_ENV" echo "CONFIGURE_ARGS=" >> "$GITHUB_ENV" if [ "${{ runner.os }}" = "Linux" ]; then - echo "INSTALL_PACKAGES=build-essential cmake git" >> "$GITHUB_ENV" + echo "INSTALL_PACKAGES=build-essential cmake git patchelf" >> "$GITHUB_ENV" elif [ "${{ runner.os }}" = "macOS" ]; then echo "INSTALL_PACKAGES=cmake" >> "$GITHUB_ENV" else diff --git a/cmake/InstallPluginRuntime.cmake b/cmake/InstallPluginRuntime.cmake index de04a43..f6371f6 100644 --- a/cmake/InstallPluginRuntime.cmake +++ b/cmake/InstallPluginRuntime.cmake @@ -1,10 +1,14 @@ # Install one plugin as a flat set of runtime shared libraries. # -# Archive/import libraries (.lib, .exp, static .a) are not installed. System -# libraries are left on the host. Imported shared libraries (vcpkg DLLs on -# Windows) are installed next to the plugin because install(RUNTIME_DEPENDENCY_SET) -# scans the built binary. A DLL is included only when that scan finds it, so -# Windows Pub/Sub does not pick up libssl while curl is built with Schannel. +# The plugin binary is installed without its import library or static archives. +# install(RUNTIME_DEPENDENCY_SET) then scans that binary and installs every +# non-system shared library it imports: vcpkg DLLs, OpenSSL from the runner +# image or Homebrew, librdkafka's libsasl/libz, the AWS SDK, and so on. +# The C/C++ runtime and Windows system DLLs stay on the host. +# +# A library is included only when the scan finds it. Windows Pub/Sub links +# Schannel curl plus OpenSSL::Crypto, so libssl is not one of its imports. +# Kafka's librdkafka does import libssl and libcrypto when it finds OpenSSL. # # Multi-config generators (Visual Studio) honor `cmake --install --config Release`. # Single-config generators (Ninja, Makefiles) use CMAKE_BUILD_TYPE, which CI sets @@ -12,6 +16,54 @@ # rules in the same build (FetchContent headers, static libraries) are not # packaged. +# Directories that hold OpenSSL (and similarly installed) shared libraries. +# FindOpenSSL's results live in the cache as LIB_EAY / SSL_EAY, often as an +# import library several levels below the bin or lib directory that contains +# the DLL or .so/.dylib the loader opens. +function(_graftcode_runtime_search_dirs out_var) + set(dirs ${${out_var}}) + set(library_vars LIB_EAY LIB_EAY_RELEASE LIB_EAY_DEBUG SSL_EAY SSL_EAY_RELEASE SSL_EAY_DEBUG) + foreach(cache_name IN LISTS library_vars) + if(DEFINED CACHE{${cache_name}} AND NOT "$CACHE{${cache_name}}" STREQUAL "") + set(entry "$CACHE{${cache_name}}") + if(EXISTS "${entry}") + get_filename_component(dir "${entry}" DIRECTORY) + # The import library and the DLL/dylib it stands for are often in the + # same directory (MinGW, some vcpkg layouts). + list(APPEND dirs "${dir}") + foreach(_i RANGE 0 6) + if(EXISTS "${dir}/bin") + list(APPEND dirs "${dir}/bin") + endif() + if(EXISTS "${dir}/lib") + list(APPEND dirs "${dir}/lib") + endif() + get_filename_component(parent "${dir}" DIRECTORY) + if(parent STREQUAL dir) + break() + endif() + set(dir "${parent}") + endforeach() + endif() + endif() + endforeach() + foreach(candidate IN ITEMS + "C:/Program Files/OpenSSL/bin" + "C:/Program Files/OpenSSL-Win64/bin" + "/opt/homebrew/opt/openssl@3/lib" + "/opt/homebrew/opt/openssl/lib" + "/usr/local/opt/openssl@3/lib" + "/usr/local/opt/openssl/lib") + if(EXISTS "${candidate}") + list(APPEND dirs "${candidate}") + endif() + endforeach() + if(dirs) + list(REMOVE_DUPLICATES dirs) + endif() + set(${out_var} "${dirs}" PARENT_SCOPE) +endfunction() + function(graftcode_install_plugin target) if(NOT TARGET "${target}") message(FATAL_ERROR "graftcode_install_plugin: target '${target}' does not exist") @@ -33,6 +85,7 @@ function(graftcode_install_plugin target) ) endforeach() endif() + _graftcode_runtime_search_dirs(dep_dirs) # Flat install: consumers extract the archive next to the host executable. if(APPLE) @@ -56,25 +109,137 @@ function(graftcode_install_plugin target) PRE_EXCLUDE_REGEXES [[api-ms-win-.*]] [[ext-ms-.*]] + # Platform C/C++ runtime. Matched against the SONAME, so libssl and + # libcrypto under /usr/lib or Homebrew are still installed. + [[libc\.so.*]] + [[libm\.so.*]] + [[libdl\.so.*]] + [[libpthread\.so.*]] + [[librt\.so.*]] + [[libgcc_s\.so.*]] + [[libstdc\+\+\.so.*]] + [[ld-linux.*]] + [[linux-vdso.*]] + [[libSystem\..*]] + [[libc\+\+\..*]] + [[libc\+\+abi\..*]] POST_EXCLUDE_REGEXES - # Windows system and toolchain directories. vcpkg copies are elsewhere. + # Windows system and toolchain directories. vcpkg and OpenSSL copies + # live elsewhere and must be packaged. [[.*[Ss]ystem32[/\\].*]] [[.*[Ss]ys[Ww][Oo][Ww]64[/\\].*]] [[.*[Ww]in[Ss]x[Ss][/\\].*]] [[^[A-Za-z]:[/\\][Ww]indows[/\\].*]] [[.*[Ww]indows [Kk]its[/\\].*]] [[.*Microsoft Visual Studio[/\\].*]] - # Linux and macOS system loaders. Homebrew is the kafka macOS OpenSSL. - [[^/lib/.*]] - [[^/lib64/.*]] - [[^/usr/lib/.*]] - [[^/usr/lib64/.*]] - [[^/usr/local/.*]] [[^/System/.*]] - [[^/opt/homebrew/.*]] RUNTIME DESTINATION . COMPONENT plugin-runtime LIBRARY DESTINATION . COMPONENT plugin-runtime ) + + # CMake copies runtime dependencies with their original load paths. + # Point each bundled ELF/Mach-O library at the flat directory so libssl + # finds libcrypto beside it on a machine that does not have that OpenSSL. + if(UNIX AND NOT APPLE) + # file(RPATH_SET) only rewrites an existing ELF RPATH/RUNPATH. System + # OpenSSL and libsasl have none, so patchelf has to add $ORIGIN. + install(CODE [[ + find_program(_graftcode_patchelf patchelf) + if(NOT _graftcode_patchelf) + message(FATAL_ERROR "patchelf is required to set $ORIGIN on bundled shared libraries") + endif() + file(GLOB _graftcode_runtime_libs + "${CMAKE_INSTALL_PREFIX}/*.so" + "${CMAKE_INSTALL_PREFIX}/*.so.*") + foreach(_graftcode_lib IN LISTS _graftcode_runtime_libs) + if(IS_SYMLINK "${_graftcode_lib}") + continue() + endif() + execute_process( + COMMAND "${_graftcode_patchelf}" --set-rpath "\$ORIGIN" "${_graftcode_lib}" + RESULT_VARIABLE _graftcode_rc + ERROR_VARIABLE _graftcode_err + ) + if(NOT _graftcode_rc EQUAL 0) + message(FATAL_ERROR "patchelf --set-rpath failed for ${_graftcode_lib}: ${_graftcode_err}") + endif() + endforeach() + ]] COMPONENT plugin-runtime) + elseif(APPLE) + install(CODE [[ + file(GLOB _graftcode_runtime_libs "${CMAKE_INSTALL_PREFIX}/*.dylib") + set(_graftcode_bundled "") + foreach(_graftcode_lib IN LISTS _graftcode_runtime_libs) + cmake_path(GET _graftcode_lib FILENAME _graftcode_name) + list(APPEND _graftcode_bundled "${_graftcode_name}") + endforeach() + foreach(_graftcode_lib IN LISTS _graftcode_runtime_libs) + if(IS_SYMLINK "${_graftcode_lib}") + continue() + endif() + cmake_path(GET _graftcode_lib FILENAME _graftcode_name) + execute_process( + COMMAND install_name_tool -id "@loader_path/${_graftcode_name}" "${_graftcode_lib}" + RESULT_VARIABLE _graftcode_rc + ERROR_VARIABLE _graftcode_err + ) + if(NOT _graftcode_rc EQUAL 0) + execute_process(COMMAND codesign --remove-signature "${_graftcode_lib}") + execute_process( + COMMAND install_name_tool -id "@loader_path/${_graftcode_name}" "${_graftcode_lib}" + RESULT_VARIABLE _graftcode_rc + ERROR_VARIABLE _graftcode_err + ) + endif() + if(NOT _graftcode_rc EQUAL 0) + message(FATAL_ERROR "install_name_tool -id failed for ${_graftcode_lib}: ${_graftcode_err}") + endif() + execute_process( + COMMAND otool -L "${_graftcode_lib}" + OUTPUT_VARIABLE _graftcode_otool + RESULT_VARIABLE _graftcode_rc + ) + if(NOT _graftcode_rc EQUAL 0) + message(FATAL_ERROR "otool -L failed for ${_graftcode_lib}") + endif() + string(REPLACE "\n" ";" _graftcode_lines "${_graftcode_otool}") + foreach(_graftcode_line IN LISTS _graftcode_lines) + string(STRIP "${_graftcode_line}" _graftcode_line) + if(_graftcode_line MATCHES "^(.+) \\(compatibility version") + set(_graftcode_dep "${CMAKE_MATCH_1}") + cmake_path(GET _graftcode_dep FILENAME _graftcode_dep_name) + list(FIND _graftcode_bundled "${_graftcode_dep_name}" _graftcode_found) + if(NOT _graftcode_found EQUAL -1 AND NOT _graftcode_dep STREQUAL "@loader_path/${_graftcode_dep_name}") + execute_process( + COMMAND install_name_tool -change "${_graftcode_dep}" "@loader_path/${_graftcode_dep_name}" "${_graftcode_lib}" + RESULT_VARIABLE _graftcode_rc + ERROR_VARIABLE _graftcode_err + ) + if(NOT _graftcode_rc EQUAL 0) + execute_process(COMMAND codesign --remove-signature "${_graftcode_lib}") + execute_process( + COMMAND install_name_tool -change "${_graftcode_dep}" "@loader_path/${_graftcode_dep_name}" "${_graftcode_lib}" + RESULT_VARIABLE _graftcode_rc + ERROR_VARIABLE _graftcode_err + ) + endif() + if(NOT _graftcode_rc EQUAL 0) + message(FATAL_ERROR "install_name_tool -change failed for ${_graftcode_lib}: ${_graftcode_err}") + endif() + endif() + endif() + endforeach() + execute_process( + COMMAND codesign --force --sign - "${_graftcode_lib}" + RESULT_VARIABLE _graftcode_rc + ERROR_VARIABLE _graftcode_err + ) + if(NOT _graftcode_rc EQUAL 0) + message(FATAL_ERROR "codesign failed for ${_graftcode_lib}: ${_graftcode_err}") + endif() + endforeach() + ]] COMPONENT plugin-runtime) + endif() endfunction() From a69ad6c4e28a20a9a5c51b9351d75edde2dbc3d2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 07:30:30 +0000 Subject: [PATCH 4/7] Link plugins with the static MSVC CRT and static third-party libraries. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set /MT before project() so FetchContent dependencies use the same CRT, and select the Windows static vcpkg triplet so curl, the AWS SDK, Azure AMQP, and Kafka's OpenSSL match it. Unix Kafka links PIC OpenSSL statically and keeps zlib, zstd, and Cyrus SASL shared. Co-authored-by: Michał Komor --- .github/workflows/release.yml | 9 ++-- README.md | 2 +- cmake/StaticPluginRuntime.cmake | 54 +++++++++++++++++++ kafka/BUILD_NOTES.md | 7 ++- kafka/CMakeLists.txt | 8 ++- kafka/KafkaPlugin/CMakeLists.txt | 13 +++++ kafka/Readme.md | 3 +- kafka/vcpkg.json | 10 ++++ pubsub/BUILD_NOTES.md | 2 +- pubsub/CMakeLists.txt | 9 +++- pubsub/PubSubPlugin/CMakeLists.txt | 5 ++ pubsub/PubSubPluginTest/CMakeLists.txt | 7 ++- pubsub/Readme.md | 4 +- rabbitmq/CMakeLists.txt | 23 ++++---- rabbitmq/RabbitmqPlugin/CMakeLists.txt | 2 +- rabbitmq/RabbitmqPluginTest/CMakeLists.txt | 5 +- rabbitmq/Readme.md | 2 +- samples/pubsub/README.md | 3 +- samples/sqs/README.md | 3 +- servicebus/CMakeLists.txt | 10 ++-- servicebus/Readme.md | 2 + .../ServiceBusPluginTest/CMakeLists.txt | 2 +- sqs/CMakeLists.txt | 9 +++- sqs/Readme.md | 2 +- sqs/SqsPluginTest/CMakeLists.txt | 2 +- 25 files changed, 156 insertions(+), 42 deletions(-) create mode 100644 cmake/StaticPluginRuntime.cmake create mode 100644 kafka/vcpkg.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4a5fc97..df128d1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -124,12 +124,15 @@ jobs: kafka) echo "DIRECTORY=kafka" >> "$GITHUB_ENV" echo "ARTIFACT_PATH=kafka/build/KafkaPlugin" >> "$GITHUB_ENV" - echo "CONFIGURE_ARGS=" >> "$GITHUB_ENV" if [ "${{ runner.os }}" = "Linux" ]; then - echo "INSTALL_PACKAGES=build-essential cmake git libssl-dev libsasl2-dev zlib1g-dev patchelf" >> "$GITHUB_ENV" + echo "CONFIGURE_ARGS=" >> "$GITHUB_ENV" + echo "INSTALL_PACKAGES=build-essential cmake git libssl-dev libsasl2-dev zlib1g-dev libzstd-dev patchelf" >> "$GITHUB_ENV" elif [ "${{ runner.os }}" = "macOS" ]; then - echo "INSTALL_PACKAGES=cmake openssl" >> "$GITHUB_ENV" + echo "CONFIGURE_ARGS=" >> "$GITHUB_ENV" + echo "INSTALL_PACKAGES=cmake openssl zstd" >> "$GITHUB_ENV" else + # Static OpenSSL/zlib built with /MT. The Program Files OpenSSL is /MD. + echo "CONFIGURE_ARGS=-DCMAKE_TOOLCHAIN_FILE=${VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" >> "$GITHUB_ENV" echo "INSTALL_PACKAGES=cmake" >> "$GITHUB_ENV" fi ;; diff --git a/README.md b/README.md index 31737d1..263c0b8 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Use RabbitMQ or Kafka when you already run that broker. Use Service Bus or SQS w ## Use a plugin 1. Build the plugin (see its Readme). The artifact is a shared library: `SqsPlugin.dll` on Windows, or `libSqsPlugin.so` / `.dylib` on Linux and macOS. -2. Put `gg`, the hosted module, the plugin library, and any runtime DLLs in one folder. `name` in the config is the library file name without the extension (`libSqsPlugin` when the file is `libSqsPlugin.so`). +2. Put `gg`, the hosted module, and the plugin library in one folder, plus any extra shared libraries from that plugin's release archive. Windows archives are the plugin DLL alone (static VC++ runtime, no VC++ Redistributable). Linux Kafka still ships `libsasl2` (and the `libcrypto` it imports), plus `libz` and `libzstd`. `name` in the config is the library file name without the extension (`libSqsPlugin` when the file is `libSqsPlugin.so`). 3. Start the Gateway with `--config`. A verified SQS layout is in [samples/sqs](samples/sqs/README.md): `gg netapp.dll --config sqsplugin.json`. 4. Open `http://localhost:81/GV`, install the generated package, and call `GraftConfig.SetConfig` with the same channel settings. `host` there is the Gateway HTTP address, not the broker. diff --git a/cmake/StaticPluginRuntime.cmake b/cmake/StaticPluginRuntime.cmake new file mode 100644 index 0000000..f86746d --- /dev/null +++ b/cmake/StaticPluginRuntime.cmake @@ -0,0 +1,54 @@ +# Static MSVC CRT and, on Windows, a matching static vcpkg triplet. +# +# Include this file after cmake_minimum_required() and call +# graftcode_prepare_static_runtime() before project(), then +# graftcode_apply_static_crt() after project(). +# +# /MT links vcruntime and the C++ standard library into the plugin, so +# loading it does not require the VC++ Redistributable. The Universal CRT +# (ucrtbase.dll) stays a Windows system DLL. +# +# x64-windows-static (and the arm64/x86 equivalents) build vcpkg ports as +# static libraries with that same CRT. Linux and macOS vcpkg triplets are +# already static; glibc, libstdc++, and libSystem stay on the host. + +macro(graftcode_prepare_static_runtime) + # FetchContent projects often pin an older cmake_minimum_required, which + # would otherwise leave CMP0091 unset and ignore CMAKE_MSVC_RUNTIME_LIBRARY. + set(CMAKE_POLICY_DEFAULT_CMP0091 NEW) + set(CMAKE_MSVC_RUNTIME_LIBRARY + "MultiThreaded$<$:Debug>" + CACHE STRING "Static MSVC CRT (/MT, /MTd for Debug)" FORCE) + + if(CMAKE_HOST_WIN32 AND NOT DEFINED VCPKG_TARGET_TRIPLET) + set(_graftcode_arch "x64") + if(CMAKE_GENERATOR_PLATFORM MATCHES "ARM64|arm64" + OR "$ENV{PROCESSOR_ARCHITECTURE}" STREQUAL "ARM64") + set(_graftcode_arch "arm64") + elseif(CMAKE_GENERATOR_PLATFORM MATCHES "Win32" + OR "$ENV{PROCESSOR_ARCHITECTURE}" STREQUAL "X86") + set(_graftcode_arch "x86") + endif() + set(VCPKG_TARGET_TRIPLET "${_graftcode_arch}-windows-static" + CACHE STRING "Static vcpkg triplet matching /MT" FORCE) + unset(_graftcode_arch) + endif() +endmacro() + +# project() fills CMAKE__FLAGS* with /MD. Rewrite those so dependencies +# that do not honor CMP0091 still compile against the static CRT. +macro(graftcode_apply_static_crt) + if(MSVC) + foreach(_graftcode_flag_var + CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE + CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO + CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE + CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO) + if(DEFINED ${_graftcode_flag_var}) + string(REPLACE "/MDd" "/MTd" ${_graftcode_flag_var} "${${_graftcode_flag_var}}") + string(REPLACE "/MD" "/MT" ${_graftcode_flag_var} "${${_graftcode_flag_var}}") + endif() + endforeach() + unset(_graftcode_flag_var) + endif() +endmacro() diff --git a/kafka/BUILD_NOTES.md b/kafka/BUILD_NOTES.md index 346512d..f5ce420 100644 --- a/kafka/BUILD_NOTES.md +++ b/kafka/BUILD_NOTES.md @@ -1,5 +1,10 @@ # Kafka plugin build notes -- Dependencies: CMake `FetchContent` only (nlohmann/json + librdkafka). No vcpkg. +- librdkafka, nlohmann/json, LZ4 (bundled), and OpenSSL are linked statically. Windows also links zlib statically. +- Windows OpenSSL and zlib come from vcpkg (`*-windows-static`, `/MT`). The Program Files OpenSSL is `/MD` and is not used. Pass `-DCMAKE_TOOLCHAIN_FILE=/scripts/buildsystems/vcpkg.cmake`. +- Linux and macOS link `libssl.a` and `libcrypto.a` into the plugin (`libssl-dev` or Homebrew `openssl`). The plugin does not import `libssl.so`. +- `libz` and `libzstd` stay shared on Linux and macOS. Those distro static archives are not built with `-fPIC`, so they cannot be linked into the plugin. +- Cyrus SASL stays a shared library on Linux (`libsasl2.so`). Its mechanisms are loaded as plugins at runtime. That library imports `libcrypto.so.3`, so the Linux archive still contains `libcrypto.so.3` for SASL even though the plugin's own TLS is static. Windows SASL is `secur32`. macOS builds that do not find Cyrus SASL omit it. +- librdkafka's libcurl OIDC helper is off, so the plugin does not depend on `libcurl`. - Smoke test links factory exports; full RPC needs a live broker (`samples/kafka`). - First configure needs network/git for FetchContent downloads. diff --git a/kafka/CMakeLists.txt b/kafka/CMakeLists.txt index 6761891..2948927 100644 --- a/kafka/CMakeLists.txt +++ b/kafka/CMakeLists.txt @@ -3,8 +3,14 @@ cmake_minimum_required(VERSION ${CMAKE_MIN}) set(CMAKE_POLICY_VERSION_MINIMUM ${CMAKE_MIN}) cmake_policy(VERSION ${CMAKE_MIN}) +set(CMAKE_POLICY_DEFAULT_CMP0135 NEW) +include("${CMAKE_CURRENT_SOURCE_DIR}/../cmake/StaticPluginRuntime.cmake") +graftcode_prepare_static_runtime() + project("KafkaPlugin" VERSION 1.0.0) +graftcode_apply_static_crt() + set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -12,8 +18,6 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) add_definitions(-DUNICODE) enable_testing() -set(CMAKE_POLICY_DEFAULT_CMP0135 NEW) - if(MSVC) add_compile_options(/utf-8) endif() diff --git a/kafka/KafkaPlugin/CMakeLists.txt b/kafka/KafkaPlugin/CMakeLists.txt index 58671ea..206e7a5 100644 --- a/kafka/KafkaPlugin/CMakeLists.txt +++ b/kafka/KafkaPlugin/CMakeLists.txt @@ -24,6 +24,19 @@ FetchContent_MakeAvailable(json) set(RDKAFKA_BUILD_STATIC ON CACHE BOOL "Build librdkafka as static library" FORCE) set(RDKAFKA_BUILD_EXAMPLES OFF CACHE BOOL "Skip librdkafka examples" FORCE) set(RDKAFKA_BUILD_TESTS OFF CACHE BOOL "Skip librdkafka tests" FORCE) +# Bundled LZ4. External liblz4 is a shared library on Homebrew and some distros. +set(ENABLE_LZ4_EXT OFF CACHE BOOL "Use librdkafka's bundled LZ4" FORCE) +# libcurl would be another shared dependency, and OIDC is unused by this plugin. +set(WITH_CURL OFF CACHE BOOL "Do not link libcurl into librdkafka" FORCE) + +# Unix OpenSSL static archives are PIC and link into the plugin. Distro +# libz.a and libzstd.a are not, so zlib and zstd stay shared. Windows uses +# the vcpkg static triplet (libcrypto.lib / zlib.lib built with /MT). +# OPENSSL_USE_STATIC_LIBS would look for libcrypto_static, which vcpkg does +# not install, so it is Unix-only. +if(UNIX AND NOT WIN32) + set(OPENSSL_USE_STATIC_LIBS ON) +endif() FetchContent_Declare( librdkafka diff --git a/kafka/Readme.md b/kafka/Readme.md index a8f7b24..4ac9711 100644 --- a/kafka/Readme.md +++ b/kafka/Readme.md @@ -2,7 +2,7 @@ This plugin carries Graftcode Gateway calls over Apache Kafka. It implements `Hypertube::Native::Interfaces::ITransport` and `GraftcodeGateway::IServer`, and exports `CreateTransportChannel` / `DestroyTransportChannel` and `CreateServer` / `DestroyServer`. -It is written in C++. CMake FetchContent builds nlohmann/json `v3.12.0` and librdkafka `v2.8.0` (static `rdkafka++`). There is no vcpkg manifest. The first configure needs network access for those downloads. +It is written in C++. CMake FetchContent builds nlohmann/json `v3.12.0` and librdkafka `v2.8.0` (static `rdkafka++`). OpenSSL, zlib, and LZ4 are linked statically. On Windows, configure with the vcpkg toolchain so OpenSSL and zlib are the static `/MT` builds from `kafka/vcpkg.json` (the OpenSSL in `Program Files` is `/MD`). On Linux, `libsasl2`, `libz`, and `libzstd` stay shared: Cyrus SASL loads mechanism modules at runtime (and imports `libcrypto.so.3`), and the distro static zlib and zstd archives are not built with `-fPIC`. The first configure needs network access. Kafka has no native request/reply. The plugin uses headers: @@ -21,6 +21,7 @@ git clone https://github.com/grft-dev/graftcode-extensions.git cd graftcode-extensions/kafka cmake -S . -B build -DCMAKE_BUILD_TYPE=Release +# Windows: add -DCMAKE_TOOLCHAIN_FILE=/scripts/buildsystems/vcpkg.cmake cmake --build build --config Release ``` diff --git a/kafka/vcpkg.json b/kafka/vcpkg.json new file mode 100644 index 0000000..554f54a --- /dev/null +++ b/kafka/vcpkg.json @@ -0,0 +1,10 @@ +{ + "name": "graftcode-kafka-plugin", + "version": "1.0.0", + "description": "Static OpenSSL and zlib for the Kafka plugin on Windows (/MT). Unix builds use the system static archives instead.", + "builtin-baseline": "1460b31b08c42cc2e9ac2c79f45ec8707e2675e2", + "dependencies": [ + "openssl", + "zlib" + ] +} diff --git a/pubsub/BUILD_NOTES.md b/pubsub/BUILD_NOTES.md index 0bc7901..836e0c9 100644 --- a/pubsub/BUILD_NOTES.md +++ b/pubsub/BUILD_NOTES.md @@ -4,7 +4,7 @@ Dependency choice: vcpkg `curl`, `openssl`, and `nlohmann-json` (same toolchain `google-cloud-cpp` (FetchContent or the vcpkg `pubsub` feature) pulls gRPC, protobuf, and Abseil. That superbuild is too heavy for this repo's six-platform CI matrix: multi-gigabyte RAM spikes and long compiles make the jobs unreliable. The REST client covers the emulator (`PUBSUB_EMULATOR_HOST`) and production (service-account JWT or a bearer token) without that stack. -- Windows curl uses Schannel (vcpkg's default `ssl` feature). The plugin links `OpenSSL::Crypto` for RS256 JWTs, which loads `libcrypto` but not `libssl`. Production HTTPS does not need `libssl-3-x64.dll`. Switching curl to OpenSSL would require a CA bundle or `CURLSSLOPT_NATIVE_CA`, because Schannel is what verifies Google's certificates today. +- Windows builds use the static vcpkg triplet (`x64-windows-static` / `arm64-windows-static`) and `/MT`. curl, OpenSSL, and zlib are linked into `PubSubPlugin.dll`. curl still uses Schannel, so certificate verification stays on the Windows store. `OpenSSL::Crypto` is linked for RS256 JWTs. The DLL does not import `vcruntime140`, `libcurl`, `libcrypto`, or `libssl`. - Hermetic `ctest` does not need an emulator. Invalid config, base64, JWT signing, and factory tests always run. - `PubSubLive.RpcRoundTrip` runs only when `PUBSUB_EMULATOR_HOST` or `PUBSUB_LIVE` is set. - First configure with the vcpkg toolchain needs network access so vcpkg can fetch ports. The manifest baseline matches `sqs/vcpkg.json`. diff --git a/pubsub/CMakeLists.txt b/pubsub/CMakeLists.txt index cef2f9c..4e41e71 100644 --- a/pubsub/CMakeLists.txt +++ b/pubsub/CMakeLists.txt @@ -3,13 +3,18 @@ cmake_minimum_required(VERSION ${CMAKE_MIN}) set(CMAKE_POLICY_VERSION_MINIMUM ${CMAKE_MIN}) cmake_policy(VERSION ${CMAKE_MIN}) -project("PubSubPlugin" VERSION 1.0.0) - set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(CMAKE_POLICY_DEFAULT_CMP0135 NEW) +include("${CMAKE_CURRENT_SOURCE_DIR}/../cmake/StaticPluginRuntime.cmake") +graftcode_prepare_static_runtime() + +project("PubSubPlugin" VERSION 1.0.0) + +graftcode_apply_static_crt() + add_definitions(-DUNICODE) enable_testing() diff --git a/pubsub/PubSubPlugin/CMakeLists.txt b/pubsub/PubSubPlugin/CMakeLists.txt index 8ffdc0e..208712e 100644 --- a/pubsub/PubSubPlugin/CMakeLists.txt +++ b/pubsub/PubSubPlugin/CMakeLists.txt @@ -23,3 +23,8 @@ target_link_libraries(${target_name} PUBLIC nlohmann_json::nlohmann_json Threads::Threads ) + +get_target_property(_curl_type CURL::libcurl TYPE) +if(_curl_type STREQUAL "STATIC_LIBRARY") + target_compile_definitions(${target_name} PRIVATE CURL_STATICLIB) +endif() diff --git a/pubsub/PubSubPluginTest/CMakeLists.txt b/pubsub/PubSubPluginTest/CMakeLists.txt index e5636dd..d6a5ba6 100644 --- a/pubsub/PubSubPluginTest/CMakeLists.txt +++ b/pubsub/PubSubPluginTest/CMakeLists.txt @@ -14,7 +14,7 @@ FetchContent_Declare( GIT_REPOSITORY https://github.com/google/googletest.git GIT_TAG v1.17.0 ) -set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) +set(gtest_force_shared_crt OFF CACHE BOOL "" FORCE) FetchContent_MakeAvailable(googletest) find_package(CURL REQUIRED) @@ -35,6 +35,11 @@ target_link_libraries(PubSubPluginTests PRIVATE Threads::Threads ) +get_target_property(_curl_type CURL::libcurl TYPE) +if(_curl_type STREQUAL "STATIC_LIBRARY") + target_compile_definitions(PubSubPluginTests PRIVATE CURL_STATICLIB) +endif() + add_executable(PubSubPluginSmoke smoke.cpp) target_link_libraries(PubSubPluginSmoke PRIVATE PubSubPlugin) target_include_directories(PubSubPluginSmoke PRIVATE diff --git a/pubsub/Readme.md b/pubsub/Readme.md index e9d4bf9..54e959c 100644 --- a/pubsub/Readme.md +++ b/pubsub/Readme.md @@ -52,7 +52,7 @@ Output: - Windows: `pubsub/build/PubSubPlugin/PubSubPlugin.dll` — config `"name": "PubSubPlugin"` - Linux/macOS: `pubsub/build/PubSubPlugin/libPubSubPlugin.so` or `.dylib` — config `"name": "libPubSubPlugin"` -On Windows, copy `PubSubPlugin.dll` and the vcpkg runtime DLLs next to `gg.exe`: `libcurl.dll`, the libcrypto DLL (for example `libcrypto-3-x64.dll`), and the zlib DLL (`z.dll` or `zlib1.dll`). vcpkg builds curl against Schannel on Windows, so HTTPS uses the Windows certificate store and `libssl` is not a runtime dependency. `libcrypto` signs the service-account JWT. The loader does not search the build tree. +On Windows, copy `PubSubPlugin.dll` next to `gg.exe`. curl, OpenSSL, zlib, and the VC++ runtime are linked into that DLL (`/MT` and the `*-windows-static` vcpkg triplet). curl uses Schannel. No VC++ Redistributable install is required. Hermetic tests always run. The round-trip test runs only when `PUBSUB_EMULATOR_HOST` or `PUBSUB_LIVE` is set. @@ -81,7 +81,7 @@ This layout is the same shape as the SQS sample. Put these files in one folder: - `gg.exe` - the hosted module, for example `netapp.dll` - `PubSubPlugin.dll` -- the libcurl, libcrypto, and zlib DLLs from the vcpkg `bin` directory (`libssl` is not required on Windows; curl uses Schannel) +- `PubSubPlugin.dll` only. curl, libcrypto, and zlib are inside the DLL. `pubsubplugin.json` (same contents as [`samples/pubsub/pluginConfig.gateway.json`](../samples/pubsub/pluginConfig.gateway.json)): diff --git a/rabbitmq/CMakeLists.txt b/rabbitmq/CMakeLists.txt index cefa250..3229fa1 100644 --- a/rabbitmq/CMakeLists.txt +++ b/rabbitmq/CMakeLists.txt @@ -3,26 +3,25 @@ cmake_minimum_required (VERSION ${CMAKE_MIN}) set(CMAKE_POLICY_VERSION_MINIMUM ${CMAKE_MIN}) cmake_policy(VERSION ${CMAKE_MIN}) +set(CMAKE_POLICY_DEFAULT_CMP0135 NEW) +include("${CMAKE_CURRENT_SOURCE_DIR}/../cmake/StaticPluginRuntime.cmake") +graftcode_prepare_static_runtime() + project ("RabbitmqPlugin" VERSION 1.0.0) +graftcode_apply_static_crt() + set(CMAKE_CXX_STANDARD 20) -set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) add_definitions(-DUNICODE) enable_testing() -# Set build type -set (BUILDTYPE ${CMAKE_BUILD_TYPE}) -set (CMAKE_POLICY_DEFAULT_CMP0135 NEW) - -if(${OSNAME} MATCHES "Windows") - add_compile_options(/utf-8) - if(MSVC) - set(CMAKE_MSVC_RUNTIME_LIBRARY - "MultiThreaded$<$:Debug>" - CACHE STRING "MSVC runtime library" FORCE) - endif() +set(BUILDTYPE ${CMAKE_BUILD_TYPE}) + +if(MSVC) + add_compile_options(/utf-8) endif() add_subdirectory ("RabbitmqPlugin") diff --git a/rabbitmq/RabbitmqPlugin/CMakeLists.txt b/rabbitmq/RabbitmqPlugin/CMakeLists.txt index fddb69b..bc008c9 100644 --- a/rabbitmq/RabbitmqPlugin/CMakeLists.txt +++ b/rabbitmq/RabbitmqPlugin/CMakeLists.txt @@ -37,7 +37,7 @@ endif() target_link_libraries(${target_name} PUBLIC amqpcpp nlohmann_json::nlohmann_json) -if(${OSNAME} MATCHES "Windows") +if(WIN32) target_link_libraries(${target_name} PUBLIC ws2_32) endif() diff --git a/rabbitmq/RabbitmqPluginTest/CMakeLists.txt b/rabbitmq/RabbitmqPluginTest/CMakeLists.txt index 98658ad..31a0088 100644 --- a/rabbitmq/RabbitmqPluginTest/CMakeLists.txt +++ b/rabbitmq/RabbitmqPluginTest/CMakeLists.txt @@ -15,7 +15,8 @@ FetchContent_Declare( ) # For Windows: Prevent overriding the parent project's compiler/linker settings -set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) +# Static /MT in the parent. ON would leave gtest on the DLL CRT and LNK2038. +set(gtest_force_shared_crt OFF CACHE BOOL "" FORCE) FetchContent_MakeAvailable(googletest) target_include_directories(RabbitMqPluginTests PUBLIC @@ -24,7 +25,7 @@ target_include_directories(RabbitMqPluginTests PUBLIC ) target_link_libraries(RabbitMqPluginTests PRIVATE gtest_main amqpcpp nlohmann_json::nlohmann_json) -if(${OSNAME} MATCHES "Windows") +if(WIN32) target_link_libraries(RabbitMqPluginTests PRIVATE ws2_32) endif() diff --git a/rabbitmq/Readme.md b/rabbitmq/Readme.md index e4790b4..3f7e288 100644 --- a/rabbitmq/Readme.md +++ b/rabbitmq/Readme.md @@ -24,7 +24,7 @@ cmake --build build --config Release Output: -- Windows: `rabbitmq/build/RabbitmqPlugin/RabbitmqPlugin.dll` — config `"name": "RabbitmqPlugin"` +- Windows: `rabbitmq/build/RabbitmqPlugin/RabbitmqPlugin.dll` — config `"name": "RabbitmqPlugin"`. AMQP-CPP and the VC++ runtime are linked in (`/MT`); the VC++ Redistributable is not required. - Linux/macOS: `rabbitmq/build/RabbitmqPlugin/libRabbitmqPlugin.so` or `.dylib` — config `"name": "libRabbitmqPlugin"` Download `gg` from https://github.com/grft-dev/graftcode-gateway/releases/. diff --git a/samples/pubsub/README.md b/samples/pubsub/README.md index afb9470..64b48f3 100644 --- a/samples/pubsub/README.md +++ b/samples/pubsub/README.md @@ -28,8 +28,7 @@ Colocate these next to each other: - `gg.exe` - `netapp.dll` (the hosted module) -- `PubSubPlugin.dll` -- the libcurl, libcrypto, and zlib runtime DLLs from `pubsub/build/vcpkg_installed//bin` (typically `libcurl.dll`, `libcrypto-3-x64.dll`, and `z.dll`). Windows curl is built with Schannel, so `libssl-3-x64.dll` is not loaded. +- `PubSubPlugin.dll` (curl, OpenSSL, zlib, and the VC++ runtime are linked in) Copy `pluginConfig.gateway.json` to that folder as `pubsubplugin.json` and run: diff --git a/samples/sqs/README.md b/samples/sqs/README.md index 559f98c..4880878 100644 --- a/samples/sqs/README.md +++ b/samples/sqs/README.md @@ -34,8 +34,7 @@ Colocate these next to each other: - `gg.exe` - `netapp.dll` (the hosted module) -- `SqsPlugin.dll` -- the AWS SDK runtime DLLs +- `SqsPlugin.dll` (the AWS SDK and the VC++ runtime are linked in) Copy `pluginConfig.gateway.json` to that folder as `sqsplugin.json` and run: diff --git a/servicebus/CMakeLists.txt b/servicebus/CMakeLists.txt index 1f3ea8b..5402958 100644 --- a/servicebus/CMakeLists.txt +++ b/servicebus/CMakeLists.txt @@ -3,8 +3,14 @@ cmake_minimum_required (VERSION ${CMAKE_MIN}) set(CMAKE_POLICY_VERSION_MINIMUM ${CMAKE_MIN}) cmake_policy(VERSION ${CMAKE_MIN}) +set(CMAKE_POLICY_DEFAULT_CMP0135 NEW) +include("${CMAKE_CURRENT_SOURCE_DIR}/../cmake/StaticPluginRuntime.cmake") +graftcode_prepare_static_runtime() + project ("ServiceBusPlugin" VERSION 1.0.0) +graftcode_apply_static_crt() + set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -12,9 +18,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) add_definitions(-DUNICODE) enable_testing() -# Set build type -set (BUILDTYPE ${CMAKE_BUILD_TYPE}) -set (CMAKE_POLICY_DEFAULT_CMP0135 NEW) +set(BUILDTYPE ${CMAKE_BUILD_TYPE}) if(MSVC) add_compile_options(/utf-8) diff --git a/servicebus/Readme.md b/servicebus/Readme.md index 2b2311d..05666f5 100644 --- a/servicebus/Readme.md +++ b/servicebus/Readme.md @@ -45,6 +45,8 @@ As a result, you will receive: - `servicebus/build/ServiceBusPlugin/ServiceBusPlugin.dll` - or `servicebus/build/ServiceBusPlugin/libServiceBusPlugin.so` / `.dylib` on Linux/macOS +Windows uses `/MT` and the `*-windows-static` vcpkg triplet, so the Azure SDK and the VC++ runtime are linked into `ServiceBusPlugin.dll`. The VC++ Redistributable is not required. + If the generated library is `libServiceBusPlugin.*`, use plugin name: `libServiceBusPlugin`. ## 5) Download GG diff --git a/servicebus/ServiceBusPluginTest/CMakeLists.txt b/servicebus/ServiceBusPluginTest/CMakeLists.txt index 8bb7efe..4ad2caf 100644 --- a/servicebus/ServiceBusPluginTest/CMakeLists.txt +++ b/servicebus/ServiceBusPluginTest/CMakeLists.txt @@ -16,7 +16,7 @@ FetchContent_Declare( ) # For Windows: Prevent overriding the parent project's compiler/linker settings -set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) +set(gtest_force_shared_crt OFF CACHE BOOL "" FORCE) FetchContent_MakeAvailable(googletest) find_package(azure-core-amqp-cpp CONFIG REQUIRED) diff --git a/sqs/CMakeLists.txt b/sqs/CMakeLists.txt index 33ee2d2..a6f5461 100644 --- a/sqs/CMakeLists.txt +++ b/sqs/CMakeLists.txt @@ -3,13 +3,18 @@ cmake_minimum_required(VERSION ${CMAKE_MIN}) set(CMAKE_POLICY_VERSION_MINIMUM ${CMAKE_MIN}) cmake_policy(VERSION ${CMAKE_MIN}) -project("SqsPlugin" VERSION 1.0.0) - set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(CMAKE_POLICY_DEFAULT_CMP0135 NEW) +include("${CMAKE_CURRENT_SOURCE_DIR}/../cmake/StaticPluginRuntime.cmake") +graftcode_prepare_static_runtime() + +project("SqsPlugin" VERSION 1.0.0) + +graftcode_apply_static_crt() + add_definitions(-DUNICODE) enable_testing() diff --git a/sqs/Readme.md b/sqs/Readme.md index 29c9657..1816399 100644 --- a/sqs/Readme.md +++ b/sqs/Readme.md @@ -38,7 +38,7 @@ Output: - Windows: `sqs/build/SqsPlugin/SqsPlugin.dll` — config `"name": "SqsPlugin"` - Linux/macOS: `sqs/build/SqsPlugin/libSqsPlugin.so` or `.dylib` — config `"name": "libSqsPlugin"` -On Windows, copy `SqsPlugin.dll` and the AWS SDK runtime DLLs next to `gg.exe`. The loader does not search the build tree. +On Windows, copy `SqsPlugin.dll` next to `gg.exe`. The AWS SDK and the VC++ runtime are linked into that DLL (`/MT` and the `*-windows-static` vcpkg triplet), so the VC++ Redistributable is not required. Download `gg` from https://github.com/grft-dev/graftcode-gateway/releases/. diff --git a/sqs/SqsPluginTest/CMakeLists.txt b/sqs/SqsPluginTest/CMakeLists.txt index e5052bf..67d6375 100644 --- a/sqs/SqsPluginTest/CMakeLists.txt +++ b/sqs/SqsPluginTest/CMakeLists.txt @@ -14,7 +14,7 @@ FetchContent_Declare( GIT_REPOSITORY https://github.com/google/googletest.git GIT_TAG v1.17.0 ) -set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) +set(gtest_force_shared_crt OFF CACHE BOOL "" FORCE) FetchContent_MakeAvailable(googletest) find_package(aws-cpp-sdk-core CONFIG REQUIRED) From d9705044ad8317f92606d9dd30ba5f6cbf351ba2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 07:43:56 +0000 Subject: [PATCH 5/7] Build Kafka zlib and zstd as static PIC libraries. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Distro libz.a and libzstd.a are not position-independent, so Linux packages were shipping those shared libraries. FetchContent now builds both into the plugin. libsasl2, and the libcrypto it imports, stay shared because Cyrus SASL loads mechanism modules at runtime. Co-authored-by: Michał Komor --- .github/workflows/release.yml | 4 +- README.md | 2 +- kafka/BUILD_NOTES.md | 21 ++++++--- kafka/KafkaPlugin/CMakeLists.txt | 81 ++++++++++++++++++++++++++++++-- kafka/Readme.md | 2 +- kafka/vcpkg.json | 5 +- 6 files changed, 97 insertions(+), 18 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index df128d1..3a17f76 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -126,10 +126,10 @@ jobs: echo "ARTIFACT_PATH=kafka/build/KafkaPlugin" >> "$GITHUB_ENV" if [ "${{ runner.os }}" = "Linux" ]; then echo "CONFIGURE_ARGS=" >> "$GITHUB_ENV" - echo "INSTALL_PACKAGES=build-essential cmake git libssl-dev libsasl2-dev zlib1g-dev libzstd-dev patchelf" >> "$GITHUB_ENV" + echo "INSTALL_PACKAGES=build-essential cmake git libssl-dev libsasl2-dev patchelf" >> "$GITHUB_ENV" elif [ "${{ runner.os }}" = "macOS" ]; then echo "CONFIGURE_ARGS=" >> "$GITHUB_ENV" - echo "INSTALL_PACKAGES=cmake openssl zstd" >> "$GITHUB_ENV" + echo "INSTALL_PACKAGES=cmake openssl" >> "$GITHUB_ENV" else # Static OpenSSL/zlib built with /MT. The Program Files OpenSSL is /MD. echo "CONFIGURE_ARGS=-DCMAKE_TOOLCHAIN_FILE=${VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake" >> "$GITHUB_ENV" diff --git a/README.md b/README.md index 263c0b8..79ad98d 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Use RabbitMQ or Kafka when you already run that broker. Use Service Bus or SQS w ## Use a plugin 1. Build the plugin (see its Readme). The artifact is a shared library: `SqsPlugin.dll` on Windows, or `libSqsPlugin.so` / `.dylib` on Linux and macOS. -2. Put `gg`, the hosted module, and the plugin library in one folder, plus any extra shared libraries from that plugin's release archive. Windows archives are the plugin DLL alone (static VC++ runtime, no VC++ Redistributable). Linux Kafka still ships `libsasl2` (and the `libcrypto` it imports), plus `libz` and `libzstd`. `name` in the config is the library file name without the extension (`libSqsPlugin` when the file is `libSqsPlugin.so`). +2. Put `gg`, the hosted module, and the plugin library in one folder, plus any extra shared libraries from that plugin's release archive. Windows archives are the plugin DLL alone (static VC++ runtime, no VC++ Redistributable). Third-party libraries are linked into the plugin. The Linux Kafka archive still contains `libsasl2` (Cyrus SASL loads mechanism modules at runtime) and the `libcrypto` that library imports. `name` in the config is the library file name without the extension (`libSqsPlugin` when the file is `libSqsPlugin.so`). 3. Start the Gateway with `--config`. A verified SQS layout is in [samples/sqs](samples/sqs/README.md): `gg netapp.dll --config sqsplugin.json`. 4. Open `http://localhost:81/GV`, install the generated package, and call `GraftConfig.SetConfig` with the same channel settings. `host` there is the Gateway HTTP address, not the broker. diff --git a/kafka/BUILD_NOTES.md b/kafka/BUILD_NOTES.md index f5ce420..f3ace41 100644 --- a/kafka/BUILD_NOTES.md +++ b/kafka/BUILD_NOTES.md @@ -1,10 +1,19 @@ # Kafka plugin build notes -- librdkafka, nlohmann/json, LZ4 (bundled), and OpenSSL are linked statically. Windows also links zlib statically. -- Windows OpenSSL and zlib come from vcpkg (`*-windows-static`, `/MT`). The Program Files OpenSSL is `/MD` and is not used. Pass `-DCMAKE_TOOLCHAIN_FILE=/scripts/buildsystems/vcpkg.cmake`. -- Linux and macOS link `libssl.a` and `libcrypto.a` into the plugin (`libssl-dev` or Homebrew `openssl`). The plugin does not import `libssl.so`. -- `libz` and `libzstd` stay shared on Linux and macOS. Those distro static archives are not built with `-fPIC`, so they cannot be linked into the plugin. -- Cyrus SASL stays a shared library on Linux (`libsasl2.so`). Its mechanisms are loaded as plugins at runtime. That library imports `libcrypto.so.3`, so the Linux archive still contains `libcrypto.so.3` for SASL even though the plugin's own TLS is static. Windows SASL is `secur32`. macOS builds that do not find Cyrus SASL omit it. -- librdkafka's libcurl OIDC helper is off, so the plugin does not depend on `libcurl`. +Third-party code is linked into the plugin. The release archive is the plugin +library plus, on Linux only, Cyrus SASL: + +- librdkafka and nlohmann/json are static (FetchContent). LZ4 and Snappy are the copies compiled into librdkafka. +- zlib (`v1.3.1`) and zstd (`v1.5.6`) are FetchContent static libraries built with position-independent code. Distro `libz.a` and `libzstd.a` are not PIC, so they are not used. +- OpenSSL is static. Unix builds use the distro or Homebrew `libssl.a` / `libcrypto.a` (those archives are PIC). Windows uses vcpkg `*-windows-static` from `kafka/vcpkg.json` because the OpenSSL under `Program Files` is `/MD`. Pass `-DCMAKE_TOOLCHAIN_FILE=/scripts/buildsystems/vcpkg.cmake` on Windows. +- librdkafka's libcurl OIDC helper is off. + +Shared libraries that stay in the Linux archive, and why: + +- `libsasl2.so` loads SASL mechanism modules at runtime. A static `libsasl2.a` would still need those modules from the host, so the shared library stays. +- `libcrypto.so.3` is imported by that `libsasl2.so`, not by the plugin. The plugin's own TLS uses the static OpenSSL, and `--exclude-libs` keeps those symbols inside the plugin. + +Windows SASL is `secur32` (system). macOS builds that do not find Cyrus SASL omit it. The MSVC CRT is `/MT`, so the VC++ Redistributable is not required. + - Smoke test links factory exports; full RPC needs a live broker (`samples/kafka`). - First configure needs network/git for FetchContent downloads. diff --git a/kafka/KafkaPlugin/CMakeLists.txt b/kafka/KafkaPlugin/CMakeLists.txt index 206e7a5..52c8f90 100644 --- a/kafka/KafkaPlugin/CMakeLists.txt +++ b/kafka/KafkaPlugin/CMakeLists.txt @@ -29,15 +29,79 @@ set(ENABLE_LZ4_EXT OFF CACHE BOOL "Use librdkafka's bundled LZ4" FORCE) # libcurl would be another shared dependency, and OIDC is unused by this plugin. set(WITH_CURL OFF CACHE BOOL "Do not link libcurl into librdkafka" FORCE) -# Unix OpenSSL static archives are PIC and link into the plugin. Distro -# libz.a and libzstd.a are not, so zlib and zstd stay shared. Windows uses -# the vcpkg static triplet (libcrypto.lib / zlib.lib built with /MT). -# OPENSSL_USE_STATIC_LIBS would look for libcrypto_static, which vcpkg does -# not install, so it is Unix-only. +# Distro libz.a / libzstd.a are not built PIC, so they cannot be linked into +# this shared plugin. Build both from source with position-independent code +# and hand those targets to librdkafka. OpenSSL on Unix is the distro static +# archive (it is PIC). On Windows, OpenSSL comes from the vcpkg static triplet; +# OPENSSL_USE_STATIC_LIBS would look for libcrypto_static, which vcpkg does not +# ship. +set(ZLIB_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) +set(SKIP_INSTALL_ALL ON) +FetchContent_Declare( + zlib + GIT_REPOSITORY https://github.com/madler/zlib.git + GIT_TAG v1.3.1 + GIT_SHALLOW TRUE +) +FetchContent_MakeAvailable(zlib) +if(TARGET zlib) + set_target_properties(zlib PROPERTIES EXCLUDE_FROM_ALL TRUE) +endif() +set_target_properties(zlibstatic PROPERTIES POSITION_INDEPENDENT_CODE ON) +if(NOT TARGET ZLIB::ZLIB) + add_library(ZLIB::ZLIB ALIAS zlibstatic) +endif() +set(ZLIB_INCLUDE_DIR "${zlib_SOURCE_DIR}" CACHE PATH "" FORCE) +set(ZLIB_LIBRARY zlibstatic CACHE FILEPATH "" FORCE) + +set(ZSTD_BUILD_SHARED OFF CACHE BOOL "" FORCE) +set(ZSTD_BUILD_STATIC ON CACHE BOOL "" FORCE) +set(ZSTD_BUILD_PROGRAMS OFF CACHE BOOL "" FORCE) +set(ZSTD_BUILD_TESTS OFF CACHE BOOL "" FORCE) +set(ZSTD_BUILD_CONTRIB OFF CACHE BOOL "" FORCE) +set(ZSTD_LEGACY_SUPPORT OFF CACHE BOOL "" FORCE) +if(MSVC) + set(ZSTD_USE_STATIC_RUNTIME ON CACHE BOOL "" FORCE) +endif() +FetchContent_Declare( + zstd + GIT_REPOSITORY https://github.com/facebook/zstd.git + GIT_TAG v1.5.6 + GIT_SHALLOW TRUE + SOURCE_SUBDIR build/cmake +) +FetchContent_MakeAvailable(zstd) +set(ZSTD_INCLUDE_DIR "${zstd_SOURCE_DIR}/lib" CACHE PATH "" FORCE) +set(ZSTD_LIBRARY_RELEASE libzstd_static CACHE FILEPATH "" FORCE) +set(ZSTD_LIBRARY_DEBUG libzstd_static CACHE FILEPATH "" FORCE) + if(UNIX AND NOT WIN32) set(OPENSSL_USE_STATIC_LIBS ON) endif() +# librdkafka install(EXPORT) rejects zlibstatic because that target is not in +# its export set. Drop EXPORT while configuring librdkafka; the plugin does not +# install those targets. +function(install) + if(ARGV0 STREQUAL "EXPORT") + return() + endif() + set(_graftcode_skip_next OFF) + set(_graftcode_kept "") + foreach(_graftcode_arg IN LISTS ARGV) + if(_graftcode_skip_next) + set(_graftcode_skip_next OFF) + continue() + endif() + if(_graftcode_arg STREQUAL "EXPORT") + set(_graftcode_skip_next ON) + continue() + endif() + list(APPEND _graftcode_kept "${_graftcode_arg}") + endforeach() + _install(${_graftcode_kept}) +endfunction() + FetchContent_Declare( librdkafka GIT_REPOSITORY https://github.com/confluentinc/librdkafka.git @@ -58,6 +122,13 @@ target_link_libraries(${target_name} PUBLIC if(TARGET rdkafka) target_link_libraries(${target_name} PUBLIC rdkafka) endif() +# librdkafka links zstd as PRIVATE, which a static archive does not forward. +target_link_libraries(${target_name} PRIVATE libzstd_static zlibstatic) +if(UNIX AND NOT APPLE) + # Keep the static OpenSSL copy inside this .so. libsasl2 loads its own + # libcrypto.so.3; exporting both sets of symbols from one process breaks TLS. + target_link_options(${target_name} PRIVATE "LINKER:--exclude-libs,ALL") +endif() if(WIN32 OR CMAKE_SYSTEM_NAME STREQUAL "Windows") target_link_libraries(${target_name} PUBLIC ws2_32) diff --git a/kafka/Readme.md b/kafka/Readme.md index 4ac9711..761f8fe 100644 --- a/kafka/Readme.md +++ b/kafka/Readme.md @@ -2,7 +2,7 @@ This plugin carries Graftcode Gateway calls over Apache Kafka. It implements `Hypertube::Native::Interfaces::ITransport` and `GraftcodeGateway::IServer`, and exports `CreateTransportChannel` / `DestroyTransportChannel` and `CreateServer` / `DestroyServer`. -It is written in C++. CMake FetchContent builds nlohmann/json `v3.12.0` and librdkafka `v2.8.0` (static `rdkafka++`). OpenSSL, zlib, and LZ4 are linked statically. On Windows, configure with the vcpkg toolchain so OpenSSL and zlib are the static `/MT` builds from `kafka/vcpkg.json` (the OpenSSL in `Program Files` is `/MD`). On Linux, `libsasl2`, `libz`, and `libzstd` stay shared: Cyrus SASL loads mechanism modules at runtime (and imports `libcrypto.so.3`), and the distro static zlib and zstd archives are not built with `-fPIC`. The first configure needs network access. +It is written in C++. CMake FetchContent builds nlohmann/json `v3.12.0`, librdkafka `v2.8.0` (static `rdkafka++`), zlib `v1.3.1`, and zstd `v1.5.6`, all linked into the plugin. OpenSSL is static too: on Windows configure with the vcpkg toolchain so it is the `/MT` build from `kafka/vcpkg.json` (the OpenSSL in `Program Files` is `/MD`); on Linux and macOS the build uses the PIC static archive from the system or Homebrew. The Linux archive still contains `libsasl2` because Cyrus SASL loads mechanism modules at runtime, and `libcrypto.so.3` because that `libsasl2` imports it. The plugin's own TLS does not. The first configure needs network access. Kafka has no native request/reply. The plugin uses headers: diff --git a/kafka/vcpkg.json b/kafka/vcpkg.json index 554f54a..0303f5e 100644 --- a/kafka/vcpkg.json +++ b/kafka/vcpkg.json @@ -1,10 +1,9 @@ { "name": "graftcode-kafka-plugin", "version": "1.0.0", - "description": "Static OpenSSL and zlib for the Kafka plugin on Windows (/MT). Unix builds use the system static archives instead.", + "description": "Static OpenSSL for the Kafka plugin on Windows (/MT). zlib and zstd are FetchContent static libraries.", "builtin-baseline": "1460b31b08c42cc2e9ac2c79f45ec8707e2675e2", "dependencies": [ - "openssl", - "zlib" + "openssl" ] } From 795ae84c00960590db3fabd71846d82928bbfd26 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 08:00:25 +0000 Subject: [PATCH 6/7] Stop Kafka from rewriting install(CODE) on macOS. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The install() override split bracket arguments on semicolons, so the generated cmake_install.cmake no longer parsed. librdkafka's export is removed in the fetched sources instead. Co-authored-by: Michał Komor --- cmake/skip-rdkafka-export.cmake | 18 ++++++++++++++++++ kafka/KafkaPlugin/CMakeLists.txt | 26 ++++---------------------- 2 files changed, 22 insertions(+), 22 deletions(-) create mode 100644 cmake/skip-rdkafka-export.cmake diff --git a/cmake/skip-rdkafka-export.cmake b/cmake/skip-rdkafka-export.cmake new file mode 100644 index 0000000..f9c0956 --- /dev/null +++ b/cmake/skip-rdkafka-export.cmake @@ -0,0 +1,18 @@ +# Drop librdkafka's install(EXPORT). It pulls zlibstatic into an export set +# this plugin does not ship. Run from the librdkafka source root. +file(READ "CMakeLists.txt" _graftcode_text) +string(REPLACE + "install( + EXPORT \"\${targets_export_name}\" + NAMESPACE \"\${namespace}\" + DESTINATION \"\${config_install_dir}\" +)" + "" + _graftcode_text "${_graftcode_text}") +file(WRITE "CMakeLists.txt" "${_graftcode_text}") + +foreach(_graftcode_list IN ITEMS "src/CMakeLists.txt" "src-cpp/CMakeLists.txt") + file(READ "${_graftcode_list}" _graftcode_text) + string(REPLACE " EXPORT \"\${targets_export_name}\"\n" "" _graftcode_text "${_graftcode_text}") + file(WRITE "${_graftcode_list}" "${_graftcode_text}") +endforeach() diff --git a/kafka/KafkaPlugin/CMakeLists.txt b/kafka/KafkaPlugin/CMakeLists.txt index 52c8f90..305ffb0 100644 --- a/kafka/KafkaPlugin/CMakeLists.txt +++ b/kafka/KafkaPlugin/CMakeLists.txt @@ -80,33 +80,15 @@ if(UNIX AND NOT WIN32) endif() # librdkafka install(EXPORT) rejects zlibstatic because that target is not in -# its export set. Drop EXPORT while configuring librdkafka; the plugin does not -# install those targets. -function(install) - if(ARGV0 STREQUAL "EXPORT") - return() - endif() - set(_graftcode_skip_next OFF) - set(_graftcode_kept "") - foreach(_graftcode_arg IN LISTS ARGV) - if(_graftcode_skip_next) - set(_graftcode_skip_next OFF) - continue() - endif() - if(_graftcode_arg STREQUAL "EXPORT") - set(_graftcode_skip_next ON) - continue() - endif() - list(APPEND _graftcode_kept "${_graftcode_arg}") - endforeach() - _install(${_graftcode_kept}) -endfunction() - +# its export set. Strip the export in the fetched sources. Do not wrap +# install(): re-listing its arguments splits install(CODE) on semicolons and +# writes a cmake_install.cmake that macOS cannot parse. FetchContent_Declare( librdkafka GIT_REPOSITORY https://github.com/confluentinc/librdkafka.git GIT_TAG v2.8.0 GIT_SHALLOW TRUE + PATCH_COMMAND ${CMAKE_COMMAND} -P "${CMAKE_SOURCE_DIR}/../cmake/skip-rdkafka-export.cmake" ) FetchContent_MakeAvailable(librdkafka) From 5caf0b635298e998f1912f7795ad8e7e2213ea32 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 08:28:55 +0000 Subject: [PATCH 7/7] Document static release archives and plugin config examples. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Client READMEs now state that each archive is the plugin library, with /MT on Windows and static third-party libraries elsewhere. Linux Kafka still ships libsasl2. Each plugin's gateway JSON is in the root README, and the Pub/Sub client config is inlined from the sample. Co-authored-by: Michał Komor --- README.md | 93 ++++++++++++++++++++++++++++++++++++++++++-- kafka/BUILD_NOTES.md | 5 +-- kafka/Readme.md | 4 +- pubsub/Readme.md | 30 ++++++++++++-- rabbitmq/Readme.md | 2 + servicebus/Readme.md | 4 +- sqs/Readme.md | 7 ++-- 7 files changed, 128 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 79ad98d..e306f1e 100644 --- a/README.md +++ b/README.md @@ -16,13 +16,98 @@ OpenTelemetry / Application Insights lives under [observability/opentelemetry](o Use RabbitMQ or Kafka when you already run that broker. Use Service Bus or SQS when the deployment is already on Azure or AWS. Use Pub/Sub when the deployment is already on Google Cloud. SQS and Pub/Sub each need one reply destination per client process; Service Bus can share one session-enabled reply queue. +## Release archives + +Each release archive is flat and contains the plugin shared library. Third-party libraries are linked into that library. A client does not install OpenSSL, curl, zlib, the AWS SDK, the Azure SDK, or the VC++ Redistributable to load these plugins. + +- **Windows.** The archive is the plugin DLL only. The MSVC CRT is static (`/MT`). `ucrtbase.dll` and other Windows system libraries stay on the machine. +- **Linux and macOS.** The archive is the plugin `.so` or `.dylib` only. glibc, libstdc++, libc++, and libSystem stay on the host. +- **Linux Kafka.** The archive also contains `libsasl2.so.2` and `libsasl2.so.2.0.25`. Cyrus SASL loads mechanism modules at runtime, so that library stays shared. Copy those files next to `libKafkaPlugin.so`. + +`name` in the config is the library file name without the extension: `SqsPlugin` for `SqsPlugin.dll`, `libSqsPlugin` for `libSqsPlugin.so`. + ## Use a plugin -1. Build the plugin (see its Readme). The artifact is a shared library: `SqsPlugin.dll` on Windows, or `libSqsPlugin.so` / `.dylib` on Linux and macOS. -2. Put `gg`, the hosted module, and the plugin library in one folder, plus any extra shared libraries from that plugin's release archive. Windows archives are the plugin DLL alone (static VC++ runtime, no VC++ Redistributable). Third-party libraries are linked into the plugin. The Linux Kafka archive still contains `libsasl2` (Cyrus SASL loads mechanism modules at runtime) and the `libcrypto` that library imports. `name` in the config is the library file name without the extension (`libSqsPlugin` when the file is `libSqsPlugin.so`). -3. Start the Gateway with `--config`. A verified SQS layout is in [samples/sqs](samples/sqs/README.md): `gg netapp.dll --config sqsplugin.json`. -4. Open `http://localhost:81/GV`, install the generated package, and call `GraftConfig.SetConfig` with the same channel settings. `host` there is the Gateway HTTP address, not the broker. +1. Download the release archive for the platform, or build the plugin (see its Readme). +2. Put `gg`, the hosted module, and the plugin library in one folder. For Linux Kafka, also put `libsasl2.so.2` and `libsasl2.so.2.0.25` from the same archive there. +3. Start the Gateway with `--config` and one of the JSON files below. A verified SQS layout is in [samples/sqs](samples/sqs/README.md): `gg netapp.dll --config sqsplugin.json`. +4. Open `http://localhost:81/GV`, install the generated package, and call `GraftConfig.SetConfig` with the same channel settings inside `plugin`. `host` next to `runtime` is the Gateway HTTP address, not the broker. How the Gateway loads a plugin is described in the "Plugin server config" section of the [Graftcode Gateway](https://github.com/grft-dev/graftcode-gateway) README. +## Gateway config + +The Gateway `--config` file is the plugin object. The calling runtime wraps that same object in `configurations` → package name → `plugin`. Linux and macOS use a `lib` prefix on `name` (`libRabbitmqPlugin`). Field lists are in each plugin Readme. + +RabbitMQ (`rabbitmq/Readme.md`): + +```json +{ + "name": "RabbitmqPlugin", + "host": "localhost", + "port": 5672, + "queue": "myqueue", + "replyQueue": "myqueue.reply", + "user": "guest", + "password": "guest", + "vhost": "/", + "rpcTimeoutMs": 30000 +} +``` + +Service Bus (`servicebus/Readme.md`): + +```json +{ + "name": "ServiceBusPlugin", + "connectionString": "Endpoint=sb://.servicebus.windows.net/;SharedAccessKeyName=;SharedAccessKey=", + "queue": "myqueue", + "replyQueue": "myqueue.reply", + "rpcTimeoutMs": 30000 +} +``` + +Kafka (`kafka/Readme.md`): + +```json +{ + "name": "KafkaPlugin", + "brokers": "localhost:9092", + "requestTopic": "graft.requests", + "replyTopic": "graft.replies", + "groupId": "graft-gateway", + "rpcTimeoutMs": 30000 +} +``` + +SQS (`sqs/Readme.md`, LocalStack): + +```json +{ + "name": "SqsPlugin", + "region": "us-east-1", + "endpointOverride": "http://localhost:4566", + "accessKeyId": "test", + "secretAccessKey": "test", + "requestQueueUrl": "http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/graft-requests", + "waitTimeSeconds": 10, + "visibilityTimeoutSeconds": 60, + "verifySsl": false +} +``` + +Pub/Sub (`pubsub/Readme.md`, emulator): + +```json +{ + "name": "PubSubPlugin", + "projectId": "graftcode-local", + "emulatorHost": "localhost:8085", + "requestTopic": "graft-requests", + "ackDeadlineSeconds": 60, + "autoCreate": true, + "verifySsl": false +} +``` + Part of the [Graftcode](https://github.com/grft-dev/graftcode) project. diff --git a/kafka/BUILD_NOTES.md b/kafka/BUILD_NOTES.md index f3ace41..1ba3df1 100644 --- a/kafka/BUILD_NOTES.md +++ b/kafka/BUILD_NOTES.md @@ -8,10 +8,7 @@ library plus, on Linux only, Cyrus SASL: - OpenSSL is static. Unix builds use the distro or Homebrew `libssl.a` / `libcrypto.a` (those archives are PIC). Windows uses vcpkg `*-windows-static` from `kafka/vcpkg.json` because the OpenSSL under `Program Files` is `/MD`. Pass `-DCMAKE_TOOLCHAIN_FILE=/scripts/buildsystems/vcpkg.cmake` on Windows. - librdkafka's libcurl OIDC helper is off. -Shared libraries that stay in the Linux archive, and why: - -- `libsasl2.so` loads SASL mechanism modules at runtime. A static `libsasl2.a` would still need those modules from the host, so the shared library stays. -- `libcrypto.so.3` is imported by that `libsasl2.so`, not by the plugin. The plugin's own TLS uses the static OpenSSL, and `--exclude-libs` keeps those symbols inside the plugin. +The Linux archive also contains `libsasl2.so.2` and `libsasl2.so.2.0.25`. Cyrus SASL loads mechanism modules at runtime, so that library stays shared. On Ubuntu 22.04 it does not import `libcrypto`; if the build machine's `libsasl2` does, the package step copies `libcrypto.so.3` as well. The plugin's own TLS uses the static OpenSSL, and `--exclude-libs` keeps those symbols inside the plugin. Windows SASL is `secur32` (system). macOS builds that do not find Cyrus SASL omit it. The MSVC CRT is `/MT`, so the VC++ Redistributable is not required. diff --git a/kafka/Readme.md b/kafka/Readme.md index 761f8fe..519c3a0 100644 --- a/kafka/Readme.md +++ b/kafka/Readme.md @@ -2,7 +2,9 @@ This plugin carries Graftcode Gateway calls over Apache Kafka. It implements `Hypertube::Native::Interfaces::ITransport` and `GraftcodeGateway::IServer`, and exports `CreateTransportChannel` / `DestroyTransportChannel` and `CreateServer` / `DestroyServer`. -It is written in C++. CMake FetchContent builds nlohmann/json `v3.12.0`, librdkafka `v2.8.0` (static `rdkafka++`), zlib `v1.3.1`, and zstd `v1.5.6`, all linked into the plugin. OpenSSL is static too: on Windows configure with the vcpkg toolchain so it is the `/MT` build from `kafka/vcpkg.json` (the OpenSSL in `Program Files` is `/MD`); on Linux and macOS the build uses the PIC static archive from the system or Homebrew. The Linux archive still contains `libsasl2` because Cyrus SASL loads mechanism modules at runtime, and `libcrypto.so.3` because that `libsasl2` imports it. The plugin's own TLS does not. The first configure needs network access. +It is written in C++. CMake FetchContent builds nlohmann/json `v3.12.0`, librdkafka `v2.8.0` (static `rdkafka++`), zlib `v1.3.1`, and zstd `v1.5.6`, all linked into the plugin. OpenSSL is static too: on Windows configure with the vcpkg toolchain so it is the `/MT` build from `kafka/vcpkg.json` (the OpenSSL in `Program Files` is `/MD`); on Linux and macOS the build uses the PIC static archive from the system or Homebrew. LZ4 and Snappy are the copies compiled into librdkafka. The first configure needs network access. + +Windows and macOS release archives contain only `KafkaPlugin.dll` or `libKafkaPlugin.dylib`. The Linux archive contains `libKafkaPlugin.so` plus `libsasl2.so.2` and `libsasl2.so.2.0.25`. Cyrus SASL loads mechanism modules at runtime, so `libsasl2` stays shared; copy those two files next to the plugin. The plugin's own TLS uses the static OpenSSL. On Windows the MSVC CRT is static (`/MT`), so the VC++ Redistributable is not required. Clients do not install OpenSSL, zlib, zstd, or librdkafka separately. Kafka has no native request/reply. The plugin uses headers: diff --git a/pubsub/Readme.md b/pubsub/Readme.md index 54e959c..8204a53 100644 --- a/pubsub/Readme.md +++ b/pubsub/Readme.md @@ -2,7 +2,9 @@ This plugin carries Graftcode Gateway calls over Google Cloud Pub/Sub. It implements `Hypertube::Native::Interfaces::ITransport` and `GraftcodeGateway::IServer`, and exports `CreateTransportChannel` / `DestroyTransportChannel` and `CreateServer` / `DestroyServer`. -It is written in C++. Dependencies are libcurl, OpenSSL, and nlohmann/json, installed with the same vcpkg toolchain file as the SQS and Service Bus plugins. +It is written in C++. Dependencies are libcurl, OpenSSL, and nlohmann/json, installed with the same vcpkg toolchain file as the SQS and Service Bus plugins. They are linked into the plugin. + +The release archive contains only the plugin shared library (`PubSubPlugin.dll`, `libPubSubPlugin.so`, or `libPubSubPlugin.dylib`). On Windows the build uses `/MT` and the `*-windows-static` vcpkg triplet, and curl uses Schannel. The VC++ Redistributable is not required. Clients do not install curl, OpenSSL, or zlib separately. ## Why not google-cloud-cpp @@ -52,7 +54,7 @@ Output: - Windows: `pubsub/build/PubSubPlugin/PubSubPlugin.dll` — config `"name": "PubSubPlugin"` - Linux/macOS: `pubsub/build/PubSubPlugin/libPubSubPlugin.so` or `.dylib` — config `"name": "libPubSubPlugin"` -On Windows, copy `PubSubPlugin.dll` next to `gg.exe`. curl, OpenSSL, zlib, and the VC++ runtime are linked into that DLL (`/MT` and the `*-windows-static` vcpkg triplet). curl uses Schannel. No VC++ Redistributable install is required. +On Windows, copy `PubSubPlugin.dll` next to `gg.exe`. Nothing else from this plugin is required. Hermetic tests always run. The round-trip test runs only when `PUBSUB_EMULATOR_HOST` or `PUBSUB_LIVE` is set. @@ -81,7 +83,6 @@ This layout is the same shape as the SQS sample. Put these files in one folder: - `gg.exe` - the hosted module, for example `netapp.dll` - `PubSubPlugin.dll` -- `PubSubPlugin.dll` only. curl, libcrypto, and zlib are inside the DLL. `pubsubplugin.json` (same contents as [`samples/pubsub/pluginConfig.gateway.json`](../samples/pubsub/pluginConfig.gateway.json)): @@ -115,6 +116,29 @@ var calculator = new graft.nuget.netapp.Calculator(); calculator.Add(1, 2); ``` +```json +{ + "configurations": { + "graft.nuget.netapp": { + "runtime": "netcore", + "host": "localhost:80", + "stateless": true, + "plugin": { + "name": "PubSubPlugin", + "projectId": "graftcode-local", + "emulatorHost": "localhost:8085", + "requestTopic": "graft-requests", + "replyTopic": "graft-replies", + "rpcTimeoutMs": 30000, + "ackDeadlineSeconds": 60, + "autoCreate": true, + "verifySsl": false + } + } + } +} +``` + On Google Cloud, omit `emulatorHost` and `verifySsl`. Provide credentials with one of: - `accessToken`, or `PUBSUB_ACCESS_TOKEN` diff --git a/rabbitmq/Readme.md b/rabbitmq/Readme.md index 3f7e288..2060974 100644 --- a/rabbitmq/Readme.md +++ b/rabbitmq/Readme.md @@ -4,6 +4,8 @@ This plugin carries Graftcode Gateway calls over RabbitMQ (AMQP 0-9-1). It imple It is written in C++. CMake FetchContent builds [AMQP-CPP](https://github.com/CopernicaMarketingSoftware/AMQP-CPP) `v4.3.27` and nlohmann/json `v3.12.0` into the plugin. There is no vcpkg manifest. +The release archive contains only the plugin shared library (`RabbitmqPlugin.dll`, `libRabbitmqPlugin.so`, or `libRabbitmqPlugin.dylib`). AMQP-CPP is linked into that library. On Windows the MSVC CRT is static (`/MT`), so the VC++ Redistributable is not required. Clients do not install OpenSSL, curl, zlib, or another runtime for this plugin. + Request/reply uses AMQP properties: - The client sets `correlation-id` and `reply-to` (the configured reply queue). diff --git a/servicebus/Readme.md b/servicebus/Readme.md index 05666f5..f1e397f 100644 --- a/servicebus/Readme.md +++ b/servicebus/Readme.md @@ -9,6 +9,8 @@ factory symbols (`CreateTransportChannel` / `DestroyTransportChannel` and It is written in C++ and talks to Azure Service Bus over its native AMQP 1.0 protocol using the Azure SDK for C++ AMQP library (`azure-core-amqp`), acquired through vcpkg. +The release archive contains only the plugin shared library (`ServiceBusPlugin.dll`, `libServiceBusPlugin.so`, or `libServiceBusPlugin.dylib`). The Azure SDK is linked into that library. On Windows the build uses `/MT` and the `*-windows-static` vcpkg triplet, so the VC++ Redistributable is not required. Clients do not install the Azure SDK, OpenSSL, or curl separately. + ## 1) Clone repository ```bash @@ -45,8 +47,6 @@ As a result, you will receive: - `servicebus/build/ServiceBusPlugin/ServiceBusPlugin.dll` - or `servicebus/build/ServiceBusPlugin/libServiceBusPlugin.so` / `.dylib` on Linux/macOS -Windows uses `/MT` and the `*-windows-static` vcpkg triplet, so the Azure SDK and the VC++ runtime are linked into `ServiceBusPlugin.dll`. The VC++ Redistributable is not required. - If the generated library is `libServiceBusPlugin.*`, use plugin name: `libServiceBusPlugin`. ## 5) Download GG diff --git a/sqs/Readme.md b/sqs/Readme.md index 1816399..f205cf9 100644 --- a/sqs/Readme.md +++ b/sqs/Readme.md @@ -2,7 +2,9 @@ This plugin carries Graftcode Gateway calls over Amazon Simple Queue Service. It implements `Hypertube::Native::Interfaces::ITransport` and `GraftcodeGateway::IServer`, and exports `CreateTransportChannel` / `DestroyTransportChannel` and `CreateServer` / `DestroyServer`. -It is written in C++ and uses the AWS SDK for C++ (`aws-sdk-cpp[sqs]`) plus `nlohmann-json`, both from vcpkg. +It is written in C++ and uses the AWS SDK for C++ (`aws-sdk-cpp[sqs]`) plus `nlohmann-json`, both from vcpkg. Both are linked into the plugin. + +The release archive contains only the plugin shared library (`SqsPlugin.dll`, `libSqsPlugin.so`, or `libSqsPlugin.dylib`). On Windows the build uses `/MT` and the `*-windows-static` vcpkg triplet, so the VC++ Redistributable is not required. Clients do not install the AWS SDK, OpenSSL, curl, or zlib separately. Binary Graft payloads are Base64-encoded in the SQS message body with the prefix `graftcode-base64:`. RPC metadata is message attributes: @@ -38,7 +40,7 @@ Output: - Windows: `sqs/build/SqsPlugin/SqsPlugin.dll` — config `"name": "SqsPlugin"` - Linux/macOS: `sqs/build/SqsPlugin/libSqsPlugin.so` or `.dylib` — config `"name": "libSqsPlugin"` -On Windows, copy `SqsPlugin.dll` next to `gg.exe`. The AWS SDK and the VC++ runtime are linked into that DLL (`/MT` and the `*-windows-static` vcpkg triplet), so the VC++ Redistributable is not required. +On Windows, copy `SqsPlugin.dll` next to `gg.exe`. Nothing else from this plugin is required. Download `gg` from https://github.com/grft-dev/graftcode-gateway/releases/. @@ -72,7 +74,6 @@ This layout was run locally against LocalStack. Put these files in one folder: - `gg.exe` - the hosted module, for example `netapp.dll` - `SqsPlugin.dll` -- the AWS SDK runtime DLLs from the plugin build `sqsplugin.json` (same contents as [`samples/sqs/pluginConfig.gateway.json`](../samples/sqs/pluginConfig.gateway.json)):