From 3802da7c943d4734cb17d6c87521676dc8c2ee3f Mon Sep 17 00:00:00 2001 From: Matheus Pimenta Date: Wed, 21 Oct 2026 17:37:56 +0100 Subject: [PATCH] [RFC-0014] Introduce SPIFFE Broker API Signed-off-by: Matheus Pimenta --- .github/workflows/e2e.yaml | 22 + docs/README.md | 18 +- docs/spec/v2/helmreleases.md | 5 + go.mod | 15 +- go.sum | 35 +- hack/ci/kubeconfig-e2e.sh | 451 ++++++++++++++++ hack/ci/kubeconfig-primary-kind.yaml | 12 + hack/ci/spire-broker.sh | 486 ++++++++++++++++++ internal/controller/helmrelease_controller.go | 17 +- .../controller/helmrelease_controller_test.go | 28 +- internal/features/features.go | 6 +- main.go | 41 +- 12 files changed, 1059 insertions(+), 77 deletions(-) create mode 100755 hack/ci/kubeconfig-e2e.sh create mode 100644 hack/ci/kubeconfig-primary-kind.yaml create mode 100644 hack/ci/spire-broker.sh diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 33928977a..6dd3f714b 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -89,3 +89,25 @@ jobs: kubectl -n helm-system get helmreleases -oyaml || true kubectl -n helm-system get all helm ls -n helm-system --all + kubeconfig-remote: + runs-on: ubuntu-latest + permissions: + contents: read # for reading the repository code. + steps: + - name: Test suite setup + # The custom kind config sets the ServiceAccount issuer URL used by the + # remote cluster's OIDC authenticator. + uses: fluxcd/gha-workflows/.github/actions/setup-kubernetes@948f0babb1fce92d805ea0755a3d7ec1cc4f3176 # v0.16.0 + with: + go-version: 1.27.x + kind-config: hack/ci/kubeconfig-primary-kind.yaml + - name: Build container image + run: | + make docker-build IMG=test/helm-controller:latest \ + BUILD_PLATFORMS=linux/amd64 \ + BUILD_ARGS="--load" + - name: Run remote kubeconfig e2e + # Reuses the "kind" cluster the setup action created and adds a second one. + run: hack/ci/kubeconfig-e2e.sh + env: + IMG: test/helm-controller:latest diff --git a/docs/README.md b/docs/README.md index 456548027..181d04faa 100644 --- a/docs/README.md +++ b/docs/README.md @@ -36,6 +36,8 @@ | `--requeue-dependency` | duration | The interval at which failing dependencies are reevaluated. (default 30s) | | `--snapshot-digest-algo` | string | The algorithm to use to calculate the digest of Helm release storage snapshots. (default "sha256") | | `--token-cache-max-size` | int | The maximum amount of entries in the LRU cache used for tokens. (default 100, enabled) | +| `--spiffe-broker-endpoint` | string | The gRPC URL of the SPIFFE Broker endpoint. | +| `--spiffe-broker-id` | strings | SPIFFE ID of the SPIFFE Broker endpoint. Can be repeated. | | `--token-cache-max-duration` | duration | The maximum duration for which a token would be considered unexpired. This is capped at 1h. (default 1h) | | `--watch-all-namespaces` | boolean | Watch for custom resources in all namespaces, if set to false it will only watch the runtime namespace. (default true) | | `--watch-configs-label-selector` | string | Watch for ConfigMaps and Secrets with matching labels (default 'reconcile.fluxcd.io/watch=Enabled'). | @@ -72,6 +74,21 @@ understands it as the trust domain of the controller's own X509-SVID. Invalid values cause helm-controller to fail at startup. +### SPIFFE Broker API + +When `--spiffe-broker-endpoint` and at least one `--spiffe-broker-id` are set, +helm-controller acquires an X.509-SVID from the SPIFFE Workload API and uses it to +authenticate to the SPIFFE Broker API. The `SPIFFE_ENDPOINT_SOCKET` environment +variable must point to the Workload API socket. + +The Broker is used to acquire JWT-SVIDs for object-level workload identity when +a ServiceAccount sets `credential.auth.fluxcd.io/provider` to `spiffe`. The +Broker endpoint is a TCP gRPC target, e.g. +`dns:///spire-server.spire.svc.cluster.local:8443`. + +Each `--spiffe-broker-id` value is interpreted like the SPIFFE IDs used by the +mTLS flags. Invalid values cause helm-controller to fail at startup. + ### Feature Gates | Name | Default Value | Description | @@ -86,6 +103,5 @@ Invalid values cause helm-controller to fail at startup. | `DisableChartDigestTracking` | `false` | Disables the tracking of digest changes for Helm OCI charts. When set to `true`, the controller will not append the OCI digest to the chart version in `Chart.yaml`. | | `DisableConfigWatchers` | `false` | Disables the watchers for ConfigMaps and Secrets. | | `ExternalArtifact` | `true` | Enables support for the [ExternalArtifact](https://github.com/fluxcd/source-controller/blob/main/docs/spec/v1/externalartifacts.md) source type. | -| `ObjectLevelWorkloadIdentity` | `false` | Enables the use of object-level workload identity for the controller. | | `OOMWatch` | `false` | Enables the OOM watcher, which will gracefully shut down the controller when the memory usage exceeds the configured limit. This is disabled by default. | | `UseHelm3Defaults` | `false` | Restores Helm v3 defaults. When enabled, uses client-side apply for installation, Helm's legacy readiness logic, and the `nohooks` post-render strategy instead of the Helm v4 defaults (server-side apply, kstatus-based health checking, and `combined` post-render strategy). | diff --git a/docs/spec/v2/helmreleases.md b/docs/spec/v2/helmreleases.md index 96e54b612..76ace14ac 100644 --- a/docs/spec/v2/helmreleases.md +++ b/docs/spec/v2/helmreleases.md @@ -1290,6 +1290,11 @@ this feature, see the following docs: - [GKE](/flux/integrations/gcp/#for-google-kubernetes-engine) - [Generic](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#configuring-the-api-server) +For a complete guide to the ServiceAccount annotations used for object-level +workload identity, including the `spiffe` credential provider, see the +[ServiceAccount annotations](/flux/integrations/serviceaccount-annotations) +docs. + Example for an EKS cluster: ```yaml diff --git a/go.mod b/go.mod index 8c78587f8..34a779fcb 100644 --- a/go.mod +++ b/go.mod @@ -21,13 +21,13 @@ require ( github.com/fluxcd/pkg/apis/acl v0.12.0 github.com/fluxcd/pkg/apis/event v0.32.0 github.com/fluxcd/pkg/apis/kustomize v1.23.0 - github.com/fluxcd/pkg/apis/meta v1.36.1-0.20261021122630-1ab1573eaf5b - github.com/fluxcd/pkg/auth v0.63.1-0.20261021122630-1ab1573eaf5b - github.com/fluxcd/pkg/cache v0.17.1-0.20261021122630-1ab1573eaf5b + github.com/fluxcd/pkg/apis/meta v1.37.0 + github.com/fluxcd/pkg/auth v0.65.0 + github.com/fluxcd/pkg/cache v0.18.0 github.com/fluxcd/pkg/chartutil v1.32.0 - github.com/fluxcd/pkg/http/fetch v0.28.1-0.20261021122630-1ab1573eaf5b - github.com/fluxcd/pkg/runtime v0.120.1-0.20261021122630-1ab1573eaf5b - github.com/fluxcd/pkg/spiffe v0.0.0-20261021122630-1ab1573eaf5b + github.com/fluxcd/pkg/http/fetch v0.29.0 + github.com/fluxcd/pkg/runtime v0.121.0 + github.com/fluxcd/pkg/spiffe v0.1.0 github.com/fluxcd/pkg/ssa v0.80.0 github.com/fluxcd/pkg/testserver v0.15.0 github.com/fluxcd/source-controller/api v1.9.6 @@ -114,7 +114,7 @@ require ( github.com/extism/go-sdk v1.7.1 // indirect github.com/fatih/color v1.19.0 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect - github.com/fluxcd/pkg/apis/crypto v0.4.1-0.20261021122630-1ab1573eaf5b // indirect + github.com/fluxcd/pkg/apis/crypto v0.5.0 // indirect github.com/fluxcd/pkg/tar v1.3.0 // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/fxamacker/cbor/v2 v2.9.3 // indirect @@ -137,6 +137,7 @@ require ( github.com/go-openapi/swag/stringutils v0.29.1 // indirect github.com/go-openapi/swag/typeutils v0.29.1 // indirect github.com/go-openapi/swag/yamlutils v0.29.1 // indirect + github.com/gobuffalo/flect v1.0.3 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/gofrs/flock v0.13.0 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect diff --git a/go.sum b/go.sum index ed8535636..8b5944509 100644 --- a/go.sum +++ b/go.sum @@ -152,26 +152,26 @@ github.com/fluxcd/cli-utils v1.4.0 h1:zlTiCWcVDQvvwZXILYeu5rpbnGt1sS+w32y3+rkOpi github.com/fluxcd/cli-utils v1.4.0/go.mod h1:OupUOj2boEKTbiguMbGMz5Z/d59JUcb1c7pPESe8Pjw= github.com/fluxcd/pkg/apis/acl v0.12.0 h1:UWpZDqKxpKDW38wdXIMHV8Uv3POPu2vUrra0JH1frH0= github.com/fluxcd/pkg/apis/acl v0.12.0/go.mod h1:PJrGIu5Bw/6/siO95uhS+pi5oKSU+gVJqHqsPHb8Kok= -github.com/fluxcd/pkg/apis/crypto v0.4.1-0.20261021122630-1ab1573eaf5b h1:/n7p1+Yyu88dVjg3ezqCM0p96VUwMxtjgSb6cELFiQg= -github.com/fluxcd/pkg/apis/crypto v0.4.1-0.20261021122630-1ab1573eaf5b/go.mod h1:bEwkDgBjLXMVJ2BwmVbbj32PkirmNSicabgl2Jvw7Dk= +github.com/fluxcd/pkg/apis/crypto v0.5.0 h1:buy9YCvrD8IpKqOVCSV9k2wlF2iVPaETLOZBqsbyfVs= +github.com/fluxcd/pkg/apis/crypto v0.5.0/go.mod h1:bEwkDgBjLXMVJ2BwmVbbj32PkirmNSicabgl2Jvw7Dk= github.com/fluxcd/pkg/apis/event v0.32.0 h1:3zyjdUspwQN+EQJPlUXwCvCMbeqtB+azlNmeenAcbuE= github.com/fluxcd/pkg/apis/event v0.32.0/go.mod h1:ALma8dTqo+NbI+z4BawSzt0pAPjetCSNNUSSS/9p0VU= github.com/fluxcd/pkg/apis/kustomize v1.23.0 h1:tT/sUURyh6ixQG4+Y/ZoUu+hh5/DQLRraWBgy42ubqQ= github.com/fluxcd/pkg/apis/kustomize v1.23.0/go.mod h1:Jm/LmEsqJIldhhTmhkORAq8JKVZK3ZV2Q88rntjGxEI= -github.com/fluxcd/pkg/apis/meta v1.36.1-0.20261021122630-1ab1573eaf5b h1:g/s+m/jU7cPJ+8IIIXbDD19G7TQAKbiUvqTt5GuywlM= -github.com/fluxcd/pkg/apis/meta v1.36.1-0.20261021122630-1ab1573eaf5b/go.mod h1:1MLvK2Xu3UdTKBzg+Q1Oj7H9PHeiKsUB1UW3uwxAwiM= -github.com/fluxcd/pkg/auth v0.63.1-0.20261021122630-1ab1573eaf5b h1:uqRL6xKRFwUi7AXrPj7BSRAqQYz7rxNMNjSK2W6YqB0= -github.com/fluxcd/pkg/auth v0.63.1-0.20261021122630-1ab1573eaf5b/go.mod h1:gWbB4MzuHWalVyI0o4lE84aINidor9+S63ithckm+94= -github.com/fluxcd/pkg/cache v0.17.1-0.20261021122630-1ab1573eaf5b h1:wi9U1NDlqYmbVUMntuKU71vOqnSFSslDEMbbnBnzJDs= -github.com/fluxcd/pkg/cache v0.17.1-0.20261021122630-1ab1573eaf5b/go.mod h1:OW3fncHhPnPZFZav5FdEa//cyMHSndI0JS+IM69LXOU= +github.com/fluxcd/pkg/apis/meta v1.37.0 h1:gVwadRI8BrRrl4Rw0ZBJ/VSWh7qNpf7ShmTjDhcVf30= +github.com/fluxcd/pkg/apis/meta v1.37.0/go.mod h1:1MLvK2Xu3UdTKBzg+Q1Oj7H9PHeiKsUB1UW3uwxAwiM= +github.com/fluxcd/pkg/auth v0.65.0 h1:AIA+RM4MOyc+iGCMgfuohH1TQsGMBB7Msay7yBLg7IE= +github.com/fluxcd/pkg/auth v0.65.0/go.mod h1:ozzC9/AiqvfeC8BVyFRsUTQAn34gasyElX4aCBHV83c= +github.com/fluxcd/pkg/cache v0.18.0 h1:MllbYkQYJ5zJ5nK52mDWpSov4kSXiCsLJoPeV43y6os= +github.com/fluxcd/pkg/cache v0.18.0/go.mod h1:OW3fncHhPnPZFZav5FdEa//cyMHSndI0JS+IM69LXOU= github.com/fluxcd/pkg/chartutil v1.32.0 h1:rqBhgiGsrS4am/jsCfKYlQvPQ+jATug10BtO95UHdq0= github.com/fluxcd/pkg/chartutil v1.32.0/go.mod h1:pBqCvJVDLpKKnU+k9A+TnceZn+DnnivfVGhzNzHJXZU= -github.com/fluxcd/pkg/http/fetch v0.28.1-0.20261021122630-1ab1573eaf5b h1:q+StvNk04P/dx7IMhw4oKmcfU870OpTIkaZMxgcn5u8= -github.com/fluxcd/pkg/http/fetch v0.28.1-0.20261021122630-1ab1573eaf5b/go.mod h1:SOq8vrzru2zipskTEguXzKHabn2ASyXESQ/DY93FafY= -github.com/fluxcd/pkg/runtime v0.120.1-0.20261021122630-1ab1573eaf5b h1:/S4RzBts7XoNrxXM0hUjmVzZ+fxrXcZ6VyHJfaiWIII= -github.com/fluxcd/pkg/runtime v0.120.1-0.20261021122630-1ab1573eaf5b/go.mod h1:Cw0CQ/tVKS9lgJoyKVouZr1qVnoJcYnSHw3QDzIOOjA= -github.com/fluxcd/pkg/spiffe v0.0.0-20261021122630-1ab1573eaf5b h1:I7Z3cEgoM+EqYpLkr0xI3a1z+Qh1WbZKDCOd8EeGYJg= -github.com/fluxcd/pkg/spiffe v0.0.0-20261021122630-1ab1573eaf5b/go.mod h1:tZ+T8ljAM9ZE0VriHSWM4vfvhlq5zv5bNcPKqwZK1lk= +github.com/fluxcd/pkg/http/fetch v0.29.0 h1:qg9TI6ZaRrJTKqewTjLlcquGQ8H6hFAcMcshnMUQbWw= +github.com/fluxcd/pkg/http/fetch v0.29.0/go.mod h1:SOq8vrzru2zipskTEguXzKHabn2ASyXESQ/DY93FafY= +github.com/fluxcd/pkg/runtime v0.121.0 h1:xFBJ+8To7YEaenBvFZ843MmKST0fQTaALsA+qH0ZzVE= +github.com/fluxcd/pkg/runtime v0.121.0/go.mod h1:8JWIvsBXvHzSOYfuiPNDB6cmHTjWF5m2WyCB1Tiy8Ek= +github.com/fluxcd/pkg/spiffe v0.1.0 h1:BpfZHrine91A/tcGjSs6oejT0VyQpDx4nal2KLfE1Fc= +github.com/fluxcd/pkg/spiffe v0.1.0/go.mod h1:IKBOcFXvk4PwdaE86+d25iB3m+l7iYKUxWl2yVDX/Co= github.com/fluxcd/pkg/ssa v0.80.0 h1:GgakpCFz4pY+unpOscPYlc5eVLUsB7hw9TmWSNr3yp8= github.com/fluxcd/pkg/ssa v0.80.0/go.mod h1:mUwKP2+KZXm1mal1gSs9SjgcexyqzXbPfHKxdFv5ntM= github.com/fluxcd/pkg/tar v1.3.0 h1:PbL5jNN5q/F6nXA8aCH+msM75BS6lSnPX2RNd48f8ls= @@ -239,6 +239,8 @@ github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpv github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gobuffalo/flect v1.0.3 h1:xeWBM2nui+qnVvNM4S3foBhCAL2XgPU+a7FdpelbTq4= +github.com/gobuffalo/flect v1.0.3/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= @@ -408,12 +410,17 @@ github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3A github.com/spiffe/go-spiffe/v2 v2.9.0 h1:g5qTg3qIP1HXV+GzuyEfSJFVxUCpEP5Ui1OINVJHPB4= github.com/spiffe/go-spiffe/v2 v2.9.0/go.mod h1:w2CLWKLMTX/PPYUEUPv3ltH0RXsw5S8suwNF46w9/Aw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 h1:ZF+QBjOI+tILZjBaFj3HgFonKXUcwgJ4djLb6i42S3Q= diff --git a/hack/ci/kubeconfig-e2e.sh b/hack/ci/kubeconfig-e2e.sh new file mode 100755 index 000000000..774734513 --- /dev/null +++ b/hack/ci/kubeconfig-e2e.sh @@ -0,0 +1,451 @@ +#!/usr/bin/env bash +# End-to-end test for .spec.kubeConfig.configMapRef between two kind clusters. +# +# The primary cluster runs the Flux controllers and the remote cluster is the +# target. The same remote cluster trusts both the Kubernetes ServiceAccount +# issuer from the primary cluster and a SPIFFE OIDC issuer backed by the SPIFFE +# Broker API. Two tenants assert that each identity can access its own namespace +# but not the other tenant's namespace. +set -euo pipefail + +IMG="${IMG:?IMG must be set}" +CLUSTER="${CLUSTER:-kind}" +REMOTE_CLUSTER="${REMOTE_CLUSTER:-kubeconfig-remote}" +APP_NS="kubeconfig-e2e" +KUBECONFIG_AUDIENCE="kubeconfig-e2e" +SPIFFE_TRUST_DOMAIN="example.org" +SPIFFE_OIDC_PORT="8083" + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +WORKDIR="$(mktemp -d)" +CREATED_REMOTE=false + +NS=helm-system +CONTROLLER=helm-controller +JWT_OBJ_TYPE=helmreleases.helm.toolkit.fluxcd.io + +log() { printf '\n\033[1;34m==> %s\033[0m\n' "$*"; } +info() { printf ' %s\n' "$*"; } +fail() { printf '\033[1;31mFAIL: %s\033[0m\n' "$*" >&2; exit 1; } + +cleanup() { + local code=$? + if [ "${code}" -ne 0 ]; then + log "FAILURE (exit ${code}) — dumping diagnostics" + kubectl --context "kind-${CLUSTER}" -n "${NS}" logs "deploy/${CONTROLLER}" --tail=120 2>/dev/null || true + kubectl --context "kind-${CLUSTER}" -n "${APP_NS}" get helmreleases -o yaml 2>/dev/null || true + kubectl --context "kind-${REMOTE_CLUSTER}" get all -A 2>/dev/null || true + fi + if [ -n "${KEEP:-}" ]; then + log "KEEP set — leaving clusters and ${WORKDIR} for inspection" + exit "${code}" + fi + if [ "${CREATED_REMOTE}" = "true" ]; then + kind delete cluster --name "${REMOTE_CLUSTER}" >/dev/null 2>&1 || true + fi + rm -rf "${WORKDIR}" + exit "${code}" +} +trap cleanup EXIT + +tools=(docker kind kubectl kustomize openssl) +for bin in "${tools[@]}"; do + command -v "${bin}" >/dev/null 2>&1 || fail "required tool '${bin}' not found" +done + +source "${ROOT_DIR}/hack/ci/spire-broker.sh" + +kubectl config use-context "kind-${CLUSTER}" >/dev/null +kubectl wait --for=condition=Ready nodes --all --timeout=2m + +log "Deploying ${CONTROLLER} in the primary cluster" +kind load docker-image "${IMG}" --name "${CLUSTER}" +make -C "${ROOT_DIR}" dev-deploy IMG="${IMG}" >/dev/null +kubectl -n "${NS}" rollout status "deploy/source-controller" --timeout=2m +kubectl -n "${NS}" rollout status "deploy/${CONTROLLER}" --timeout=2m + +log "Preparing the primary cluster OIDC discovery" +kubectl create clusterrolebinding oidc-reviewer \ + --clusterrole=system:service-account-issuer-discovery \ + --group=system:unauthenticated >/dev/null 2>&1 || true + +############################################################################## +log "Generating a TLS certificate for the SPIFFE OIDC provider" +############################################################################## +openssl req -x509 -newkey rsa:2048 -nodes -days 1 \ + -subj "/CN=kubeconfig-oidc-ca" \ + -keyout "${WORKDIR}/oidc-ca.key" \ + -out "${WORKDIR}/oidc-ca.crt" >/dev/null 2>&1 +openssl req -newkey rsa:2048 -nodes \ + -subj "/CN=${CLUSTER}-control-plane" \ + -keyout "${WORKDIR}/oidc-tls.key" \ + -out "${WORKDIR}/oidc-tls.csr" >/dev/null 2>&1 +cat > "${WORKDIR}/oidc-tls.ext" </dev/null 2>&1 + +kubectl create namespace spire --dry-run=client -o yaml | kubectl apply -f - >/dev/null +kubectl -n spire create secret tls spire-oidc-tls \ + --cert="${WORKDIR}/oidc-tls.crt" \ + --key="${WORKDIR}/oidc-tls.key" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null + +############################################################################## +log "Installing SPIRE with the Broker API" +############################################################################## +SPIRE_NAMESPACE=spire +SPIRE_TRUST_DOMAIN="${SPIFFE_TRUST_DOMAIN}" +SPIRE_SERVER_IMAGE="${SPIRE_SERVER_IMAGE:-ghcr.io/spiffe/spire-server:1.15.5}" +SPIRE_AGENT_IMAGE="${SPIRE_AGENT_IMAGE:-ghcr.io/spiffe/spire-agent:1.15.5}" +SPIRE_OIDC_IMAGE="${SPIRE_OIDC_IMAGE:-ghcr.io/spiffe/oidc-discovery-provider:1.15.5}" +SPIRE_CLUSTER="${CLUSTER}" +SPIRE_OIDC_ISSUER="http://${CLUSTER}-control-plane:8082" +SPIRE_JWT_ISSUER="https://${CLUSTER}-control-plane:${SPIFFE_OIDC_PORT}" +SPIRE_BROKER_ENDPOINT="dns:///spire-broker.spire:8443" +SPIRE_BROKER_ID="spiffe://${SPIRE_TRUST_DOMAIN}/spire/agent/k8s_psat/broker-cluster/" +BROKER_SA="$(kubectl -n "${NS}" get deploy "${CONTROLLER}" -o jsonpath='{.spec.template.spec.serviceAccountName}')" +[ -n "${BROKER_SA}" ] || BROKER_SA=default +JWT_OBJ_NAME="hr-spiffe-a-own" +JWT_SPIFFE_ID="spiffe://${SPIRE_TRUST_DOMAIN}/kubeconfig/${JWT_OBJ_NAME}" +install_spire_broker + +log "Deploying the HTTPS SPIFFE OIDC provider" +cat > "${WORKDIR}/spire-oidc-tls.conf" </dev/null +kubectl -n spire apply -f - >/dev/null </dev/null +done + +############################################################################## +log "Creating the remote kind cluster with both JWT authenticators" +############################################################################## +docker cp "${CLUSTER}-control-plane:/etc/kubernetes/pki/ca.crt" "${WORKDIR}/primary-ca.crt" +cat > "${WORKDIR}/authentication.yaml" < "${WORKDIR}/remote-kind.yaml" </dev/null 2>&1 || true +kind create cluster --name "${REMOTE_CLUSTER}" --config "${WORKDIR}/remote-kind.yaml" --wait 3m +CREATED_REMOTE=true +kubectl config use-context "kind-${REMOTE_CLUSTER}" >/dev/null +docker cp "${REMOTE_CLUSTER}-control-plane:/etc/kubernetes/pki/ca.crt" "${WORKDIR}/remote-ca.crt" + +############################################################################## +log "Creating remote tenant namespaces and RBAC" +############################################################################## +kubectl create namespace tenant-a --dry-run=client -o yaml | kubectl apply -f - >/dev/null +kubectl create namespace tenant-b --dry-run=client -o yaml | kubectl apply -f - >/dev/null +for tenant in a b; do + kubectl -n "tenant-${tenant}" apply -f - >/dev/null </dev/null +kubectl create namespace "${APP_NS}" --dry-run=client -o yaml | kubectl apply -f - >/dev/null + +for tenant in a b; do + other=$([ "${tenant}" = a ] && echo b || echo a) + for provider in kubernetes spiffe; do + kubectl -n "${APP_NS}" apply -f - >/dev/null </dev/null </dev/null </dev/null </dev/null +kubectl -n "${NS}" patch "deploy/${CONTROLLER}" --type=strategic -p "$(cat </dev/null +kubectl -n "${NS}" rollout status "deploy/${CONTROLLER}" --timeout=2m + +############################################################################## +log "Creating HelmRelease resources for both providers and tenants" +############################################################################## +for tenant in a b; do + other=$([ "${tenant}" = a ] && echo b || echo a) + for provider in kubernetes spiffe; do + for scope in own cross; do + target_tenant="${tenant}" + [ "${scope}" = cross ] && target_tenant="${other}" + name="hr-${provider}-${tenant}-${scope}" + kubectl -n "${APP_NS}" apply -f - >/dev/null <=6.0.0 <7.0.0" + sourceRef: + kind: HelmRepository + name: podinfo + targetNamespace: tenant-${target_tenant} + storageNamespace: tenant-${target_tenant} + install: + createNamespace: false + kubeConfig: + configMapRef: + name: kc-${provider}-${tenant} +EOF + done + done +done + +wait_ready() { + local name="$1" + kubectl -n "${APP_NS}" wait "helmrelease/${name}" --for=condition=Ready --timeout=5m +} + +expect_denied() { + local name="$1" status msg + for _ in $(seq 1 60); do + status="$(kubectl -n "${APP_NS}" get "helmrelease/${name}" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" + msg="$(kubectl -n "${APP_NS}" get "helmrelease/${name}" -o jsonpath='{.status.conditions[?(@.type=="Ready")].message}' 2>/dev/null || true)" + if [ "${status}" = "True" ]; then + fail "${name} unexpectedly succeeded" + fi + if [ "${status}" = "False" ] && printf '%s' "${msg}" | grep -qiE 'forbidden|cannot|denied|unauthorized'; then + info "${name} correctly denied" + return 0 + fi + sleep 2 + done + fail "${name} did not fail with an authorization error in time (status=${status:-unknown}; message=${msg:-})" +} + +for tenant in a b; do + for provider in kubernetes spiffe; do + name="hr-${provider}-${tenant}-own" + wait_ready "${name}" + kubectl --context "kind-${REMOTE_CLUSTER}" -n "tenant-${tenant}" \ + wait --for=condition=Available deployment \ + -l "helm.toolkit.fluxcd.io/name=${name}" --timeout=5m + info "${name} applied to tenant-${tenant}" + expect_denied "hr-${provider}-${tenant}-cross" + done +done + +log "SUCCESS: remote kubeconfig access is isolated per tenant for both providers" diff --git a/hack/ci/kubeconfig-primary-kind.yaml b/hack/ci/kubeconfig-primary-kind.yaml new file mode 100644 index 000000000..73aadbe3d --- /dev/null +++ b/hack/ci/kubeconfig-primary-kind.yaml @@ -0,0 +1,12 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + kubeadmConfigPatches: + - | + kind: ClusterConfiguration + apiServer: + certSANs: + - "kind-control-plane" + extraArgs: + service-account-issuer: "https://kind-control-plane:6443" diff --git a/hack/ci/spire-broker.sh b/hack/ci/spire-broker.sh new file mode 100644 index 000000000..8a874ec73 --- /dev/null +++ b/hack/ci/spire-broker.sh @@ -0,0 +1,486 @@ +#!/usr/bin/env bash +# Installs SPIRE with the SPIFFE Broker API and OIDC discovery provider in the +# current kubectl context. Expects an already-running single-node kind cluster. +# +# Required environment: +# SPIRE_NAMESPACE, SPIRE_TRUST_DOMAIN, SPIRE_SERVER_IMAGE, SPIRE_AGENT_IMAGE, +# SPIRE_OIDC_IMAGE, SPIRE_CLUSTER, SPIRE_OIDC_ISSUER, BROKER_SA, +# NS, APP_NS, JWT_OBJ_TYPE, JWT_OBJ_NAME, JWT_SPIFFE_ID +# +# Optional environment: +# SPIRE_JWT_ISSUER (defaults to SPIRE_OIDC_ISSUER) is the JWT issuer URL +# embedded in JWT-SVIDs. Set it when the trusted OIDC endpoint differs from +# the local discovery provider, e.g. an HTTPS endpoint on another port. +set -euo pipefail + +install_spire_broker() { +log "Installing SPIRE with the SPIFFE Broker API and OIDC discovery" +############################################################################## +kubectl create namespace "${SPIRE_NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f - >/dev/null + +kubectl apply -f - >/dev/null </dev/null +} + +create_entry "spiffe://${SPIRE_TRUST_DOMAIN}/oidc-discovery-provider" \ + -selector "k8s:ns:${SPIRE_NAMESPACE}" \ + -selector "k8s:sa:spire-agent" \ + -selector "k8s:container-name:oidc-discovery-provider" +create_entry "spiffe://${SPIRE_TRUST_DOMAIN}/ns/${NS}/sa/${BROKER_SA}" \ + -selector "k8s:ns:${NS}" \ + -selector "k8s:sa:${BROKER_SA}" +create_entry "${JWT_SPIFFE_ID}" \ + -selector "k8s:resource:${JWT_OBJ_TYPE}" \ + -selector "k8s:namespace:${APP_NS}" \ + -selector "k8s:name:${JWT_OBJ_NAME}" + +kubectl apply -f - >/dev/null < 0 || len(authorizedEventServers) > 0 { + var spiffeBroker fluxbroker.Client + if len(authorizedArtifactServers) > 0 || len(authorizedEventServers) > 0 || + spiffeBrokerEndpoint != "" || len(spiffeBrokerIDs) > 0 { source, err := workloadapi.NewX509Source(ctx) if err != nil { setupLog.Error(err, "unable to create SPIFFE X.509 source") @@ -374,6 +370,21 @@ func main() { } setupLog.Info("event server will use inter-controller mTLS via SPIFFE", "spiffeIDs", authorizedEventServers) } + + if spiffeBrokerEndpoint != "" || len(spiffeBrokerIDs) > 0 { + spiffeBroker, err = fluxbroker.New(spiffeBrokerEndpoint, spiffeBrokerIDs, source) + if err != nil { + setupLog.Error(err, "unable to create SPIFFE Broker client") + os.Exit(1) + } + defer func() { + if err := spiffeBroker.Close(); err != nil { + setupLog.Error(err, "unable to close SPIFFE Broker client") + } + }() + setupLog.Info("object-level workload identity will use the SPIFFE Broker API", + "endpoint", spiffeBrokerEndpoint, "spiffeIDs", spiffeBrokerIDs) + } } eventRecorderOpts := []events.RecorderOption{events.WithManager(mgr)} @@ -431,12 +442,12 @@ func main() { KubeConfigOpts: kubeConfigOpts, FieldManager: controllerName, DefaultKubeConfigServiceAccount: defaultKubeConfigServiceAccount, - ObjectLevelWorkloadIdentity: objectLevelWorkloadIdentity, DefaultToRetryOnFailure: defaultToRetryOnFailure, DisableChartDigestTracking: disableChartDigestTracking, AdditiveCELDependencyCheck: additiveCELDependencyCheck, DirectSourceFetch: directSourceFetch, TokenCache: tokenCache, + SPIFFEBroker: spiffeBroker, DependencyRequeueInterval: requeueDependency, ArtifactFetchRetries: httpRetry, ArtifactFetchTimeout: httpTimeout,