Node Agent Component Tests #523

Workflow file for this run

.github/workflows/component-tests.yaml at aa9af8c

	# =============================================================================
	# Node Agent Component Tests
	# =============================================================================
	#
	# Architecture:
	# There are TWO independent artifacts in play:
	#
	# 1. Node-agent container image — the eBPF runtime agent deployed INTO the
	# Kind cluster via Helm. Lives in pkg/, cmd/, Makefile, Dockerfile, etc.
	# Changes here require an image rebuild before tests can validate them.
	#
	# 2. Component test binary — a Go test suite compiled on-the-fly from
	# tests/component_test.go via `go test`. Runs OUTSIDE the cluster on the
	# CI runner. It drives the cluster by creating k8s resources, exec-ing
	# into pods, and querying Alertmanager for alerts.
	# Changes here do NOT require a node-agent image rebuild.
	#
	# Rebuild logic (on push):
	# - If ONLY files under tests/ or .github/ changed → skip image build,
	# run tests immediately against the existing 'latest' image.
	# - If ANY agent code changed (pkg/, cmd/, go.mod, Makefile, …) → rebuild
	# the node-agent image first, then run tests against the freshly built image.
	#
	# Manual trigger (workflow_dispatch):
	# - Use the `build_image` checkbox to force an image rebuild.
	# - Supply NODE_AGENT_TAG / STORAGE_TAG to pin specific pre-built images.
	# =============================================================================

	name: Node Agent Component Tests
	on:
	push:
	branches:
	- feat/signature-verification
	- feat/tamperalert
	- feat/tamper-detection
	workflow_dispatch:
	inputs:
	build_image:
	description: 'Build and push a new container image for the test'
	type: boolean
	required: false
	default: false
	STORAGE_TAG:
	description: 'Storage image tag (must match the tag built by storage/build)'
	type: string
	required: true
	default: 'latest'
	NODE_AGENT_TAG:
	description: 'Node-agent image tag (must match the tag built by node-agent/build)'
	type: string
	required: true
	default: 'latest'
	STORAGE_REF:
	description: 'Commit SHA of k8sstormcenter/storage — the SAME ref the storage image (STORAGE_TAG) was built from, used for the go.mod replace (leave empty to resolve current main HEAD at runtime)'
	type: string
	required: false
	default: ''
	SOURCE_REF:
	description: 'node-agent ref to take the component tests from (e.g. a clean ct/** branch). tests/ code is overlaid from here so the source branch never needs fork workflow files. Empty = use this branch as-is.'
	type: string
	required: false
	default: ''

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	# Default to read-only at the workflow level (least privilege per Scorecard).
	# Jobs that need elevated scopes override below.
	permissions: read-all

	jobs:
	# -------------------------------------------------------------------
	# Detect what changed to decide whether an image rebuild is needed.
	# On push: compare HEAD with HEAD~1.
	# On workflow_dispatch: always outputs false (rebuild controlled by input).
	# -------------------------------------------------------------------
	detect-changes:
	runs-on: ubuntu-latest
	outputs:
	needs_rebuild: ${{ steps.check.outputs.needs_rebuild }}
	steps:
	- name: Checkout code
	uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
	with:
	fetch-depth: 2

	- name: Check for agent code changes
	id: check
	run: \|
	if [ "${{ github.event_name }}" != "push" ]; then
	echo "Not a push event — rebuild decision deferred to workflow inputs"
	echo "needs_rebuild=false" >> "$GITHUB_OUTPUT"
	exit 0
	fi

	CHANGED=$(git diff --name-only HEAD~1 HEAD)
	echo "=== Changed files ==="
	echo "$CHANGED"
	echo ""

	# Agent code = anything outside tests/ and .github/
	# These are the paths that end up in the node-agent container image.
	AGENT_CHANGES=$(echo "$CHANGED" \| grep -vE '^(tests/\|\.github/)' \|\| true)

	if [ -n "$AGENT_CHANGES" ]; then
	echo "=== Agent code changed (rebuild needed) ==="
	echo "$AGENT_CHANGES"
	echo "needs_rebuild=true" >> "$GITHUB_OUTPUT"
	else
	echo "=== Only test/workflow files changed — no rebuild needed ==="
	echo "needs_rebuild=false" >> "$GITHUB_OUTPUT"
	fi

	# -------------------------------------------------------------------
	# Build and push the node-agent container image.
	# Triggers when:
	# - Manual dispatch with build_image=true, OR
	# - Push event where agent code changed (detected above)
	# -------------------------------------------------------------------
	build-and-push-image:
	needs: [detect-changes]
	if: >-
	(github.event_name == 'workflow_dispatch' && inputs.build_image == true) \|\|
	(needs.detect-changes.outputs.needs_rebuild == 'true')
	runs-on: ubuntu-latest
	permissions:
	contents: read
	packages: write
	id-token: write
	steps:
	- name: Checkout code
	uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
	with:
	# Build the node-agent image from the code under test (SOURCE_REF),
	# not this harness branch. The workflow logic stays fork-ci's; only the
	# source tree that gets compiled comes from SOURCE_REF.
	ref: ${{ inputs.SOURCE_REF \|\| github.ref }}
	- name: Login to GitHub Container Registry
	uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
	with:
	registry: ghcr.io
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}

	- name: Install IG
	run: \|
	sudo apt-get update
	sudo apt-get install -y jq curl
	IG_ARCH=amd64
	IG_VERSION=$(curl -s https://api.github.com/repos/inspektor-gadget/inspektor-gadget/releases/latest \| jq -r .tag_name)
	echo "Installing IG version: ${IG_VERSION}"
	curl -sL https://github.com/inspektor-gadget/inspektor-gadget/releases/download/${IG_VERSION}/ig-linux-${IG_ARCH}-${IG_VERSION}.tar.gz \| sudo tar -C /usr/local/bin -xzf - ig
	sudo chmod +x /usr/local/bin/ig

	# Resolve the storage commit SHA once and use the same one for the
	# image build AND the test runner (output downstream). Without this,
	# the docker image and the test binary can compile against different
	# storage versions when their go.mod replace directives drift.
	- name: Resolve storage ref
	id: resolve-storage
	env:
	STORAGE_REF_INPUT: ${{ inputs.STORAGE_REF }}
	run: \|
	STORAGE_REF="${STORAGE_REF_INPUT}"
	if [ -z "${STORAGE_REF}" ]; then
	STORAGE_REF=$(git ls-remote https://github.com/k8sstormcenter/storage refs/heads/main \| awk '{print $1}')
	echo "Resolved k8sstormcenter/storage main to: ${STORAGE_REF}"
	else
	echo "Using supplied STORAGE_REF: ${STORAGE_REF}"
	fi
	echo "storage_ref=${STORAGE_REF}" >> "$GITHUB_OUTPUT"
	echo "storage_short=${STORAGE_REF:0:7}" >> "$GITHUB_OUTPUT"

	- name: Set up Go
	uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
	with:
	go-version: "1.25"

	- name: Pin storage version for image build
	env:
	STORAGE_REF: ${{ steps.resolve-storage.outputs.storage_ref }}
	GOFLAGS: "-mod=mod"
	# The fork storage module has no public checksum-DB entry, so tidy's
	# sum.golang.org lookup 500s. Skip sumdb verification for it.
	GOSUMDB: "off"
	GONOSUMCHECK: "*"
	run: \|
	echo "Replacing github.com/kubescape/storage with github.com/k8sstormcenter/storage@${STORAGE_REF}"
	go mod edit -replace "github.com/kubescape/storage=github.com/k8sstormcenter/storage@${STORAGE_REF}"
	go mod tidy
	echo "Resolved storage version:"
	grep "k8sstormcenter/storage" go.sum \| head -1

	- name: Build the Image and Push to GHCR
	id: build-and-push-image
	run: \|
	COMMIT_HASH=$(git rev-parse --short HEAD)
	STORAGE_SHORT="${{ steps.resolve-storage.outputs.storage_short }}"
	# Image tag encodes both node-agent and storage SHAs so the same
	# source pair always produces the same artifact and can be cached.
	export IMAGE_TAG=test-${COMMIT_HASH}-s${STORAGE_SHORT}
	export IMAGE_REPO=ghcr.io/${{ github.repository_owner }}/node-agent
	echo "image_repo=${IMAGE_REPO}" >> "$GITHUB_OUTPUT"
	export IMAGE_NAME=ghcr.io/${{ github.repository_owner }}/node-agent:${IMAGE_TAG}
	echo "image_tag=${IMAGE_TAG}" >> "$GITHUB_OUTPUT"
	make docker-build TAG=${IMAGE_TAG} IMAGE=${IMAGE_REPO} && make docker-push TAG=${IMAGE_TAG} IMAGE=${IMAGE_REPO}
	outputs:
	image_tag: ${{ steps.build-and-push-image.outputs.image_tag }}
	image_repo: ${{ steps.build-and-push-image.outputs.image_repo }}
	storage_ref: ${{ steps.resolve-storage.outputs.storage_ref }}

	# -------------------------------------------------------------------
	# Component tests.
	#
	# These are Go tests compiled from tests/component_test.go — they are
	# NOT part of the node-agent container image. The test binary runs on
	# the CI runner and talks to the Kind cluster via the k8s API.
	#
	# Dependency logic:
	# - If build-and-push-image ran → waits for it, uses the freshly
	# built image tag.
	# - If build-and-push-image was skipped (tests-only change) → runs
	# immediately with the default 'latest' image.
	# - If build-and-push-image failed → tests do NOT run (no point
	# testing against a stale image when code changed).
	# -------------------------------------------------------------------
	component-tests:
	needs: [detect-changes, build-and-push-image]
	# Run when build succeeded or was skipped; don't run if build failed.
	if: >-
	always() && !cancelled() &&
	(needs.build-and-push-image.result == 'success' \|\| needs.build-and-push-image.result == 'skipped')
	runs-on: ubuntu-latest
	continue-on-error: true
	strategy:
	fail-fast: false
	matrix:
	test: [
	Test_01_BasicAlertTest,
	Test_02_AllAlertsFromMaliciousApp,
	Test_03_BasicLoadActivities,
	Test_04_MemoryLeak,
	Test_05_MemoryLeak_10K_Alerts,
	Test_06_KillProcessInTheMiddle,
	Test_07_RuleBindingApplyTest,
	Test_08_ContainerProfilePatching,
	Test_10_MalwareDetectionTest,
	Test_11_EndpointTest,
	Test_14_RulePoliciesTest,
	Test_15_CompletedApCannotBecomeReadyAgain,
	Test_16_ApNotStuckOnRestart,
	Test_17_ApCompletedToPartialUpdateTest,
	Test_18_ShortLivedJobTest,
	Test_19_AlertOnPartialProfileTest,
	Test_20_AlertOnPartialThenLearnProcessTest,
	Test_21_AlertOnPartialThenLearnNetworkTest,
	Test_22_AlertOnPartialNetworkProfileTest,
	Test_23_RuleCooldownTest,
	Test_24_ProcessTreeDepthTest,
	Test_27_ApplicationProfileOpens,
	Test_28_UserDefinedNetworkNeighborhood,
	Test_32_UnexpectedProcessArguments,
	Test_33_AnalyzeOpensWildcardAnchoring,
	Test_34_NetworkNeighborsCIDRCollapse,
	Test_35_ExecTTYFieldTest,
	Test_36_MultiContainerPerContainerBinding,
	Test_29_SignedContainerProfile,
	Test_31_TamperDetectionAlert,
	Test_37_SignedBundleOverlay,
	Test_38_SignedRulesBundleOverlay,
	Test_39_SignedRulesUntrustedSignerRejected,
	Test_40_TrustPolicyFailClosed,
	Test_41_SignedBundleNamespaceFreedom,
	Test_42_SignatureStrippedFragmentRejected,
	Test_43_RelativeOpenPathResolution,
	Test_44_TrustAnchorModificationRequiresElevatedRBAC,
	Test_45_RuleSigningZeroAdmittedDetectionOutage,
	Test_46_TrustPolicyReloadLifecycle,
	Test_47_StoredSpecDivergenceInert,
	Test_48_MultiSubtypeGroupedProfileDocument,
	Test_12_AuthoredMultiSubtypeProfile,
	Test_13_NaturalLearningLifecycle,
	Test_25_RuleCooldownExactThreshold,
	Test_30_IgnoreExcludeAndLearningDuration,
	Test_49_EphemeralContainerFullTreatment
	]
	steps:
	- name: Checkout code
	uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4

	- name: Overlay component tests from SOURCE_REF
	if: ${{ inputs.SOURCE_REF != '' }}
	env:
	SOURCE_REF: ${{ inputs.SOURCE_REF }}
	run: \|
	set -euo pipefail
	# Pull the component-test code from the clean source branch (which
	# carries no fork workflow files) and run it under this overlay, so
	# upstream-bound test branches never need .github of their own. The
	# workflow, matrix and deploy steps stay this branch's; only the test
	# code comes from SOURCE_REF.
	git fetch origin "${SOURCE_REF}" --depth=1
	# Overlay the WHOLE tree from the code under test, then restore OUR
	# .github (the harness). Piecemeal per-file overlays drift: a stale
	# tests/testutils, tests/chart, go.mod, etc. from this branch breaks
	# compilation or deploys the wrong rules. One folder is ours (.github);
	# everything else comes from SOURCE_REF.
	git checkout FETCH_HEAD -- .
	git checkout HEAD -- .github
	echo "Overlaid full tree from ${SOURCE_REF}; kept harness .github"

	- name: Set up Kind
	run: \|
	curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.31.0/kind-linux-amd64
	chmod +x ./kind
	./kind create cluster
	curl -LO "https://dl.k8s.io/release/v1.35.0/bin/linux/amd64/kubectl"
	chmod +x ./kubectl
	sudo mv ./kubectl /usr/local/bin/kubectl
	- name: Install Helm and Kubectl
	run: \|
	curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
	chmod 700 get_helm.sh
	sudo ./get_helm.sh
	- name: Install Prometheus and Node Exporter
	run: \|
	helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
	helm repo update
	helm upgrade --install prometheus prometheus-community/kube-prometheus-stack --set grafana.enabled=false --namespace monitoring --create-namespace --set prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues=false,prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false --set prometheus.prometheusSpec.maximumStartupDurationSeconds=300 --wait --timeout 5m
	# Check that the prometheus pod is running
	kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=prometheus -n monitoring --timeout=300s

	# Image selection logic:
	# - If the build job ran and produced a tag → use it.
	# - Otherwise fall back to the workflow_dispatch input or 'latest'.
	- name: Install Node Agent Chart
	run: \|
	STORAGE_TAG="${{ inputs.STORAGE_TAG \|\| 'latest' }}"
	# Storage is built to the fork registry (ghcr), not the chart-default
	# quay.io/kubescape. Without overriding the repository the deploy pulls
	# quay.io/kubescape/storage:<fork-tag>, which does not exist there →
	# ImagePullBackOff and every test leg fails at Install.
	STORAGE_REPO="ghcr.io/${{ github.repository_owner }}/storage"
	echo "Storage image: ${STORAGE_REPO}:${STORAGE_TAG}"

	# Prefer freshly built image; fall back to input or default.
	IMAGE_TAG="${{ needs.build-and-push-image.outputs.image_tag \|\| inputs.NODE_AGENT_TAG \|\| 'latest' }}"
	IMAGE_REPO="${{ needs.build-and-push-image.outputs.image_repo \|\| 'ghcr.io/k8sstormcenter/node-agent' }}"
	echo "Node Agent image: ${IMAGE_REPO}:${IMAGE_TAG}"

	# Log whether we're using a freshly built image or a pre-existing one.
	if [ -n "${{ needs.build-and-push-image.outputs.image_tag }}" ]; then
	echo ">>> Using FRESHLY BUILT image from this workflow run"
	else
	echo ">>> Using PRE-EXISTING image (no agent code changes detected)"
	fi

	helm upgrade --install kubescape ./tests/chart --set clusterName=`kubectl config current-context` --set nodeAgent.image.tag=${IMAGE_TAG} --set nodeAgent.image.repository=${IMAGE_REPO} --set storage.image.tag=${STORAGE_TAG} --set storage.image.repository=${STORAGE_REPO} -n kubescape --create-namespace --wait --timeout 5m --debug
	# Check that the node-agent pod is running
	kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=node-agent -n kubescape --timeout=300s
	sleep 5
	- name: Run Port Forwarding
	run: \|
	./tests/scripts/port-forward.sh

	# The test binary is compiled from source here — it is NOT part of
	# the node-agent container image. Only changes under tests/ affect
	# it; agent code changes (pkg/, cmd/, …) require an image rebuild
	# but do NOT change the test binary.
	- name: Set up Go
	env:
	CGO_ENABLED: 0
	uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
	with:
	go-version: "1.25"
	- name: Set unlimited memlock limit
	run: \|
	sudo sh -c "ulimit -l unlimited"
	- name: Update storage dependency
	env:
	STORAGE_REF: ${{ needs.build-and-push-image.outputs.storage_ref \|\| inputs.STORAGE_REF }}
	GONOSUMCHECK: "*"
	GOFLAGS: "-mod=mod"
	GOSUMDB: "off"
	run: \|
	if [ -z "${STORAGE_REF}" ]; then
	STORAGE_REF=$(git ls-remote https://github.com/k8sstormcenter/storage refs/heads/main \| awk '{print $1}')
	echo "Resolved k8sstormcenter/storage main to: ${STORAGE_REF}"
	fi
	echo "Replacing github.com/kubescape/storage with github.com/k8sstormcenter/storage@${STORAGE_REF}"
	go mod edit -replace "github.com/kubescape/storage=github.com/k8sstormcenter/storage@${STORAGE_REF}"
	go mod tidy
	echo "Resolved storage version:"
	grep "k8sstormcenter/storage" go.sum \| head -1
	- name: Run test
	run: \|
	cd tests && go test -v ./... -run ${{ matrix.test }} --timeout=20m --tags=component
	- name: Print node agent & storage logs
	if: always()
	run: \|
	echo "Node agent logs"
	kubectl logs $(kubectl get pods -n kubescape -o name \| grep node-agent) -n kubescape -c node-agent
	echo "-----------------------------------------"
	echo "Storage logs"
	kubectl logs $(kubectl get pods -n kubescape -o name \| grep storage) -n kubescape

	trigger-integration-tests:
	needs: component-tests
	if: >-
	github.event_name == 'workflow_dispatch' &&
	inputs.STORAGE_TAG != '' &&
	inputs.NODE_AGENT_TAG != ''
	runs-on: ubuntu-latest
	steps:
	- name: Trigger storage integration tests
	env:
	GH_TOKEN: ${{ secrets.CROSS_REPO_PAT }}
	run: \|
	STORAGE_TAG="${{ inputs.STORAGE_TAG }}"
	NODE_AGENT_TAG="${{ inputs.NODE_AGENT_TAG }}"
	echo "Triggering storage integration tests"
	echo " node_agent_image=ghcr.io/${{ github.repository_owner }}/node-agent:${NODE_AGENT_TAG}"
	echo " storage_image=ghcr.io/${{ github.repository_owner }}/storage:${STORAGE_TAG}"
	gh workflow run manual-integration-tests.yml \
	--repo "${{ github.repository_owner }}/storage" \
	--ref "${{ github.ref_name }}" \
	-f branch="${{ github.ref_name }}" \
	-f branch_helm_chart=main \
	-f node_agent_image="ghcr.io/${{ github.repository_owner }}/node-agent:${NODE_AGENT_TAG}" \
	-f storage_image="ghcr.io/${{ github.repository_owner }}/storage:${STORAGE_TAG}"

Navigation Menu

Uh oh!

Node Agent Component Tests #523

Workflow file

Node Agent Component Tests #523

Uh oh!

Workflow file for this run