Report the endpoints actually called, not the ones declared #250

Workflow file for this run

	name: CI

	on:
	push:
	branches: [main]
	tags: ['v*']
	paths-ignore:
	pull_request:
	branches: [main]
	workflow_dispatch:
	inputs:
	ollama_mlx:
	description: >-
	Run the Ollama tests on macOS against the MLX build of Gemma 4.
	The hosted macOS runner leaves Ollama about 4GB, which is under
	what gemma4:e2b-mlx needs, so this needs a larger runner.
	type: boolean
	default: false

	jobs:
	lint:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	- uses: actions/setup-node@v7
	with:
	node-version: '22.x'
	- run: npm install
	- name: ESLint
	shell: bash
	run: \|
	set -o pipefail
	npm run lint 2>&1 \| tee out.txt \|\| {
	tail -60 out.txt \| while read -r line; do
	echo "::error::ESLint: $line"
	done
	exit 1
	}

	test:
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, windows-latest, macos-latest]
	node-version: ['22.x', '24.x']
	steps:
	- uses: actions/checkout@v7
	- uses: actions/setup-node@v7
	with:
	node-version: ${{ matrix.node-version }}
	- run: npm install
	- name: Generation, CLI and Bruno tests
	shell: bash
	run: \|
	set -o pipefail
	npm test 2>&1 \| tee out.txt \|\| {
	(grep -A 18 "^not ok" out.txt \| head -80; echo "--- tail ---"; tail -20 out.txt) \| while read -r line; do
	echo "::error::Generation, CLI and Bruno tests: $line"
	done
	exit 1
	}
	- name: Editor UI tests
	shell: bash
	run: \|
	set -o pipefail
	npm run test:ui 2>&1 \| tee out.txt \|\| {
	(grep -A 18 "^not ok" out.txt \| head -80; echo "--- tail ---"; tail -20 out.txt) \| while read -r line; do
	echo "::error::Editor UI tests: $line"
	done
	exit 1
	}
	- name: Node-RED integration tests
	shell: bash
	run: \|
	set -o pipefail
	npm run test:nodered 2>&1 \| tee out.txt \|\| {
	(grep -A 18 "^not ok" out.txt \| head -80; echo "--- tail ---"; tail -20 out.txt) \| while read -r line; do
	echo "::error::Node-RED integration tests: $line"
	done
	exit 1
	}
	- name: Command line picker tests
	if: runner.os == 'Linux'
	run: npm run test:wizard

	- name: Response scanning tests
	run: npm run test:errorwords

	- name: Bruno conversion equivalence tests
	run: npm run test:converters

	- name: Cross format equivalence tests
	run: npm run test:formats

	- name: Flow lint tests
	shell: bash
	run: \|
	set -o pipefail
	npm run test:nrlint 2>&1 \| tee out.txt \|\| {
	(grep -A 18 "^not ok" out.txt \| head -80; echo "--- tail ---"; tail -20 out.txt) \| while read -r line; do
	echo "::error::Flow lint tests: $line"
	done
	exit 1
	}

	- name: Plugin package tests
	shell: bash
	run: \|
	set -o pipefail
	npm run test:plugin 2>&1 \| tee out.txt \|\| {
	(grep -A 18 "^not ok" out.txt \| head -80; echo "--- tail ---"; tail -20 out.txt) \| while read -r line; do
	echo "::error::Plugin package tests: $line"
	done
	exit 1
	}

	browser:
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	include:
	- os: ubuntu-latest
	browser: chromium
	- os: ubuntu-latest
	browser: firefox
	- os: ubuntu-latest
	browser: webkit
	- os: windows-latest
	browser: msedge
	- os: macos-latest
	browser: webkit
	steps:
	- uses: actions/checkout@v7
	- uses: actions/setup-node@v7
	with:
	node-version: '22.x'
	- run: npm install
	- name: Install the browser
	run: npx playwright install --with-deps ${{ matrix.browser == 'msedge' && 'msedge' \|\| matrix.browser }}
	- name: Editor tests in ${{ matrix.browser }}
	id: browser-tests
	shell: bash
	env:
	BROWSERS: ${{ matrix.browser }}
	run: npm run test:browser 2>&1 \| tee browser.log

	- name: Show the browser log
	if: failure() && steps.browser-tests.outcome != 'success'
	shell: bash
	run: \|
	echo "===== browser.log ====="
	cat browser.log \|\| echo "no log produced"
	echo "===== end ====="
	# The failures, not the tail: a passing test that happens to run last
	# would otherwise be all the annotation shows.
	(grep -E "^ *not ok" browser.log \| head -20; \
	echo '--- detail ---'; \
	grep -A 14 -E "^ *not ok" browser.log \| grep -E \
	"error:\|Expected\|actual:\|expected:\|AssertionError\|Timeout\|at " \| head -40) \
	\| while read -r line; do
	echo "::error::${{ matrix.browser }}: ${line}"
	done
	exit 1


	live:
	runs-on: ubuntu-latest
	needs: test
	permissions:
	contents: write
	steps:
	- uses: actions/checkout@v7
	- uses: actions/setup-node@v7
	with:
	node-version: '22.x'
	- run: npm install
	- name: Check the bundled httpbingo definition against the live service
	run: npm run check:httpbingo

	- name: Compare the Bruno starter guide against Node-RED
	run: npm run check:bruno-guide

	- name: Call real public APIs through Node-RED
	env:
	LIVE_RESULTS: tests/live-results/latest.json
	run: npm run test:live

	- name: Keep the comparison results as an artifact
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: live-comparisons
	path: tests/live-results/
	if-no-files-found: ignore

	- name: Call a corpus of public APIs through Node-RED
	env:
	LIVE_CORPUS: '1'
	run: npm run test:live

	node-floor:
	# The declared floor has to be a runtime where the editor actually works,
	# in every browser, not merely one where Node-RED boots.
	runs-on: ubuntu-latest
	needs: test
	timeout-minutes: 30
	strategy:
	fail-fast: false
	matrix:
	browser: [chromium, firefox, webkit]
	steps:
	- uses: actions/checkout@v7
	- uses: actions/setup-node@v7
	with:
	node-version: '22.x'
	- run: npm install
	- run: npx playwright install --with-deps ${{ matrix.browser }}

	- name: Check the declared node floor in ${{ matrix.browser }}
	shell: bash
	env:
	BROWSERS: ${{ matrix.browser }}
	RED_VERSION: 4.1.13
	run: npm run check:node-floor

	corpus-matrix:
	# One job per definition. A public API that is down or rate limiting should
	# cost one red square, not the whole build, and twenty of them in sequence
	# would take longer than everything else combined.
	runs-on: ubuntu-latest
	needs: test
	outputs:
	definitions: ${{ steps.read.outputs.definitions }}
	steps:
	- uses: actions/checkout@v7
	- id: read
	run: \|
	echo "definitions=$(node -e "
	const c = require('./tests/specs/corpus.json');
	process.stdout.write(JSON.stringify(c.definitions.map(d => (
	{ id: d.id, spec: d.spec }))));
	")" >> "$GITHUB_OUTPUT"

	corpus:
	runs-on: ubuntu-latest
	needs: corpus-matrix
	timeout-minutes: 20
	strategy:
	fail-fast: false
	max-parallel: 20
	matrix:
	definition: ${{ fromJson(needs.corpus-matrix.outputs.definitions) }}
	steps:
	- uses: actions/checkout@v7
	- uses: actions/setup-node@v7
	with:
	node-version: '22.x'
	- run: npm install

	- name: Call ${{ matrix.definition.spec }} for real
	shell: bash
	env:
	SPEC: ${{ matrix.definition.spec }}
	run: \|
	set -o pipefail
	node tests/test-corpus.js 2>&1 \| tee corpus.log \|\| {
	# Only the verdict lines. A dump of an eight megabyte response
	# would otherwise be the whole annotation, which is what happened
	# with the apis.guru catalogue.
	grep -E "^FAIL" corpus.log \| head -20 \| cut -c1-300 \| while read -r line; do
	echo "::error::${{ matrix.definition.id }}: ${line}"
	done
	exit 1
	}

	corpus-progress:
	# Records what the corpus jobs found and retires the definitions that
	# passed, so the next run spends its time on ones not yet proven.
	runs-on: ubuntu-latest
	# The default token is read only for contents, and pushing without this
	# fails with a bare exit code 128.
	permissions:
	contents: write
	needs: corpus
	if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main'
	steps:
	- uses: actions/checkout@v7
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	- uses: actions/setup-node@v7
	with:
	node-version: '22.x'

	- name: Record the results
	env:
	GITHUB_TOKEN: ${{ github.token }}
	run: node tests/record-progress.js "${{ github.run_id }}"

	- name: Commit the progress
	run: \|
	set -e
	git config user.name "github-actions[bot]"
	git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
	git add tests/progress.md tests/specs/corpus.json
	git diff --cached --quiet && exit 0
	git commit -m "Record corpus progress [skip ci]"
	git pull --rebase origin main \|\| true
	git push origin HEAD:main

	ollama:
	# An MLX build needs the Metal stack on Apple silicon, so the Gemma 4 run
	# goes to macOS. The hosted macOS runner only leaves Ollama around 4GB
	# though, and gemma4:e2b-mlx asks for 6GB, so that path is opt in and the
	# everyday run stays on Linux with a model that fits.
	runs-on: ${{ inputs.ollama_mlx && 'macos-latest' \|\| 'ubuntu-latest' }}
	needs: test
	# A multi gigabyte pull plus local inference, so far longer than a unit
	# test run.
	timeout-minutes: 60
	env:
	OLLAMA_MODEL: ${{ inputs.ollama_mlx && 'gemma4:e2b-mlx' \|\| 'gemma3:4b' }}
	steps:
	- uses: actions/checkout@v7
	- uses: actions/setup-node@v7
	with:
	node-version: '22.x'
	- run: npm install

	- name: Install Ollama
	run: \|
	set -e
	if [ "$RUNNER_OS" = "macOS" ]; then
	brew install --cask ollama-app \|\| brew install ollama
	else
	curl -fsSL https://ollama.com/install.sh \| sh
	fi

	- name: Start Ollama and pull the model
	run: \|
	set -e
	OLLAMA_BIN="$(command -v ollama \|\| echo /Applications/Ollama.app/Contents/Resources/ollama)"
	echo "OLLAMA_BIN=$OLLAMA_BIN" >> "$GITHUB_ENV"
	"$OLLAMA_BIN" serve > ollama.log 2>&1 &
	for i in $(seq 1 90); do
	curl -sf http://127.0.0.1:11434/api/version && break
	sleep 2
	done
	curl -sf http://127.0.0.1:11434/api/version \|\| {
	echo '::error::ollama: the server never came up'
	tail -40 ollama.log
	exit 1
	}
	"$OLLAMA_BIN" pull "$OLLAMA_MODEL"
	"$OLLAMA_BIN" list

	- name: Exercise every Ollama endpoint through Node-RED
	shell: bash
	env:
	OLLAMA_URL: http://127.0.0.1:11434
	run: \|
	set -o pipefail
	npm run test:ollama 2>&1 \| tee ollama-tests.log \|\| {
	(grep -E "^not ok\| error:" ollama-tests.log \| head -25; \
	echo '--- server log ---'; tail -30 ollama.log) \| while read -r line; do
	echo "::error::ollama: ${line}"
	done
	exit 1
	}

	samples:
	runs-on: ubuntu-latest
	needs: test
	steps:
	- uses: actions/checkout@v7
	- uses: actions/setup-node@v7
	with:
	node-version: '22.x'
	- run: npm install

	- name: Generate the JavaScript for every endpoint
	run: npm run samples

	- uses: actions/upload-artifact@v7
	id: samples-artifact
	with:
	name: generated-code
	path: samples

	- name: Report where to download the generated code
	run: \|
	echo "::notice::Generated code: ${{ steps.samples-artifact.outputs.artifact-url }}"
	echo "Generated code: ${{ steps.samples-artifact.outputs.artifact-url }}" \
	>> "$GITHUB_STEP_SUMMARY"

	coverage:
	runs-on: ubuntu-latest
	needs: test
	steps:
	- uses: actions/checkout@v7
	- uses: actions/setup-node@v7
	with:
	node-version: '22.x'
	- run: npm install

	- name: Measure coverage
	shell: bash
	run: \|
	set -o pipefail
	npm run coverage 2>&1 \| tee coverage.log \|\| {
	(grep -E "^not ok" coverage.log \| head -25; \
	echo '--- detail ---'; \
	grep -A 12 "^not ok" coverage.log \| head -80) \| while read -r line; do
	echo "::error::coverage: ${line}"
	done
	exit 1
	}
	{
	echo '## Coverage'
	echo
	echo '```'
	sed -n '/Coverage summary/,/^=\{10,\}$/p' coverage.log
	echo '```'
	} >> "$GITHUB_STEP_SUMMARY"

	- uses: actions/upload-artifact@v7
	id: coverage-artifact
	with:
	name: coverage-report
	path: coverage

	- name: Report where to download the coverage report
	run: \|
	echo "::notice::Coverage report: ${{ steps.coverage-artifact.outputs.artifact-url }}"
	echo "Coverage report: ${{ steps.coverage-artifact.outputs.artifact-url }}" \
	>> "$GITHUB_STEP_SUMMARY"

	release:
	runs-on: ubuntu-latest
	needs: [lint, test, browser, live, coverage, ollama, samples, node-floor]
	if: startsWith(github.ref, 'refs/tags/v')
	permissions:
	contents: write
	steps:
	- uses: actions/checkout@v7

	- uses: actions/setup-node@v7
	with:
	node-version: '22.x'
	registry-url: https://registry.npmjs.org

	- run: npm install

	- name: Build the tarball
	run: \|
	set -e
	npm pack
	mv node-red-flowgen-*.tgz node-red-flowgen.tgz
	tar tzf node-red-flowgen.tgz

	- name: Check the tag matches package.json
	run: \|
	set -e
	PKG_VERSION=$(node -p "require('./package.json').version")
	TAG_VERSION="${GITHUB_REF_NAME#v}"
	if [ "$PKG_VERSION" != "$TAG_VERSION" ]; then
	echo "package.json is $PKG_VERSION but the tag is $TAG_VERSION" >&2
	exit 1
	fi

	- name: Remove older releases and tags
	env:
	GH_TOKEN: ${{ github.token }}
	run: \|
	set -e
	gh release list --repo "$GITHUB_REPOSITORY" --limit 200 \
	--json tagName --jq '.[].tagName' > releases.txt \|\| true
	while read -r tag; do
	[ -z "$tag" ] && continue
	[ "$tag" = "$GITHUB_REF_NAME" ] && continue
	gh release delete "$tag" --repo "$GITHUB_REPOSITORY" --yes --cleanup-tag \|\| true
	done < releases.txt
	git ls-remote --tags origin \| awk '{print $2}' \| sed 's\|refs/tags/\|\|' \
	\| grep -v '\^{}' \| sort -u > tags.txt \|\| true
	while read -r tag; do
	[ -z "$tag" ] && continue
	[ "$tag" = "$GITHUB_REF_NAME" ] && continue
	git push origin ":refs/tags/$tag" \|\| true
	done < tags.txt

	- name: Publish the release with the tarball
	env:
	GH_TOKEN: ${{ github.token }}
	run: \|
	set -e
	gh release create "$GITHUB_REF_NAME" node-red-flowgen.tgz \
	--repo "$GITHUB_REPOSITORY" \
	--title "$GITHUB_REF_NAME" \
	--notes "" \
	\|\| gh release upload "$GITHUB_REF_NAME" node-red-flowgen.tgz \
	--repo "$GITHUB_REPOSITORY" --clobber

	- name: Publish to npm
	env:
	NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
	run: \|
	set -e
	if [ -z "$NODE_AUTH_TOKEN" ]; then
	echo "::warning::NPM_TOKEN is not set; skipping npm publish"
	exit 0
	fi
	npm publish --access public

Navigation Menu

Report the endpoints actually called, not the ones declared #250

Workflow file

Report the endpoints actually called, not the ones declared #250

Uh oh!

Workflow file for this run