feat(argo-workflows): Allow enrollment to prepare nodes for monitoring after inspection

.github/workflows/lint-workflow-templates.yaml

@@ -40,6 +40,28 @@ jobs:
           kubectl create namespace argo-events
           kubectl -n argo-events apply -f workflows/argo-events/workflowtemplates/
 
+      - name: Deploy lint stubs for externally-provided templateRefs
+        run: |
+          # Some templates reference WorkflowTemplates that live outside this
+          # repo (site-specific extensions installed from undercloud-deploy,
+          # opted into per site). `argo lint` resolves templateRef names against
+          # the cluster, so those references fail to resolve here unless a
+          # placeholder is present. Any stub placed in the lint-stubs directory
+          # is deployed so the reference resolves; the stub bodies are
+          # deliberately minimal and are never shipped. Add a new external
+          # reference by dropping a file in that directory -- no change here.
+          stub_dir="workflows/argo-events/_lint-stubs"
+          shopt -s nullglob
+          stubs=("$stub_dir"/*.yaml)
+          if [ ${#stubs[@]} -gt 0 ]; then
+            for stub in "${stubs[@]}"; do
+              echo "Deploying lint stub: $stub"
+              kubectl -n argo-events apply -f "$stub"
+            done
+          else
+            echo "No lint stubs to deploy."
+          fi
+
       - name: Argo Lint Workflow Template
         run: |
           set +e

charts/argocd-understack/templates/application-obm-monitoring.yaml

@@ -0,0 +1,31 @@
+{{- if eq (include "understack.isEnabled" (list $.Values.site "obm_monitoring")) "true" }}
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: {{ printf "%s-%s" $.Release.Name "obm-monitoring" }}
+  finalizers:
+  - resources-finalizer.argocd.argoproj.io
+  annotations:
+    argocd.argoproj.io/compare-options: ServerSideDiff=true,IncludeMutationWebhook=true
+{{- include "understack.appLabelsBlock" $ | nindent 2 }}
+spec:
+  destination:
+    # argo-events is owned by the argo_events Application, so no CreateNamespace.
+    namespace: argo-events
+    server: {{ $.Values.cluster_server }}
+  project: understack
+  sources:
+  - path: {{ include "understack.deploy_path" $ }}/obm-monitoring
+    repoURL: {{ include "understack.deploy_url" $ }}
+    targetRevision: {{ include "understack.deploy_ref" $ }}
+  syncPolicy:
+    automated:
+      enabled: true
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - ServerSideApply=true
+    - RespectIgnoreDifferences=true
+    - ApplyOutOfSyncOnly=true
+{{- end }}

charts/argocd-understack/values.yaml

@@ -583,6 +583,17 @@ site:
     # @default -- false
     enabled: false
 
+  # -- Out-of-band monitoring WorkflowTemplates. Deploys the site's
+  #    `<cluster>/obm-monitoring` Kustomize overlay from the deployment
+  #    repository into the existing `argo-events` namespace. The overlay and its
+  #    application code/images live in the deployment repository; this chart only
+  #    renders the ArgoCD Application pointer. A site opts in by setting this to
+  #    true in its deploy.yaml.
+  obm_monitoring:
+    # -- Enable/disable deploying OBM monitoring WorkflowTemplates
+    # @default -- false
+    enabled: false
+
   # -- Nautobot Celery workers (site-level, connects to global Nautobot)
   nautobot_worker:
     # -- Enable/disable deploying Nautobot workers at the site level

docs/deploy-guide/components/index.md

@@ -92,6 +92,7 @@ enablement defaults, validation, and troubleshooting notes.
 | [nautobot-api-tokens](./nautobot-api-tokens.md) | global |
 | [nautobot-job-queues](./nautobot-job-queues.md) | global |
 | [nautobotop](./nautobotop.md) | global |
+| [obm-monitoring](./obm-monitoring.md) | site |
 | [openebs](./openebs.md) | global, site |
 | [openstack](./openstack.md) | site |
 | [openstack-exporter](./openstack-exporter.md) | site |

docs/deploy-guide/components/obm-monitoring.md

@@ -0,0 +1,59 @@
+---
+source_text: ArgoCD renders only the sources declared directly in the Application
+  template.
+deploy_overrides:
+  helm:
+    mode: none
+  kustomize:
+    mode: only_source
+---
+
+# obm-monitoring
+
+Out-of-band monitoring WorkflowTemplates, deployed from a deploy-repo Kustomize
+overlay into the existing `argo-events` namespace.
+
+## Deployment Scope
+
+- Cluster scope: site
+- Values key: `site.obm_monitoring`
+- ArgoCD Application template: `charts/argocd-understack/templates/application-obm-monitoring.yaml`
+
+## How ArgoCD Builds It
+
+{{ component_argocd_builds() }}
+
+## How to Enable
+
+Enable this component under the scope that matches your deployment model:
+
+```yaml title="$CLUSTER_NAME/deploy.yaml"
+site:
+  obm_monitoring:
+    enabled: true
+```
+
+## Deployment Repo Content
+
+{{ secrets_disclaimer }}
+
+The Application deploys the site overlay at
+`$CLUSTER_NAME/obm-monitoring/` from the deployment repository into the
+`argo-events` namespace. That namespace, its `workflow` service account, and the
+shared executor permissions are owned by the `argo-events` component, so keep
+`site.argo_events` and `site.argo_workflows` enabled alongside this one.
+
+Required or commonly required items:
+
+- `kustomization.yaml`: Set `namespace: argo-events` and include the shared
+  WorkflowTemplates base plus any site-specific resources.
+- API ExternalSecret: Provide the monitoring platform API credentials the
+  WorkflowTemplates read at submission time.
+
+Optional additions:
+
+- `Additional overlays or patches`: Pin container images or add site-specific
+  parameters, RBAC, or Secrets as the automation grows.
+
+Installing the WorkflowTemplates does not submit a workflow; submission remains
+an explicit, operator-driven step.

properdocs.yml

@@ -285,6 +285,7 @@ nav:
       - deploy-guide/components/nautobot.md
       - deploy-guide/components/nautobotop.md
       - deploy-guide/components/nautobot-worker.md
+      - deploy-guide/components/obm-monitoring.md
       - deploy-guide/components/neutron.md
       - deploy-guide/components/nova.md
       - deploy-guide/components/octavia.md

python/understack-workflows/README.md

@@ -8,3 +8,22 @@ use inside of Argo Workflows.
 This script takes an OpenStack Project ID and ensures the proper
 operation happens against the Nautobot Tenants. Operations include
 create, update and delete.
+
+
+## enroll-server post-inspection runbooks
+
+Optionally pass an ordered JSON array of runbook names or UUIDs:
+
+```sh
+enroll-server --ip-address 192.0.2.10 \
+  --post-inspection-runbooks '["first-runbook", "second-runbook"]'
+```
+
+For Argo, use the `enroll-server` WorkflowTemplate parameter
+`post_inspection_runbooks` with the same JSON array. It defaults to `[]`.
+
+Runbooks execute sequentially after agent inspection and final BIOS
+configuration, before RAID configuration, firmware updates, and `provide`.
+Each requires at least one matching node/runbook trait (Ironic API 1.112 or
+newer), uses the runbook's ramdisk setting, and waits for `manageable`.
+A failed lookup, trait check, or cleaning operation stops enrollment.

python/understack-workflows/pyproject.toml

@@ -22,7 +22,7 @@ classifiers = [
 dependencies = [
     "pydantic>=2,<3",
     "pynautobot>=3,<4",
-    "python-ironicclient>=6,<7",
+    "python-ironicclient>=6.3.0,<7",
     "sushy>=5.3.0,<6",
     "kubernetes==33.1.0",
     "netapp-ontap==9.17.1.0",

python/understack-workflows/tests/test_enroll_server.py

@@ -1,9 +1,11 @@
+import argparse
 import secrets
 from types import SimpleNamespace
 from typing import cast
 from unittest.mock import MagicMock
 from unittest.mock import call
 
+import pytest
 from ironicclient.common.apiclient import exceptions as ironic_exceptions
 from ironicclient.v1.node import Node
 
@@ -596,3 +598,198 @@ def test_get_lldp_connected_interfaces_absent_from_lldp():
     result = get_lldp_connected_interfaces(interfaces, parsed_lldp={})
 
     assert result == []
+
+
+@pytest.mark.parametrize("runbooks", [None, [], ["first", "second"]])
+def test_post_inspection_runbook_order(mocker, runbooks):
+    node = SimpleNamespace(uuid="node-123")
+    bmc = make_bmc(mocker)
+    mocker.patch.object(enroll_server, "bmc_for_ip_address", return_value=bmc)
+    mocker.patch.object(
+        enroll_server, "initialize_bmc", return_value=make_device_info()
+    )
+    mocker.patch.object(
+        enroll_server.ironic_node, "create_or_update", return_value=node
+    )
+    mocker.patch.object(enroll_server.ironic_node, "inspect_out_of_band")
+    mocker.patch.object(
+        enroll_server.ironic_node, "get_node_inventory", return_value={}
+    )
+    mocker.patch.object(
+        enroll_server.ironic_node, "pxe_enabled_bios_name", return_value="NIC.1"
+    )
+    events = MagicMock()
+    for name in ("agent_inspection", "update_dell_bios_settings", "configure_raid"):
+        events.attach_mock(mocker.patch.object(enroll_server, name), name)
+    for name in ("run_post_inspection_runbook", "apply_firmware_updates", "transition"):
+        events.attach_mock(mocker.patch.object(enroll_server.ironic_node, name), name)
+
+    result = enroll_server.enroll(
+        ip_address=bmc.ip_address,
+        firmware_update=True,
+        raid_configure=True,
+        old_password=None,
+        post_inspection_runbooks=runbooks,
+    )
+
+    expected = [
+        call.update_dell_bios_settings(bmc, pxe_interface="NIC.1"),
+        call.agent_inspection(node, virtual_media=False),
+        call.update_dell_bios_settings(bmc, pxe_interface="NIC.1"),
+    ]
+    expected.extend(
+        call.run_post_inspection_runbook(node, runbook) for runbook in runbooks or []
+    )
+    expected.extend(
+        [
+            call.configure_raid(node, {}),
+            call.apply_firmware_updates(node),
+            call.transition(node, target_state="provide", expected_state="available"),
+        ]
+    )
+    assert events.mock_calls == expected
+    assert result == node.uuid
+
+
+def test_post_inspection_runbook_uses_uuid_and_waits(mocker):
+    client = mocker.patch.object(enroll_server.ironic_node, "IronicClient").return_value
+    client.get_runbook.return_value = SimpleNamespace(
+        uuid="runbook-123", traits=["CUSTOM_DELL", "CUSTOM_OTHER"], disable_ramdisk=True
+    )
+    client.get_node_traits.return_value = ["CUSTOM_DELL"]
+    node = cast(Node, SimpleNamespace(uuid="node-123"))
+
+    enroll_server.ironic_node.run_post_inspection_runbook(node, "post-inspection")
+
+    client.get_runbook.assert_called_once_with("post-inspection")
+    client.get_node_traits.assert_called_once_with(node.uuid)
+    client.set_node_provision_state.assert_called_once_with(
+        node.uuid,
+        "clean",
+        clean_steps=None,
+        runbook="runbook-123",
+        disable_ramdisk=None,
+    )
+    client.wait_for_node_provision_state.assert_called_once_with(
+        node.uuid,
+        "manageable",
+        timeout=enroll_server.ironic_node.NODE_STATE_TIMEOUT_SECS,
+    )
+
+
+@pytest.mark.parametrize("traits", [[], ["CUSTOM_OTHER"], None])
+def test_post_inspection_runbook_rejects_ineligible_node(mocker, traits):
+    client = mocker.patch.object(enroll_server.ironic_node, "IronicClient").return_value
+    client.get_runbook.return_value = SimpleNamespace(uuid="runbook-123", traits=traits)
+    client.get_node_traits.return_value = ["CUSTOM_DELL"]
+    node = cast(Node, SimpleNamespace(uuid="node-123"))
+
+    with pytest.raises(ValueError, match=r"traits metadata|matching node trait"):
+        enroll_server.ironic_node.run_post_inspection_runbook(node, "post-inspection")
+
+    client.set_node_provision_state.assert_not_called()
+
+
+def test_post_inspection_runbook_failure_propagates(mocker):
+    client = mocker.patch.object(enroll_server.ironic_node, "IronicClient").return_value
+    client.get_runbook.return_value = SimpleNamespace(
+        uuid="runbook-123", traits=["CUSTOM_DELL"]
+    )
+    client.get_node_traits.return_value = ["CUSTOM_DELL"]
+    client.wait_for_node_provision_state.side_effect = RuntimeError("clean failed")
+    node = cast(Node, SimpleNamespace(uuid="node-123"))
+
+    with pytest.raises(RuntimeError, match="clean failed"):
+        enroll_server.ironic_node.run_post_inspection_runbook(node, "post-inspection")
+
+
+def test_post_inspection_runbook_cli(mocker):
+    mocker.patch(
+        "sys.argv",
+        [
+            "enroll-server",
+            "--ip-address",
+            "10.0.0.10",
+            "--post-inspection-runbooks",
+            '["first", "second"]',
+        ],
+    )
+    mocker.patch.object(enroll_server.helpers, "setup_logger")
+    enroll = mocker.patch.object(enroll_server, "enroll")
+
+    enroll_server.main()
+
+    assert enroll.call_args.kwargs["post_inspection_runbooks"] == ["first", "second"]
+
+
+@pytest.mark.parametrize(
+    "value", ["invalid", '"first"', "{}", "null", "[null]", "[1]", '[""]', '[" "]']
+)
+def test_post_inspection_runbooks_reject_invalid_cli(value):
+    with pytest.raises(argparse.ArgumentTypeError, match="JSON array"):
+        enroll_server.parse_runbooks(value)
+
+
+def test_post_inspection_runbooks_stop_on_failure(mocker):
+    node = SimpleNamespace(uuid="node-123")
+    mocker.patch.object(enroll_server, "bmc_for_ip_address")
+    mocker.patch.object(
+        enroll_server, "initialize_bmc", return_value=make_device_info()
+    )
+    mocker.patch.object(
+        enroll_server.ironic_node, "create_or_update", return_value=node
+    )
+    mocker.patch.object(enroll_server.ironic_node, "inspect_out_of_band")
+    mocker.patch.object(
+        enroll_server.ironic_node, "get_node_inventory", return_value={}
+    )
+    mocker.patch.object(
+        enroll_server.ironic_node, "pxe_enabled_bios_name", return_value="NIC.1"
+    )
+    mocker.patch.object(enroll_server, "agent_inspection")
+    mocker.patch.object(enroll_server, "update_dell_bios_settings")
+    runbook = mocker.patch.object(
+        enroll_server.ironic_node,
+        "run_post_inspection_runbook",
+        side_effect=RuntimeError("clean failed"),
+    )
+    raid = mocker.patch.object(enroll_server, "configure_raid")
+    firmware = mocker.patch.object(enroll_server.ironic_node, "apply_firmware_updates")
+    transition = mocker.patch.object(enroll_server.ironic_node, "transition")
+
+    with pytest.raises(RuntimeError, match="clean failed"):
+        enroll_server.enroll(
+            ip_address="10.0.0.10",
+            firmware_update=True,
+            raid_configure=True,
+            old_password=None,
+            post_inspection_runbooks=["first", "second"],
+        )
+
+    runbook.assert_called_once_with(node, "first")
+    raid.assert_not_called()
+    firmware.assert_not_called()
+    transition.assert_not_called()
+
+
+def test_main_emits_node_uuid_only_after_success(mocker, tmp_path):
+    output = tmp_path / "node-uuid"
+    mocker.patch(
+        "sys.argv",
+        [
+            "enroll-server",
+            "--ip-address",
+            "192.0.2.1",
+            "--node-uuid-output",
+            str(output),
+        ],
+    )
+    mocker.patch.object(enroll_server.helpers, "setup_logger")
+    enroll = mocker.patch.object(enroll_server, "enroll", return_value="node-123")
+    enroll_server.main()
+    assert output.read_text() == "node-123\n"
+    output.unlink()
+    enroll.side_effect = RuntimeError("enrollment failed")
+    with pytest.raises(RuntimeError, match="enrollment failed"):
+        enroll_server.main()
+    assert not output.exists()