diff --git a/.github/scripts/check-public.sh b/.github/scripts/check-public.sh index 9e5b182..77d8c80 100755 --- a/.github/scripts/check-public.sh +++ b/.github/scripts/check-public.sh @@ -12,7 +12,7 @@ fi # IPv4 addresses other than 192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24, # loopback and 0.0.0.0. -ips=$(grep -rnoE --exclude-dir=.git '\b([0-9]{1,3}\.){3}[0-9]{1,3}\b' . \ +ips=$(grep -rnoE --exclude-dir=.git --exclude-dir=__pycache__ '\b([0-9]{1,3}\.){3}[0-9]{1,3}\b' . \ | grep -vE ':(192\.0\.2|198\.51\.100|203\.0\.113)\.[0-9]+$|:127\.[0-9.]+$|:0\.0\.0\.0$' || true) if [ -n "$ips" ]; then echo "$ips" diff --git a/CHANGELOG.md b/CHANGELOG.md index be8bb21..232b069 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ project adheres to [Semantic Versioning](https://semver.org/). ### Added -- The connector SDK (`unified_connector.py`) and the `kv` example gitops connector. +- The connector SDK (the `unified_connectors` package) and the `kv` example gitops connector. - SDK helpers for inventory connectors: settings from the source's config as strings or JSON, and per-system credential sets. - `centreon`: Centreon monitored hosts as an Ansible inventory, with their services when asked. - `cisco_ucs`: Cisco UCS Manager blades as an Ansible inventory, servers and spares apart. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ef26d70..de3c443 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,10 +55,12 @@ vmware_vcenter/ └── fixtures/ # recorded API responses, anonymised, so tests run offline ``` -`unified_connector.py` at the root is the shared SDK. Inventory connectors read their settings with -its `settings`, `as_list`, `as_bool` and `as_int` (unified-api passes every `config:` value as a -string) and find per-system credentials with `credential_sets`. Code two connectors need goes in a -module at the root too, never in a vendor folder. +`unified_connectors/` is the shared SDK: a package every connector imports after putting the +repository root on its path, laid out like a connector otherwise (its code in +`unified_connectors.py`, a README, `tests/`). Inventory connectors read their settings with its +`settings`, `as_list`, `as_bool` and `as_int` (unified-api passes every `config:` value as a +string) and find per-system credentials with `credential_sets`. Code two connectors need goes in +the SDK too, never in a vendor folder. ## Connector rules diff --git a/centreon/centreon.py b/centreon/centreon.py index 052a5ad..35fb459 100755 --- a/centreon/centreon.py +++ b/centreon/centreon.py @@ -72,9 +72,9 @@ import urllib.request from concurrent.futures import ThreadPoolExecutor -# The SDK sits at the repository root, one folder up. +# The SDK, the unified_connectors package, sits at the repository root. sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from unified_connector import ConnectorError, as_bool, as_int, as_list, settings +from unified_connectors import ConnectorError, as_bool, as_int, as_list, settings PAGE_SIZE = 1000 # A configuration page costs about the same at 5000 rows as at 1000. diff --git a/cisco_ucs/cisco_ucs.py b/cisco_ucs/cisco_ucs.py index bb98a72..a060171 100755 --- a/cisco_ucs/cisco_ucs.py +++ b/cisco_ucs/cisco_ucs.py @@ -39,9 +39,9 @@ import xml.etree.ElementTree as ET from urllib.parse import urlparse -# The SDK sits at the repository root, one folder up. +# The SDK, the unified_connectors package, sits at the repository root. sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from unified_connector import ( +from unified_connectors import ( ConnectorError, as_bool, as_int, diff --git a/dell_avamar/dell_avamar.py b/dell_avamar/dell_avamar.py index 0b6e682..caae1f8 100755 --- a/dell_avamar/dell_avamar.py +++ b/dell_avamar/dell_avamar.py @@ -42,9 +42,9 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from urllib.parse import urlencode, urlparse -# The SDK sits at the repository root, one folder up. +# The SDK, the unified_connectors package, sits at the repository root. sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from unified_connector import ( +from unified_connectors import ( ConnectorError, as_bool, as_int, diff --git a/device42/device42.py b/device42/device42.py index 79a237a..cf021e8 100755 --- a/device42/device42.py +++ b/device42/device42.py @@ -63,9 +63,9 @@ from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed -# The SDK sits at the repository root, one folder up. +# The SDK, the unified_connectors package, sits at the repository root. sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from unified_connector import ConnectorError, as_bool, as_int, settings +from unified_connectors import ConnectorError, as_bool, as_int, settings log = logging.getLogger("device42") diff --git a/infinidat_infinibox/infinidat_infinibox.py b/infinidat_infinibox/infinidat_infinibox.py index 6ea125a..a1829ff 100755 --- a/infinidat_infinibox/infinidat_infinibox.py +++ b/infinidat_infinibox/infinidat_infinibox.py @@ -53,9 +53,9 @@ import urllib.request from urllib.parse import urlencode, urlparse -# The SDK sits at the repository root, one folder up. +# The SDK, the unified_connectors package, sits at the repository root. sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from unified_connector import ConnectorError, as_bool, as_int, settings +from unified_connectors import ConnectorError, as_bool, as_int, settings PAGE_SIZE = 1000 GROUP_PREFIX = "iboxarray_" diff --git a/kv/kv.py b/kv/kv.py index 6abf4ee..aad5c17 100755 --- a/kv/kv.py +++ b/kv/kv.py @@ -15,9 +15,9 @@ import os import sys -# The SDK sits at the repository root, one folder up. +# The SDK, the unified_connectors package, sits at the repository root. sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from unified_connector import ConnectorError, run +from unified_connectors import ConnectorError, run # What this connector manages: each type of object, the field that identifies # one, and what must exist first. Entries live in a namespace, so namespaces diff --git a/pure_flasharray/pure_flasharray.py b/pure_flasharray/pure_flasharray.py index 5b4ecdc..f951a5e 100755 --- a/pure_flasharray/pure_flasharray.py +++ b/pure_flasharray/pure_flasharray.py @@ -43,9 +43,9 @@ from concurrent.futures import ThreadPoolExecutor from urllib.parse import quote, urlparse -# The SDK sits at the repository root, one folder up. +# The SDK, the unified_connectors package, sits at the repository root. sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from unified_connector import ( +from unified_connectors import ( ConnectorError, as_bool, as_int, diff --git a/unified_connectors/README.md b/unified_connectors/README.md new file mode 100644 index 0000000..701c63a --- /dev/null +++ b/unified_connectors/README.md @@ -0,0 +1,51 @@ +# unified_connectors + +The shared SDK every connector in this repository imports. Standard library only, Python 3.11+. + +```python +# The SDK, the unified_connectors package, sits at the repository root. +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from unified_connectors import ( + ConnectorError, + as_bool, + as_list, + credential_sets, + settings, +) +``` + +## Inventory connectors (unified-api) + +| Helper | What it does | +| ------ | ------------ | +| `settings()` | The source's `config:` as a dict, from `SOURCE_CONFIG`; refuses anything but a JSON object | +| `as_list(value, default)` | A list from a comma-separated string (`"a, b"`) or a JSON list | +| `as_bool(value, default)` | `true`, `1` and `yes` in any case are true | +| `as_int(value, default)` | A whole number, or a clear error | +| `credential_sets(prefix, required, optional)` | Every complete `__` set, unified-api's `CREDENTIAL_` form first | + +unified-api passes every `config:` value as a string, so a connector reads `domains: "a,b"` with +`as_list` and `insecure: "true"` with `as_bool`. + +## Gitops connectors (unified-gitops) + +A gitops connector declares the kinds it manages and hands two functions to `run`: + +```python +run( + kinds={ + "namespace": {"key": "name"}, + "entry": {"key": "name", "depends_on": ["namespace"]}, + }, + list_objects=list_objects, # (kind, config, credentials) -> the objects that exist now + apply=apply, # (kind, op, obj, config, credentials) -> create, update or delete one +) +``` + +`run` answers `describe`, `list --kind K` and `apply --kind K`, reads `APPLICATION_CONFIG` and +the `CREDENTIAL_*` variables, refuses to write unless `UNIFIED_ALLOW_WRITE=1`, and prints JSON on +stdout with exit 0 only after a whole answer. `kv/` is a complete example. + +## Errors + +Raise `ConnectorError("...")` for a failure worth one clean line on stderr; the connector exits 1. diff --git a/unified_connectors/__init__.py b/unified_connectors/__init__.py new file mode 100644 index 0000000..63bd80c --- /dev/null +++ b/unified_connectors/__init__.py @@ -0,0 +1,23 @@ +"""The shared SDK every connector imports; the code is in unified_connectors.py.""" + +from .unified_connectors import ( + PROTOCOL, + ConnectorError, + as_bool, + as_int, + as_list, + credential_sets, + run, + settings, +) + +__all__ = [ + "PROTOCOL", + "ConnectorError", + "as_bool", + "as_int", + "as_list", + "credential_sets", + "run", + "settings", +] diff --git a/tests/test_unified_connector.py b/unified_connectors/tests/test_unified_connectors.py similarity index 94% rename from tests/test_unified_connector.py rename to unified_connectors/tests/test_unified_connectors.py index 8820c25..c57b42d 100644 --- a/tests/test_unified_connector.py +++ b/unified_connectors/tests/test_unified_connectors.py @@ -3,9 +3,9 @@ import pytest -ROOT = Path(__file__).resolve().parent.parent +FOLDER = Path(__file__).resolve().parent.parent spec = importlib.util.spec_from_file_location( - "unified_connector", ROOT / "unified_connector.py" + "unified_connectors", FOLDER / "unified_connectors.py" ) sdk = importlib.util.module_from_spec(spec) spec.loader.exec_module(sdk) diff --git a/unified_connector.py b/unified_connectors/unified_connectors.py similarity index 99% rename from unified_connector.py rename to unified_connectors/unified_connectors.py index 787bb31..4f1efa2 100644 --- a/unified_connector.py +++ b/unified_connectors/unified_connectors.py @@ -3,7 +3,7 @@ A gitops connector describes the kinds of object it manages, writes two functions and hands them to run: - from unified_connector import ConnectorError, run + from unified_connectors import ConnectorError, run def list_objects(kind, config, credentials): return [{"name": "a"}] # what exists now diff --git a/vmware_vcenter/vmware_vcenter.py b/vmware_vcenter/vmware_vcenter.py index c2839c5..064dd2a 100755 --- a/vmware_vcenter/vmware_vcenter.py +++ b/vmware_vcenter/vmware_vcenter.py @@ -52,9 +52,9 @@ from pyVim.connect import Disconnect, SmartConnect from pyVmomi import vim, vmodl -# The SDK sits at the repository root, one folder up. +# The SDK, the unified_connectors package, sits at the repository root. sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from unified_connector import ( +from unified_connectors import ( ConnectorError, as_bool, as_int,