diff --git a/docs/how-to/dashboards_and_quality_gates.rst b/docs/how-to/dashboards_and_quality_gates.rst index 21f828ec3..6f9dba5af 100644 --- a/docs/how-to/dashboards_and_quality_gates.rst +++ b/docs/how-to/dashboards_and_quality_gates.rst @@ -76,6 +76,21 @@ The documentation build writes ``metrics.json`` via ``score_metrics``, and the ` The dashboard charts and the CI gate both use the same computed metrics. +``metrics.json`` Content +~~~~~~~~~~~~~~~~~~~~~~~~ + +The exported file contains a versioned set of fields: + +- ``schema_version`` – schema version identifier. +- ``generated_by`` – identifier of the producer that generated this file. +- ``overall_metrics`` – requirement coverage aggregated over all requirement types. +- ``metrics_by_type`` – the same coverage numbers per requirement type. +- ``tests`` – testcase linkage statistics, including broken test references. +- ``verification_metrics`` – per-component verification status and coverage metrics + for use by verification reports. Contains status distribution (valid, valid_inspected, + invalid) and test coverage (fully_covered, partially_covered, not_covered) for + both requirements and architecture elements, organized by component ID. + Inputs for Linkage Metrics -------------------------- diff --git a/scripts_bazel/traceability_metrics_schema.json b/scripts_bazel/traceability_metrics_schema.json index dea866a8c..5d7d6df74 100644 --- a/scripts_bazel/traceability_metrics_schema.json +++ b/scripts_bazel/traceability_metrics_schema.json @@ -40,6 +40,17 @@ }, "tests": { "$ref": "#/$defs/testMetrics" + }, + "verification_metrics": { + "type": "object", + "description": "Per-component verification status and coverage metrics for verification reports.", + "additionalProperties": { + "$ref": "#/$defs/componentVerificationMetrics" + }, + "propertyNames": { + "type": "string", + "minLength": 1 + } } }, "$defs": { @@ -134,6 +145,67 @@ "type": "number", "minimum": 0, "maximum": 100 + }, + "componentVerificationMetrics": { + "type": "object", + "description": "Verification metrics for a single component.", + "additionalProperties": false, + "properties": { + "requirements": { + "$ref": "#/$defs/statusAndCoverageMetrics" + }, + "architecture": { + "$ref": "#/$defs/statusAndCoverageMetrics" + } + } + }, + "statusAndCoverageMetrics": { + "type": "object", + "description": "Status distribution and test coverage metrics.", + "additionalProperties": false, + "properties": { + "status": { + "type": "object", + "description": "Count by status value.", + "additionalProperties": false, + "properties": { + "valid": { + "type": "integer", + "minimum": 0 + }, + "valid_inspected": { + "type": "integer", + "minimum": 0 + }, + "invalid": { + "type": "integer", + "minimum": 0 + } + } + }, + "coverage": { + "type": "object", + "description": "Test coverage by verification links.", + "additionalProperties": false, + "properties": { + "fully_covered": { + "type": "integer", + "minimum": 0, + "description": "Items with fully_verifies_back link." + }, + "partially_covered": { + "type": "integer", + "minimum": 0, + "description": "Items with only partially_verifies_back link." + }, + "not_covered": { + "type": "integer", + "minimum": 0, + "description": "Items with no verification links." + } + } + } + } } } } diff --git a/src/extensions/score_metrics/BUILD b/src/extensions/score_metrics/BUILD index c909fe7ab..c08e1ef20 100644 --- a/src/extensions/score_metrics/BUILD +++ b/src/extensions/score_metrics/BUILD @@ -39,6 +39,7 @@ py_library( # TODO: Figure out if all requirements are needed or if we can break it down a bit deps = all_requirements + [ "@score_docs_as_code//src/helper_lib", + "@score_docs_as_code//src/extensions/score_sphinx_needs_templates", ], ) diff --git a/src/extensions/score_metrics/sphinx_filters.py b/src/extensions/score_metrics/sphinx_filters.py index f120157a9..66e938ca4 100644 --- a/src/extensions/score_metrics/sphinx_filters.py +++ b/src/extensions/score_metrics/sphinx_filters.py @@ -42,8 +42,35 @@ def func(needs: list[NeedItem], results: list[int], **kwargs) -> None: ... from typing import Any from score_metrics.traceability_metrics import CALCULATED_METRICS + +# Import the public version-scoping helper for filtering by valid_from +from score_sphinx_needs_templates import req_in_report_version from sphinx_needs.need_item import NeedItem +# Import shared verification metrics helpers +from src.helper_lib.verification_metrics import ( + count_coverage_filtered, + count_status_filtered, +) + + +def _arg(kwargs: dict[str, Any], name: str, default: str = "") -> str: + """Return a filter-func argument without surrounding whitespace and quotes. + + sphinx-needs splits the argument string on ``,`` and passes the parts + verbatim, so ``f("a", "b")`` arrives as ``'"a"'`` and ``' "b"'``. + """ + return str(kwargs.get(name, default)).strip().strip("\"'") + + +def _version_arg(kwargs: dict[str, Any]) -> str | None: + """Return ``arg3`` as a version, with ``_`` turned back into ``.``. + + sphinx-needs resolves the module with ``rsplit(".", 1)`` over the whole + call, so a version such as ``v1.0`` must be passed as ``v1_0``. + """ + return _arg(kwargs, "arg3").replace("_", ".") or None + def _matches_source_selector(need: NeedItem, selector: str) -> bool: """Return whether a need matches a source selector. @@ -247,3 +274,69 @@ def get_non_overlapping_link_metrics( only_code = with_code - fully results.extend([remaining, only_test, only_code, fully]) + + +def get_component_status_metrics( + needs: list[Any], results: list[int], **kwargs: Any +) -> None: + """Get status distribution for a component's requirements or architecture. + + Arguments are passed via ``arg1`` .. ``arg4``: + - ``arg1`` – component_id: The component ID (e.g., "comp__my_component") + - ``arg2`` – category: Either "requirements" or "architecture" + - ``arg3`` – version: Optional, see :func:`_version_arg` + - ``arg4`` – safety: Optional safety class, e.g. ``ASIL_B`` or ``QM`` + + Returns ``[valid, valid_inspected, invalid]`` without a safety class, + ``[valid_inspected, valid, invalid]`` for an ASIL class and + ``[valid, invalid]`` for ``QM``, where ``valid_inspected`` counts as + ``valid`` because QM Needs are not formally inspected. + """ + results.clear() + safety = _arg(kwargs, "arg4") or None + status = count_status_filtered( + needs, + _arg(kwargs, "arg1"), + _arg(kwargs, "arg2", "requirements"), + _version_arg(kwargs), + req_in_report_version, + safety, + ) + + if safety == "QM": + results.extend([status["valid"] + status["valid_inspected"], status["invalid"]]) + elif safety: + results.extend([status["valid_inspected"], status["valid"], status["invalid"]]) + else: + results.extend([status["valid"], status["valid_inspected"], status["invalid"]]) + + +def get_component_coverage_metrics( + needs: list[Any], results: list[int], **kwargs: Any +) -> None: + """Get verification coverage for a component's requirements or architecture. + + Uses the verification coverage axis (fully_verifies_back / partially_verifies_back), + not the link coverage axis (testlink / source_code_link). + + Arguments are the same as for :func:`get_component_status_metrics`. + + Returns [fully_covered, partially_covered, not_covered]. + """ + results.clear() + coverage = count_coverage_filtered( + needs, + _arg(kwargs, "arg1"), + _arg(kwargs, "arg2", "requirements"), + _version_arg(kwargs), + req_in_report_version, + _arg(kwargs, "arg4") or None, + ) + + results.extend( + [ + coverage["fully_covered"], + coverage["partially_covered"], + coverage["not_covered"], + ] + ) diff --git a/src/extensions/score_metrics/tests/test_sphinx_filters.py b/src/extensions/score_metrics/tests/test_sphinx_filters.py index e42fdf845..5e1649eb7 100644 --- a/src/extensions/score_metrics/tests/test_sphinx_filters.py +++ b/src/extensions/score_metrics/tests/test_sphinx_filters.py @@ -11,18 +11,21 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -from typing import cast +from typing import Any, cast import pytest # noqa import score_metrics.sphinx_filters as sphinx_filters +import score_sphinx_needs_templates as sphinx_needs_templates from score_metrics.sphinx_filters import ( generic_pie_items_by_tag, generic_pie_linked_items, ) from score_pytest.attribute_plugin import add_test_properties +from sphinx_needs.exceptions import NeedsInvalidFilter from sphinx_needs.need_item import NeedItem +from sphinx_needs.utils import check_and_get_external_filter_func @add_test_properties( @@ -252,3 +255,182 @@ def test_get_metrics_with_custom_type_total_considered_empty_kwargs_raises_index with pytest.raises(IndexError): sphinx_filters.get_metrics_with_first_value_total(needs=[], results=results) + + +_PROPS: dict[str, Any] = { + "partially_verifies": [], + "test_type": "interface-test", + "derivation_technique": "explorative-testing", +} + + +def _need(need_id: str, need_type: str, link: str, **fields: Any) -> NeedItem: + need: dict[str, Any] = { + "id": need_id, + "type": need_type, + link: ["comp__x"], + "fully_verifies_back": [], + "partially_verifies_back": [], + **fields, + } + return cast(NeedItem, need) + + +_COMPONENT_NEEDS = [ + _need( + "r1", + "comp_req", + "satisfied_by", + safety="ASIL_B", + status="valid_inspected", + valid_from="v1.0", + fully_verifies_back=["tc__a"], + ), + _need( + "r2", + "comp_req", + "satisfied_by", + safety="ASIL_B", + status="valid", + valid_from="v1.0", + partially_verifies_back=["tc__b"], + ), + _need( + "r3", + "comp_req", + "satisfied_by", + safety="QM", + status="valid_inspected", + valid_from="v1.0", + ), + _need( + "r4", + "comp_req", + "satisfied_by", + safety="QM", + status="invalid", + valid_from="v2.0", + ), + _need( + "a1", + "comp_arc_sta", + "belongs_to", + safety="ASIL_B", + status="valid", + partially_verifies_back=["tc__c"], + ), + _need("a2", "comp_arc_dyn", "belongs_to", safety="QM", status="valid"), +] + + +def _call_like_needpie(signature: str, needs: list[NeedItem]) -> list[int]: + """Parse and call a filter-func exactly the way ``needpie`` does.""" + if not signature.startswith("score_metrics."): + signature = f"score_metrics.sphinx_filters.{signature}" + ff = check_and_get_external_filter_func(signature) + assert ff is not None + args = ff.args.split(",") if ff.args else [] + results: list[int] = [] + ff.func( + needs=cast(Any, needs), + results=results, + **{f"arg{index + 1}": arg for index, arg in enumerate(args)}, + ) + return results + + +@add_test_properties(**_PROPS) +def test_component_metrics_accept_quoted_arguments() -> None: + """Quoted, space-separated arguments as written in a template are decoded.""" + status = 'get_component_status_metrics("comp__x", "requirements", "", "")' + coverage = 'get_component_coverage_metrics("comp__x", "architecture")' + + assert _call_like_needpie(status, _COMPONENT_NEEDS) == [1, 2, 1] + assert _call_like_needpie(coverage, _COMPONENT_NEEDS) == [0, 1, 1] + + +@add_test_properties(**_PROPS) +def test_component_metrics_split_by_safety_class() -> None: + """ASIL lists valid_inspected first; QM counts valid_inspected as valid.""" + asil = "get_component_status_metrics(comp__x, requirements, , ASIL_B)" + qm = "get_component_status_metrics(comp__x, requirements, , QM)" + qm_coverage = "get_component_coverage_metrics(comp__x, requirements, , QM)" + + assert _call_like_needpie(asil, _COMPONENT_NEEDS) == [1, 1, 0] + assert _call_like_needpie(qm, _COMPONENT_NEEDS) == [1, 1] + assert _call_like_needpie(qm_coverage, _COMPONENT_NEEDS) == [0, 0, 2] + + +@add_test_properties(**_PROPS) +def test_component_metrics_version_scopes_requirements_only() -> None: + """``v1_0`` means v1.0; it excludes later requirements, never architecture.""" + reqs = "get_component_status_metrics(comp__x, requirements, v1_0, QM)" + arcs = "get_component_status_metrics(comp__x, architecture, v1_0, QM)" + + assert _call_like_needpie(reqs, _COMPONENT_NEEDS) == [1, 0] + assert _call_like_needpie(arcs, _COMPONENT_NEEDS) == [1, 0] + + +@add_test_properties(**_PROPS) +def test_filter_func_arguments_must_not_contain_a_dot() -> None: + """sphinx-needs splits the call on its last dot, so ``v1.0`` cannot be passed.""" + with pytest.raises(NeedsInvalidFilter): + _call_like_needpie( + "get_component_status_metrics(comp__x, requirements, v1.0, QM)", + _COMPONENT_NEEDS, + ) + + +_DEFAULTS: dict[str, list[str]] = {"satisfied_by": [], "belongs_to": []} + + +def _pie_counts(rst: str, needs: list[NeedItem]) -> list[list[int]]: + """Evaluate every needpie in a ``safety_pie_row`` block like sphinx-needs.""" + pies: list[list[int]] = [] + for block in rst.split(".. needpie::")[1:]: + body = [line.strip() for line in block.splitlines()[1:]] + func = next((line for line in body if line.startswith(":filter-func:")), None) + if func: + pies.append(_call_like_needpie(func.split(" ", 1)[1], needs)) + continue + filters = [line for line in body if line and not line.startswith(":")] + filters = ( + filters[: filters.index(".. grid-item::")] + if ".. grid-item::" in filters + else filters + ) + pies.append( + [sum(bool(eval(f, {}, {**_DEFAULTS, **n})) for n in needs) for f in filters] # noqa: S307 + ) + return pies + + +@add_test_properties(**_PROPS) +def test_safety_pie_row_filter_funcs_match_inline_filters() -> None: + """The central filter functions count exactly what the inline filters count.""" + row = vars(sphinx_needs_templates)["_safety_pie_row_callable"] + in_scope = ["r1", "r2", "r3"] + req_base = f'type == "comp_req" and "comp__x" in satisfied_by and id in {in_scope}' + arc_base = 'type in ["comp_arc_sta", "comp_arc_dyn"] and "comp__x" in belongs_to' + + for safety in ("ASIL_B", "QM"): + for base, category, version in ( + (req_base, "requirements", "v1.0"), + (arc_base, "architecture", None), + ): + inline = row("C", "S", base, safety, safety, 1) + central = row( + "C", + "S", + "", + safety, + safety, + 1, + component="comp__x", + category=category, + version=version, + ) + assert ":filter-func:" in central + assert _pie_counts(central, _COMPONENT_NEEDS) == _pie_counts( + inline, _COMPONENT_NEEDS + ) diff --git a/src/extensions/score_metrics/traceability_metrics.py b/src/extensions/score_metrics/traceability_metrics.py index 07e40e0b0..3dc5f0c0d 100644 --- a/src/extensions/score_metrics/traceability_metrics.py +++ b/src/extensions/score_metrics/traceability_metrics.py @@ -28,6 +28,13 @@ from sphinx_needs.data import NeedsView, SphinxNeedsData from sphinx_needs.need_item import NeedItem +# Import shared verification metrics helpers +from src.helper_lib.verification_metrics import ( + count_coverage_filtered, + count_status_filtered, + is_need_in_component, +) + CALCULATED_METRICS: dict[str, Any] = {} @@ -119,6 +126,115 @@ def calculate_test_metrics( } +def calculate_status_and_coverage_metrics( + needs_list: list[NeedItem], +) -> dict[str, dict[str, int]]: + """Calculate status distribution and test coverage for a list of needs. + + Version scoping is done by the callers via ``req_in_report_version``. + + Returns a dict with: + - status: {valid: int, valid_inspected: int, invalid: int} + - coverage: {fully_covered: int, partially_covered: int, not_covered: int} + """ + status_counts = {"valid": 0, "valid_inspected": 0, "invalid": 0} + coverage_counts = {"fully_covered": 0, "partially_covered": 0, "not_covered": 0} + + for need in needs_list: + # Status distribution + status = str(need.get("status", "")) + if status in status_counts: + status_counts[status] += 1 + + # Test coverage (based on verifies_back links) + has_fully = bool(need.get("fully_verifies_back")) + has_partially = bool(need.get("partially_verifies_back")) + + if has_fully: + coverage_counts["fully_covered"] += 1 + elif has_partially: + coverage_counts["partially_covered"] += 1 + else: + coverage_counts["not_covered"] += 1 + + return {"status": status_counts, "coverage": coverage_counts} + + +def calculate_verification_metrics( # noqa: C901 + all_needs: NeedsView, report_version: str | None = None +) -> dict[str, dict[str, dict[str, dict[str, int]]]]: + """Calculate per-component verification metrics for verification reports. + + Args: + all_needs: All needs view from Sphinx build. + report_version: Optional version to filter requirements by valid_from. + Note: Architecture elements are NOT version-filtered as they have + no valid_from or derived_from attributes. + + Looks for needs that have belongs_to links to components and calculates + status/coverage metrics for requirements and architecture elements. + """ + verification_metrics: dict[str, dict[str, dict[str, dict[str, int]]]] = {} + + # Find all components by looking at Needs with type "comp" + component_ids: set[str] = { + n["id"] for n in all_needs.values() if n.get("type") == "comp" + } + + # Simple version filter function for use with shared helpers + def version_filter_func(need: NeedItem, version: str) -> bool: + """Simple passthrough to req_in_report_version for shared helpers.""" + from score_sphinx_needs_templates import req_in_report_version + + return req_in_report_version(need, version) + + # Calculate metrics per component + for component_id in component_ids: + # Requirements (comp_req with satisfied_by pointing to this component) + # Version filtering is applied to requirements + reqs = [ + n + for n in all_needs.values() + if is_need_in_component(n, component_id, "requirements") + ] + + if report_version: + req_metrics = { + "status": count_status_filtered( + reqs, + component_id, + "requirements", + report_version, + version_filter_func, + ), + "coverage": count_coverage_filtered( + reqs, + component_id, + "requirements", + report_version, + version_filter_func, + ), + } + else: + req_metrics = calculate_status_and_coverage_metrics(reqs) + + # Architecture elements (comp_arc_sta, comp_arc_dyn with belongs_to this component) + # Version filtering is NOT applied to architecture elements + arch = [ + n + for n in all_needs.values() + if is_need_in_component(n, component_id, "architecture") + ] + arch_metrics = calculate_status_and_coverage_metrics(arch) + + verification_metrics[component_id] = { + "requirements": req_metrics, + "architecture": arch_metrics, + } + + return verification_metrics + + def calculate_full_need_metrics(app: Sphinx, include_external: bool): """ Calculate all tracked metrics for requirements and tests. @@ -188,6 +304,7 @@ def calculate_full_need_metrics(app: Sphinx, include_external: bool): "overall_metrics": overall_metrics, "metrics_by_type": metrics_by_type, "tests": test_metrics, + "verification_metrics": calculate_verification_metrics(all_needs), } # Save the metrics in a Global Variable to enable access from other parts. # Not a great solution but it is needed, as needpie filter functions for example diff --git a/src/extensions/score_sphinx_needs_templates/__init__.py b/src/extensions/score_sphinx_needs_templates/__init__.py index d7f3f44d8..322c4a290 100644 --- a/src/extensions/score_sphinx_needs_templates/__init__.py +++ b/src/extensions/score_sphinx_needs_templates/__init__.py @@ -178,12 +178,14 @@ def __call__(self, need_type: str) -> list[NeedItem]: _needs_of_type_callable = _NeedsOfType() -def _parse_version(value: str) -> tuple[int, int, int]: +def parse_version(value: str) -> tuple[int, int, int]: """Parse a ``valid_from``/``report_version``-style milestone string. Accepts ``vMAJOR.MINOR`` or ``vMAJOR.MINOR.PATCH`` (e.g. ``v0.8`` or ``v1.0.1``), matching the format enforced by the metamodel for ``valid_from``/``valid_until``/``report_version``. + + Public API for use by other extensions that need version comparison. """ numbers = [int(part) for part in value.strip().lstrip("vV").split(".")] while len(numbers) < 3: @@ -191,7 +193,11 @@ def _parse_version(value: str) -> tuple[int, int, int]: return (numbers[0], numbers[1], numbers[2]) -class _RequirementInReportVersion: +# Keep private alias for backwards compatibility with internal code +_parse_version = parse_version + + +class RequirementInReportVersion: """Decide whether a requirement Need belongs to a ``report_version`` scope. ``feat_req`` (and ``stkh_req``) carry ``valid_from`` directly. ``comp_req`` @@ -205,6 +211,8 @@ class _RequirementInReportVersion: Calling with an empty/``None`` ``report_version`` always returns ``True``, which keeps unscoped reports (e.g. "latest") showing every requirement as before. + + Public API for use by other extensions that need version-scoped filtering. """ def __call__(self, need: NeedItem, report_version: str | None) -> bool: @@ -222,10 +230,14 @@ def __call__(self, need: NeedItem, report_version: str | None) -> bool: return any(self(feat_req, report_version) for feat_req in linked_feat_reqs) -_req_in_report_version_callable = _RequirementInReportVersion() +# Public instance for direct import +req_in_report_version = RequirementInReportVersion() +# Private alias for backwards compatibility with internal code +_req_in_report_version_callable = req_in_report_version -class _AnyRequirementInReportVersion: + +class AnyRequirementInReportVersion: """Decide whether a Feature/Component has any requirement in scope. Used to drop an entire Feature/Component section from the report when @@ -233,6 +245,8 @@ class _AnyRequirementInReportVersion: rendering an empty section. An empty/``None`` ``report_version`` always returns ``True`` (unscoped reports keep every Feature/Component, even ones without any requirement at all, as before). + + Public API for use by other extensions that need version-scoped filtering. """ def __call__(self, reqs: list[NeedItem], report_version: str | None) -> bool: @@ -241,7 +255,11 @@ def __call__(self, reqs: list[NeedItem], report_version: str | None) -> bool: return any(_req_in_report_version_callable(req, report_version) for req in reqs) -_any_req_in_report_version_callable = _AnyRequirementInReportVersion() +# Public instance for direct import +any_req_in_report_version = AnyRequirementInReportVersion() + +# Private alias for backwards compatibility with Jinja context +_any_req_in_report_version_callable = any_req_in_report_version class _ReferencedIds: @@ -356,6 +374,10 @@ class _SafetyPieRow: count through Sphinx-Needs filters, and the two can disagree on links carrying a ``[version==N]`` selector. + With ``component`` and ``category`` the status and coverage pies are + counted by the shared ``score_metrics`` filter functions instead of the + inline ``base`` filters; ``version`` then scopes the requirements. + This is a render helper instead of a Jinja macro because the MiniJinja environment has no template loader, so macros cannot be shared between templates. @@ -372,6 +394,9 @@ def __call__( with_reference: bool = False, referenced: list[str] | None = None, missing: list[str] | None = None, + component: str | None = None, + category: str | None = None, + version: str | None = None, ) -> str: scope = f'{base} and safety == "{safety}"' # (slice label, status condition) per pie slice; green "done" comes first. @@ -422,14 +447,28 @@ def grid_item(name: str, pie: list[str]) -> None: ) lines.extend(f" {line}".rstrip() for line in pie) + def counted_by(func: str) -> list[str]: + # sphinx-needs splits the call on its last ".", so encode the version. + encoded_version = (version or "").replace(".", "_") + return [ + f":filter-func: score_metrics.sphinx_filters.{func}" + f"({component}, {category}, {encoded_version}, {safety})" + ] + grid_item( "Status", [ f":labels: {', '.join(label for label, _ in statuses)}", f":colors: {status_colors}", ":legend:", - "", - *(f"{scope} and {condition}" for _, condition in statuses), + *( + counted_by("get_component_status_metrics") + if component + else [ + "", + *(f"{scope} and {condition}" for _, condition in statuses), + ] + ), ], ) grid_item( @@ -438,10 +477,16 @@ def grid_item(name: str, pie: list[str]) -> None: ":labels: fully covered, partially covered, not covered", ":colors: #37a12d, #f0a500, #ca2828", ":legend:", - "", - f"{scope} and fully_verifies_back", - f"{scope} and partially_verifies_back and not fully_verifies_back", - f"{scope} and not fully_verifies_back and not partially_verifies_back", + *( + counted_by("get_component_coverage_metrics") + if component + else [ + "", + f"{scope} and fully_verifies_back", + f"{scope} and partially_verifies_back and not fully_verifies_back", + f"{scope} and not fully_verifies_back and not partially_verifies_back", + ] + ), ], ) if with_reference: diff --git a/src/extensions/score_sphinx_needs_templates/tests/test_needs_templates.py b/src/extensions/score_sphinx_needs_templates/tests/test_needs_templates.py index 8e2a15c61..b00580225 100644 --- a/src/extensions/score_sphinx_needs_templates/tests/test_needs_templates.py +++ b/src/extensions/score_sphinx_needs_templates/tests/test_needs_templates.py @@ -32,7 +32,11 @@ def to_link_string(self) -> str: class FakeNeed(dict[str, str]): - """Need-shaped test double with independently controlled link indexes.""" + """Need-shaped test double with independently controlled link indexes. + + Note: This is a simplified test double that doesn't fully implement NeedItem. + Tests using this class with typed APIs may need type: ignore comments. + """ def __init__( self, @@ -85,11 +89,9 @@ def test_backlinks_merge_indexed_and_new_outgoing_links( ) def test_parse_version_pads_missing_patch() -> None: """A two-component milestone implies patch ``0`` for comparison purposes.""" - parse_version = vars(templates)["_parse_version"] - - assert parse_version("v0.8") == (0, 8, 0) - assert parse_version("v1.0.1") == (1, 0, 1) - assert parse_version("V2.3") == (2, 3, 0) + assert templates.parse_version("v0.8") == (0, 8, 0) + assert templates.parse_version("v1.0.1") == (1, 0, 1) + assert templates.parse_version("V2.3") == (2, 3, 0) @add_test_properties( @@ -99,11 +101,10 @@ def test_parse_version_pads_missing_patch() -> None: ) def test_req_in_report_version_unscoped_report_version_always_true() -> None: """No ``report_version`` means the report is unscoped ("latest").""" - req_in_report_version = vars(templates)["_RequirementInReportVersion"]() need = FakeNeed("feat_req__x") - assert req_in_report_version(need, None) is True - assert req_in_report_version(need, "") is True + assert templates.req_in_report_version(need, None) is True # type: ignore[reportArgumentType] + assert templates.req_in_report_version(need, "") is True # type: ignore[reportArgumentType] @add_test_properties( @@ -113,13 +114,12 @@ def test_req_in_report_version_unscoped_report_version_always_true() -> None: ) def test_req_in_report_version_direct_valid_from_boundary() -> None: """``valid_from`` is inclusive: equal or later report_version is in scope.""" - req_in_report_version = vars(templates)["_RequirementInReportVersion"]() need = FakeNeed("feat_req__x") need["valid_from"] = "v0.8" - assert req_in_report_version(need, "v0.7") is False - assert req_in_report_version(need, "v0.8") is True - assert req_in_report_version(need, "v0.9") is True + assert templates.req_in_report_version(need, "v0.7") is False # type: ignore[reportArgumentType] + assert templates.req_in_report_version(need, "v0.8") is True # type: ignore[reportArgumentType] + assert templates.req_in_report_version(need, "v0.9") is True # type: ignore[reportArgumentType] @add_test_properties( @@ -129,11 +129,10 @@ def test_req_in_report_version_direct_valid_from_boundary() -> None: ) def test_req_in_report_version_malformed_valid_from_is_excluded() -> None: """An unparsable ``valid_from`` must not raise; it just excludes the need.""" - req_in_report_version = vars(templates)["_RequirementInReportVersion"]() need = FakeNeed("feat_req__x") need["valid_from"] = "not-a-version" - assert req_in_report_version(need, "v1.0") is False + assert templates.req_in_report_version(need, "v1.0") is False # type: ignore[reportArgumentType] @add_test_properties( @@ -145,13 +144,12 @@ def test_req_in_report_version_missing_valid_from_without_derived_from_is_exclud monkeypatch: pytest.MonkeyPatch, ) -> None: """A requirement with no ``valid_from`` and nothing to inherit from is out.""" - req_in_report_version = vars(templates)["_RequirementInReportVersion"]() comp_req = FakeNeed("comp_req__x") monkeypatch.setattr( templates, "_get_available_needs", lambda: {"comp_req__x": comp_req} ) - assert req_in_report_version(comp_req, "v1.0") is False + assert templates.req_in_report_version(comp_req, "v1.0") is False # type: ignore[reportArgumentType] @add_test_properties( @@ -164,15 +162,14 @@ def test_req_in_report_version_comp_req_inherits_via_derived_from( ) -> None: """``comp_req`` has no ``valid_from`` of its own; it inherits its scope from the ``feat_req`` it is ``derived_from``.""" - req_in_report_version = vars(templates)["_RequirementInReportVersion"]() feat_req = FakeNeed("feat_req__x") feat_req["valid_from"] = "v0.8" comp_req = FakeNeed("comp_req__x", links=[FakeLink("feat_req__x")]) needs = {"feat_req__x": feat_req, "comp_req__x": comp_req} monkeypatch.setattr(templates, "_get_available_needs", lambda: needs) - assert req_in_report_version(comp_req, "v0.7") is False - assert req_in_report_version(comp_req, "v0.8") is True + assert templates.req_in_report_version(comp_req, "v0.7") is False # type: ignore[reportArgumentType] + assert templates.req_in_report_version(comp_req, "v0.8") is True # type: ignore[reportArgumentType] @add_test_properties( @@ -182,9 +179,7 @@ def test_req_in_report_version_comp_req_inherits_via_derived_from( ) def test_any_req_in_report_version_unscoped_true_even_for_empty_list() -> None: """Unscoped ("latest") reports keep Features/Components with no requirement.""" - any_req_in_report_version = vars(templates)["_AnyRequirementInReportVersion"]() - - assert any_req_in_report_version([], None) is True + assert templates.any_req_in_report_version([], None) is True @add_test_properties( @@ -194,9 +189,7 @@ def test_any_req_in_report_version_unscoped_true_even_for_empty_list() -> None: ) def test_any_req_in_report_version_scoped_empty_list_is_excluded() -> None: """A scoped report has nothing to show for a Feature/Component with no reqs.""" - any_req_in_report_version = vars(templates)["_AnyRequirementInReportVersion"]() - - assert any_req_in_report_version([], "v1.0") is False + assert templates.any_req_in_report_version([], "v1.0") is False @add_test_properties( @@ -206,13 +199,12 @@ def test_any_req_in_report_version_scoped_empty_list_is_excluded() -> None: ) def test_any_req_in_report_version_true_if_at_least_one_requirement_matches() -> None: """Test any req in report version true if at least one requirement matches.""" - any_req_in_report_version = vars(templates)["_AnyRequirementInReportVersion"]() out_of_scope = FakeNeed("feat_req__a") out_of_scope["valid_from"] = "v2.0" in_scope = FakeNeed("feat_req__b") in_scope["valid_from"] = "v0.8" - assert any_req_in_report_version([out_of_scope, in_scope], "v1.0") is True + assert templates.any_req_in_report_version([out_of_scope, in_scope], "v1.0") is True # type: ignore[reportArgumentType] @add_test_properties( @@ -222,13 +214,12 @@ def test_any_req_in_report_version_true_if_at_least_one_requirement_matches() -> ) def test_any_req_in_report_version_false_if_no_requirement_matches() -> None: """Test any req in report version false if no requirement matches.""" - any_req_in_report_version = vars(templates)["_AnyRequirementInReportVersion"]() a = FakeNeed("feat_req__a") a["valid_from"] = "v2.0" b = FakeNeed("feat_req__b") b["valid_from"] = "v3.0" - assert any_req_in_report_version([a, b], "v1.0") is False + assert templates.any_req_in_report_version([a, b], "v1.0") is False # type: ignore[reportArgumentType] def test_safety_pie_row_qm_counts_inspected_as_valid() -> None: diff --git a/src/helper_lib/verification_metrics.py b/src/helper_lib/verification_metrics.py new file mode 100644 index 000000000..1171a6c90 --- /dev/null +++ b/src/helper_lib/verification_metrics.py @@ -0,0 +1,157 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Shared verification metrics helpers for use in both metrics calculation and filtering. + +These helpers are used by: +- traceability_metrics.py for pre-calculating verification_metrics in metrics.json +- sphinx_filters.py for on-the-fly filtering in needpie directives +""" + +from typing import Any + +from sphinx_needs.need_item import NeedItem + + +def is_need_in_component(need: NeedItem, component_id: str, category: str) -> bool: + """Check if a need belongs to a component for a given category. + + Args: + need: The need item to check. + component_id: The component ID (e.g., "comp__my_component"). + category: Either "requirements" or "architecture". + + Returns: + True if the need belongs to the component for that category. + """ + need_type = str(need.get("type", "")) + + if category == "requirements": + if need_type != "comp_req": + return False + sat_by = need.get("satisfied_by", []) + if isinstance(sat_by, str): + sat_by = [sat_by] + return component_id in sat_by + + # architecture + if need_type not in ("comp_arc_sta", "comp_arc_dyn"): + return False + bel_to = need.get("belongs_to", []) + if isinstance(bel_to, str): + bel_to = [bel_to] + return component_id in bel_to + + +def _in_scope( + need: NeedItem, + component_id: str, + category: str, + version: str | None, + version_filter_func: Any, + safety: str | None, +) -> bool: + """Whether a need is counted for this component, category, version and safety.""" + if not is_need_in_component(need, component_id, category): + return False + if safety and need.get("safety") != safety: + return False + # Architecture elements have neither valid_from nor derived_from, so the + # version scope applies to requirements only. + return not ( + category == "requirements" + and version + and not version_filter_func(need, version) + ) + + +def count_status_filtered( + needs: list[Any], + component_id: str, + category: str, + version: str | None, + version_filter_func: Any, + safety: str | None = None, +) -> dict[str, int]: + """Count status distribution with version filtering. + + Args: + needs: List of need items. + component_id: The component ID. + category: Either "requirements" or "architecture". + version: Version string for filtering (only applied to requirements). + version_filter_func: Callable like req_in_report_version(need, version). + safety: Optional safety class (e.g. "ASIL_B", "QM") to restrict to. + + Returns: + Dict with status counts: {valid: int, valid_inspected: int, invalid: int} + """ + status = {"valid": 0, "valid_inspected": 0, "invalid": 0} + + for need in needs: + if not _in_scope( + need, component_id, category, version, version_filter_func, safety + ): + continue + + need_status = str(need.get("status", "")) + if need_status in status: + status[need_status] += 1 + + return status + + +def count_coverage_filtered( + needs: list[Any], + component_id: str, + category: str, + version: str | None, + version_filter_func: Any, + safety: str | None = None, +) -> dict[str, int]: + """Count verification coverage with version filtering. + + Uses the verification coverage axis (fully_verifies_back / partially_verifies_back), + not the link coverage axis (testlink / source_code_link). + + Args: + needs: List of need items. + component_id: The component ID. + category: Either "requirements" or "architecture". + version: Version string for filtering (only applied to requirements). + version_filter_func: Callable like req_in_report_version(need, version). + safety: Optional safety class (e.g. "ASIL_B", "QM") to restrict to. + + Returns: + Dict with coverage counts: {fully_covered: int, partially_covered: int, not_covered: int} + """ + coverage = {"fully_covered": 0, "partially_covered": 0, "not_covered": 0} + + for need in needs: + if not _in_scope( + need, component_id, category, version, version_filter_func, safety + ): + continue + + # Use verification coverage axis (not link coverage axis) + has_fully = bool(need.get("fully_verifies_back")) + has_partially = bool(need.get("partially_verifies_back")) + + if has_fully: + coverage["fully_covered"] += 1 + elif has_partially: + coverage["partially_covered"] += 1 + else: + coverage["not_covered"] += 1 + + return coverage diff --git a/src/needs_templates/module_verification_report.need b/src/needs_templates/module_verification_report.need index 6df98553a..160473087 100644 --- a/src/needs_templates/module_verification_report.need +++ b/src/needs_templates/module_verification_report.need @@ -200,19 +200,21 @@ Component Overview Component Requirements Statistics ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -{% set req_base = 'type == "comp_req" and "' ~ component_id ~ '" in satisfied_by and ' ~ component_req_filter %} -{{ safety_pie_row(component_title, "Requirements", req_base, "ASIL_B", "ASIL B", - component_reqs|selectattr("safety", "eq", "ASIL_B")|list|length) }} -{{ safety_pie_row(component_title, "Requirements", req_base, "QM", "QM", - component_reqs|selectattr("safety", "eq", "QM")|list|length) }} +{{ safety_pie_row(component_title, "Requirements", "", "ASIL_B", "ASIL B", + component_reqs|selectattr("safety", "eq", "ASIL_B")|list|length, + component=component_id, category="requirements", version=module_version) }} +{{ safety_pie_row(component_title, "Requirements", "", "QM", "QM", + component_reqs|selectattr("safety", "eq", "QM")|list|length, + component=component_id, category="requirements", version=module_version) }} Component Architecture Statistics ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -{% set arc_base = 'type in ["comp_arc_sta", "comp_arc_dyn"] and "' ~ component_id ~ '" in belongs_to' %} -{{ safety_pie_row(component_title, "Architecture Elements", arc_base, "ASIL_B", "ASIL B", - component_arcs|selectattr("safety", "eq", "ASIL_B")|list|length) }} -{{ safety_pie_row(component_title, "Architecture Elements", arc_base, "QM", "QM", - component_arcs|selectattr("safety", "eq", "QM")|list|length) }} +{{ safety_pie_row(component_title, "Architecture Elements", "", "ASIL_B", "ASIL B", + component_arcs|selectattr("safety", "eq", "ASIL_B")|list|length, + component=component_id, category="architecture") }} +{{ safety_pie_row(component_title, "Architecture Elements", "", "QM", "QM", + component_arcs|selectattr("safety", "eq", "QM")|list|length, + component=component_id, category="architecture") }} Requirements Traceability ^^^^^^^^^^^^^^^^^^^^^^^^^