TEST: unify Entra and API-key integration tests (#2263)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 45a85ef3-3476-4c7f-bc9e-c1b8e1a20a45

Author: romanlutz

doc/contributing/10_release_process.md

@@ -14,7 +14,7 @@ Before starting the release process, verify the codebase is in a healthy state.
 - **Check for pending changes.** Ask other PyRIT maintainers whether they have any in-flight changes that should land before the release.
 - **Verify build pipelines.** Confirm that all integration tests and end-to-end tests are passing in the CI pipelines. If any tests are failing, fix them before proceeding.
   - **Partner integration tests.** Ensure the partner integration tests are also passing. These tests validate that we are not breaking contracts with partner teams (e.g., Foundry). If any are failing, coordinate with the affected partner teams before proceeding with the release.
-  - **Azure key-based auth is disabled in our tenant.** Our Azure subscription has API-key (local) auth turned off, so Azure targets authenticate with Microsoft Entra ID (Entra auth) only. Integration tests and notebooks that exercise Azure targets with API keys are deliberately skipped; otherwise they fail with HTTP 403 `AuthenticationTypeDisabled` ("Key based authentication is disabled for this resource"). The same endpoints are covered by the Entra-auth tests in `tests/integration/targets/test_entra_auth_targets.py`, so this is expected and not a coverage gap. Do not re-enable these for our tenant. When validating a release manually, authenticate Azure targets with Entra (`az login`) rather than API keys.
+  - **Azure key-based auth is disabled in our tenant.** Our Azure subscription has API-key (local) auth turned off, so Azure target integration tests authenticate with Microsoft Entra ID. Tests that run notebooks requiring Azure API keys are deliberately skipped; otherwise they fail with HTTP 403 `AuthenticationTypeDisabled` ("Key based authentication is disabled for this resource"). Do not re-enable key auth for our tenant. When validating a release manually, authenticate Azure targets with Entra (`az login`) rather than API keys.
 - **Update scorer metrics.** Run `python .\build_scripts\evaluate_scorers.py` and commit the results so that scorer evaluation metrics are up to date.
 
 ## 2. Decide the Next Version

tests/integration/embeddings/test_openai_embedding.py

@@ -8,30 +8,42 @@
 from pyrit.auth import get_azure_openai_auth
 from pyrit.embedding import OpenAITextEmbedding
 
-_AZURE_KEY_AUTH_DISABLED_REASON = (
-    "Azure key-based (local) auth is disabled in our tenant; "
-    "covered by the Entra-auth tests (test_entra_auth_targets.py)."
-)
+_AZURE_KEY_AUTH_DISABLED_REASON = "Azure key-based (local) auth is disabled in our tenant."
 
 
-@pytest.mark.run_only_if_all_tests
 @pytest.mark.parametrize(
-    "endpoint_env,key_env,model_env",
+    ("endpoint_env", "api_key_env", "model_env"),
     [
+        pytest.param(
+            "OPENAI_EMBEDDING_ENDPOINT",
+            None,
+            "OPENAI_EMBEDDING_MODEL",
+            id="entra",
+        ),
         pytest.param(
             "OPENAI_EMBEDDING_ENDPOINT",
             "OPENAI_EMBEDDING_KEY",
             "OPENAI_EMBEDDING_MODEL",
             marks=pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON),
+            id="azure-api-key",
+        ),
+        pytest.param(
+            "PLATFORM_OPENAI_EMBEDDING_ENDPOINT",
+            "PLATFORM_OPENAI_EMBEDDING_KEY",
+            "PLATFORM_OPENAI_EMBEDDING_MODEL",
+            marks=pytest.mark.run_only_if_all_tests,
+            id="api-key",
         ),
-        ("PLATFORM_OPENAI_EMBEDDING_ENDPOINT", "PLATFORM_OPENAI_EMBEDDING_KEY", "PLATFORM_OPENAI_EMBEDDING_MODEL"),
     ],
 )
-def test_openai_embedding_with_api_key(endpoint_env: str, key_env: str, model_env: str):
-    """Test OpenAI embedding with API key authentication."""
-    api_key = os.environ[key_env]
+def test_openai_embedding(
+    endpoint_env: str,
+    api_key_env: str | None,
+    model_env: str,
+) -> None:
     endpoint = os.environ[endpoint_env]
     model = os.environ[model_env]
+    api_key = os.environ[api_key_env] if api_key_env else get_azure_openai_auth(endpoint)
 
     embedding = OpenAITextEmbedding(
         api_key=api_key,
@@ -46,26 +58,3 @@ def test_openai_embedding_with_api_key(endpoint_env: str, key_env: str, model_en
     assert len(response.data) == 1
     assert len(response.data[0].embedding) > 0
     assert response.usage.total_tokens > 0
-
-
-def test_azure_openai_embedding_with_entra_auth():
-    """Test Azure OpenAI embedding with Entra (token provider) authentication."""
-    endpoint = os.environ["OPENAI_EMBEDDING_ENDPOINT"]
-    model = os.environ["OPENAI_EMBEDDING_MODEL"]
-
-    # Get token provider for Entra auth
-    token_provider = get_azure_openai_auth(endpoint)
-
-    embedding = OpenAITextEmbedding(
-        api_key=token_provider,
-        endpoint=endpoint,
-        model_name=model,
-    )
-
-    test_text = "Testing embedding with Entra authentication."
-    response = embedding.generate_text_embedding(text=test_text)
-
-    assert response is not None
-    assert len(response.data) == 1
-    assert len(response.data[0].embedding) > 0
-    assert response.usage.total_tokens > 0

tests/integration/score/test_azure_content_filter_integration.py

@@ -12,31 +12,50 @@
 from pyrit.memory import CentralMemory, MemoryInterface
 from pyrit.score import AzureContentFilterScorer
 
-_AZURE_KEY_AUTH_DISABLED_REASON = (
-    "Azure key-based (local) auth is disabled in our tenant; "
-    "covered by the Entra-auth tests (test_entra_auth_targets.py)."
-)
+_AZURE_KEY_AUTH_DISABLED_REASON = "Azure key-based (local) auth is disabled in our tenant."
 
 
 @pytest.fixture
 def memory() -> Generator[MemoryInterface, None, None]:
     yield from get_memory_interface()
 
 
-async def test_azure_content_filter_scorer_image_integration(memory) -> None:
+@pytest.fixture(
+    params=[
+        pytest.param(None, id="entra"),
+        pytest.param(
+            "AZURE_CONTENT_SAFETY_API_KEY",
+            marks=[
+                pytest.mark.run_only_if_all_tests,
+                pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON),
+            ],
+            id="api-key",
+        ),
+    ]
+)
+def content_filter_scorer(request: pytest.FixtureRequest) -> AzureContentFilterScorer:
+    api_key_env: str | None = request.param
+    api_key = os.getenv(api_key_env) if api_key_env else None
+    if api_key_env:
+        assert api_key, f"{api_key_env} must be set to test API key auth"
+
+    with patch.dict(os.environ, {"AZURE_CONTENT_SAFETY_API_KEY": ""}):
+        return AzureContentFilterScorer(api_key=api_key)
+
+
+async def test_azure_content_filter_scorer_image_integration(
+    memory: MemoryInterface,
+    content_filter_scorer: AzureContentFilterScorer,
+) -> None:
     """
     Integration test for Azure Content Filter Scorer with image input.
 
     This test requires AZURE_CONTENT_SAFETY_API_ENDPOINT to be set.
-    Authentication uses Entra ID by default (via `az login`). Alternatively,
-    set AZURE_CONTENT_SAFETY_API_KEY for API key auth.
     Uses a sample image from the assets folder.
     """
     with patch.object(CentralMemory, "get_memory_instance", return_value=memory):
-        scorer = AzureContentFilterScorer()
-
         image_path = HOME_PATH / "assets" / "architecture_components.png"
-        scores = await scorer.score_image_async(image_path=str(image_path))
+        scores = await content_filter_scorer.score_image_async(image_path=str(image_path))
 
         # Should return scores for all harm categories
         assert len(scores) > 0, "Expected scores for harm categories"
@@ -53,21 +72,22 @@ async def test_azure_content_filter_scorer_image_integration(memory) -> None:
         assert max_score < 0.5, "Architecture diagram should have low harm scores"
 
 
-async def test_azure_content_filter_scorer_long_text_chunking_integration(memory) -> None:
+async def test_azure_content_filter_scorer_long_text_chunking_integration(
+    memory: MemoryInterface,
+    content_filter_scorer: AzureContentFilterScorer,
+) -> None:
     """
     Integration test for Azure Content Filter Scorer with text longer than 10,000 characters.
 
     This verifies that the chunking and aggregation logic works correctly with the real API.
     """
     with patch.object(CentralMemory, "get_memory_instance", return_value=memory):
-        scorer = AzureContentFilterScorer()
-
         # This should be greater than the rate limit
         base_text = "I hate you " * 10000
 
         assert len(base_text) > AzureContentFilterScorer.MAX_TEXT_LENGTH
 
-        scores = await scorer.score_text_async(text=base_text)
+        scores = await content_filter_scorer.score_text_async(text=base_text)
 
         # Should return aggregated scores (one per category)
         assert len(scores) > 0, "Expected aggregated scores for harm categories"
@@ -82,64 +102,3 @@ async def test_azure_content_filter_scorer_long_text_chunking_integration(memory
         # Long benign text should still have low scores
         max_score = max(float(s.score_value) for s in scores)
         assert max_score > 0, "text should have > 0 score"
-
-
-@pytest.mark.run_only_if_all_tests
-@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON)
-async def test_azure_content_filter_scorer_image_with_api_key(memory) -> None:
-    """
-    Integration test for Azure Content Filter Scorer image scoring with explicit API key auth.
-
-    This test requires AZURE_CONTENT_SAFETY_API_KEY to be set.
-    """
-    api_key = os.getenv("AZURE_CONTENT_SAFETY_API_KEY")
-    assert api_key, "AZURE_CONTENT_SAFETY_API_KEY must be set to test API key auth"
-
-    with patch.object(CentralMemory, "get_memory_instance", return_value=memory):
-        scorer = AzureContentFilterScorer(api_key=api_key)
-
-        image_path = HOME_PATH / "assets" / "architecture_components.png"
-        scores = await scorer.score_image_async(image_path=str(image_path))
-
-        assert len(scores) > 0, "Expected scores for harm categories"
-
-        for score in scores:
-            assert score.score_type == "float_scale"
-            assert score.score_category is not None
-            assert 0.0 <= float(score.score_value) <= 1.0
-            assert "azure_severity" in score.score_metadata
-
-        max_score = max(float(s.score_value) for s in scores)
-        assert max_score < 0.5, "Architecture diagram should have low harm scores"
-
-
-@pytest.mark.run_only_if_all_tests
-@pytest.mark.skip(reason=_AZURE_KEY_AUTH_DISABLED_REASON)
-async def test_azure_content_filter_scorer_text_with_api_key(memory) -> None:
-    """
-    Integration test for Azure Content Filter Scorer text scoring with explicit API key auth.
-
-    This test requires AZURE_CONTENT_SAFETY_API_KEY to be set.
-    """
-    api_key = os.getenv("AZURE_CONTENT_SAFETY_API_KEY")
-    assert api_key, "AZURE_CONTENT_SAFETY_API_KEY must be set to test API key auth"
-
-    with patch.object(CentralMemory, "get_memory_instance", return_value=memory):
-        scorer = AzureContentFilterScorer(api_key=api_key)
-
-        base_text = "I hate you " * 10000
-
-        assert len(base_text) > AzureContentFilterScorer.MAX_TEXT_LENGTH
-
-        scores = await scorer.score_text_async(text=base_text)
-
-        assert len(scores) > 0, "Expected aggregated scores for harm categories"
-
-        for score in scores:
-            assert score.score_type == "float_scale"
-            assert score.score_category is not None
-            assert 0.0 <= float(score.score_value) <= 1.0
-            assert "azure_severity" in score.score_metadata
-
-        max_score = max(float(s.score_value) for s in scores)
-        assert max_score > 0, "text should have > 0 score"