diff --git a/.pipelines/v2/sdk_v2-pipeline-plan.md b/.pipelines/v2/sdk_v2-pipeline-plan.md index b7288179e..666b6f3dd 100644 --- a/.pipelines/v2/sdk_v2-pipeline-plan.md +++ b/.pipelines/v2/sdk_v2-pipeline-plan.md @@ -90,9 +90,11 @@ are gated separately via `.pipelines/v1/templates/stages-sdk-v1.yml`. 10. **C++ staging step is the policy authority for native payload contents.** `steps-build-{windows,linux,macos}.yml` stage the **full runtime closure** of `foundry_local` into the `cpp-native-` artifact, with explicit - inclusions/exclusions. Downstream consumers (Python wheel build, C# pack) - copy the artifact verbatim — they do not re-filter. This keeps the - "what ships next to `foundry_local`" decision in exactly one place. + inclusions/exclusions. The Python wheel build copies the artifact contents + needed by its wheel, while C# packing applies its own explicit runtime + payload allow-list in `sdk_v2/cpp/nuget/pack.py`. That allow-list includes + the Windows import library (`foundry_local.lib`) for applications that + link against `foundry_local.dll`. Each staging step copies an explicit allow-list, not a glob: just the redistributable `foundry_local` library (`.dll` + `.pdb` + `.lib` on diff --git a/.pipelines/v2/templates/stages-build-native.yml b/.pipelines/v2/templates/stages-build-native.yml index 2a7b465c0..333d3ebae 100644 --- a/.pipelines/v2/templates/stages-build-native.yml +++ b/.pipelines/v2/templates/stages-build-native.yml @@ -321,15 +321,14 @@ stages: script: | $ErrorActionPreference = 'Stop' - $archive = Get-ChildItem '$(Pipeline.Workspace)/cpp-sdk-v2' -Recurse -Filter '*win-x64.tgz' | + $archive = Get-ChildItem '$(Pipeline.Workspace)/cpp-sdk-v2' -Recurse -Filter 'foundry-local-win-x64.zip' | Select-Object -First 1 if (-not $archive) { throw 'Windows x64 C++ SDK archive not found' } $consumerDir = Join-Path '$(Agent.TempDirectory)' 'cpp-sdk-package-test' if (Test-Path $consumerDir) { Remove-Item -Recurse -Force $consumerDir } New-Item -ItemType Directory -Force -Path $consumerDir | Out-Null - tar -xzf $archive.FullName -C $consumerDir - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + Expand-Archive -Path $archive.FullName -DestinationPath $consumerDir -Force $source = '$(Build.SourcesDirectory)/sdk_v2/cpp/examples/basic_chat/main.cc' if (-not (Test-Path $source)) { throw "C++ basic_chat sample not found: $source" } diff --git a/.pipelines/v2/templates/steps-pack-cpp-sdk.yml b/.pipelines/v2/templates/steps-pack-cpp-sdk.yml index b44519818..c61f0a7d1 100644 --- a/.pipelines/v2/templates/steps-pack-cpp-sdk.yml +++ b/.pipelines/v2/templates/steps-pack-cpp-sdk.yml @@ -1,22 +1,22 @@ -# Package per-platform C++ SDK bundles as .tgz files. +# Package per-platform C++ SDK bundles as .zip or .tgz files. # # Each bundle has this layout: # include/ -> public headers (from cpp-native-include artifact) # lib/ -> platform native binaries # # Output: -# $(Build.ArtifactStagingDirectory)/cpp-sdk-v2/*.tgz +# $(Build.ArtifactStagingDirectory)/cpp-sdk-v2/foundry-local-*.{zip,tgz} parameters: - name: outputDir type: string default: '$(Build.ArtifactStagingDirectory)/cpp-sdk-v2' - displayName: 'Directory where per-platform .tgz bundles are staged' + displayName: 'Directory where per-platform SDK bundles are staged' steps: - task: PowerShell@2 - displayName: 'Create C++ SDK platform bundles (.tgz)' + displayName: 'Create C++ SDK platform bundles (.zip/.tgz)' inputs: targetType: inline pwsh: true @@ -62,7 +62,7 @@ steps: Copy-Item -Path (Join-Path $includeSrc '*') -Destination $includeDst -Recurse -Force - # The SDK tgz should include the public wrapper headers and their public + # The SDK archive should include the public wrapper headers and their public # ms-gsl dependency. Only pipeline artifact metadata is excluded. $pathsToPrune = @( (Join-Path $includeDst '_manifest') @@ -105,14 +105,22 @@ steps: } } - $archive = Join-Path $outDir ("cpp-sdk-v2-{0}.tgz" -f $Rid) + $archiveExtension = if ($Rid -like 'win-*') { 'zip' } else { 'tgz' } + $archive = Join-Path $outDir ("foundry-local-{0}.{1}" -f $Rid, $archiveExtension) if (Test-Path $archive) { Remove-Item -Force $archive } Push-Location $root try { - tar -czf $archive include lib - if ($LASTEXITCODE -ne 0) { - throw "tar failed for RID '$Rid'" + if ($archiveExtension -eq 'zip') { + Compress-Archive -Path include, lib -DestinationPath $archive -CompressionLevel Optimal + if (-not $?) { + throw "Zip creation failed for RID '$Rid'" + } + } else { + tar -czf $archive include lib + if ($LASTEXITCODE -ne 0) { + throw "Tar creation failed for RID '$Rid'" + } } } finally { Pop-Location @@ -176,4 +184,4 @@ steps: ) Write-Host "Staged archives:" - Get-ChildItem $outDir -Filter '*.tgz' | ForEach-Object { Write-Host " $($_.Name)" } + Get-ChildItem $outDir -File | ForEach-Object { Write-Host " $($_.Name)" } diff --git a/sdk_v2/cpp/nuget/Microsoft.AI.Foundry.Local.Runtime.nuspec b/sdk_v2/cpp/nuget/Microsoft.AI.Foundry.Local.Runtime.nuspec index 01036f874..b2c4c49ea 100644 --- a/sdk_v2/cpp/nuget/Microsoft.AI.Foundry.Local.Runtime.nuspec +++ b/sdk_v2/cpp/nuget/Microsoft.AI.Foundry.Local.Runtime.nuspec @@ -7,6 +7,7 @@ Microsoft Microsoft LICENSE.txt + README.md https://github.com/microsoft/Foundry-Local Native runtime libraries for the Microsoft AI Foundry Local SDK. @@ -29,5 +30,8 @@ + + + diff --git a/sdk_v2/cpp/nuget/Privacy.md b/sdk_v2/cpp/nuget/Privacy.md new file mode 100644 index 000000000..66ddd1fb5 --- /dev/null +++ b/sdk_v2/cpp/nuget/Privacy.md @@ -0,0 +1,7 @@ +# Privacy + +Microsoft AI Foundry Local runs inference locally on the device. The runtime package does not send prompts, model inputs, or generated outputs to Microsoft as part of native library loading or inference. + +Applications using Foundry Local may independently collect telemetry or other diagnostic data. Application authors are responsible for disclosing that collection and complying with applicable privacy requirements. + +For Microsoft's general privacy statement, see [Microsoft Privacy](https://privacy.microsoft.com/privacystatement). diff --git a/sdk_v2/cpp/nuget/README.md b/sdk_v2/cpp/nuget/README.md new file mode 100644 index 000000000..dd7ea8dc6 --- /dev/null +++ b/sdk_v2/cpp/nuget/README.md @@ -0,0 +1,7 @@ +# Microsoft AI Foundry Local Runtime + +The Microsoft AI Foundry Local Runtime NuGet package provides the native runtime libraries used by the Foundry Local SDK. + +The package includes platform-specific native assets for Windows x64, Windows ARM64, Linux x64, Linux ARM64, and macOS ARM64. Windows packages also include the `foundry_local.lib` import library for applications that link against `foundry_local.dll`. + +For SDK usage and development documentation, see the [Foundry Local repository](https://github.com/microsoft/Foundry-Local). diff --git a/sdk_v2/cpp/nuget/ThirdPartyNotices.txt b/sdk_v2/cpp/nuget/ThirdPartyNotices.txt new file mode 100644 index 000000000..8d6983e09 --- /dev/null +++ b/sdk_v2/cpp/nuget/ThirdPartyNotices.txt @@ -0,0 +1,10 @@ +Microsoft AI Foundry Local Runtime +Third-Party Notices + +This package includes or depends on software from third parties. The applicable third-party licenses and notices are available from the corresponding dependency packages and source repositories: + +- ONNX Runtime: https://github.com/microsoft/onnxruntime +- ONNX Runtime GenAI: https://github.com/microsoft/onnxruntime-genai +- Microsoft Windows App SDK / Windows ML components, where applicable: https://github.com/microsoft/WindowsAppSDK + +The native runtime package is distributed under the MIT License. Dependency packages retain their respective licenses and notices. Review the dependency packages restored with this package for the complete notices applicable to each dependency version. diff --git a/sdk_v2/cpp/nuget/pack.py b/sdk_v2/cpp/nuget/pack.py index ea694af76..b6dbf2301 100644 --- a/sdk_v2/cpp/nuget/pack.py +++ b/sdk_v2/cpp/nuget/pack.py @@ -40,9 +40,9 @@ # # Vcpkg is statically linked on every platform (see sdk_v2/cpp/triplets/), so # the foundry_local shared library carries its transitive deps inside itself -# and the upstream platform-build artifact for each RID contains only the -# primary library (plus, on Windows, .pdb and .lib companions consumed by the -# Python wheel build). We copy that one file into runtimes//native/. +# and the upstream platform-build artifact for each RID contains the primary +# library plus platform-specific companions. We copy the redistributable files +# into runtimes//native/. # # On Windows the build also drops Microsoft.Windows.AI.MachineLearning.dll — the # reg-free WinML 2.x runtime — next to foundry_local.dll (the cmake post-build @@ -58,9 +58,11 @@ } # Sibling files copied into runtimes//native/ when present in the upstream -# artifact. Windows builds drop Microsoft.Windows.AI.MachineLearning.dll alongside -# foundry_local.dll; other platforms don't, so presence alone drives inclusion. +# artifact. Windows builds provide the import library needed by applications +# that link against foundry_local.dll, plus the WinML runtime DLL. Other +# platforms don't provide these files, so presence alone drives inclusion. OPTIONAL_SIBLINGS: tuple[str, ...] = ( + "foundry_local.lib", "Microsoft.Windows.AI.MachineLearning.dll", ) @@ -135,6 +137,14 @@ def stage(args: argparse.Namespace, staging: Path) -> int: else: log.warning("LICENSE.txt not found at %s", license_file) + # --- package documentation and notices --- + for document_name in ("README.md", "Privacy.md", "ThirdPartyNotices.txt"): + document_file = SCRIPT_DIR / document_name + if document_file.is_file(): + shutil.copy2(document_file, staging) + else: + log.warning("Package document not found at %s", document_file) + # --- runtimes/{rid}/native/ --- rid_count = 0 for arg_name, (rid, lib_name) in RIDS.items(): @@ -152,11 +162,8 @@ def stage(args: argparse.Namespace, staging: Path) -> int: native_dir = staging / "runtimes" / rid / "native" native_dir.mkdir(parents=True, exist_ok=True) - # The upstream artifact for each RID is the primary library plus, on - # Windows, .pdb / .lib companions used by the Python wheel build, and - # Microsoft.Windows.AI.MachineLearning.dll (the WinML 2.x runtime). - # We forward the primary library plus any present OPTIONAL_SIBLINGS; - # everything else (.pdb, .lib) stays out of the NuGet runtime payload. + # Forward the primary library plus any present redistributable siblings. + # Build symbols such as .pdb remain out of the NuGet runtime payload. shutil.copy2(lib_path, native_dir) log.info(" %s → runtimes/%s/native/%s", lib_path, rid, lib_path.name) staged = 1 diff --git a/sdk_v2/cpp/nuget/test_pack.py b/sdk_v2/cpp/nuget/test_pack.py index f44e3100c..1501c0181 100644 --- a/sdk_v2/cpp/nuget/test_pack.py +++ b/sdk_v2/cpp/nuget/test_pack.py @@ -117,6 +117,8 @@ def _run_stage(self, rid_arg: str) -> tuple[set[str], int]: nuspec_stub.write_text('', encoding="utf-8") (fake_script_dir / "build").mkdir() (fake_script_dir / "buildTransitive").mkdir() + for document_name in ("README.md", "Privacy.md", "ThirdPartyNotices.txt"): + (fake_script_dir / document_name).write_text(document_name, encoding="utf-8") # REPO_ROOT = SCRIPT_DIR.parent; make include/ optional (fake_script_dir.parent / "include").mkdir(exist_ok=True) @@ -155,6 +157,47 @@ def test_linux_arm64_staged_correctly(self): self.assertEqual(rid_count, 1) self._check_native_file_present(layout, "linux-arm64", "libfoundry_local.so") + def test_windows_import_library_is_staged(self): + with tempfile.TemporaryDirectory() as tmp_str: + tmp = Path(tmp_str) + artifact_dir = _make_fake_artifact(tmp / "artifacts", "win_x64") + (artifact_dir / "foundry_local.lib").write_bytes(b"fake-import-library") + output_dir = tmp / "out" + staging_dir = tmp / "_staging" + staging_dir.mkdir() + + fake_script_dir = tmp / "nuget_stub" + fake_script_dir.mkdir() + (fake_script_dir / "Microsoft.AI.Foundry.Local.Runtime.nuspec").write_text( + '', encoding="utf-8" + ) + (fake_script_dir / "build").mkdir() + (fake_script_dir / "buildTransitive").mkdir() + (fake_script_dir.parent / "include").mkdir(exist_ok=True) + (fake_script_dir.parent / "LICENSE.txt").write_text("MIT", encoding="utf-8") + + original_script_dir = pack.SCRIPT_DIR + original_repo_root = pack.REPO_ROOT + try: + pack.SCRIPT_DIR = fake_script_dir + pack.REPO_ROOT = fake_script_dir.parent + args = _fake_args(output_dir=output_dir, win_x64=artifact_dir) + pack.stage(args, staging_dir) + finally: + pack.SCRIPT_DIR = original_script_dir + pack.REPO_ROOT = original_repo_root + + self.assertTrue( + (staging_dir / "runtimes" / "win-x64" / "native" / "foundry_local.lib").is_file() + ) + + def test_package_documents_are_staged_at_root(self): + layout, rid_count = self._run_stage("linux_x64") + self.assertEqual(rid_count, 1) + for document_name in ("README.md", "Privacy.md", "ThirdPartyNotices.txt"): + with self.subTest(document=document_name): + self.assertIn(document_name, layout) + def test_all_platforms_staged_together(self): """Staging all five platforms at once yields rid_count == 5.""" with tempfile.TemporaryDirectory() as tmp_str: diff --git a/sdk_v2/cpp/src/inferencing/session/session.cc b/sdk_v2/cpp/src/inferencing/session/session.cc index e8f90ee69..ba1352043 100644 --- a/sdk_v2/cpp/src/inferencing/session/session.cc +++ b/sdk_v2/cpp/src/inferencing/session/session.cc @@ -8,14 +8,17 @@ #include "inferencing/generative/embeddings/embeddings_session.h" #include "inferencing/model_load_manager.h" #include "inferencing/session/session_manager.h" +#include "items/message_item.h" #include "manager.h" #include "model.h" #include "telemetry/telemetry.h" #include "telemetry/telemetry_action_tracker.h" #include "utils.h" +#include #include +#include #include namespace fl { @@ -94,6 +97,54 @@ void Session::AddToolDefinition(ToolDefinition tool_def) { tool_definitions_.push_back(std::move(tool_def)); } +void Session::ValidateRequestItems(const Request& request) const { + // Only chat tasks are validated: other tasks either have no IO descriptor (embeddings) or accept + // transport items that the descriptor does not advertise (the ASR streaming QUEUE item). + const auto& task = catalog_model_.Info().task; + if (task != "chat-completion" && task != "vision-language-chat") { + return; + } + + // The model's task metadata is the source of truth for which input modalities are accepted. + const auto io_info = catalog_model_.GetInputOutputInfo(); + + // An item type is accepted only if it matches one of the advertised inputs. + auto check = [&](flItemType type) { + const bool supported = std::any_of(io_info.inputs, io_info.inputs + io_info.num_inputs, + [type](const Item* input) { return input->type == type; }); + if (!supported) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, + fmt::format("{} input is not supported by model task '{}'", + Item::TypeName(type), catalog_model_.Info().task)); + } + }; + + // Walk every request item, unwrapping containers so the check always lands on a modality item. + for (const auto* item : request.items) { + if (!item) { + continue; + } + + switch (item->type) { + case FOUNDRY_LOCAL_ITEM_MESSAGE: + // The message wrapper itself is not a modality; validate the parts it carries. + for (const auto& part : static_cast(*item).content) { + if (part.view) { + check(part.view->type); + } + } + break; + case FOUNDRY_LOCAL_ITEM_TOOL_CALL: + case FOUNDRY_LOCAL_ITEM_TOOL_RESULT: + // Tool plumbing, not model input. + break; + default: + check(item->type); + break; + } + } +} + void Session::ProcessRequest(const Request& request, Response& response) { // Serialize requests unless the derived class opted into concurrency. std::unique_lock lock(*request_mutex_, std::defer_lock); @@ -105,6 +156,8 @@ void Session::ProcessRequest(const Request& request, Response& response) { tracker.SetModelId(CatalogModel().Id()); try { + ValidateRequestItems(request); + ProcessRequestImpl(request, response); tracker.SetStatus(ActionStatus::kSuccess); diff --git a/sdk_v2/cpp/src/inferencing/session/session.h b/sdk_v2/cpp/src/inferencing/session/session.h index f3ec0e5f1..1e5326639 100644 --- a/sdk_v2/cpp/src/inferencing/session/session.h +++ b/sdk_v2/cpp/src/inferencing/session/session.h @@ -144,6 +144,10 @@ class Session { const KeyValuePairs& SessionOptions() const { return session_options_; } private: + /// Reject items (and message content parts) whose type the model's task does not advertise as an + /// input. Currently applies to chat tasks only. + void ValidateRequestItems(const Request& request) const; + const fl::Model& catalog_model_; ILogger& logger_; ITelemetry& telemetry_; diff --git a/sdk_v2/cpp/src/items/item.cc b/sdk_v2/cpp/src/items/item.cc index b7c0641b1..ac7d994a8 100644 --- a/sdk_v2/cpp/src/items/item.cc +++ b/sdk_v2/cpp/src/items/item.cc @@ -51,4 +51,35 @@ std::unique_ptr Item::Create(flItemType type) { } } +std::string_view Item::TypeName(flItemType type) noexcept { + switch (type) { + case FOUNDRY_LOCAL_ITEM_UNKNOWN: + return "UNKNOWN"; + case FOUNDRY_LOCAL_ITEM_BYTES: + return "BYTES"; + case FOUNDRY_LOCAL_ITEM_TENSOR: + return "TENSOR"; + case FOUNDRY_LOCAL_ITEM_TEXT: + return "TEXT"; + case FOUNDRY_LOCAL_ITEM_MESSAGE: + return "MESSAGE"; + case FOUNDRY_LOCAL_ITEM_IMAGE: + return "IMAGE"; + case FOUNDRY_LOCAL_ITEM_AUDIO: + return "AUDIO"; + case FOUNDRY_LOCAL_ITEM_SPEECH_SEGMENT: + return "SPEECH_SEGMENT"; + case FOUNDRY_LOCAL_ITEM_SPEECH_RESULT: + return "SPEECH_RESULT"; + case FOUNDRY_LOCAL_ITEM_TOOL_CALL: + return "TOOL_CALL"; + case FOUNDRY_LOCAL_ITEM_TOOL_RESULT: + return "TOOL_RESULT"; + case FOUNDRY_LOCAL_ITEM_QUEUE: + return "QUEUE"; + default: + return "UNKNOWN"; + } +} + } // namespace fl diff --git a/sdk_v2/cpp/src/items/item.h b/sdk_v2/cpp/src/items/item.h index 2ba69f3b2..0e3f1ca59 100644 --- a/sdk_v2/cpp/src/items/item.h +++ b/sdk_v2/cpp/src/items/item.h @@ -5,6 +5,7 @@ #include #include +#include #include "util/key_value_pairs.h" @@ -51,6 +52,9 @@ struct Item { /// Factory: creates the correct derived type for the given flItemType. static std::unique_ptr Create(flItemType type); + /// Return the stable symbolic name for an item type. + static std::string_view TypeName(flItemType type) noexcept; + protected: explicit Item(flItemType type) : type(type) {} diff --git a/sdk_v2/cpp/src/model.cc b/sdk_v2/cpp/src/model.cc index 2575d3922..a4bccb110 100644 --- a/sdk_v2/cpp/src/model.cc +++ b/sdk_v2/cpp/src/model.cc @@ -424,6 +424,29 @@ const StaticIOCache& ChatCompletionIO() { return cache; } +const StaticIOCache& VisionLanguageChatIO() { + static const StaticIOCache cache = [] { + StaticIOCache c; + c.input_items.push_back(Item::Create(FOUNDRY_LOCAL_ITEM_MESSAGE)); + c.input_items.push_back(std::make_unique("", FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON)); + c.input_items.push_back(Item::Create(FOUNDRY_LOCAL_ITEM_IMAGE)); + c.output_items.push_back(Item::Create(FOUNDRY_LOCAL_ITEM_MESSAGE)); + c.output_items.push_back(std::make_unique("", FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON)); + + for (const auto& item : c.input_items) { + c.input_ptrs.push_back(item.get()); + } + + for (const auto& item : c.output_items) { + c.output_ptrs.push_back(item.get()); + } + + return c; + }(); + + return cache; +} + const StaticIOCache& AutomaticSpeechRecognitionIO() { static const StaticIOCache cache = [] { StaticIOCache c; @@ -464,6 +487,10 @@ Model::IOInfo Model::GetInputOutputInfo() const { return IOInfoFromCache(ChatCompletionIO()); } + if (task == "vision-language-chat") { + return IOInfoFromCache(VisionLanguageChatIO()); + } + if (task == "automatic-speech-recognition") { return IOInfoFromCache(AutomaticSpeechRecognitionIO()); } diff --git a/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc b/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc index c6511c769..281f8198a 100644 --- a/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc +++ b/sdk_v2/cpp/test/internal_api/chat/chat_session_test.cc @@ -8,6 +8,8 @@ #include "exception.h" #include "inferencing/model_load_manager.h" #include "inferencing/generative/chat/search_options.h" +#include "items/audio_item.h" +#include "items/image_item.h" #include "items/text_item.h" #include "ep_detection/ep_detector.h" #include "logger.h" @@ -65,8 +67,11 @@ class ChatSessionTest : public ::testing::Test { static inline std::unique_ptr load_manager_; static inline GenAIModelInstance* model_ = nullptr; static inline fl::test::FakeServiceBindings svc_; - static inline Model catalog_model_ = Model::FromModelInfo( - ModelInfo{}, "", svc_.download_manager, svc_.model_load_manager); + static inline Model catalog_model_ = [] { + ModelInfo info; + info.task = "chat-completion"; + return Model::FromModelInfo(std::move(info), "", svc_.download_manager, svc_.model_load_manager); + }(); TelemetryLogger null_telemetry_{"test", fl::test::NullLog()}; fl::test::NullSessionManager null_session_manager_; }; @@ -139,6 +144,46 @@ TEST_F(ChatSessionTest, RunBasic) { EXPECT_EQ(session.GetHistory()[1].GetSimpleText(), text); } +TEST_F(ChatSessionTest, ChatCompletionRejectsAudioInput) { + ChatSession session(GetCatalogModel(), GetModel(), *logger_, null_telemetry_); + + std::vector> parts; + parts.push_back(std::make_unique(std::vector(32000), "pcm")); + + Request request; + request.AddOwnedItem(std::make_unique(FOUNDRY_LOCAL_ROLE_USER, std::move(parts))); + + Response response; + try { + session.ProcessRequest(request, response); + FAIL() << "Expected unsupported audio input to be rejected"; + } catch (const fl::Exception& e) { + EXPECT_EQ(e.code(), FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT); + EXPECT_NE(std::string(e.what()).find("AUDIO input is not supported"), std::string::npos); + EXPECT_NE(std::string(e.what()).find("chat-completion"), std::string::npos); + } +} + +TEST_F(ChatSessionTest, ChatCompletionRejectsImageInput) { + ChatSession session(GetCatalogModel(), GetModel(), *logger_, null_telemetry_); + + std::vector> parts; + parts.push_back(std::make_unique(std::vector{1}, "png")); + + Request request; + request.AddOwnedItem(std::make_unique(FOUNDRY_LOCAL_ROLE_USER, std::move(parts))); + + Response response; + try { + session.ProcessRequest(request, response); + FAIL() << "Expected image input to be rejected by a chat-completion model"; + } catch (const fl::Exception& e) { + EXPECT_EQ(e.code(), FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT); + EXPECT_NE(std::string(e.what()).find("IMAGE input is not supported"), std::string::npos); + EXPECT_NE(std::string(e.what()).find("chat-completion"), std::string::npos); + } +} + TEST_F(ChatSessionTest, RunWithStreaming) { ChatSession session(GetCatalogModel(), GetModel(), *logger_, null_telemetry_); diff --git a/sdk_v2/cpp/test/internal_api/item_test.cc b/sdk_v2/cpp/test/internal_api/item_test.cc index f77470366..02d5284ec 100644 --- a/sdk_v2/cpp/test/internal_api/item_test.cc +++ b/sdk_v2/cpp/test/internal_api/item_test.cc @@ -33,6 +33,12 @@ using namespace fl; +TEST(ItemTypeNameTest, ReturnsSymbolicNames) { + EXPECT_EQ(Item::TypeName(FOUNDRY_LOCAL_ITEM_IMAGE), "IMAGE"); + EXPECT_EQ(Item::TypeName(FOUNDRY_LOCAL_ITEM_AUDIO), "AUDIO"); + EXPECT_EQ(Item::TypeName(static_cast(-1)), "UNKNOWN"); +} + // ======================================================================== // Item::Create factory — every derived type // ======================================================================== diff --git a/sdk_v2/cpp/test/internal_api/model_io_info_test.cc b/sdk_v2/cpp/test/internal_api/model_io_info_test.cc index 34fcfd421..4ab731ee5 100644 --- a/sdk_v2/cpp/test/internal_api/model_io_info_test.cc +++ b/sdk_v2/cpp/test/internal_api/model_io_info_test.cc @@ -52,6 +52,31 @@ TEST(ModelIOInfoTest, ChatCompletion_ReturnsMessageAndOpenAIJsonOutputs) { EXPECT_EQ(static_cast(io.outputs[1])->text_type, FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON); } +// ======================================================================== +// Vision-language-chat task +// ======================================================================== + +TEST(ModelIOInfoTest, VisionLanguageChat_ReturnsImageInput) { + auto model = MakeModelWithTask("vision-language-chat"); + auto io = model.GetInputOutputInfo(); + + ASSERT_EQ(io.num_inputs, 3u); + EXPECT_EQ(io.inputs[0]->type, FOUNDRY_LOCAL_ITEM_MESSAGE); + EXPECT_EQ(io.inputs[1]->type, FOUNDRY_LOCAL_ITEM_TEXT); + EXPECT_EQ(static_cast(io.inputs[1])->text_type, FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON); + EXPECT_EQ(io.inputs[2]->type, FOUNDRY_LOCAL_ITEM_IMAGE); +} + +TEST(ModelIOInfoTest, VisionLanguageChat_ReturnsChatOutputs) { + auto model = MakeModelWithTask("vision-language-chat"); + auto io = model.GetInputOutputInfo(); + + ASSERT_EQ(io.num_outputs, 2u); + EXPECT_EQ(io.outputs[0]->type, FOUNDRY_LOCAL_ITEM_MESSAGE); + EXPECT_EQ(io.outputs[1]->type, FOUNDRY_LOCAL_ITEM_TEXT); + EXPECT_EQ(static_cast(io.outputs[1])->text_type, FOUNDRY_LOCAL_TEXT_ITEM_TYPE_OPENAI_JSON); +} + // ======================================================================== // Automatic speech recognition task // ========================================================================