diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index 47cbf270f..1a070e397 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -205,6 +205,7 @@ set(FOUNDRY_LOCAL_SOURCES src/catalog/azure_catalog_models.cc src/catalog/catalog_cache.cc src/catalog/catalog_client.cc + src/catalog/local_model_catalog.cc src/catalog/local_model_scanner.cc src/inferencing/generative/audio/audio_generator.cc src/inferencing/generative/audio/audio_session.cc @@ -270,6 +271,7 @@ set(FOUNDRY_LOCAL_SOURCES src/util/path_safety.cc src/util/region_fallback.cc src/util/sha256.cc + src/util/time_utils.cc src/util/zip_extract.cc ${FOUNDRY_LOCAL_PLATFORM_SOURCES} ${FOUNDRY_LOCAL_INTERNAL_HEADERS} diff --git a/sdk_v2/cpp/include/foundry_local/foundry_local_c.h b/sdk_v2/cpp/include/foundry_local/foundry_local_c.h index fa1f63738..f5c4b9287 100644 --- a/sdk_v2/cpp/include/foundry_local/foundry_local_c.h +++ b/sdk_v2/cpp/include/foundry_local/foundry_local_c.h @@ -60,7 +60,7 @@ * Incremented with each release. * Used to request the API function table via FoundryLocalGetApi. * ----------------------------------------------------------------------- */ -#define FOUNDRY_LOCAL_API_VERSION 1 +#define FOUNDRY_LOCAL_API_VERSION 2 /* ----------------------------------------------------------------------- * Platform export macros (C version) @@ -202,6 +202,11 @@ typedef enum flDeviceType { FOUNDRY_LOCAL_DEVICE_NPU = 3 } flDeviceType; +typedef enum flCatalogType { + FOUNDRY_LOCAL_CATALOG_PUBLIC = 0, + FOUNDRY_LOCAL_CATALOG_LOCAL = 1, +} flCatalogType; + /// Tensor element data types. Values match ONNX TensorProto.DataType. typedef enum flTensorDataType { FOUNDRY_LOCAL_TENSOR_UNDEFINED = 0, @@ -256,6 +261,12 @@ typedef enum flTensorDataType { #define FOUNDRY_LOCAL_MODEL_PROP_TOOL_CALL_END_STR "tool_call_end" ///< optional tool call end marker token #define FOUNDRY_LOCAL_MODEL_PROP_REASONING_START_STR "reasoning_start" ///< optional reasoning/think start marker token #define FOUNDRY_LOCAL_MODEL_PROP_REASONING_END_STR "reasoning_end" ///< optional reasoning/think end marker token +#define FOUNDRY_LOCAL_MODEL_PROP_DEVICE_TYPE_STR "device_type" ///< CPU, GPU, or NPU +#define FOUNDRY_LOCAL_MODEL_PROP_EP_STR "execution_provider" ///< optional execution provider +#define FOUNDRY_LOCAL_MODEL_PROP_ENTITY_TYPE_STR "entity_type" ///< fixed to "Model" for BYOM +#define FOUNDRY_LOCAL_MODEL_PROP_AUTHOR_STR "author" ///< optional +#define FOUNDRY_LOCAL_MODEL_PROP_QUANTIZATION_STR "quantization" ///< optional +#define FOUNDRY_LOCAL_MODEL_PROP_CREATION_TIME_STR "creation_time" ///< ISO-8601 UTC timestamp /* flModelInfo Int properties. Comments provide details on the type and expected values. */ #define FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT "supports_tool_calling" ///< optional bool (not set or -1=unknown, 0=false, 1=true) @@ -265,6 +276,7 @@ typedef enum flTensorDataType { #define FOUNDRY_LOCAL_MODEL_PROP_CREATED_AT_UNIX_INT "created_at_unix" ///< Unix timestamp. default=0 #define FOUNDRY_LOCAL_MODEL_PROP_IS_TEST_MODEL_INT "is_test_model" ///< bool (0=false, 1=true) #define FOUNDRY_LOCAL_MODEL_PROP_CONTEXT_LENGTH_INT "context_length" ///< optional int64_t +#define FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_HYBRID_REASONING_INT "supports_hybrid_reasoning" ///< optional bool #define FOUNDRY_LOCAL_MODEL_PROP_INPUT_MODALITIES_STR "input_modalities" ///< optional, comma-separated #define FOUNDRY_LOCAL_MODEL_PROP_OUTPUT_MODALITIES_STR "output_modalities" ///< optional, comma-separated @@ -709,6 +721,10 @@ typedef struct flApi { bool FL_API_T(Manager_IsShutdownRequested, _In_ const flManager* manager); // End V1 + FL_API_STATUS(Manager_GetCatalogByType, _In_ const flManager* manager, flCatalogType catalog_type, + _Outptr_ flCatalog** out_catalog); + + // End V2 /* Append new function pointers at the end for future versions and add marker for the end of each version */ } flApi; @@ -941,8 +957,9 @@ struct flCatalogApi { /// Returned string is owned by the catalog and valid for the catalog's lifetime. FL_API_STATUS(GetName, _In_ const flCatalog* catalog, _Out_ const char** out_name); - // Catalog owns model list. Cached for efficiency. - // Models are mutable for load/unload/remove operations. Model info is immutable though. + /// The caller owns each returned model list and must release it with ModelList_Release. The model handles in a list + /// are borrowed from the catalog and remain address-valid until the owning manager is destroyed. Releasing a list + /// does not invalidate its model handles. Models are mutable for load/unload/remove operations; model info is immutable. FL_API_STATUS(GetModels, _In_ const flCatalog* catalog, _Outptr_ flModelList** out_models); FL_API_STATUS(GetModel, _In_ const flCatalog* catalog, _In_ const char* alias, _Outptr_ flModel** out_model); @@ -972,6 +989,20 @@ struct flCatalogApi { _In_opt_ const char* model_name, int32_t max_versions, _Outptr_ flModelList** out_models); // End V1 + /// Register a model in a local catalog without taking ownership of its assets. + /// `model_path` must identify a model directory containing genai_config.json. + /// `model_id` must use the canonical `:` format and be unique in the local catalog. + /// The metadata is copied; model identity and location are taken only from the explicit arguments. + FL_API_STATUS(RegisterModel, _In_ flCatalog* catalog, _In_ const char* model_path, + _In_ const char* model_id, _In_ const flModelInfo* metadata, + _Outptr_ flModel** out_model); + /// Unregister by alias or model ID without deleting model assets. + /// Future catalog queries exclude the registration, but outstanding model handles and their immutable metadata remain + /// valid until the owning manager is destroyed. Operations that require the retired registration, including Download + /// and Load, return FOUNDRY_LOCAL_ERROR_INVALID_USAGE; query and cleanup operations such as Unload remain valid. + FL_API_STATUS(UnregisterModel, _In_ flCatalog* catalog, _In_ const char* alias_or_model_id); + + // End V2 }; /* --- Model API --------------------------------------------------------- */ @@ -988,8 +1019,8 @@ struct flModelApi { /* Model handle operations. Catalog owns Model instances. */ FL_API_STATUS(IsCached, _In_ const flModel* model, _Out_ int* out_cached); - /// Returned path string is owned by the model and valid until the model is released or its cache state - /// changes via RemoveFromCache. + /// Returned path string is owned by the model and valid until the owning manager is destroyed or the model's cache + /// state changes via Download or RemoveFromCache. FL_API_STATUS(GetPath, _In_ const flModel* model, _Out_ const char** out_path); FL_API_STATUS(Download, _In_ flModel* model, _In_opt_ flProgressCallback callback, _In_opt_ void* user_data); @@ -1041,6 +1072,13 @@ struct flModelApi { int64_t FL_API_T(Info_GetIntProperty, _In_ const flModelInfo* info, _In_ const char* key, int64_t default_value); // End V1 + /// Create a caller-owned mutable ModelInfo. Release it with ReleaseModelInfo. + FL_API_STATUS(CreateModelInfo, _Outptr_ flModelInfo** out_info); + void FL_API_T(ReleaseModelInfo, _Frees_ptr_opt_ flModelInfo* info); + FL_API_STATUS(Info_SetStringProperty, _In_ flModelInfo* info, _In_ const char* key, _In_ const char* value); + FL_API_STATUS(Info_SetIntProperty, _In_ flModelInfo* info, _In_ const char* key, int64_t value); + + // End V2 }; #ifdef __cplusplus diff --git a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h index cf96fa752..57fd7490b 100644 --- a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h +++ b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h @@ -63,6 +63,9 @@ namespace detail { /// Returns nullptr if the library does not support the requested API version. inline const flApi* api() { static const flApi* p = FoundryLocalGetApi(FOUNDRY_LOCAL_API_VERSION); + if (!p) { + throw std::runtime_error("Foundry Local runtime does not support the API version requested by this header"); + } return p; } @@ -298,14 +301,31 @@ struct Runtime { std::optional execution_provider; }; +enum class CatalogType { + Public = FOUNDRY_LOCAL_CATALOG_PUBLIC, + Local = FOUNDRY_LOCAL_CATALOG_LOCAL, +}; + // =========================================================================== -// ModelInfo — non-owning read-only view +// ModelInfo — owning mutable value or non-owning read-only view // =========================================================================== -/// Non-owning view over an opaque flModelInfo. Lifetime is tied to the owning Model/Catalog. Immutable. +/// Opaque model metadata. Default construction creates an owning mutable value for registration. +/// Construction from `const flModelInfo&` creates a non-owning read-only view tied to its Model. class ModelInfo { public: - explicit ModelInfo(const flModelInfo& info) noexcept : info_(&info) {} + ModelInfo(); + explicit ModelInfo(const flModelInfo& info) noexcept : handle_(&info) {} + + ModelInfo(const ModelInfo&) = delete; + ModelInfo& operator=(const ModelInfo&) = delete; + ModelInfo(ModelInfo&&) noexcept = default; + ModelInfo& operator=(ModelInfo&&) noexcept = default; + + ModelInfo& SetStringProperty(const char* key, const char* value); + ModelInfo& SetIntProperty(const char* key, int64_t value); + + const flModelInfo* native_handle() const noexcept { return handle_.get(); } // Core identity. std::string_view Id() const noexcept; @@ -376,7 +396,7 @@ class ModelInfo { private: static std::string_view safe(const char* s) noexcept { return s ? s : ""; } - const flModelInfo* info_; + detail::Base handle_; }; // =========================================================================== @@ -695,6 +715,9 @@ class IModel { // Model — concrete IModel implementation using composition // =========================================================================== +/// Non-owning wrapper over a catalog-owned model. The Manager that supplied the catalog must outlive this object and +/// any ModelInfo view obtained from it. Unregistering a local model removes it from future catalog queries without +/// invalidating existing wrappers; operations that require the retired registration, including Download and Load, fail. class Model final : public IModel { public: /// Mutable construction (from catalog lookups that return flModel*). @@ -735,6 +758,8 @@ class Model final : public IModel { // ModelList // =========================================================================== +/// Owning wrapper for a native model-list allocation. Its Model entries are non-owning views into catalog storage, so +/// the Manager that supplied the catalog must outlive the list and any Model wrapper retained from it. class ModelList { public: ModelList(flModelList& model_list); @@ -780,12 +805,22 @@ class ICatalog { /// returns every variant. `max_versions` selects the latest X versions per /// variant name (defaults to 50, matching the web service contract); pass 0 /// or a negative value for no per-variant cap. Each call performs a fresh - /// query and the returned model handles remain valid until the next - /// GetModelVersions call for the same alias or until the catalog is destroyed. - /// Queries for different aliases do not invalidate each other's results. + /// query and the returned model handles remain valid until the owning Manager + /// is destroyed. Repeated queries do not invalidate earlier results. virtual ModelList GetModelVersions(const std::string& model_alias, const std::string& variant_name = {}, int max_versions = 50) = 0; + + /// Register existing local model assets. `model_id` must use `:`; metadata is copied. + /// The catalog does not take ownership of `model_path` and never deletes its contents. + virtual std::unique_ptr RegisterModel(const std::string&, const std::string&, const ModelInfo&) { + throw Error("models can only be registered in a local catalog", FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT); + } + /// Unregister without deleting model assets. Existing model wrappers and immutable metadata views remain valid, but + /// operations that require the retired registration, including Download and Load, fail with invalid usage. + virtual void UnregisterModel(const std::string&) { + throw Error("models can only be unregistered from a local catalog", FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT); + } }; // =========================================================================== @@ -794,7 +829,7 @@ class ICatalog { class Catalog final : public ICatalog { public: - /// Adopt an already-created catalog handle (owning). + /// Wrap an already-created manager-owned catalog handle (non-owning). /// Most users should obtain a catalog via Manager::GetCatalog() rather than constructing one directly. explicit Catalog(flCatalog& catalog) : handle_(&catalog) {} @@ -811,6 +846,9 @@ class Catalog final : public ICatalog { ModelList GetModelVersions(const std::string& model_alias, const std::string& variant_name = {}, int max_versions = 50) override; + std::unique_ptr RegisterModel(const std::string& model_path, const std::string& model_id, + const ModelInfo& metadata) override; + void UnregisterModel(const std::string& alias_or_model_id) override; private: detail::Base handle_; @@ -840,6 +878,7 @@ class Manager { /// Get the catalog for querying models. Creates on first call, caches internally. ICatalog& GetCatalog() const; + ICatalog& GetCatalog(CatalogType type) const; /// Start the embedded web service. void StartWebService(); @@ -872,10 +911,16 @@ class Manager { bool IsShutdownRequested() const; private: + struct CatalogCollection { + mutable std::unique_ptr public_; + mutable std::unique_ptr local_; + }; + detail::Base handle_; Configuration config_; - mutable std::unique_ptr catalog_; + CatalogCollection catalogs_; mutable std::unique_ptr catalog_once_{std::make_unique()}; + mutable std::unique_ptr local_catalog_once_{std::make_unique()}; }; // =========================================================================== diff --git a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.inline.h b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.inline.h index 8a59085d5..b9045a5b2 100644 --- a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.inline.h +++ b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.inline.h @@ -204,9 +204,22 @@ inline ICatalog& Manager::GetCatalog() const { std::call_once(*catalog_once_, [this]() { flCatalog* cat = nullptr; Check(detail::api()->Manager_GetCatalog(handle_.get(), &cat)); - catalog_ = std::unique_ptr(new Catalog(*cat)); + catalogs_.public_ = std::unique_ptr(new Catalog(*cat)); }); - return *catalog_; + return *catalogs_.public_; +} + +inline ICatalog& Manager::GetCatalog(CatalogType type) const { + if (type == CatalogType::Public) { + return GetCatalog(); + } + + std::call_once(*local_catalog_once_, [this, type]() { + flCatalog* catalog = nullptr; + Check(detail::api()->Manager_GetCatalogByType(handle_.get(), static_cast(type), &catalog)); + catalogs_.local_ = std::make_unique(*catalog); + }); + return *catalogs_.local_; } inline void Manager::StartWebService() { @@ -295,32 +308,50 @@ inline flManager* detail::CreateManager(const Configuration& config) { // ModelInfo // =========================================================================== +inline ModelInfo::ModelInfo() + : handle_([] { + flModelInfo* info = nullptr; + Check(detail::model_api()->CreateModelInfo(&info)); + return info; + }(), + detail::model_api()->ReleaseModelInfo) {} + +inline ModelInfo& ModelInfo::SetStringProperty(const char* key, const char* value) { + Check(detail::model_api()->Info_SetStringProperty(handle_.get_mutable(), key, value)); + return *this; +} + +inline ModelInfo& ModelInfo::SetIntProperty(const char* key, int64_t value) { + Check(detail::model_api()->Info_SetIntProperty(handle_.get_mutable(), key, value)); + return *this; +} + inline std::string_view ModelInfo::Id() const noexcept { - return safe(detail::model_api()->Info_GetId(info_)); + return safe(detail::model_api()->Info_GetId(handle_.get())); } inline std::string_view ModelInfo::Name() const noexcept { - return safe(detail::model_api()->Info_GetName(info_)); + return safe(detail::model_api()->Info_GetName(handle_.get())); } inline int ModelInfo::Version() const noexcept { - return detail::model_api()->Info_GetVersion(info_); + return detail::model_api()->Info_GetVersion(handle_.get()); } inline std::string_view ModelInfo::Alias() const noexcept { - return safe(detail::model_api()->Info_GetAlias(info_)); + return safe(detail::model_api()->Info_GetAlias(handle_.get())); } inline std::string_view ModelInfo::Uri() const noexcept { - return safe(detail::model_api()->Info_GetUri(info_)); + return safe(detail::model_api()->Info_GetUri(handle_.get())); } inline flDeviceType ModelInfo::DeviceType() const noexcept { - return detail::model_api()->Info_GetDeviceType(info_); + return detail::model_api()->Info_GetDeviceType(handle_.get()); } inline std::optional ModelInfo::ExecutionProvider() const noexcept { - const char* v = detail::model_api()->Info_GetExecutionProvider(info_); + const char* v = detail::model_api()->Info_GetExecutionProvider(handle_.get()); return v ? std::optional{v} : std::nullopt; } @@ -333,7 +364,7 @@ inline std::optional ModelInfo::GetRuntime() const noexcept { } inline std::optional ModelInfo::GetPromptTemplate(const char* key) const noexcept { - const flKeyValuePairs* kvps = detail::model_api()->Info_GetPromptTemplates(info_); + const flKeyValuePairs* kvps = detail::model_api()->Info_GetPromptTemplates(handle_.get()); if (!kvps) { return std::nullopt; } @@ -342,7 +373,7 @@ inline std::optional ModelInfo::GetPromptTemplate(const char* } inline std::optional ModelInfo::GetModelSetting(const char* key) const noexcept { - const flKeyValuePairs* kvps = detail::model_api()->Info_GetModelSettings(info_); + const flKeyValuePairs* kvps = detail::model_api()->Info_GetModelSettings(handle_.get()); if (!kvps) { return std::nullopt; } @@ -351,7 +382,7 @@ inline std::optional ModelInfo::GetModelSetting(const char* ke } inline std::optional ModelInfo::GetModelSettings() const noexcept { - const flKeyValuePairs* kvps = detail::model_api()->Info_GetModelSettings(info_); + const flKeyValuePairs* kvps = detail::model_api()->Info_GetModelSettings(handle_.get()); if (!kvps) { return std::nullopt; } @@ -359,12 +390,12 @@ inline std::optional ModelInfo::GetModelSettings() const noexcept } inline std::optional ModelInfo::GetStringProperty(const char* key) const noexcept { - const char* v = detail::model_api()->Info_GetStringProperty(info_, key); + const char* v = detail::model_api()->Info_GetStringProperty(handle_.get(), key); return v ? std::optional{v} : std::nullopt; } inline int64_t ModelInfo::GetIntProperty(const char* key, int64_t default_value) const noexcept { - return detail::model_api()->Info_GetIntProperty(info_, key, default_value); + return detail::model_api()->Info_GetIntProperty(handle_.get(), key, default_value); } // --- Typed property accessors --- @@ -626,6 +657,18 @@ inline ModelList Catalog::GetModelVersions(const std::string& model_alias, return ModelList(*models); } +inline std::unique_ptr Catalog::RegisterModel(const std::string& model_path, const std::string& model_id, + const ModelInfo& metadata) { + flModel* model = nullptr; + Check(detail::catalog_api()->RegisterModel(handle_.get_mutable(), model_path.c_str(), model_id.c_str(), + metadata.native_handle(), &model)); + return std::make_unique(*model); +} + +inline void Catalog::UnregisterModel(const std::string& alias_or_model_id) { + Check(detail::catalog_api()->UnregisterModel(handle_.get_mutable(), alias_or_model_id.c_str())); +} + // =========================================================================== // Item // =========================================================================== diff --git a/sdk_v2/cpp/src/c_api.cc b/sdk_v2/cpp/src/c_api.cc index 6e89e534f..59189637c 100644 --- a/sdk_v2/cpp/src/c_api.cc +++ b/sdk_v2/cpp/src/c_api.cc @@ -71,7 +71,8 @@ struct flCatalog { // --- Manager --- struct flManager { fl::Manager& impl; - std::unique_ptr catalog; // stores the flCatalog wrapper around impl.GetCatalog() + std::unique_ptr public_catalog; + std::unique_ptr local_catalog; mutable std::vector urls_cache; }; @@ -327,8 +328,9 @@ FL_API_STATUS_IMPL(Manager_CreateImpl, const flConfiguration* config, flManager* } auto& mgr = fl::Manager::Create(*cfg); - auto wrapper = std::make_unique(flManager{mgr, nullptr, {}}); - wrapper->catalog = std::make_unique(flCatalog{mgr.GetCatalog()}); + auto wrapper = std::make_unique(flManager{mgr, nullptr, nullptr, {}}); + wrapper->public_catalog = std::make_unique(flCatalog{mgr.GetCatalog(fl::CatalogType::kPublic)}); + wrapper->local_catalog = std::make_unique(flCatalog{mgr.GetCatalog(fl::CatalogType::kLocal)}); *out_manager = wrapper.release(); return nullptr; API_IMPL_END @@ -349,11 +351,31 @@ FL_API_STATUS_IMPL(Manager_GetCatalogImpl, const flManager* manager, flCatalog** return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "null argument"); } - *out_catalog = manager->catalog.get(); + *out_catalog = manager->public_catalog.get(); return nullptr; API_IMPL_END } +FL_API_STATUS_IMPL(Manager_GetCatalogByTypeImpl, const flManager* manager, flCatalogType catalog_type, + flCatalog** out_catalog) { + API_IMPL_BEGIN + if (!manager || !out_catalog) { + return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "null argument"); + } + + switch (catalog_type) { + case FOUNDRY_LOCAL_CATALOG_PUBLIC: + *out_catalog = manager->public_catalog.get(); + return nullptr; + case FOUNDRY_LOCAL_CATALOG_LOCAL: + *out_catalog = manager->local_catalog.get(); + return nullptr; + default: + return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "unknown catalog type"); + } + API_IMPL_END +} + FL_API_STATUS_IMPL(Manager_WebServiceStartImpl, flManager* manager) { API_IMPL_BEGIN if (!manager) { @@ -714,6 +736,29 @@ FL_API_STATUS_IMPL(Catalog_GetModelVersionsImpl, const flCatalog* catalog, API_IMPL_END } +FL_API_STATUS_IMPL(Catalog_RegisterModelImpl, flCatalog* catalog, const char* model_path, + const char* model_id, const flModelInfo* metadata, flModel** out_model) { + API_IMPL_BEGIN + if (!catalog || !model_path || !model_id || !metadata || !out_model) { + return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "null argument"); + } + + *out_model = AsHandle(catalog->impl.RegisterModel(model_path, model_id, *AsImpl(metadata))); + return nullptr; + API_IMPL_END +} + +FL_API_STATUS_IMPL(Catalog_UnregisterModelImpl, flCatalog* catalog, const char* alias_or_model_id) { + API_IMPL_BEGIN + if (!catalog || !alias_or_model_id) { + return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "null argument"); + } + + catalog->impl.UnregisterModel(alias_or_model_id); + return nullptr; + API_IMPL_END +} + static const flCatalogApi g_catalog_api = { Catalog_GetNameImpl, Catalog_GetModelsImpl, @@ -723,6 +768,8 @@ static const flCatalogApi g_catalog_api = { Catalog_GetCachedModelsImpl, Catalog_GetLoadedModelsImpl, Catalog_GetModelVersionsImpl, + Catalog_RegisterModelImpl, + Catalog_UnregisterModelImpl, }; // ======================================================================== @@ -837,15 +884,6 @@ FL_API_STATUS_IMPL(Model_RemoveFromCacheImpl, flModel* model) { } auto* impl = AsImpl(model); - if (!impl->IsCached()) { - return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "model is not cached locally"); - } - - if (impl->IsLoaded()) { - return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, - "cannot remove a loaded model from cache; unload it first"); - } - impl->RemoveFromCache(); return nullptr; API_IMPL_END @@ -969,6 +1007,40 @@ static int64_t FL_API_CALL Info_GetIntPropertyImpl(const flModelInfo* info, return AsImpl(info)->GetPropertyWithDefault(key, default_value); } +FL_API_STATUS_IMPL(ModelInfo_CreateImpl, flModelInfo** out_info) { + API_IMPL_BEGIN + if (!out_info) { + return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "out_info must not be null"); + } + *out_info = AsHandle(new fl::ModelInfo()); + return nullptr; + API_IMPL_END +} + +static void FL_API_CALL ModelInfo_ReleaseImpl(flModelInfo* info) FL_NO_EXCEPTION { + delete AsImpl(info); +} + +FL_API_STATUS_IMPL(Info_SetStringPropertyImpl, flModelInfo* info, const char* key, const char* value) { + API_IMPL_BEGIN + if (!info || !key || !value) { + return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "null argument"); + } + AsImpl(info)->SetPropertyStr(key, value); + return nullptr; + API_IMPL_END +} + +FL_API_STATUS_IMPL(Info_SetIntPropertyImpl, flModelInfo* info, const char* key, int64_t value) { + API_IMPL_BEGIN + if (!info || !key) { + return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "null argument"); + } + AsImpl(info)->SetPropertyInt(key, value); + return nullptr; + API_IMPL_END +} + static const flModelApi g_model_api = { Model_GetInfoImpl, Model_GetInputOutputInfoImpl, @@ -993,6 +1065,10 @@ static const flModelApi g_model_api = { Info_GetModelSettingsImpl, Info_GetStringPropertyImpl, Info_GetIntPropertyImpl, + ModelInfo_CreateImpl, + ModelInfo_ReleaseImpl, + Info_SetStringPropertyImpl, + Info_SetIntPropertyImpl, }; // ======================================================================== @@ -1842,51 +1918,37 @@ static const flModelApi* FL_API_CALL GetModelApiImpl() FL_NO_EXCEPTION { return &g_model_api; } -// ======================================================================== -// Root API function table (version 1) -// ======================================================================== - -static const flApi g_api_v1 = { - /* Status */ +static const flApi g_api = { Status_CreateImpl, Status_ReleaseImpl, Status_GetErrorCodeImpl, Status_GetErrorMessageImpl, - - /* Manager lifecycle */ Manager_CreateImpl, Manager_ReleaseImpl, Manager_GetCatalogImpl, Manager_WebServiceStartImpl, Manager_WebServiceUrlsImpl, Manager_WebServiceStopImpl, - - /* Sub-API accessors */ GetCatalogApiImpl, GetConfigurationApiImpl, GetItemApiImpl, GetInferenceApiImpl, GetModelApiImpl, - - /* KeyValuePairs */ CreateKeyValuePairsImpl, AddKeyValuePairImpl, GetKeyValueImpl, GetKeyValuePairsImpl, RemoveKeyValuePairImpl, KeyValuePairs_ReleaseImpl, - - /* ModelList */ ModelList_ReleaseImpl, ModelList_SizeImpl, ModelList_GetAtImpl, - - /* EP detection */ Manager_GetDiscoverableEpsImpl, Manager_DownloadAndRegisterEpsImpl, Manager_IsEpDownloadInProgressImpl, Manager_ShutdownImpl, Manager_IsShutdownRequestedImpl, + Manager_GetCatalogByTypeImpl, }; // ======================================================================== @@ -1896,8 +1958,8 @@ static const flApi g_api_v1 = { extern "C" { FL_EXPORT const flApi* FL_API_CALL FoundryLocalGetApi(uint32_t version) FL_NO_EXCEPTION { - if (version == 0 || version <= FOUNDRY_LOCAL_API_VERSION) { - return &g_api_v1; + if (version <= FOUNDRY_LOCAL_API_VERSION) { + return &g_api; } return nullptr; diff --git a/sdk_v2/cpp/src/catalog.h b/sdk_v2/cpp/src/catalog.h index 3868f74b6..5d41b4717 100644 --- a/sdk_v2/cpp/src/catalog.h +++ b/sdk_v2/cpp/src/catalog.h @@ -2,13 +2,21 @@ // Licensed under the MIT License. #pragma once +#include "exception.h" #include "model.h" +#include + #include #include namespace fl { +enum class CatalogType { + kPublic, + kLocal, +}; + /// Abstract catalog interface for querying available models. /// Mirrors the C API's flCatalogApi surface. class ICatalog { @@ -19,6 +27,8 @@ class ICatalog { /// For Azure catalogs this is the catalog URI. virtual const std::string& GetName() const = 0; + virtual CatalogType GetType() const { return CatalogType::kPublic; } + /// Lists all models in the catalog. virtual std::vector ListModels() const = 0; @@ -58,6 +68,15 @@ class ICatalog { /// Lists only models that are currently loaded into a runtime. virtual std::vector GetLoadedModels() const = 0; + virtual Model* RegisterModel(const std::string& /*model_path*/, const std::string& /*model_id*/, + const ModelInfo& /*metadata*/) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "models can only be registered in a local catalog"); + } + + virtual void UnregisterModel(const std::string& /*alias_or_model_id*/) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "models can only be unregistered from a local catalog"); + } + /// Invalidate the cached model list so the next query re-fetches. /// Called after EP registration changes, since the available device filters /// may now include additional execution providers. diff --git a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc index 63776436c..853aefd9d 100644 --- a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc @@ -6,7 +6,6 @@ #include "catalog/local_model_scanner.h" #include "model.h" #include "model_info.h" -#include "utils.h" #include #include @@ -20,20 +19,6 @@ namespace fl { namespace { -ModelInfo MakeByomModelInfo(const std::string& model_id) { - auto [name, version] = Utils::SplitModelNameAndVersion(model_id); - - ModelInfo info; - info.model_id = model_id; - info.name = name; - info.alias = name; - info.uri = "local://" + name; - info.version = version; - info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR] = "Local"; - info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_MODEL_TYPE_STR] = "ONNX"; - return info; -} - std::vector DeduplicateByModelId(std::vector model_infos) { std::vector deduplicated; deduplicated.reserve(model_infos.size()); @@ -48,6 +33,13 @@ std::vector DeduplicateByModelId(std::vector model_infos) return deduplicated; } +void RemoveLegacyLocalEntries(std::vector& model_infos) { + std::erase_if(model_infos, [](const auto& info) { + const auto* provider = info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR); + return provider && *provider == "Local"; + }); +} + } // namespace AzureModelCatalog::AzureModelCatalog(std::vector>> catalog_urls, @@ -115,37 +107,26 @@ AzureModelCatalog::CatalogResult AzureModelCatalog::GetLiveCatalogOrLocalSnapsho CatalogCache cache(cache_dir_, logger_); cache.Load(); auto cached = cache.GetCachedModels(); + auto snapshot_model_infos = cached ? std::move(*cached) : std::vector{}; + RemoveLegacyLocalEntries(snapshot_model_infos); return { - .model_infos = cached ? DeduplicateByModelId(std::move(*cached)) : std::vector{}, + .model_infos = DeduplicateByModelId(std::move(snapshot_model_infos)), .source = CatalogSource::kSnapshot, }; } -std::vector AzureModelCatalog::AddLocalModels(std::vector& model_infos, - const LocalModels& local_models) const { +std::vector AzureModelCatalog::CreateModelsWithLocalPaths(const std::vector& model_infos, + const LocalModels& local_models) const { std::vector models; - models.reserve(model_infos.size() + local_models.size()); + models.reserve(model_infos.size()); - std::unordered_set model_ids; - model_ids.reserve(model_infos.size() + local_models.size()); for (const auto& info : model_infos) { - model_ids.insert(info.model_id); - auto local_model = local_models.find(info.model_id); auto local_path = local_model != local_models.end() ? local_model->second : std::string{}; models.push_back(model_factory_(ModelInfo(info), std::move(local_path))); } - for (const auto& [model_id, local_path] : local_models) { - if (!model_ids.insert(model_id).second) { - continue; - } - - model_infos.push_back(MakeByomModelInfo(model_id)); - models.push_back(model_factory_(ModelInfo(model_infos.back()), local_path)); - } - return models; } @@ -162,7 +143,7 @@ std::vector AzureModelCatalog::FetchModels() const { logger_.Log(LogLevel::Information, fmt::format("Found {} locally cached models.", cached_model_ids.size())); auto catalog_result = GetLiveCatalogOrLocalSnapshot(cached_model_ids); - auto models = AddLocalModels(catalog_result.model_infos, local_models); + auto models = CreateModelsWithLocalPaths(catalog_result.model_infos, local_models); logger_.Log(LogLevel::Information, fmt::format("Populated model info for {} models.", models.size())); diff --git a/sdk_v2/cpp/src/catalog/azure_model_catalog.h b/sdk_v2/cpp/src/catalog/azure_model_catalog.h index b7bb3bc6e..df75f70f2 100644 --- a/sdk_v2/cpp/src/catalog/azure_model_catalog.h +++ b/sdk_v2/cpp/src/catalog/azure_model_catalog.h @@ -60,7 +60,8 @@ class AzureModelCatalog : public BaseModelCatalog { static constexpr const char* kDefaultCatalogFilter = "''"; CatalogResult GetLiveCatalogOrLocalSnapshot(const std::vector& cached_model_ids) const; - std::vector AddLocalModels(std::vector& model_infos, const LocalModels& local_models) const; + std::vector CreateModelsWithLocalPaths(const std::vector& model_infos, + const LocalModels& local_models) const; std::vector>> catalog_urls_; std::string cache_dir_; diff --git a/sdk_v2/cpp/src/catalog/base_model_catalog.cc b/sdk_v2/cpp/src/catalog/base_model_catalog.cc index 29365fe54..d569fe287 100644 --- a/sdk_v2/cpp/src/catalog/base_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/base_model_catalog.cc @@ -16,7 +16,9 @@ namespace fl { BaseModelCatalog::BaseModelCatalog(std::string name, ILogger& logger) - : name_(std::move(name)), logger_(logger) {} + : BaseModelCatalog(std::move(name), CatalogType::kPublic, logger) {} +BaseModelCatalog::BaseModelCatalog(std::string name, CatalogType type, ILogger& logger) + : name_(std::move(name)), type_(type), logger_(logger) {} BaseModelCatalog::~BaseModelCatalog() = default; void BaseModelCatalog::PopulateModels(std::vector variants) const { @@ -48,18 +50,42 @@ void BaseModelCatalog::PopulateModels(std::vector variants) const { } // On refresh: merge new models into stable storage. Existing models keep their addresses. - // New aliases are appended. Existing aliases are left unchanged (their Model* stays valid). if (populated_) { - // Build a set of existing aliases for fast lookup. std::unordered_map existing_aliases; - for (auto& m : models_) { - existing_aliases[m->Alias()] = m.get(); + for (auto& stored : models_) { + if (stored.active) { + existing_aliases[stored.model->Alias()] = stored.model.get(); + } + } + + if (IsAuthoritativeSnapshot()) { + for (auto& stored : models_) { + if (!stored.active) { + continue; + } + + const auto incoming = alias_to_model.find(stored.model->Alias()); + if (incoming == alias_to_model.end()) { + if (!stored.model->TryDeactivateForRefresh()) { + continue; + } + stored.active = false; + existing_aliases.erase(stored.model->Alias()); + continue; + } + + if (!stored.model->TryReconcileVariants(incoming->second)) { + alias_to_model.erase(incoming); + continue; + } + alias_to_model.erase(incoming); + } } size_t new_count = 0; for (auto& [alias, model] : alias_to_model) { if (!existing_aliases.contains(alias)) { - models_.push_back(std::make_unique(std::move(model))); + models_.push_back({std::make_unique(std::move(model)), true}); ++new_count; } } @@ -77,7 +103,7 @@ void BaseModelCatalog::PopulateModels(std::vector variants) const { // Initial population: move all models into stable storage. models_.reserve(alias_to_model.size()); for (auto& [alias, model] : alias_to_model) { - models_.push_back(std::make_unique(std::move(model))); + models_.push_back({std::make_unique(std::move(model)), true}); } logger_.Log(LogLevel::Debug, @@ -98,15 +124,20 @@ void BaseModelCatalog::IntegrateVariants(std::vector variants) const { // Build a lookup of existing aliases -> containers so we can merge new // variants in O(1) per incoming variant. std::unordered_map alias_to_existing; - for (auto& m : models_) { - alias_to_existing[m->Alias()] = m.get(); + for (auto& stored : models_) { + if (stored.active) { + alias_to_existing[stored.model->Alias()] = stored.model.get(); + } } // Track existing model_ids in a single set so the dedup check is O(1) and // doesn't require walking each container's variants per incoming variant. std::unordered_set existing_ids; - for (auto& m : models_) { - for (auto* v : m->Variants()) { + for (auto& stored : models_) { + if (!stored.active) { + continue; + } + for (auto* v : stored.model->Variants()) { existing_ids.insert(v->Info().model_id); } } @@ -151,7 +182,7 @@ void BaseModelCatalog::IntegrateVariants(std::vector variants) const { container.SelectDefaultVariant(); - models_.push_back(std::make_unique(std::move(container))); + models_.push_back({std::make_unique(std::move(container)), true}); ++added_aliases; added_variants += alias_variants.size(); } @@ -169,7 +200,12 @@ void BaseModelCatalog::IntegrateVariants(std::vector variants) const { void BaseModelCatalog::RebuildIndex() const { auto new_index = std::make_shared(); - for (auto& m : models_) { + for (auto& stored : models_) { + if (!stored.active) { + continue; + } + + auto& m = stored.model; new_index->alias_index[m->Alias()] = m.get(); for (auto* variant : m->Variants()) { @@ -224,8 +260,8 @@ void BaseModelCatalog::EnsurePopulated(bool allow_refresh) const { // not worth the complexity to optimise.) std::lock_guard lock(mutex_); - bool needs_refresh = allow_refresh && - std::chrono::steady_clock::now() >= next_refresh_at_; + const bool needs_refresh = IsAuthoritativeSnapshot() || + (allow_refresh && std::chrono::steady_clock::now() >= next_refresh_at_); if (populated_ && !needs_refresh) { return; @@ -249,8 +285,10 @@ std::vector BaseModelCatalog::ListModels() const { std::lock_guard lock(mutex_); std::vector result; result.reserve(models_.size()); - for (auto& m : models_) { - result.push_back(m.get()); + for (auto& stored : models_) { + if (stored.active) { + result.push_back(stored.model.get()); + } } return result; @@ -346,8 +384,12 @@ std::vector BaseModelCatalog::GetCachedModels() const { std::lock_guard lock(mutex_); std::vector result; - for (auto& m : models_) { - for (auto* variant : m->Variants()) { + for (auto& stored : models_) { + if (!stored.active) { + continue; + } + + for (auto* variant : stored.model->Variants()) { if (variant->IsCached()) { result.push_back(variant); } @@ -362,15 +404,71 @@ std::vector BaseModelCatalog::GetLoadedModels() const { std::lock_guard lock(mutex_); std::vector result; - for (auto& m : models_) { - if (m->IsLoaded()) { - result.push_back(m.get()); + for (auto& stored : models_) { + if (stored.active && stored.model->IsLoaded()) { + result.push_back(stored.model.get()); } } return result; } +Model* BaseModelCatalog::AppendActiveModel(Model model) { + EnsurePopulated(); + std::lock_guard lock(mutex_); + auto container = std::make_unique(Model::MakeContainer(std::move(model))); + container->SelectDefaultVariant(); + auto* result = container.get(); + models_.push_back({std::move(container), true}); + RebuildIndex(); + return result; +} + +bool BaseModelCatalog::RetireModel(const std::string& alias_or_model_id) { + EnsurePopulated(); + std::lock_guard lock(mutex_); + for (auto& stored : models_) { + if (!stored.active) { + continue; + } + + bool matches = stored.model->Alias() == alias_or_model_id; + for (auto* variant : stored.model->Variants()) { + matches = matches || variant->Id() == alias_or_model_id; + } + + if (matches) { + stored.active = false; + stored.model->Deactivate(); + RebuildIndex(); + return true; + } + } + + return false; +} + +bool BaseModelCatalog::CommitUnregister(Model* model, const std::string& alias_or_model_id) { + std::lock_guard lock(mutex_); + for (auto& stored : models_) { + if (!stored.active || stored.model.get() != model) { + continue; + } + + if (stored.model->Alias() == alias_or_model_id) { + stored.active = false; + stored.model->Deactivate(); + } else if (!stored.model->RetireVariant(alias_or_model_id)) { + stored.active = false; + } + + RebuildIndex(); + return true; + } + + return false; +} + std::vector BaseModelCatalog::GetModelVersions(const std::string& model_alias, const std::string& variant_name, int max_versions) { diff --git a/sdk_v2/cpp/src/catalog/base_model_catalog.h b/sdk_v2/cpp/src/catalog/base_model_catalog.h index 97411e395..a422fdd1f 100644 --- a/sdk_v2/cpp/src/catalog/base_model_catalog.h +++ b/sdk_v2/cpp/src/catalog/base_model_catalog.h @@ -33,6 +33,7 @@ class BaseModelCatalog : public ICatalog { ~BaseModelCatalog() override; const std::string& GetName() const override { return name_; } + CatalogType GetType() const override { return type_; } // ICatalog implementations — query/lookup layer std::vector ListModels() const override; @@ -47,6 +48,17 @@ class BaseModelCatalog : public ICatalog { void InvalidateCache() override; protected: + BaseModelCatalog(std::string name, CatalogType type, ILogger& logger); + + /// Append a newly registered active model. Inactive tombstones are never revived. + Model* AppendActiveModel(Model model); + + /// Remove a model from catalog lookup while retaining its storage for pointer safety. + bool RetireModel(const std::string& alias_or_model_id); + + /// Commit an unregister operation against the exact container whose lifecycle lock is held. + bool CommitUnregister(Model* model, const std::string& alias_or_model_id); + /// Derived classes implement this to fetch model variants from their source. /// Returns the full variant list. Base class handles caching and indexing. /// Maps to C# FetchModelInfoAsync. @@ -73,6 +85,9 @@ class BaseModelCatalog : public ICatalog { return {}; } + /// Authoritative catalogs reconcile removals and replacements from every fetched snapshot. + virtual bool IsAuthoritativeSnapshot() const { return false; } + private: /// Lookup indices into the stable models_ storage. /// Rebuilt on refresh. Does not own any Model instances. @@ -82,9 +97,13 @@ class BaseModelCatalog : public ICatalog { std::unordered_map name_index; // name -> latest version Model* }; - /// Stable model storage. unique_ptr ensures addresses never change. - /// Models are only appended, never removed — external Model* pointers remain valid. - mutable std::vector> models_; + struct StoredModel { + std::unique_ptr model; + bool active = true; + }; + + /// Stable append-only storage. Inactive models are tombstones retained for pointer safety. + mutable std::vector models_; /// Lookup indices, rebuilt on each populate/refresh. /// Guarded by std::atomic_load/store free functions so readers get a consistent @@ -127,6 +146,7 @@ class BaseModelCatalog : public ICatalog { mutable std::vector> version_query_models_; std::string name_; + CatalogType type_; ILogger& logger_; }; diff --git a/sdk_v2/cpp/src/catalog/catalog_client.cc b/sdk_v2/cpp/src/catalog/catalog_client.cc index 0910965b4..731d8bed4 100644 --- a/sdk_v2/cpp/src/catalog/catalog_client.cc +++ b/sdk_v2/cpp/src/catalog/catalog_client.cc @@ -46,6 +46,9 @@ std::vector FetchAllModelInfosWithCachedModels( } catch (...) { logger.Log(LogLevel::Warning, "catalog: failed to fetch cached model IDs — unknown error"); } + + // IDs the public source does not recognize are intentionally omitted. Arbitrary models copied into the + // cache must be explicitly registered in the local catalog instead of appearing in the public catalog. } return result; diff --git a/sdk_v2/cpp/src/catalog/catalog_client.h b/sdk_v2/cpp/src/catalog/catalog_client.h index 142dfc728..5c5940293 100644 --- a/sdk_v2/cpp/src/catalog/catalog_client.h +++ b/sdk_v2/cpp/src/catalog/catalog_client.h @@ -49,7 +49,8 @@ class ICatalogClient { } }; -/// Fetch the current catalog and resolve cached model IDs that are no longer in the latest response. +/// Production helper that combines a catalog fetch with resolution of cached versions known to the public source. +/// Unknown cache entries are omitted; BYOM models require explicit local-catalog registration. std::vector FetchAllModelInfosWithCachedModels( ICatalogClient& client, const std::vector& cached_model_ids, diff --git a/sdk_v2/cpp/src/catalog/local_model_catalog.cc b/sdk_v2/cpp/src/catalog/local_model_catalog.cc new file mode 100644 index 000000000..c5490968d --- /dev/null +++ b/sdk_v2/cpp/src/catalog/local_model_catalog.cc @@ -0,0 +1,444 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "catalog/local_model_catalog.h" + +#include "exception.h" +#include "inferencing/generative/genai_config.h" +#include "util/file_lock.h" +#include "util/time_utils.h" + +#include +#include + +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include +#endif + +namespace fl { +namespace { + +constexpr const char* kRegistrationIdProperty = "_local_registration_id"; +constexpr const char* kLegacyModelPathProperty = "model_path"; +constexpr const char* kLegacyAliasProperty = "alias"; +constexpr const char* kLegacyVersionProperty = "version"; +const std::regex kModelNamePattern("[A-Za-z0-9][A-Za-z0-9._-]*"); +std::mutex kLocalCatalogMutationMutex; +const std::unordered_set kSupportedTasks = { + "automatic-speech-recognition", + "chat-completion", + "embeddings", + "vision-language-chat", +}; + +bool HasParentTraversal(const std::filesystem::path& path) { + return std::any_of(path.begin(), path.end(), [](const auto& component) { return component == ".."; }); +} + +struct ParsedModelId { + std::string name; + int version; +}; + +ParsedModelId ParseModelId(const std::string& model_id) { + const auto separator = model_id.find(':'); + if (separator == std::string::npos || separator == 0 || separator + 1 == model_id.size() || + model_id.find(':', separator + 1) != std::string::npos) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "model_id must have format :"); + } + + auto name = model_id.substr(0, separator); + if (!std::regex_match(name, kModelNamePattern)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "model_id name must match [a-zA-Z0-9][a-zA-Z0-9._-]*"); + } + + const auto version_text = model_id.substr(separator + 1); + if ((version_text.size() > 1 && version_text.front() == '0') || + !std::all_of(version_text.begin(), version_text.end(), [](char c) { return c >= '0' && c <= '9'; })) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "model_id version must be a canonical non-negative integer"); + } + + int version = 0; + const auto [end, error] = std::from_chars(version_text.data(), version_text.data() + version_text.size(), version); + if (error != std::errc{} || end != version_text.data() + version_text.size()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "model_id version is out of range"); + } + + return {std::move(name), version}; +} + +void RemoveLegacyRegistrationProperties(ModelInfo& info) { + info.string_properties.erase(kLegacyModelPathProperty); + info.string_properties.erase(kLegacyAliasProperty); + info.string_properties.erase(kRegistrationIdProperty); + info.string_properties.erase(kLegacyVersionProperty); + info.int_properties.erase(kLegacyModelPathProperty); + info.int_properties.erase(kLegacyAliasProperty); + info.int_properties.erase(kRegistrationIdProperty); + info.int_properties.erase(kLegacyVersionProperty); +} + +bool IsLegacyIntegerProperty(std::string_view key) { + return key == FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT || + key == FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT || + key == FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_HYBRID_REASONING_INT || + key == FOUNDRY_LOCAL_MODEL_PROP_FILESIZE_MB_INT || + key == FOUNDRY_LOCAL_MODEL_PROP_MAX_OUTPUT_TOKENS_INT || + key == FOUNDRY_LOCAL_MODEL_PROP_CREATED_AT_UNIX_INT || + key == FOUNDRY_LOCAL_MODEL_PROP_IS_TEST_MODEL_INT || + key == FOUNDRY_LOCAL_MODEL_PROP_CONTEXT_LENGTH_INT; +} + +ModelInfo ModelInfoFromLegacyPropertyBagJson(const nlohmann::json& json) { + if (!json.is_object()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "legacy model info properties must contain an object"); + } + + ModelInfo info; + for (const auto& [key, value] : json.items()) { + if (value.is_string()) { + auto text = value.get(); + if (IsLegacyIntegerProperty(key)) { + int64_t integer = 0; + const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), integer); + if (error != std::errc{} || end != text.data() + text.size()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "invalid legacy integer model info property: " + key); + } + + info.SetPropertyInt(key, integer); + } else { + info.SetPropertyStr(key, std::move(text)); + } + } else if (value.is_number_integer()) { + info.SetPropertyInt(key, value.get()); + } else { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, + "legacy model info property values must be strings or integers"); + } + } + + return info; +} + +nlohmann::json RegistrationToJson(const LocalModelCatalog::Registration& registration) { + return { + {"model_info", ModelInfoToJson(registration.info)}, + {"model_path", registration.model_path}, + {"registration_id", registration.registration_id}, + }; +} + +} // namespace + +LocalModelCatalog::LocalModelCatalog(std::filesystem::path app_data_dir, ModelFactory model_factory, ILogger& logger) + : BaseModelCatalog("local", CatalogType::kLocal, logger), + catalog_dir_(std::move(app_data_dir) / "catalogs" / "local"), + index_path_(catalog_dir_ / "local_models.json"), + lock_path_(catalog_dir_ / "local_models.lock"), + model_factory_(std::move(model_factory)), + logger_(logger) {} + +std::vector LocalModelCatalog::FetchModels() const { + std::lock_guard guard(registration_mutex_); + FileLock file_lock(lock_path_); + const auto registrations = LoadRegistrations(); + + std::vector models; + models.reserve(registrations.size()); + for (const auto& registration : registrations) { + models.push_back(CreateModel(registration)); + } + + return models; +} + +Model* LocalModelCatalog::RegisterModel(const std::string& model_path_value, const std::string& model_id, + const ModelInfo& metadata) { + std::lock_guard mutation_guard(kLocalCatalogMutationMutex); + + if (model_path_value.empty()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "model_path is required"); + } + + const auto parsed_id = ParseModelId(model_id); + + const std::filesystem::path supplied_path(model_path_value); + if (HasParentTraversal(supplied_path)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "model_path must not contain '..' path components"); + } + + std::error_code ec; + if (!std::filesystem::is_directory(supplied_path, ec)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "model_path must be an existing directory"); + } + + const auto model_path = std::filesystem::absolute(supplied_path).lexically_normal(); + const auto config_path = model_path / "genai_config.json"; + if (!std::filesystem::is_regular_file(config_path, ec)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "model_path must contain a regular genai_config.json file"); + } + + GenAIConfig::LoadFromFile(config_path.string()); + + const auto* task = metadata.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_TASK_STR); + if (!task || task->empty()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "task is required"); + } + if (!kSupportedTasks.contains(*task)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "unsupported task: " + *task); + } + + ListModels(); + Registration registration; + { + std::lock_guard guard(registration_mutex_); + FileLock file_lock(lock_path_); + auto registrations = LoadRegistrations(); + const auto duplicate = std::find_if(registrations.begin(), registrations.end(), [&](const auto& existing) { + return existing.info.model_id == model_id; + }); + if (duplicate != registrations.end()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "model_id is already registered: " + model_id); + } + + registration = {ResolveMetadata(metadata, model_id, parsed_id.name, parsed_id.version), model_path.string(), + std::to_string(std::chrono::high_resolution_clock::now().time_since_epoch().count())}; + while (std::any_of(registrations.begin(), registrations.end(), [&](const auto& existing) { + return existing.registration_id == registration.registration_id; + })) { + registration.registration_id += "-1"; + } + + registrations.push_back(registration); + SaveRegistrations(registrations); + } + + ListModels(); + auto* model = GetModelVariant(registration.info.model_id); + const auto runtime_id = "local/" + registration.registration_id; + if (!model || model->RuntimeId() != runtime_id) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "registered model was not available after catalog refresh"); + } + + return model; +} + +void LocalModelCatalog::UnregisterModel(const std::string& alias_or_model_id) { + std::lock_guard mutation_guard(kLocalCatalogMutationMutex); + + if (alias_or_model_id.empty()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "alias_or_model_id must not be empty"); + } + + ListModels(); + auto* model = GetModel(alias_or_model_id); + if (!model) { + auto* variant = GetModelVariant(alias_or_model_id); + if (variant) { + model = GetModel(variant->Alias()); + } + } + if (!model) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "model not found: " + alias_or_model_id); + } + + model->BeginUnregister(); + bool unregister_in_progress = true; + try { + for (const auto* variant : model->Variants()) { + if (variant->IsLoaded()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "cannot unregister a loaded model; unload it first"); + } + } + + const bool unregister_alias = model->Alias() == alias_or_model_id; + if (!unregister_alias && !model->PrepareRetireVariant(alias_or_model_id)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "model not found: " + alias_or_model_id); + } + + { + std::lock_guard guard(registration_mutex_); + FileLock file_lock(lock_path_); + auto registrations = LoadRegistrations(); + const auto end = std::remove_if(registrations.begin(), registrations.end(), [&](const auto& registration) { + return registration.info.alias == alias_or_model_id || registration.info.model_id == alias_or_model_id; + }); + if (end == registrations.end()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "model not found: " + alias_or_model_id); + } + + registrations.erase(end, registrations.end()); + SaveRegistrations(registrations); + } + + if (!CommitUnregister(model, alias_or_model_id)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "unregistered model was not present in the in-memory catalog"); + } + + model->EndUnregister(); + unregister_in_progress = false; + ListModels(); + } catch (...) { + if (unregister_in_progress) { + model->EndUnregister(); + } + throw; + } +} + +ModelInfo LocalModelCatalog::ResolveMetadata(const ModelInfo& metadata, const std::string& model_id, + const std::string& name, int version) const { + auto resolved = metadata; + RemoveLegacyRegistrationProperties(resolved); + + resolved.alias = name; + resolved.name = name; + resolved.version = version; + resolved.model_id = model_id; + resolved.uri.clear(); + if (!resolved.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_PUBLISHER_STR)) { + resolved.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_PUBLISHER_STR, "local"); + } + resolved.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR, "LocalRegistration"); + resolved.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_ENTITY_TYPE_STR, "Model"); + resolved.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_MODEL_TYPE_STR, "ONNX"); + + const auto now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + resolved.SetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_CREATED_AT_UNIX_INT, now); + resolved.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_CREATION_TIME_STR, FormatUtcTimestamp(now)); + + return resolved; +} + +std::vector LocalModelCatalog::LoadRegistrations() const { + std::ifstream stream(index_path_, std::ios::binary); + if (!stream) { + return {}; + } + + std::vector registrations; + try { + nlohmann::json root; + stream >> root; + const auto schema_version = root.value("version", 0); + if (!root.is_object() || (schema_version != 1 && schema_version != 2) || !root.contains("models") || + !root["models"].is_array()) { + logger_.Log(LogLevel::Warning, "Ignoring malformed local model registration index: " + index_path_.string()); + return {}; + } + + for (const auto& item : root["models"]) { + try { + if (!item.is_object() || !item.contains("model_path") || !item["model_path"].is_string()) { + continue; + } + + ModelInfo info; + std::string model_id; + if (schema_version == 2) { + if (!item.contains("model_info") || !item["model_info"].is_object()) { + continue; + } + + info = ModelInfoFromJson(item["model_info"]); + model_id = info.model_id; + } else { + if (!item.contains("model_id") || !item["model_id"].is_string() || !item.contains("properties")) { + continue; + } + + info = ModelInfoFromLegacyPropertyBagJson(item["properties"]); + model_id = item["model_id"].get(); + } + + std::string registration_id; + if (item.contains("registration_id") && item["registration_id"].is_string()) { + registration_id = item["registration_id"].get(); + } else if (const auto* legacy_registration_id = info.GetPropertyStr(kRegistrationIdProperty)) { + registration_id = *legacy_registration_id; + } + if (registration_id.empty()) { + continue; + } + + const auto parsed_id = ParseModelId(model_id); + std::filesystem::path model_path = item["model_path"].get(); + if (model_path.empty() || HasParentTraversal(model_path)) { + continue; + } + model_path = std::filesystem::absolute(model_path).lexically_normal(); + + RemoveLegacyRegistrationProperties(info); + info.alias = parsed_id.name; + info.name = parsed_id.name; + info.version = parsed_id.version; + info.model_id = model_id; + info.uri.clear(); + + const auto duplicate = std::find_if(registrations.begin(), registrations.end(), [&](const auto& existing) { + return existing.info.model_id == info.model_id || existing.registration_id == registration_id; + }); + if (duplicate == registrations.end()) { + registrations.push_back({std::move(info), model_path.string(), std::move(registration_id)}); + } + } catch (const std::exception& ex) { + logger_.Log(LogLevel::Warning, std::string("Ignoring malformed local model registration: ") + ex.what()); + } + } + } catch (const std::exception& ex) { + logger_.Log(LogLevel::Warning, std::string("Ignoring unreadable local model registration index: ") + ex.what()); + } + + return registrations; +} + +void LocalModelCatalog::SaveRegistrations(const std::vector& registrations) const { + std::filesystem::create_directories(catalog_dir_); + nlohmann::json models = nlohmann::json::array(); + for (const auto& registration : registrations) { + models.push_back(RegistrationToJson(registration)); + } + + const nlohmann::json root = {{"version", 2}, {"catalog_name", "local"}, {"models", std::move(models)}}; + const auto temp_path = index_path_.string() + ".tmp"; + { + std::ofstream stream(temp_path, std::ios::binary | std::ios::trunc); + if (!stream) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "failed to write local model registration index"); + } + + stream << root.dump(2) << '\n'; + if (!stream) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "failed to write local model registration index"); + } + } + +#ifdef _WIN32 + if (!MoveFileExW(std::filesystem::path(temp_path).wstring().c_str(), index_path_.wstring().c_str(), + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) { + std::filesystem::remove(temp_path); + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "failed to commit local model registration index"); + } +#else + std::error_code ec; + std::filesystem::rename(temp_path, index_path_, ec); + if (ec) { + std::filesystem::remove(temp_path); + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "failed to commit local model registration index: " + ec.message()); + } +#endif +} + +Model LocalModelCatalog::CreateModel(const Registration& registration) const { + return model_factory_(registration.info, registration.model_path, "local/" + registration.registration_id); +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/catalog/local_model_catalog.h b/sdk_v2/cpp/src/catalog/local_model_catalog.h new file mode 100644 index 000000000..c6ab87611 --- /dev/null +++ b/sdk_v2/cpp/src/catalog/local_model_catalog.h @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "catalog/base_model_catalog.h" + +#include +#include +#include + +namespace fl { + +/// Mutable, persistent catalog for models registered from arbitrary local directories. +class LocalModelCatalog final : public BaseModelCatalog { + public: + using ModelFactory = + std::function; + + LocalModelCatalog(std::filesystem::path app_data_dir, ModelFactory model_factory, ILogger& logger); + + Model* RegisterModel(const std::string& model_path, const std::string& model_id, + const ModelInfo& metadata) override; + void UnregisterModel(const std::string& alias_or_model_id) override; + + struct Registration { + ModelInfo info; + std::string model_path; + std::string registration_id; + }; + + protected: + std::vector FetchModels() const override; + bool IsAuthoritativeSnapshot() const override { return true; } + + private: + ModelInfo ResolveMetadata(const ModelInfo& metadata, const std::string& model_id, + const std::string& name, int version) const; + std::vector LoadRegistrations() const; + void SaveRegistrations(const std::vector& registrations) const; + Model CreateModel(const Registration& registration) const; + + std::filesystem::path catalog_dir_; + std::filesystem::path index_path_; + std::filesystem::path lock_path_; + ModelFactory model_factory_; + ILogger& logger_; + mutable std::mutex registration_mutex_; +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc index 879f7c8cc..1ee701032 100644 --- a/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc +++ b/sdk_v2/cpp/src/inferencing/generative/genai_model_instance.cc @@ -34,12 +34,15 @@ GenAIModelInstance::GenAIModelInstance(std::string model_id, "failed to create OGA config for model ", model_id_, ": ", e.what()); } - // Apply EP override to the OGA config + // Every explicit EP overrides providers from genai_config.json. CPU is OGA's default when the provider list is + // empty, and EPtoGenAI intentionally has no CPU name, so CPU clears the list without appending a provider. if (ep_ != ExecutionProvider::kDefault) { try { oga_config->ClearProviders(); - std::string_view provider_str = EPUtils::EPtoGenAI(ep_); - oga_config->AppendProvider(provider_str.data()); + if (ep_ != ExecutionProvider::kCPU) { + std::string_view provider_str = EPUtils::EPtoGenAI(ep_); + oga_config->AppendProvider(provider_str.data()); + } // Disable CUDA graph for CUDA EP (matches C# behavior) if (ep_ == ExecutionProvider::kCUDA) { diff --git a/sdk_v2/cpp/src/inferencing/model_load_manager.cc b/sdk_v2/cpp/src/inferencing/model_load_manager.cc index a6e2defa8..a13c7441c 100644 --- a/sdk_v2/cpp/src/inferencing/model_load_manager.cc +++ b/sdk_v2/cpp/src/inferencing/model_load_manager.cc @@ -47,6 +47,36 @@ std::string_view RequiredEpForModelId(std::string_view model_id) { return {}; } +/// Returns whether the provider is DirectML, which is supplied by WinML rather than a downloadable EP bootstrapper. +bool IsDmlProvider(std::string_view provider) { + return provider == "dml" || provider == "DML" || provider == "DmlExecutionProvider" || + provider == "DMLExecutionProvider"; +} + +/// Returns the required EP registration name for a provider declared in genai_config.json. +std::string RequiredEpForConfigProvider(std::string_view provider) { + if (provider.empty() || IsDmlProvider(provider)) { + return {}; + } + + auto ep = EPUtils::StringtoEP(provider); + if (provider == "WebGpuExecutionProvider") { + ep = ExecutionProvider::kWebGPU; + } + + if (ep == ExecutionProvider::kCPU) { + return {}; + } + if (ep != ExecutionProvider::kUnknown) { + return std::string(EPUtils::EPtoRegistrationName(ep)); + } + + // Configs may name WinML/OGA providers that are intentionally outside the public override enum, such as + // MIGraphXExecutionProvider and RyzenAILightExecutionProvider. Canonical registration names can still be guarded. + constexpr std::string_view suffix = "ExecutionProvider"; + return provider.ends_with(suffix) ? std::string(provider) : std::string{}; +} + } // namespace // --------------------------------------------------------------------------- @@ -118,6 +148,10 @@ ModelLoadManager::LoadResult ModelLoadManager::LoadModel(std::string_view model_ // Determine execution provider auto resolved_ep = ep_override; + if (resolved_ep == ExecutionProvider::kUnknown) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "unknown execution provider override"); + } + if (resolved_ep == ExecutionProvider::kDefault) { // Auto-select EP for generic-gpu models: DML models are compatible with // CUDA and WebGPU, so try those in order when available. @@ -132,11 +166,15 @@ ModelLoadManager::LoadResult ModelLoadManager::LoadModel(std::string_view model_ } } - std::string_view required_ep; - if (resolved_ep != ExecutionProvider::kDefault && resolved_ep != ExecutionProvider::kCPU) { + std::string required_ep; + if (resolved_ep == ExecutionProvider::kCPU) { + // Explicit CPU overrides both the provider in genai_config.json and model-ID hints. + } else if (resolved_ep != ExecutionProvider::kDefault) { required_ep = EPUtils::EPtoRegistrationName(resolved_ep); } else { - required_ep = RequiredEpForModelId(id_str); + const auto config_provider = genai_config.DefaultProvider(); + required_ep = config_provider.empty() ? std::string(RequiredEpForModelId(id_str)) + : RequiredEpForConfigProvider(config_provider); } // OGA can crash or hang if a model is loaded with an unregistered EP. diff --git a/sdk_v2/cpp/src/inferencing/session/session.cc b/sdk_v2/cpp/src/inferencing/session/session.cc index 85ba011bf..1f80b1849 100644 --- a/sdk_v2/cpp/src/inferencing/session/session.cc +++ b/sdk_v2/cpp/src/inferencing/session/session.cc @@ -51,7 +51,7 @@ std::unique_ptr Session::Create(const fl::Model& model) { } auto& lm = mgr.GetModelLoadManager(); - auto* loaded = lm.GetLoadedModel(model.Id()); + auto* loaded = lm.GetLoadedModel(model.RuntimeId()); if (!loaded) { FL_LOG_AND_THROW(logger, FOUNDRY_LOCAL_ERROR_INTERNAL, "loaded model not found in load manager"); } diff --git a/sdk_v2/cpp/src/manager.cc b/sdk_v2/cpp/src/manager.cc index 6eb2ea527..f8aaf125c 100644 --- a/sdk_v2/cpp/src/manager.cc +++ b/sdk_v2/cpp/src/manager.cc @@ -11,6 +11,7 @@ #include "catalog.h" #include "catalog/azure_model_catalog.h" +#include "catalog/local_model_catalog.h" #include "download/download_manager.h" #if FOUNDRY_LOCAL_HAS_EP_BOOTSTRAPPERS #include "ep_detection/cuda_ep_bootstrapper.h" @@ -306,27 +307,31 @@ Manager::Manager(const Configuration& config) : config_(config) { model_load_manager_ = std::make_unique(*ep_detector_, *logger_); session_manager_ = std::make_unique(*logger_); const bool disable_nonessential_telemetry = - config_.disable_nonessential_telemetry || - IsAdditionalOptionEnabled(config_, "DisableNonessentialTelemetry"); + config_.disable_nonessential_telemetry || IsAdditionalOptionEnabled(config_, "DisableNonessentialTelemetry"); const bool telemetry_hard_disabled = TelemetryEnvironment::IsCiEnvironment() || TelemetryEnvironment::IsTelemetryDisabledByEnvVar(); telemetry_ = std::make_unique(config_.app_name, *logger_, disable_nonessential_telemetry); try { - telemetry_->RecordProcessInfo( - BuildProcessInfo(BuildTelemetryMetadata(config_.app_name), - !disable_nonessential_telemetry && !telemetry_hard_disabled)); + telemetry_->RecordProcessInfo(BuildProcessInfo(BuildTelemetryMetadata(config_.app_name), + !disable_nonessential_telemetry && !telemetry_hard_disabled)); } catch (const std::exception& ex) { - logger_->Log( - LogLevel::Warning, - fmt::format("telemetry ProcessInfo failed during Manager initialization: {}", ex.what())); + logger_->Log(LogLevel::Warning, + fmt::format("telemetry ProcessInfo failed during Manager initialization: {}", ex.what())); } catch (...) { logger_->Log(LogLevel::Warning, "telemetry ProcessInfo failed during Manager initialization."); } - catalog_ = std::make_unique( + + public_catalog_ = std::make_unique( config_.catalog_urls, download_manager_->GetCacheDirectory(), [this](ModelInfo info, std::string local_path) { return CreateModel(std::move(info), std::move(local_path)); }, *ep_detector_, *logger_, config_.external_service_url.has_value(), config_.catalog_region.value_or("auto"), disable_region_fallback); + local_catalog_ = std::make_unique( + *config_.app_data_dir, + [this](ModelInfo info, std::string local_path, std::string runtime_model_id) { + return CreateLocalModel(std::move(info), std::move(local_path), std::move(runtime_model_id)); + }, + *logger_); } Manager::~Manager() { @@ -354,7 +359,8 @@ Manager::~Manager() { session_manager_.reset(); model_load_manager_.reset(); download_manager_.reset(); - catalog_.reset(); + local_catalog_.reset(); + public_catalog_.reset(); telemetry_.reset(); OgaShutdown(); @@ -435,7 +441,20 @@ void Manager::Destroy() { s_instance_.reset(); } -ICatalog& Manager::GetCatalog() { return *catalog_; } +ICatalog& Manager::GetCatalog() { + return *public_catalog_; +} + +ICatalog& Manager::GetCatalog(CatalogType type) { + switch (type) { + case CatalogType::kPublic: + return *public_catalog_; + case CatalogType::kLocal: + return *local_catalog_; + default: + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "unknown catalog type"); + } +} void Manager::StartWebService() { if (web_service_running_) { @@ -450,8 +469,10 @@ void Manager::StartWebService() { ActionTracker tracker(Action::kCoreServiceStart, *telemetry_); #ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE - web_service_ = std::make_unique(*catalog_, *logger_, *config_.model_cache_dir, *model_load_manager_, - *session_manager_, *telemetry_, [this]() { Shutdown(); }); + web_service_ = std::make_unique(*public_catalog_, *logger_, *config_.model_cache_dir, + *model_load_manager_, + *session_manager_, *telemetry_, + [this]() { Shutdown(); }); auto endpoints = config_.web_service_endpoints; if (endpoints.empty()) { @@ -546,6 +567,11 @@ Model Manager::CreateModel(ModelInfo info, std::string local_path) { return Model::FromModelInfo(std::move(info), std::move(local_path), *download_manager_, *model_load_manager_); } +Model Manager::CreateLocalModel(ModelInfo info, std::string local_path, std::string runtime_model_id) { + return Model::FromLocalRegistration(std::move(info), std::move(local_path), *download_manager_, + *model_load_manager_, std::move(runtime_model_id)); +} + DownloadManager& Manager::GetDownloadManager() { return *download_manager_; } ModelLoadManager& Manager::GetModelLoadManager() { return *model_load_manager_; } @@ -568,7 +594,7 @@ EpDownloadResult Manager::DownloadAndRegisterEps(const std::vector* // least one EP registered — including partial success, where result.success is false because // another EP failed — so the next catalog query re-fetches with the updated filters. if (!result.registered_eps.empty()) { - catalog_->InvalidateCache(); + public_catalog_->InvalidateCache(); } // Warn if any EPs failed to download or register, but keep going: CPU is always available and diff --git a/sdk_v2/cpp/src/manager.h b/sdk_v2/cpp/src/manager.h index a496d865e..eccfc2aaa 100644 --- a/sdk_v2/cpp/src/manager.h +++ b/sdk_v2/cpp/src/manager.h @@ -7,8 +7,10 @@ #include "logger.h" #include +#include #include #include +#include #include #include @@ -25,6 +27,7 @@ namespace fl { // Forward declarations class ICatalog; +enum class CatalogType; class DownloadManager; class ITelemetry; class Model; @@ -49,6 +52,7 @@ class Manager { /// The catalog is owned by the manager and shared across all consumers /// (web service, C API, etc.) so model state (e.g. IsLoaded) is consistent. ICatalog& GetCatalog(); + ICatalog& GetCatalog(CatalogType type); /// Get the configuration used to create this manager. const Configuration& GetConfiguration() const; @@ -137,7 +141,8 @@ class Manager { std::unique_ptr logger_; std::unique_ptr ep_detector_; std::unique_ptr telemetry_; - std::unique_ptr catalog_; + std::unique_ptr public_catalog_; + std::unique_ptr local_catalog_; std::unique_ptr download_manager_; std::unique_ptr model_load_manager_; std::unique_ptr session_manager_; @@ -151,6 +156,7 @@ class Manager { private: Model CreateModel(ModelInfo info, std::string local_path); + Model CreateLocalModel(ModelInfo info, std::string local_path, std::string runtime_model_id); static std::mutex s_mutex_; static std::unique_ptr s_instance_; diff --git a/sdk_v2/cpp/src/model.cc b/sdk_v2/cpp/src/model.cc index 6b2a81372..3a308a728 100644 --- a/sdk_v2/cpp/src/model.cc +++ b/sdk_v2/cpp/src/model.cc @@ -15,6 +15,7 @@ #include #include +#include namespace fl { @@ -104,14 +105,43 @@ bool CompareModelsForSort(const Model& m1, const Model& m2) { Model::~Model() = default; +Model::Model(ModelInfo info, + std::string local_path, + DownloadManager& download_manager, + ModelLoadManager& model_load_manager, + std::string runtime_model_id, + bool external_registration) + : info_(std::make_unique(std::move(info))), + cached_(!local_path.empty()), + local_path_(std::move(local_path)), + runtime_model_id_(std::move(runtime_model_id)), + external_registration_(external_registration), + download_manager_(&download_manager), + model_load_manager_(&model_load_manager) {} + +Model::Model(ContainerTag, Model first_variant) { + if (!first_variant.info_) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "MakeContainer requires an initialized leaf Model"); + } + + variants_.push_back(std::make_unique(std::move(first_variant))); + selected_variant_.store(variants_.back().get(), std::memory_order_release); +} + Model::Model(Model&& other) noexcept : info_(std::move(other.info_)), cached_(other.cached_.load()), + active_(other.active_.load()), local_path_(std::move(other.local_path_)), + runtime_model_id_(std::move(other.runtime_model_id_)), + external_registration_(other.external_registration_), download_manager_(other.download_manager_), model_load_manager_(other.model_load_manager_), variants_(std::move(other.variants_)), - selected_variant_(other.selected_variant_.load(std::memory_order_relaxed)) { + retired_variants_(std::move(other.retired_variants_)), + selected_variant_(other.selected_variant_.load(std::memory_order_relaxed)), + selection_is_explicit_(other.selection_is_explicit_) { + // The mutex and unregistering state are intentionally not moved; moving while unregistering is invalid. // After vector move, selected_variant_ still points into the transferred buffer. other.download_manager_ = nullptr; other.model_load_manager_ = nullptr; @@ -122,11 +152,17 @@ Model& Model::operator=(Model&& other) noexcept { if (this != &other) { info_ = std::move(other.info_); cached_.store(other.cached_.load()); + active_.store(other.active_.load()); local_path_ = std::move(other.local_path_); + runtime_model_id_ = std::move(other.runtime_model_id_); + external_registration_ = other.external_registration_; download_manager_ = other.download_manager_; model_load_manager_ = other.model_load_manager_; variants_ = std::move(other.variants_); + retired_variants_ = std::move(other.retired_variants_); selected_variant_.store(other.selected_variant_.load(std::memory_order_relaxed), std::memory_order_relaxed); + selection_is_explicit_ = other.selection_is_explicit_; + unregistering_ = false; other.download_manager_ = nullptr; other.model_load_manager_ = nullptr; other.selected_variant_.store(nullptr, std::memory_order_relaxed); @@ -143,17 +179,18 @@ Model Model::FromModelInfo(ModelInfo info, std::string local_path, DownloadManager& download_manager, ModelLoadManager& model_load_manager) { - Model model; - model.info_ = std::move(info); - model.download_manager_ = &download_manager; - model.model_load_manager_ = &model_load_manager; - - if (!local_path.empty()) { - model.cached_ = true; - model.local_path_ = std::move(local_path); - } + auto runtime_model_id = info.model_id; + return Model(std::move(info), std::move(local_path), download_manager, model_load_manager, + std::move(runtime_model_id), false); +} - return model; +Model Model::FromLocalRegistration(ModelInfo info, + std::string local_path, + DownloadManager& download_manager, + ModelLoadManager& model_load_manager, + std::string runtime_model_id) { + return Model(std::move(info), std::move(local_path), download_manager, model_load_manager, + std::move(runtime_model_id), true); } // --------------------------------------------------------------------------- @@ -161,16 +198,16 @@ Model Model::FromModelInfo(ModelInfo info, // --------------------------------------------------------------------------- Model Model::MakeContainer(Model first_variant) { - Model container; - container.variants_.push_back(std::make_unique(std::move(first_variant))); - container.selected_variant_.store(container.variants_.back().get(), std::memory_order_release); - return container; + return Model(ContainerTag{}, std::move(first_variant)); } void Model::AddVariant(Model variant) { if (!IsContainer()) { FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "AddVariant called on a non-container Model; use MakeContainer first"); } + if (!variant.info_) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "AddVariant requires an initialized leaf Model"); + } std::lock_guard lock(state_mutex_); @@ -182,6 +219,121 @@ void Model::AddVariant(Model variant) { variants_.insert(pos, std::make_unique(std::move(variant))); } +bool Model::TryReconcileVariants(Model& incoming) { + if (!IsContainer() || !incoming.IsContainer() || Alias() != incoming.Alias()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "TryReconcileVariants requires containers with the same alias"); + } + + std::unique_lock lifecycle_lock(lifecycle_mutex_, std::try_to_lock); + if (!lifecycle_lock.owns_lock()) { + return false; + } + + std::scoped_lock lock(state_mutex_, incoming.state_mutex_); + variants_.reserve(variants_.size() + incoming.variants_.size()); + retired_variants_.reserve(retired_variants_.size() + variants_.size()); + + auto* previous_selection = selected_variant_.load(std::memory_order_acquire); + const auto preserve_selection = selection_is_explicit_; + + for (auto current = variants_.begin(); current != variants_.end();) { + const auto match = std::find_if(incoming.variants_.begin(), incoming.variants_.end(), [&](const auto& candidate) { + return (*current)->Info().model_id == candidate->Info().model_id && + (*current)->RuntimeId() == candidate->RuntimeId(); + }); + if (match != incoming.variants_.end()) { + incoming.variants_.erase(match); + ++current; + continue; + } + + (*current)->Deactivate(); + retired_variants_.push_back(std::move(*current)); + current = variants_.erase(current); + } + + for (auto& variant : incoming.variants_) { + variants_.push_back(std::move(variant)); + } + incoming.variants_.clear(); + + std::sort(variants_.begin(), variants_.end(), [](const auto& left, const auto& right) { + return CompareModelsForSort(*left, *right); + }); + + const auto retained_selection = std::find_if(variants_.begin(), variants_.end(), [&](const auto& variant) { + return variant.get() == previous_selection; + }); + const auto cached_selection = std::find_if(variants_.begin(), variants_.end(), [](const auto& variant) { + return variant->IsCached(); + }); + const auto selected = preserve_selection && retained_selection != variants_.end() + ? retained_selection + : (cached_selection != variants_.end() ? cached_selection : variants_.begin()); + selection_is_explicit_ = preserve_selection && retained_selection != variants_.end(); + selected_variant_.store(selected->get(), std::memory_order_release); + return true; +} + +bool Model::TryDeactivateForRefresh() { + std::unique_lock lifecycle_lock(lifecycle_mutex_, std::try_to_lock); + if (!lifecycle_lock.owns_lock()) { + return false; + } + + Deactivate(); + return true; +} + +bool Model::PrepareRetireVariant(const std::string& model_id) { + if (!IsContainer()) { + return false; + } + + std::lock_guard lock(state_mutex_); + const auto variant = std::find_if(variants_.begin(), variants_.end(), [&](const auto& candidate) { + return candidate->Info().model_id == model_id; + }); + if (variant == variants_.end()) { + return false; + } + + retired_variants_.reserve(retired_variants_.size() + 1); + return true; +} + +bool Model::RetireVariant(const std::string& model_id) { + std::lock_guard lock(state_mutex_); + const auto variant = std::find_if(variants_.begin(), variants_.end(), [&](const auto& candidate) { + return candidate->Info().model_id == model_id; + }); + if (variant == variants_.end()) { + return !variants_.empty(); + } + + auto* retired = variant->get(); + retired->Deactivate(); + retired_variants_.push_back(std::move(*variant)); + variants_.erase(variant); + + if (variants_.empty()) { + selection_is_explicit_ = false; + selected_variant_.store(retired, std::memory_order_release); + return false; + } + + if (selected_variant_.load(std::memory_order_acquire) == retired) { + const auto cached = std::find_if(variants_.begin(), variants_.end(), [](const auto& candidate) { + return candidate->IsCached(); + }); + const auto selected = cached != variants_.end() ? cached : variants_.begin(); + selection_is_explicit_ = false; + selected_variant_.store(selected->get(), std::memory_order_release); + } + + return true; +} + bool Model::CompareBestFirst(const Model& a, const Model& b) { return CompareModelsForSort(a, b); } @@ -196,11 +348,13 @@ void Model::SelectDefaultVariant() { for (auto& v : variants_) { if (v->IsCached()) { + selection_is_explicit_ = false; selected_variant_.store(v.get(), std::memory_order_release); return; } } + selection_is_explicit_ = false; selected_variant_.store(variants_.front().get(), std::memory_order_release); } @@ -213,7 +367,7 @@ const std::string& Model::Id() const { return sv->Id(); } - return info_.model_id; + return Info().model_id; } const std::string& Model::Alias() const { @@ -221,7 +375,7 @@ const std::string& Model::Alias() const { return sv->Alias(); } - return info_.alias; + return Info().alias; } const ModelInfo& Model::Info() const { @@ -229,7 +383,11 @@ const ModelInfo& Model::Info() const { return sv->Info(); } - return info_; + if (!info_) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "cannot access metadata on a moved-from Model"); + } + + return *info_; } std::vector Model::Variants() const { @@ -255,7 +413,13 @@ bool Model::IsCached() const { return sv->IsCached(); } - return cached_; + if (external_registration_) { + std::error_code ec; + return std::filesystem::is_directory(local_path_, ec) && + std::filesystem::is_regular_file(std::filesystem::path(local_path_) / "genai_config.json", ec); + } + + return active_ && cached_; } bool Model::IsLoaded() const { @@ -266,7 +430,7 @@ bool Model::IsLoaded() const { // ModelLoadManager owns the authoritative loaded-instance map. The pointer is set at // construction and never reassigned, so querying it here stays in sync with paths that // bypass Model::Load/Unload (e.g., Manager::Shutdown -> ModelLoadManager::UnloadAll). - return model_load_manager_->GetLoadedModel(info_.model_id) != nullptr; + return model_load_manager_->GetLoadedModel(runtime_model_id_) != nullptr; } // --------------------------------------------------------------------------- @@ -279,6 +443,17 @@ void Model::Download(std::function progress_cb) { return; } + if (!active_) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "model is no longer registered"); + } + + if (external_registration_) { + if (progress_cb) { + progress_cb(100.0f); + } + return; + } + // Already cached (scanner found the model on disk during catalog construction). // No need to re-derive the path via DownloadManager — local_path_ is authoritative. bool already_cached; @@ -296,7 +471,7 @@ void Model::Download(std::function progress_cb) { return; } - auto path = download_manager_->DownloadModel(info_, std::move(progress_cb)); + auto path = download_manager_->DownloadModel(Info(), std::move(progress_cb)); { std::lock_guard lock(state_mutex_); local_path_ = std::move(path); @@ -320,9 +495,24 @@ void Model::Load(ExecutionProvider ep) { return; } + std::lock_guard lifecycle_lock(lifecycle_mutex_); + + if (!active_ || unregistering_) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "model is no longer registered"); + } + + const auto& info = Info(); + if (external_registration_ && ep == ExecutionProvider::kDefault && !info.execution_provider.empty()) { + ep = EPUtils::StringtoEP(info.execution_provider); + if (ep == ExecutionProvider::kUnknown) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, + "unknown execution provider for local model: " + info.execution_provider); + } + } + // LoadModel is idempotent — it returns kModelAlreadyLoaded if the id is already // in the load manager's map, so no need for a local short-circuit. - auto result = model_load_manager_->LoadModel(local_path_, info_.model_id, ep); + auto result = model_load_manager_->LoadModel(local_path_, runtime_model_id_, ep); if (result.status == ModelLoadManager::LoadStatus::kModelNotFound) { FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "model not found at path: " + local_path_); @@ -336,7 +526,7 @@ void Model::Unload() { } // UnloadModel is idempotent — returns false if the id isn't loaded. - model_load_manager_->UnloadModel(info_.model_id); + model_load_manager_->UnloadModel(runtime_model_id_); } void Model::RemoveFromCache() { @@ -345,6 +535,11 @@ void Model::RemoveFromCache() { return; } + if (external_registration_) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "local registrations are not cache entries; call Catalog::UnregisterModel instead"); + } + std::string path; { std::lock_guard lock(state_mutex_); @@ -370,6 +565,78 @@ void Model::RemoveFromCache() { } } +void Model::Deactivate() { + active_.store(false); + if (IsContainer()) { + for (auto* variant : Variants()) { + variant->Deactivate(); + } + } +} + +void Model::BeginUnregister() { + if (IsContainer()) { + lifecycle_mutex_.lock(); + if (!active_ || unregistering_) { + lifecycle_mutex_.unlock(); + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "model is no longer registered"); + } + unregistering_ = true; + + try { + std::lock_guard state_lock(state_mutex_); + unregistering_variants_.clear(); + unregistering_variants_.reserve(variants_.size()); + for (const auto& variant : variants_) { + unregistering_variants_.push_back(variant.get()); + } + } catch (...) { + unregistering_ = false; + lifecycle_mutex_.unlock(); + throw; + } + + size_t locked_count = 0; + try { + for (; locked_count < unregistering_variants_.size(); ++locked_count) { + unregistering_variants_[locked_count]->BeginUnregister(); + } + } catch (...) { + while (locked_count > 0) { + unregistering_variants_[--locked_count]->EndUnregister(); + } + unregistering_variants_.clear(); + unregistering_ = false; + lifecycle_mutex_.unlock(); + throw; + } + return; + } + + lifecycle_mutex_.lock(); + if (!active_ || unregistering_) { + lifecycle_mutex_.unlock(); + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "model is no longer registered"); + } + + unregistering_ = true; +} + +void Model::EndUnregister() { + if (!unregistering_variants_.empty()) { + for (auto* variant : unregistering_variants_) { + variant->EndUnregister(); + } + unregistering_variants_.clear(); + unregistering_ = false; + lifecycle_mutex_.unlock(); + return; + } + + unregistering_ = false; + lifecycle_mutex_.unlock(); +} + void Model::SelectVariant(const Model& variant) { if (!IsContainer()) { FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, @@ -377,8 +644,10 @@ void Model::SelectVariant(const Model& variant) { "with all variants available."); } + std::lock_guard lock(state_mutex_); for (auto& v : variants_) { if (v.get() == &variant) { + selection_is_explicit_ = true; selected_variant_.store(v.get(), std::memory_order_release); return; } diff --git a/sdk_v2/cpp/src/model.h b/sdk_v2/cpp/src/model.h index 5c142c830..34f576aa3 100644 --- a/sdk_v2/cpp/src/model.h +++ b/sdk_v2/cpp/src/model.h @@ -28,13 +28,15 @@ class ModelLoadManager; // Model leaves that share the same alias and delegates all operations // to a selected variant. Additional variants are added via AddVariant. // -// The C API type (flModel) inherits from this with zero extra members, -// following the same pattern as flItem : fl::Item. +// The C API exposes this through the unrelated opaque flModel handle type; +// c_api_types.h provides the internal pointer conversions. // ----------------------------------------------------------------------- class Model { public: - Model() = default; + // Every successfully constructed Model is either a leaf with immutable metadata or a container with a selected, + // metadata-bearing leaf. A moved-from Model may only be destroyed or assigned a new value. + Model() = delete; ~Model(); Model(Model&& other) noexcept; Model& operator=(Model&& other) noexcept; @@ -51,6 +53,13 @@ class Model { DownloadManager& download_manager, ModelLoadManager& model_load_manager); + /// Create an in-place externally registered model. Assets are never deleted by this Model. + static Model FromLocalRegistration(ModelInfo info, + std::string local_path, + DownloadManager& download_manager, + ModelLoadManager& model_load_manager, + std::string runtime_model_id); + // --- Container construction --- /// Create a container Model wrapping the given variant as its first (and selected) variant. @@ -66,6 +75,21 @@ class Model { /// container has its full variant set. void AddVariant(Model variant); + /// Reconcile this container with a fresh authoritative container while preserving leaves whose model ID and + /// private runtime ID are unchanged. Returns false without changing either container when unregister is in progress. + /// Removed/replaced leaves are deactivated but retained in container-owned storage for outstanding handle safety. + bool TryReconcileVariants(Model& incoming); + + /// Deactivate this container for an authoritative refresh unless unregister currently owns its lifecycle lock. + bool TryDeactivateForRefresh(); + + /// Reserve storage for retiring one variant. Call before persisting an unregister operation so the later + /// in-memory commit cannot fail while transferring ownership of an outstanding model handle. + bool PrepareRetireVariant(const std::string& model_id); + + /// Retire one variant after PrepareRetireVariant succeeds. Returns true when active variants remain. + bool RetireVariant(const std::string& model_id); + /// Choose the default selected variant from the current sorted variant list: /// first cached variant if any, else the best variant. /// Requires IsContainer() to be true. @@ -130,6 +154,13 @@ class Model { void Unload(); void RemoveFromCache(); + /// Mark this model and its variants inactive while retaining pointer validity. + void Deactivate(); + + /// Serialize unregister with Load(); EndUnregister releases the lock after success or rollback. + void BeginUnregister(); + void EndUnregister(); + /// Select a specific variant within this container. Throws if the variant is /// not part of this model, or if this is a leaf. /// @@ -147,9 +178,23 @@ class Model { /// one-shot operations, so callers reading the path concurrently with download /// or removal of the same Model are out of contract. const std::string& LocalPath() const { return local_path_; } + const std::string& RuntimeId() const { + Model* selected = selected_variant_.load(std::memory_order_acquire); + return selected ? selected->RuntimeId() : runtime_model_id_; + } private: - // Leaf data (default/empty for containers). + struct ContainerTag {}; + + Model(ModelInfo info, + std::string local_path, + DownloadManager& download_manager, + ModelLoadManager& model_load_manager, + std::string runtime_model_id, + bool external_registration); + Model(ContainerTag, Model first_variant); + + // Leaf data (empty for containers). Construction guarantees this is non-null for every leaf. // cached_ is atomic — flipped concurrently by the download path. // Loaded state is NOT stored here; it is queried from ModelLoadManager so the load // manager remains the single source of truth (Manager::Shutdown clears its map without @@ -158,9 +203,14 @@ class Model { // cleared by RemoveFromCache(). Its mutation is guarded by state_mutex_; the reader-safety // contract is that the path is published before cached_ flips true (and cleared after cached_ // flips false), so any reader that gates on IsCached() observes a complete path. - ModelInfo info_; + std::unique_ptr info_; std::atomic cached_{false}; + // Logical tombstone state. Retired models remain allocated so outstanding catalog-owned handles stay address-valid, + // but operations that would use the registration reject them. + std::atomic active_{true}; std::string local_path_; + std::string runtime_model_id_; + bool external_registration_ = false; // Non-owning service bindings for leaf operations. Set once at construction and never // reassigned; guaranteed non-null because FromModelInfo takes them by reference. @@ -170,11 +220,18 @@ class Model { // Container data (empty/null for leaves). unique_ptr keeps Model addresses // stable across vector growth/reordering. std::vector> variants_; + std::vector> retired_variants_; std::atomic selected_variant_{nullptr}; // non-null = this is a container + bool selection_is_explicit_ = false; // guarded by state_mutex_ // Guards variants_ across reader/writer threads (catalog refresh adding variants // while another thread enumerates via Variants()). mutable std::mutex state_mutex_; + + // Leaf lifecycle state. A Model must not be moved while an unregister operation holds this lock. + mutable std::mutex lifecycle_mutex_; + bool unregistering_ = false; + std::vector unregistering_variants_; // exact container snapshot locked by BeginUnregister }; } // namespace fl diff --git a/sdk_v2/cpp/src/model_info.cc b/sdk_v2/cpp/src/model_info.cc index cc2f68f16..25a2ceb9d 100644 --- a/sdk_v2/cpp/src/model_info.cc +++ b/sdk_v2/cpp/src/model_info.cc @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. #include "model_info.h" +#include "util/string_utils.h" #include #include @@ -28,15 +29,16 @@ std::string DeviceTypeToString(DeviceType dt) { namespace { DeviceType DeviceTypeFromString(const std::string& s) { - if (s == "CPU") { + const auto lowered = ToLower(s); + if (lowered == "cpu") { return DeviceType::kCPU; } - if (s == "GPU") { + if (lowered == "gpu") { return DeviceType::kGPU; } - if (s == "NPU") { + if (lowered == "npu") { return DeviceType::kNPU; } @@ -106,6 +108,24 @@ ModelInfo ModelInfoFromJson(const nlohmann::json& j) { info.detected_region = j["detectedRegion"].get(); } + // Preserve the complete extensible ModelInfo state in addition to the catalog-compatible named fields below. + // Keeping string and integer properties separate retains their types even when both maps contain the same key. + if (j.contains("stringProperties") && j["stringProperties"].is_object()) { + for (const auto& [key, value] : j["stringProperties"].items()) { + if (value.is_string()) { + info.SetPropertyStr(key, value.get()); + } + } + } + + if (j.contains("intProperties") && j["intProperties"].is_object()) { + for (const auto& [key, value] : j["intProperties"].items()) { + if (value.is_number_integer()) { + info.SetPropertyInt(key, value.get()); + } + } + } + // String properties — named top-level fields → string_properties map ReadStringProp(j, "providerType", info.string_properties, FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR); ReadStringProp(j, "modelType", info.string_properties, FOUNDRY_LOCAL_MODEL_PROP_MODEL_TYPE_STR); @@ -208,6 +228,22 @@ nlohmann::json ModelInfoToJson(const ModelInfo& info) { j["detectedRegion"] = info.detected_region; } + if (!info.string_properties.empty()) { + nlohmann::json properties = nlohmann::json::object(); + for (const auto& [key, value] : info.string_properties) { + properties[key] = value; + } + j["stringProperties"] = std::move(properties); + } + + if (!info.int_properties.empty()) { + nlohmann::json properties = nlohmann::json::object(); + for (const auto& [key, value] : info.int_properties) { + properties[key] = value; + } + j["intProperties"] = std::move(properties); + } + // providerType — required in C#, defaults to empty const auto* provider = info.GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR); j["providerType"] = provider ? *provider : ""; @@ -342,4 +378,20 @@ nlohmann::json ModelInfoToJson(const ModelInfo& info) { return j; } +void ModelInfo::SetPropertyStr(std::string key, std::string value) { + if (key == FOUNDRY_LOCAL_MODEL_PROP_TASK_STR) { + task = value; + } else if (key == FOUNDRY_LOCAL_MODEL_PROP_EP_STR) { + execution_provider = value; + } else if (key == FOUNDRY_LOCAL_MODEL_PROP_DEVICE_TYPE_STR) { + device_type = DeviceTypeFromString(value); + } + + string_properties[std::move(key)] = std::move(value); +} + +void ModelInfo::SetPropertyInt(std::string key, int64_t value) { + int_properties[std::move(key)] = value; +} + } // namespace fl diff --git a/sdk_v2/cpp/src/model_info.h b/sdk_v2/cpp/src/model_info.h index 09f279545..adea98dce 100644 --- a/sdk_v2/cpp/src/model_info.h +++ b/sdk_v2/cpp/src/model_info.h @@ -55,6 +55,12 @@ struct ModelInfo { std::map> string_properties; std::map> int_properties; + /// Set a string property while keeping typed fields synchronized with well-known keys. + void SetPropertyStr(std::string key, std::string value); + + /// Set an int property. + void SetPropertyInt(std::string key, int64_t value); + /// Look up a string property by key, returning nullptr if missing. const std::string* GetPropertyStr(std::string_view key) const { auto it = string_properties.find(key); diff --git a/sdk_v2/cpp/src/platform/posix/cross_process_file_lock.cc b/sdk_v2/cpp/src/platform/posix/cross_process_file_lock.cc index 4d32a69f3..f939fc4ab 100644 --- a/sdk_v2/cpp/src/platform/posix/cross_process_file_lock.cc +++ b/sdk_v2/cpp/src/platform/posix/cross_process_file_lock.cc @@ -6,13 +6,11 @@ #include "platform/cross_process_file_lock.h" #include "exception.h" #include "logger.h" +#include "util/time_utils.h" #include #include -#include -#include -#include #include #include @@ -31,11 +29,7 @@ constexpr const char* kLockFileName = ".download.lock"; std::string FormatProcessInfo() { auto pid = static_cast(getpid()); auto t = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); - std::tm tm{}; - gmtime_r(&t, &tm); - std::ostringstream oss; - oss << "PID:" << pid << ",Time:" << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ") << '\n'; - return oss.str(); + return "PID:" + std::to_string(pid) + ",Time:" + FormatUtcTimestamp(t) + '\n'; } } // namespace diff --git a/sdk_v2/cpp/src/platform/windows/cross_process_file_lock.cc b/sdk_v2/cpp/src/platform/windows/cross_process_file_lock.cc index 6f5e44cf2..2f9cb2fae 100644 --- a/sdk_v2/cpp/src/platform/windows/cross_process_file_lock.cc +++ b/sdk_v2/cpp/src/platform/windows/cross_process_file_lock.cc @@ -6,13 +6,11 @@ #include "platform/cross_process_file_lock.h" #include "exception.h" #include "logger.h" +#include "util/time_utils.h" #include #include -#include -#include -#include #include #define WIN32_LEAN_AND_MEAN @@ -30,11 +28,7 @@ constexpr const char* kLockFileName = ".download.lock"; std::string FormatProcessInfo() { auto pid = static_cast(_getpid()); auto t = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); - std::tm tm{}; - gmtime_s(&tm, &t); - std::ostringstream oss; - oss << "PID:" << pid << ",Time:" << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ") << '\n'; - return oss.str(); + return "PID:" + std::to_string(pid) + ",Time:" + FormatUtcTimestamp(t) + '\n'; } } // namespace diff --git a/sdk_v2/cpp/src/util/time_utils.cc b/sdk_v2/cpp/src/util/time_utils.cc new file mode 100644 index 000000000..043dfc988 --- /dev/null +++ b/sdk_v2/cpp/src/util/time_utils.cc @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "util/time_utils.h" + +#include +#include + +namespace fl { + +std::string FormatUtcTimestamp(std::time_t unix_time) { + std::tm utc{}; +#ifdef _WIN32 + if (gmtime_s(&utc, &unix_time) != 0) { + return {}; + } +#else + if (!gmtime_r(&unix_time, &utc)) { + return {}; + } +#endif + + std::ostringstream stream; + stream << std::put_time(&utc, "%Y-%m-%dT%H:%M:%SZ"); + return stream.str(); +} + +} // namespace fl \ No newline at end of file diff --git a/sdk_v2/cpp/src/util/time_utils.h b/sdk_v2/cpp/src/util/time_utils.h new file mode 100644 index 000000000..85c1c8bfe --- /dev/null +++ b/sdk_v2/cpp/src/util/time_utils.h @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include +#include + +namespace fl { + +/// Formats a Unix timestamp as an ISO 8601 UTC timestamp. Returns an empty string if conversion fails. +std::string FormatUtcTimestamp(std::time_t unix_time); + +} // namespace fl \ No newline at end of file diff --git a/sdk_v2/cpp/test/CMakeLists.txt b/sdk_v2/cpp/test/CMakeLists.txt index 5b53c4bbc..1a5dffd7c 100644 --- a/sdk_v2/cpp/test/CMakeLists.txt +++ b/sdk_v2/cpp/test/CMakeLists.txt @@ -37,9 +37,11 @@ add_executable(foundry_local_tests internal_api/http_download_test.cc internal_api/http_retry_test.cc internal_api/item_test.cc + internal_api/local_model_catalog_test.cc internal_api/local_model_scanner_test.cc internal_api/model_info_test.cc internal_api/model_info_accessors_test.cc + internal_api/model_construction_test.cc internal_api/model_io_info_test.cc internal_api/model_load_manager_test.cc internal_api/model_sorting_test.cc @@ -55,6 +57,7 @@ add_executable(foundry_local_tests internal_api/azure_model_catalog_test.cc internal_api/telemetry_test.cc internal_api/tensor_test.cc + internal_api/time_utils_test.cc internal_api/chat/search_options_test.cc internal_api/toolcalling/tool_call_stream_accumulator_test.cc internal_api/toolcalling/tool_call_utils_test.cc @@ -148,6 +151,7 @@ add_executable(sdk_integration_tests sdk_api/embeddings_test.cc sdk_api/ep_detection_test.cc sdk_api/chat_session_test.cc + sdk_api/bring_your_own_model_e2e_test.cc sdk_api/model_test.cc sdk_api/model_endpoints_test.cc sdk_api/chat_completions_test.cc diff --git a/sdk_v2/cpp/test/internal_api/azure_catalog_test.cc b/sdk_v2/cpp/test/internal_api/azure_catalog_test.cc index 18feea1b7..f5bfa3bc0 100644 --- a/sdk_v2/cpp/test/internal_api/azure_catalog_test.cc +++ b/sdk_v2/cpp/test/internal_api/azure_catalog_test.cc @@ -588,7 +588,7 @@ TEST(AzureCatalogClientTest, WithCachedModels_UnresolvedId_TriggersSecondFetch) EXPECT_TRUE(found_old); } -TEST(AzureCatalogClientTest, WithCachedModels_FullyUnresolved_DefersBYOEntryToCatalogMerge) { +TEST(AzureCatalogClientTest, WithCachedModels_FullyUnresolved_DoesNotCreatePublicEntry) { CpuOnlyEpDetector ep; StderrLogger logger; int http_call_count = 0; @@ -609,8 +609,9 @@ TEST(AzureCatalogClientTest, WithCachedModels_FullyUnresolved_DefersBYOEntryToCa auto result = FetchAllModelInfosWithCachedModels(client, {"custom-model:0"}, logger); EXPECT_EQ(http_call_count, 2); + ASSERT_EQ(result.size(), 1u); - EXPECT_EQ(result[0].model_id, "phi-4-mini:3"); + EXPECT_EQ(result.front().model_id, "phi-4-mini:3"); } // ======================================================================== diff --git a/sdk_v2/cpp/test/internal_api/azure_model_catalog_test.cc b/sdk_v2/cpp/test/internal_api/azure_model_catalog_test.cc index 77179bd00..804082124 100644 --- a/sdk_v2/cpp/test/internal_api/azure_model_catalog_test.cc +++ b/sdk_v2/cpp/test/internal_api/azure_model_catalog_test.cc @@ -207,7 +207,7 @@ TEST_F(AzureModelCatalogTest, AllUrlsFailUsesSnapshotMetadataAndScannedPathsWith WriteSnapshot({gpu_info, cpu_info}); const auto gpu_path = AddLocalModel("snapshot-gpu:2", "snapshot-gpu"); const auto cpu_path = AddLocalModel("snapshot-cpu:2", "snapshot-cpu"); - const auto byom_path = AddLocalModel("offline-byom:4", "offline-byom"); + AddLocalModel("offline-byom:4", "offline-byom"); AddBehavior("https://catalog-one.test", true); AddBehavior("https://catalog-two.test", true); @@ -218,7 +218,7 @@ TEST_F(AzureModelCatalogTest, AllUrlsFailUsesSnapshotMetadataAndScannedPathsWith const auto cached_models = catalog->GetCachedModels(); - ASSERT_EQ(cached_models.size(), 3u); + ASSERT_EQ(cached_models.size(), 2u); auto* gpu_model = FindVariant(cached_models, "snapshot-gpu:2"); ASSERT_NE(gpu_model, nullptr); EXPECT_EQ(gpu_model->Info().alias, "snapshot-alias"); @@ -232,9 +232,7 @@ TEST_F(AzureModelCatalogTest, AllUrlsFailUsesSnapshotMetadataAndScannedPathsWith EXPECT_EQ(cpu_model->Info().alias, "snapshot-alias"); EXPECT_EQ(cpu_model->LocalPath(), cpu_path.string()); - auto* byom_model = FindVariant(cached_models, "offline-byom:4"); - ASSERT_NE(byom_model, nullptr); - EXPECT_EQ(byom_model->LocalPath(), byom_path.string()); + EXPECT_EQ(FindVariant(cached_models, "offline-byom:4"), nullptr); CatalogCache persisted_cache(cache_directory_.string(), services_.logger); persisted_cache.Load(); @@ -246,8 +244,8 @@ TEST_F(AzureModelCatalogTest, AllUrlsFailUsesSnapshotMetadataAndScannedPathsWith EXPECT_EQ(factory_calls_, 2); } -TEST_F(AzureModelCatalogTest, AllUrlsFailWithoutSnapshotSurfacesScannedModelAsByom) { - const auto local_path = AddLocalModel("custom-model:0", "custom-model"); +TEST_F(AzureModelCatalogTest, AllUrlsFailWithoutSnapshotIgnoresUnknownScannedModel) { + AddLocalModel("custom-model:0", "custom-model"); AddBehavior("https://catalog-one.test", true); AddBehavior("https://catalog-two.test", true); auto catalog = CreateCatalog({ @@ -257,20 +255,8 @@ TEST_F(AzureModelCatalogTest, AllUrlsFailWithoutSnapshotSurfacesScannedModelAsBy const auto cached_models = catalog->GetCachedModels(); - ASSERT_EQ(cached_models.size(), 1u); - auto* byom_model = FindVariant(cached_models, "custom-model:0"); - ASSERT_NE(byom_model, nullptr); - EXPECT_EQ(byom_model->Info().name, "custom-model"); - EXPECT_EQ(byom_model->Info().alias, "custom-model"); - EXPECT_EQ(byom_model->Info().version, 0); - EXPECT_EQ(byom_model->Info().uri, "local://custom-model"); - const auto* provider = byom_model->Info().GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR); - const auto* model_type = byom_model->Info().GetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_MODEL_TYPE_STR); - ASSERT_NE(provider, nullptr); - ASSERT_NE(model_type, nullptr); - EXPECT_EQ(*provider, "Local"); - EXPECT_EQ(*model_type, "ONNX"); - EXPECT_EQ(byom_model->LocalPath(), local_path.string()); + EXPECT_TRUE(cached_models.empty()); + EXPECT_EQ(catalog->GetModelVariant("custom-model:0"), nullptr); EXPECT_FALSE(fs::exists(cache_directory_.path() / "foundry.modelinfo.json")); } @@ -290,7 +276,7 @@ TEST_F(AzureModelCatalogTest, EmptySuccessfulUrlPreventsSnapshotFallbackWhenAnot EXPECT_EQ(factory_calls_, 2); } -TEST_F(AzureModelCatalogTest, CacheOnlyUsesSnapshotAndScannedByomWithoutCreatingLiveClient) { +TEST_F(AzureModelCatalogTest, CacheOnlyUsesSnapshotPathAndIgnoresUnknownScannedModelWithoutCreatingLiveClient) { WriteSnapshot({MakeModelInfo("snapshot-model:3", "snapshot-model", 3, "snapshot-alias", "SnapshotProvider")}); AddLocalModel("snapshot-model:3", "snapshot-model"); AddLocalModel("cache-only-byom:5", "cache-only-byom"); @@ -298,13 +284,13 @@ TEST_F(AzureModelCatalogTest, CacheOnlyUsesSnapshotAndScannedByomWithoutCreating const auto cached_models = catalog->GetCachedModels(); - ASSERT_EQ(cached_models.size(), 2u); + ASSERT_EQ(cached_models.size(), 1u); EXPECT_NE(FindVariant(cached_models, "snapshot-model:3"), nullptr); - EXPECT_NE(FindVariant(cached_models, "cache-only-byom:5"), nullptr); + EXPECT_EQ(catalog->GetModelVariant("cache-only-byom:5"), nullptr); EXPECT_EQ(factory_calls_, 0); } -TEST_F(AzureModelCatalogTest, LiveAggregationDeduplicatesAndSavesResolvedAndByomMetadata) { +TEST_F(AzureModelCatalogTest, LiveAggregationDeduplicatesAndSavesOnlyResolvedPublicMetadata) { AddLocalModel("old-model:1", "old-model"); AddLocalModel("custom-model:0", "custom-model"); @@ -324,7 +310,8 @@ TEST_F(AzureModelCatalogTest, LiveAggregationDeduplicatesAndSavesResolvedAndByom const auto models = catalog->ListModels(); - ASSERT_EQ(models.size(), 3u); + ASSERT_EQ(models.size(), 2u); + EXPECT_EQ(FindVariant(models, "custom-model:0"), nullptr); EXPECT_EQ(first_behavior->fetch_by_id_calls, 1); EXPECT_EQ(second_behavior->fetch_by_id_calls, 1); @@ -332,13 +319,31 @@ TEST_F(AzureModelCatalogTest, LiveAggregationDeduplicatesAndSavesResolvedAndByom persisted_cache.Load(); const auto persisted_models = persisted_cache.GetCachedModels(); ASSERT_TRUE(persisted_models.has_value()); - ASSERT_EQ(persisted_models->size(), 3u); + ASSERT_EQ(persisted_models->size(), 2u); std::unordered_set persisted_ids; for (const auto& info : *persisted_models) { persisted_ids.insert(info.model_id); } - EXPECT_EQ(persisted_ids, - (std::unordered_set{"latest-model:2", "old-model:1", "custom-model:0"})); + EXPECT_EQ(persisted_ids, (std::unordered_set{"latest-model:2", "old-model:1"})); +} + +TEST_F(AzureModelCatalogTest, CacheOnlyIgnoresLegacySynthesizedByomSnapshotEntry) { + const auto public_info = MakeModelInfo("snapshot-model:3", "snapshot-model", 3, "snapshot-alias", + "SnapshotProvider"); + const auto legacy_byom_info = MakeModelInfo("legacy-byom:1", "legacy-byom", 1, "legacy-byom", "Local"); + WriteSnapshot({public_info, legacy_byom_info}); + AddLocalModel("snapshot-model:3", "snapshot-model"); + AddLocalModel("legacy-byom:1", "legacy-byom"); + auto catalog = CreateCatalog({{"https://must-not-be-called.test", std::nullopt}}, true); + + const auto models = catalog->ListModels(); + const auto cached_models = catalog->GetCachedModels(); + + ASSERT_EQ(models.size(), 1u); + ASSERT_EQ(cached_models.size(), 1u); + EXPECT_NE(catalog->GetModelVariant("snapshot-model:3"), nullptr); + EXPECT_EQ(catalog->GetModelVariant("legacy-byom:1"), nullptr); + EXPECT_EQ(factory_calls_, 0); } diff --git a/sdk_v2/cpp/test/internal_api/c_api_test.cc b/sdk_v2/cpp/test/internal_api/c_api_test.cc index 5fc46d302..f4f87eb78 100644 --- a/sdk_v2/cpp/test/internal_api/c_api_test.cc +++ b/sdk_v2/cpp/test/internal_api/c_api_test.cc @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. #include "internal_api/c_api_test_helpers.h" +#include "utils/temp_path.h" #include #include @@ -33,6 +34,22 @@ TEST(CApiTest, GetApiReturnsNullForFutureVersion) { EXPECT_EQ(api, nullptr); } +TEST(CApiTest, SupportedVersionsReturnSameExpandedApiTables) { + const flApi* v0 = FoundryLocalGetApi(0); + const flApi* v1 = FoundryLocalGetApi(1); + const flApi* v2 = FoundryLocalGetApi(2); + ASSERT_NE(v0, nullptr); + ASSERT_NE(v1, nullptr); + ASSERT_NE(v2, nullptr); + + EXPECT_EQ(v0, v2); + EXPECT_EQ(v1, v2); + EXPECT_EQ(v1->GetCatalogApi(), v2->GetCatalogApi()); + EXPECT_EQ(v1->GetModelApi(), v2->GetModelApi()); + EXPECT_NE(v1->GetCatalogApi()->RegisterModel, nullptr); + EXPECT_NE(v1->GetModelApi()->CreateModelInfo, nullptr); +} + TEST(CApiTest, VersionReturnsNonNull) { const char* version = FoundryLocalGetVersionString(); ASSERT_NE(version, nullptr); @@ -227,6 +244,86 @@ TEST(CApiTest, GetCatalogFromManager) { api->Manager_Release(mgr); } +TEST(CApiTest, LocalCatalogRegistersListsAndUnregistersWithoutOwningAssets) { + auto root = fl::test::TempPath::CreateTempDir("c_api_local_catalog"); + const auto model_path = root.path() / "model"; + const auto app_data_path = root.path() / "appdata"; + std::filesystem::create_directories(model_path); + std::ofstream(model_path / "genai_config.json") << R"({"model":{"type":"phi3"}})"; + + const flApi* api = GetApi(); + const flConfigurationApi* config_api = api->GetConfigurationApi(); + const flCatalogApi* catalog_api = api->GetCatalogApi(); + const flModelApi* model_api = api->GetModelApi(); + flConfiguration* config = nullptr; + ASSERT_FL_OK(api, config_api->Create("c-api-local-catalog", &config)); + ASSERT_FL_OK(api, config_api->SetAppDataDir(config, app_data_path.string().c_str())); + + flManager* manager = nullptr; + ASSERT_FL_OK(api, api->Manager_Create(config, &manager)); + flCatalog* catalog = nullptr; + ASSERT_FL_OK(api, api->Manager_GetCatalogByType(manager, FOUNDRY_LOCAL_CATALOG_LOCAL, &catalog)); + ASSERT_NE(catalog, nullptr); + + flModelInfo* metadata = nullptr; + ASSERT_FL_OK(api, model_api->CreateModelInfo(&metadata)); + ASSERT_FL_OK(api, model_api->Info_SetStringProperty(metadata, FOUNDRY_LOCAL_MODEL_PROP_TASK_STR, + "chat-completion")); + ASSERT_FL_OK(api, model_api->Info_SetIntProperty(metadata, FOUNDRY_LOCAL_MODEL_PROP_FILESIZE_MB_INT, 17)); + + flModel* registered = nullptr; + ASSERT_FL_OK(api, catalog_api->RegisterModel(catalog, model_path.string().c_str(), "c-api-model:3", metadata, + ®istered)); + model_api->ReleaseModelInfo(metadata); + ASSERT_NE(registered, nullptr); + + const flModelInfo* registered_info = nullptr; + ASSERT_FL_OK(api, model_api->GetInfo(registered, ®istered_info)); + EXPECT_STREQ(model_api->Info_GetId(registered_info), "c-api-model:3"); + EXPECT_STREQ(model_api->Info_GetName(registered_info), "c-api-model"); + EXPECT_STREQ(model_api->Info_GetAlias(registered_info), "c-api-model"); + EXPECT_EQ(model_api->Info_GetVersion(registered_info), 3); + EXPECT_STREQ(model_api->Info_GetTask(registered_info), "chat-completion"); + EXPECT_EQ(model_api->Info_GetIntProperty(registered_info, FOUNDRY_LOCAL_MODEL_PROP_FILESIZE_MB_INT, -1), 17); + + flModelList* models = nullptr; + ASSERT_FL_OK(api, catalog_api->GetModels(catalog, &models)); + ASSERT_EQ(api->ModelList_Size(models), 1u); + flModel* listed = api->ModelList_GetAt(models, 0); + ASSERT_NE(listed, nullptr); + + StatusGuard remove_status{model_api->RemoveFromCache(registered), api}; + ASSERT_NE(remove_status.s, nullptr); + EXPECT_EQ(api->Status_GetErrorCode(remove_status.s), FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + + ASSERT_FL_OK(api, catalog_api->UnregisterModel(catalog, "c-api-model")); + EXPECT_TRUE(std::filesystem::exists(model_path / "genai_config.json")); + EXPECT_FALSE(std::filesystem::exists(model_path / "model_metadata.yml")); + + ASSERT_FL_OK(api, model_api->GetInfo(registered, ®istered_info)); + EXPECT_STREQ(model_api->Info_GetId(registered_info), "c-api-model:3"); + const flModelInfo* listed_info = nullptr; + ASSERT_FL_OK(api, model_api->GetInfo(listed, &listed_info)); + EXPECT_STREQ(model_api->Info_GetId(listed_info), "c-api-model:3"); + + StatusGuard registered_load_status{model_api->Load(registered), api}; + ASSERT_NE(registered_load_status.s, nullptr); + EXPECT_EQ(api->Status_GetErrorCode(registered_load_status.s), FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + StatusGuard listed_load_status{model_api->Load(listed), api}; + ASSERT_NE(listed_load_status.s, nullptr); + EXPECT_EQ(api->Status_GetErrorCode(listed_load_status.s), FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + ASSERT_FL_OK(api, model_api->Unload(registered)); + ASSERT_FL_OK(api, model_api->Unload(listed)); + api->ModelList_Release(models); + + ASSERT_FL_OK(api, catalog_api->GetModels(catalog, &models)); + EXPECT_EQ(api->ModelList_Size(models), 0u); + api->ModelList_Release(models); + + config_api->Configuration_Release(config); + api->Manager_Release(manager); +} + TEST(CApiTest, GetCatalogNameReturnsNonEmptyString) { const flApi* api = GetApi(); ASSERT_NE(api, nullptr); diff --git a/sdk_v2/cpp/test/internal_api/local_model_catalog_test.cc b/sdk_v2/cpp/test/internal_api/local_model_catalog_test.cc new file mode 100644 index 000000000..418b9374a --- /dev/null +++ b/sdk_v2/cpp/test/internal_api/local_model_catalog_test.cc @@ -0,0 +1,357 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "catalog/local_model_catalog.h" + +#include "internal_api/test_helpers.h" +#include "utils/temp_path.h" + +#include +#include +#include + +#include +#include +#include +#include + +namespace fl::test { +namespace { + +class LocalModelCatalogTest : public ::testing::Test { + protected: + LocalModelCatalogTest() + : root_(TempPath::CreateTempDir("local_model_catalog")), + model_dir_(root_.path() / "model"), + catalog_(MakeCatalog()) { + WriteConfig(model_dir_); + } + + LocalModelCatalog MakeCatalog() { + return LocalModelCatalog( + root_.path() / "appdata", + [this](ModelInfo info, std::string path, std::string runtime_model_id) { + return Model::FromLocalRegistration(std::move(info), std::move(path), bindings_.download_manager, + bindings_.model_load_manager, std::move(runtime_model_id)); + }, + NullLog()); + } + + static void WriteConfig(const std::filesystem::path& path) { + std::filesystem::create_directories(path); + std::ofstream(path / "genai_config.json") << R"({"model":{"type":"phi3","context_length":4096}})"; + } + + ModelInfo MakeMetadata(std::string task = "chat-completion") const { + ModelInfo info; + info.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_TASK_STR, std::move(task)); + return info; + } + + Model* Register(std::string model_id = "my-model:1") { + return catalog_.RegisterModel(model_dir_.string(), model_id, MakeMetadata()); + } + + TempPath root_; + std::filesystem::path model_dir_; + FakeServiceBindings bindings_; + LocalModelCatalog catalog_; +}; + +TEST_F(LocalModelCatalogTest, RegisterPreservesCallerMetadataAndWritesOnlyAppDataIndex) { + auto info = MakeMetadata(); + info.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_INPUT_MODALITIES_STR, "text,image"); + info.SetPropertyStr(FOUNDRY_LOCAL_MODEL_PROP_OUTPUT_MODALITIES_STR, "text"); + info.SetPropertyInt(FOUNDRY_LOCAL_MODEL_PROP_FILESIZE_MB_INT, 321); + info.SetPropertyStr("custom_metadata", "preserved"); + info.SetPropertyInt("custom_count", 42); + info.prompt_templates.Add("user", "<|user|>{Content}<|end|>"); + info.model_settings.Add("temperature", "0.5"); + info.SetPropertyStr("model_path", "ignored"); + info.SetPropertyStr("alias", "ignored"); + info.SetPropertyStr("version", "ignored"); + info.SetPropertyInt("model_path", 99); + info.SetPropertyInt("alias", 99); + info.SetPropertyInt("_local_registration_id", 99); + info.SetPropertyInt("version", 99); + + auto* model = catalog_.RegisterModel(model_dir_.string(), "my-model:7", info); + + ASSERT_NE(model, nullptr); + EXPECT_EQ(model->Id(), "my-model:7"); + EXPECT_EQ(model->Info().name, "my-model"); + EXPECT_EQ(model->Info().alias, "my-model"); + EXPECT_EQ(model->Info().version, 7); + EXPECT_EQ(model->Info().task, "chat-completion"); + EXPECT_EQ(model->Info().GetPropertyWithDefault(FOUNDRY_LOCAL_MODEL_PROP_INPUT_MODALITIES_STR, std::string{}), + "text,image"); + EXPECT_EQ(model->Info().GetPropertyWithDefault(FOUNDRY_LOCAL_MODEL_PROP_OUTPUT_MODALITIES_STR, std::string{}), + "text"); + EXPECT_EQ(model->Info().GetPropertyWithDefault(FOUNDRY_LOCAL_MODEL_PROP_FILESIZE_MB_INT, int64_t{-1}), 321); + EXPECT_EQ(model->Info().GetPropertyStr("model_path"), nullptr); + EXPECT_EQ(model->Info().GetPropertyStr("alias"), nullptr); + EXPECT_EQ(model->Info().GetPropertyStr("version"), nullptr); + EXPECT_EQ(model->Info().GetPropertyInt("model_path"), nullptr); + EXPECT_EQ(model->Info().GetPropertyInt("alias"), nullptr); + EXPECT_EQ(model->Info().GetPropertyInt("_local_registration_id"), nullptr); + EXPECT_EQ(model->Info().GetPropertyInt("version"), nullptr); + EXPECT_TRUE(model->IsCached()); + EXPECT_FALSE(std::filesystem::exists(model_dir_ / "model_metadata.yml")); + + const auto index_path = root_.path() / "appdata" / "catalogs" / "local" / "local_models.json"; + ASSERT_TRUE(std::filesystem::exists(index_path)); + nlohmann::json index; + std::ifstream(index_path) >> index; + EXPECT_EQ(index["version"], 2); + ASSERT_EQ(index["models"].size(), 1u); + ASSERT_TRUE(index["models"][0].contains("model_info")); + EXPECT_EQ(index["models"][0]["model_info"]["id"], "my-model:7"); + EXPECT_EQ(index["models"][0]["model_info"]["name"], "my-model"); + EXPECT_EQ(index["models"][0]["model_info"]["version"], 7); + EXPECT_EQ(index["models"][0]["model_info"]["alias"], "my-model"); + EXPECT_EQ(index["models"][0]["model_info"]["stringProperties"]["custom_metadata"], "preserved"); + EXPECT_EQ(index["models"][0]["model_info"]["intProperties"]["custom_count"], 42); + EXPECT_TRUE(index["models"][0].contains("registration_id")); + EXPECT_FALSE(index["models"][0].contains("model_id")); + EXPECT_FALSE(index["models"][0].contains("alias")); + EXPECT_FALSE(index["models"][0].contains("properties")); + EXPECT_FALSE(index["models"][0].contains("registered_at")); + EXPECT_FALSE(index["models"][0].contains("metadata_prepared")); + + auto restored = MakeCatalog(); + auto* restored_model = restored.GetModelVariant("my-model:7"); + ASSERT_NE(restored_model, nullptr); + EXPECT_EQ(restored_model->Info().GetPropertyWithDefault("custom_metadata", std::string{}), "preserved"); + EXPECT_EQ(restored_model->Info().GetPropertyWithDefault("custom_count", int64_t{-1}), 42); + EXPECT_STREQ(restored_model->Info().prompt_templates.Find("user"), "<|user|>{Content}<|end|>"); + EXPECT_STREQ(restored_model->Info().model_settings.Find("temperature"), "0.5"); +} + +TEST_F(LocalModelCatalogTest, RegistrationRequiresExistingDirectoryAndParseableConfig) { + EXPECT_THROW(catalog_.RegisterModel((root_.path() / "missing").string(), "missing:1", MakeMetadata()), Exception); + + const auto no_config_path = root_.path() / "no-config"; + std::filesystem::create_directories(no_config_path); + EXPECT_THROW(catalog_.RegisterModel(no_config_path.string(), "no-config:1", MakeMetadata()), Exception); + + const auto malformed_path = root_.path() / "malformed"; + std::filesystem::create_directories(malformed_path); + std::ofstream(malformed_path / "genai_config.json") << R"({"model":)"; + EXPECT_THROW(catalog_.RegisterModel(malformed_path.string(), "malformed:1", MakeMetadata()), Exception); +} + +TEST_F(LocalModelCatalogTest, RegistrationRequiresSupportedTask) { + ModelInfo missing_task; + EXPECT_THROW(catalog_.RegisterModel(model_dir_.string(), "missing-task:1", missing_task), Exception); + EXPECT_THROW(catalog_.RegisterModel(model_dir_.string(), "invalid-task:1", MakeMetadata("text-generation")), + Exception); +} + +TEST_F(LocalModelCatalogTest, RegistrationRequiresCanonicalModelId) { + const std::vector invalid_ids = { + "", "missing-version", ":1", "model:", "model:1:2", + "model:-1", "model:+1", "model:01", "model:2147483648", + "model/path:1", "model name:1", + }; + + for (const auto& model_id : invalid_ids) { + EXPECT_THROW(catalog_.RegisterModel(model_dir_.string(), model_id, MakeMetadata()), Exception) << model_id; + } +} + +TEST_F(LocalModelCatalogTest, RegistrationUsesUniqueIdsAndGroupsVersionsByDerivedAlias) { + auto* first = Register("my-model:1"); + ASSERT_NE(first, nullptr); + EXPECT_THROW(Register("my-model:1"), Exception); + + auto* second = Register("my-model:2"); + ASSERT_NE(second, nullptr); + EXPECT_EQ(second->Info().version, 2); + EXPECT_NO_THROW(first->Download()); + + auto* grouped = catalog_.GetModel("my-model"); + ASSERT_NE(grouped, nullptr); + EXPECT_EQ(grouped->Variants().size(), 2u); + EXPECT_EQ(grouped->Id(), "my-model:2"); + EXPECT_NE(catalog_.GetModelVariant("my-model:1"), nullptr); + EXPECT_EQ(catalog_.GetModelVariant("my-model:2"), second); + + grouped->SelectVariant(*first); + EXPECT_EQ(grouped->Id(), "my-model:1"); + auto other = MakeCatalog(); + ASSERT_NE(other.RegisterModel(model_dir_.string(), "my-model:3", MakeMetadata()), nullptr); + ASSERT_EQ(catalog_.ListModels().size(), 1u); + EXPECT_EQ(grouped->Variants().size(), 3u); + EXPECT_EQ(grouped->Id(), "my-model:1"); + EXPECT_EQ(catalog_.GetModelVariant("my-model:1"), first); + + catalog_.UnregisterModel("my-model:1"); + EXPECT_EQ(catalog_.GetModelVariant("my-model:1"), nullptr); + EXPECT_THROW(first->Download(), Exception); + EXPECT_EQ(catalog_.GetModelVariant("my-model:2"), second); + EXPECT_NO_THROW(second->Download()); +} + +TEST_F(LocalModelCatalogTest, GetCachedModelsReturnsActiveRegisteredLeafVariants) { + Register("my-model:1"); + Register("my-model:2"); + + auto cached = catalog_.GetCachedModels(); + ASSERT_EQ(cached.size(), 2u); + EXPECT_EQ(cached[0]->Id(), "my-model:2"); + EXPECT_EQ(cached[1]->Id(), "my-model:1"); + EXPECT_FALSE(cached[0]->IsContainer()); + EXPECT_FALSE(cached[1]->IsContainer()); + + catalog_.UnregisterModel("my-model:2"); + + cached = catalog_.GetCachedModels(); + ASSERT_EQ(cached.size(), 1u); + EXPECT_EQ(cached.front()->Id(), "my-model:1"); +} + +TEST_F(LocalModelCatalogTest, RefreshDefersVariantReconciliationDuringUnregister) { + Register("my-model:1"); + auto* grouped = catalog_.GetModel("my-model"); + ASSERT_NE(grouped, nullptr); + + auto other = MakeCatalog(); + grouped->BeginUnregister(); + size_t variants_during_unregister = 0; + try { + EXPECT_NE(other.RegisterModel(model_dir_.string(), "my-model:2", MakeMetadata()), nullptr); + variants_during_unregister = catalog_.ListModels().front()->Variants().size(); + } catch (...) { + grouped->EndUnregister(); + throw; + } + grouped->EndUnregister(); + + EXPECT_EQ(variants_during_unregister, 1u); + ASSERT_EQ(catalog_.ListModels().size(), 1u); + EXPECT_EQ(grouped->Variants().size(), 2u); +} + +TEST_F(LocalModelCatalogTest, UnregisterPersistsAndPreservesAssets) { + auto* stale = Register(); + catalog_.UnregisterModel("my-model:1"); + + EXPECT_TRUE(catalog_.ListModels().empty()); + EXPECT_TRUE(std::filesystem::exists(model_dir_ / "genai_config.json")); + EXPECT_THROW(stale->Load(), Exception); + EXPECT_NO_THROW(stale->Unload()); + auto restored = MakeCatalog(); + EXPECT_TRUE(restored.ListModels().empty()); +} + +TEST_F(LocalModelCatalogTest, AliasHandleRemainsSafeAfterItsFinalVariantIsUnregisteredById) { + Register(); + auto* alias_handle = catalog_.GetModel("my-model"); + ASSERT_NE(alias_handle, nullptr); + + catalog_.UnregisterModel("my-model:1"); + + EXPECT_FALSE(alias_handle->IsLoaded()); + EXPECT_EQ(alias_handle->Info().model_id, "my-model:1"); + EXPECT_TRUE(alias_handle->Variants().empty()); + EXPECT_THROW(alias_handle->Load(), Exception); + EXPECT_NO_THROW(alias_handle->Unload()); +} + +TEST_F(LocalModelCatalogTest, UnregisterWriteFailureLeavesModelActiveAndUsable) { + auto* model = Register(); + ASSERT_NE(model, nullptr); + + const auto temp_index_path = root_.path() / "appdata" / "catalogs" / "local" / "local_models.json.tmp"; + std::filesystem::create_directory(temp_index_path); + + EXPECT_THROW(catalog_.UnregisterModel("my-model"), Exception); + EXPECT_EQ(catalog_.GetModelVariant("my-model:1"), model); + + float progress = 0.0f; + EXPECT_NO_THROW(model->Download([&progress](float value) { + progress = value; + return 0; + })); + EXPECT_EQ(progress, 100.0f); +} + +TEST_F(LocalModelCatalogTest, TwoCatalogsReconcileRegisterUnregisterAndReregister) { + auto second = MakeCatalog(); + EXPECT_TRUE(second.ListModels().empty()); + + auto* stale = Register(); + ASSERT_NE(stale, nullptr); + ASSERT_EQ(second.ListModels().size(), 1u); + + second.UnregisterModel("my-model"); + EXPECT_TRUE(catalog_.ListModels().empty()); + EXPECT_THROW(stale->Download(), Exception); + EXPECT_THROW(stale->Load(), Exception); + EXPECT_THROW(stale->RemoveFromCache(), Exception); + + auto* replacement = second.RegisterModel(model_dir_.string(), "my-model:1", MakeMetadata()); + ASSERT_NE(replacement, nullptr); + EXPECT_NE(replacement->RuntimeId(), stale->RuntimeId()); + ASSERT_EQ(catalog_.ListModels().size(), 1u); + EXPECT_NE(catalog_.GetModelVariant("my-model:1"), stale); +} + +TEST_F(LocalModelCatalogTest, LoadsLegacyRegistrationPropertiesUsingPersistedModelId) { + const auto catalog_dir = root_.path() / "appdata" / "catalogs" / "local"; + std::filesystem::create_directories(catalog_dir); + const nlohmann::json legacy_index = { + {"version", 1}, + {"catalog_name", "local"}, + {"models", + {{{"alias", "legacy-wrong-alias"}, + {"model_id", "legacy-model:4"}, + {"model_path", model_dir_.string()}, + {"properties", + {{"_local_registration_id", "legacy-registration"}, + {"alias", "legacy-wrong-alias"}, + {"model_path", model_dir_.string()}, + {"version", 99}, + {FOUNDRY_LOCAL_MODEL_PROP_TASK_STR, "chat-completion"}}}}}}, + }; + std::ofstream(catalog_dir / "local_models.json") << legacy_index.dump(2); + + auto restored = MakeCatalog(); + auto* model = restored.GetModelVariant("legacy-model:4"); + ASSERT_NE(model, nullptr); + EXPECT_EQ(model->Info().alias, "legacy-model"); + EXPECT_EQ(model->Info().version, 4); + EXPECT_EQ(model->Info().GetPropertyStr("model_path"), nullptr); + EXPECT_EQ(model->Info().GetPropertyStr("alias"), nullptr); + EXPECT_EQ(model->Info().GetPropertyInt("version"), nullptr); + + ASSERT_NE(restored.RegisterModel(model_dir_.string(), "new-model:1", MakeMetadata()), nullptr); + nlohmann::json migrated_index; + std::ifstream(catalog_dir / "local_models.json") >> migrated_index; + EXPECT_EQ(migrated_index["version"], 2); + ASSERT_EQ(migrated_index["models"].size(), 2u); + EXPECT_TRUE(migrated_index["models"][0].contains("model_info")); + EXPECT_FALSE(migrated_index["models"][0].contains("properties")); + EXPECT_NE(restored.GetModelVariant("legacy-model:4"), nullptr); +} + +TEST_F(LocalModelCatalogTest, PublicCatalogContractRejectsRegistration) { + class ReadOnlyCatalog final : public ICatalog { + public: + const std::string& GetName() const override { return name_; } + std::vector ListModels() const override { return {}; } + Model* GetModel(const std::string&) const override { return nullptr; } + Model* GetModelVariant(const std::string&) const override { return nullptr; } + Model* GetLatestVersion(const Model*) const override { return nullptr; } + std::vector GetModelVersions(const std::string&, const std::string&, int) override { return {}; } + std::vector GetCachedModels() const override { return {}; } + std::vector GetLoadedModels() const override { return {}; } + + private: + std::string name_ = "public"; + } catalog; + + EXPECT_THROW(catalog.RegisterModel(model_dir_.string(), "my-model:1", MakeMetadata()), Exception); +} + +} // namespace +} // namespace fl::test diff --git a/sdk_v2/cpp/test/internal_api/model_construction_test.cc b/sdk_v2/cpp/test/internal_api/model_construction_test.cc new file mode 100644 index 000000000..6fceef0a0 --- /dev/null +++ b/sdk_v2/cpp/test/internal_api/model_construction_test.cc @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "exception.h" +#include "internal_api/test_helpers.h" +#include "model.h" +#include "model_info.h" + +#include + +#include +#include +#include + +using namespace fl; + +static_assert(!std::is_default_constructible_v); +static_assert(std::is_nothrow_move_constructible_v); +static_assert(std::is_nothrow_move_assignable_v); + +namespace { + +Model MakeLeaf(std::string model_id = "test-model", std::string task = "chat-completion") { + static fl::test::FakeServiceBindings svc; + ModelInfo info; + info.model_id = std::move(model_id); + info.name = "test"; + info.version = 1; + info.alias = "test-alias"; + info.task = std::move(task); + return Model::FromModelInfo(std::move(info), "", svc.download_manager, svc.model_load_manager); +} + +} // namespace + +TEST(ModelConstructionTest, LeafHasMetadataImmediately) { + auto model = MakeLeaf(); + + EXPECT_EQ(model.Info().model_id, "test-model"); + EXPECT_EQ(model.Info().alias, "test-alias"); + EXPECT_EQ(model.Info().task, "chat-completion"); +} + +TEST(ModelConstructionTest, LocalRegistrationHasMetadataImmediately) { + static fl::test::FakeServiceBindings svc; + ModelInfo info; + info.model_id = "local-model:1"; + info.name = "local-model"; + info.version = 1; + info.alias = "local-model"; + info.task = "chat-completion"; + + auto model = Model::FromLocalRegistration(std::move(info), "model-path", svc.download_manager, + svc.model_load_manager, "local/registration-id"); + + EXPECT_EQ(model.Info().model_id, "local-model:1"); + EXPECT_EQ(model.Info().task, "chat-completion"); + EXPECT_EQ(model.RuntimeId(), "local/registration-id"); +} + +TEST(ModelConstructionTest, ContainerHasSelectedMetadataImmediately) { + auto container = Model::MakeContainer(MakeLeaf()); + + ASSERT_TRUE(container.IsContainer()); + EXPECT_EQ(container.Info().model_id, "test-model"); + EXPECT_EQ(container.Info().task, "chat-completion"); +} + +TEST(ModelConstructionTest, MoveConstructionPreservesContainerMetadata) { + auto source = Model::MakeContainer(MakeLeaf()); + + Model moved(std::move(source)); + + ASSERT_TRUE(moved.IsContainer()); + EXPECT_EQ(moved.Info().model_id, "test-model"); + EXPECT_EQ(moved.Info().task, "chat-completion"); +} + +TEST(ModelConstructionTest, MoveAssignmentPreservesContainerMetadata) { + auto source = Model::MakeContainer(MakeLeaf()); + auto destination = MakeLeaf("destination-model"); + + destination = std::move(source); + + ASSERT_TRUE(destination.IsContainer()); + EXPECT_EQ(destination.Info().model_id, "test-model"); + EXPECT_EQ(destination.Info().task, "chat-completion"); +} + +TEST(ModelConstructionTest, MakeContainerRejectsContainerAsVariant) { + auto nested = Model::MakeContainer(MakeLeaf()); + + EXPECT_THROW(Model::MakeContainer(std::move(nested)), fl::Exception); +} + +TEST(ModelConstructionTest, AddVariantRejectsContainerAndPreservesExistingSelection) { + auto container = Model::MakeContainer(MakeLeaf()); + auto nested = Model::MakeContainer(MakeLeaf("nested-model")); + + EXPECT_THROW(container.AddVariant(std::move(nested)), fl::Exception); + + ASSERT_TRUE(container.IsContainer()); + EXPECT_EQ(container.Variants().size(), 1u); + EXPECT_EQ(container.Info().model_id, "test-model"); +} diff --git a/sdk_v2/cpp/test/internal_api/model_info_test.cc b/sdk_v2/cpp/test/internal_api/model_info_test.cc index b91ce886e..0082ca349 100644 --- a/sdk_v2/cpp/test/internal_api/model_info_test.cc +++ b/sdk_v2/cpp/test/internal_api/model_info_test.cc @@ -4,7 +4,6 @@ // Round-trip tests for ModelInfo JSON serialization/deserialization. // #include "model_info.h" - #include #include #include @@ -112,6 +111,9 @@ TEST(ModelInfoRoundTrip, AllMetadataFieldsSurviveRoundTrip) { original.string_properties[FOUNDRY_LOCAL_MODEL_PROP_MIN_FL_VERSION_STR] = "0.5.0"; original.string_properties[FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR] = "FoundryLocal"; original.string_properties[FOUNDRY_LOCAL_MODEL_PROP_MODEL_TYPE_STR] = "ONNX"; + original.string_properties[FOUNDRY_LOCAL_MODEL_PROP_INPUT_MODALITIES_STR] = "text,image"; + original.string_properties["custom_property"] = "custom value"; + original.string_properties["shared_property"] = "string value"; // Reasoning fields original.string_properties[FOUNDRY_LOCAL_MODEL_PROP_REASONING_START_STR] = ""; @@ -126,6 +128,9 @@ TEST(ModelInfoRoundTrip, AllMetadataFieldsSurviveRoundTrip) { // Int properties original.int_properties[FOUNDRY_LOCAL_MODEL_PROP_FILESIZE_MB_INT] = 4096; original.int_properties[FOUNDRY_LOCAL_MODEL_PROP_MAX_OUTPUT_TOKENS_INT] = 8192; + original.int_properties[FOUNDRY_LOCAL_MODEL_PROP_CONTEXT_LENGTH_INT] = 32768; + original.int_properties["custom_count"] = 42; + original.int_properties["shared_property"] = 7; // Prompt templates original.prompt_templates.Add("system", "<|system|>\n{Content}<|end|>"); @@ -137,6 +142,8 @@ TEST(ModelInfoRoundTrip, AllMetadataFieldsSurviveRoundTrip) { // Round-trip nlohmann::json j = ModelInfoToJson(original); + EXPECT_EQ(j["stringProperties"]["custom_property"], "custom value"); + EXPECT_EQ(j["intProperties"]["custom_count"], 42); ModelInfo restored = ModelInfoFromJson(j); // Core identity @@ -158,6 +165,9 @@ TEST(ModelInfoRoundTrip, AllMetadataFieldsSurviveRoundTrip) { EXPECT_EQ(restored.string_properties.at(FOUNDRY_LOCAL_MODEL_PROP_MIN_FL_VERSION_STR), "0.5.0"); EXPECT_EQ(restored.string_properties.at(FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR), "FoundryLocal"); EXPECT_EQ(restored.string_properties.at(FOUNDRY_LOCAL_MODEL_PROP_MODEL_TYPE_STR), "ONNX"); + EXPECT_EQ(restored.string_properties.at(FOUNDRY_LOCAL_MODEL_PROP_INPUT_MODALITIES_STR), "text,image"); + EXPECT_EQ(restored.string_properties.at("custom_property"), "custom value"); + EXPECT_EQ(restored.string_properties.at("shared_property"), "string value"); // Reasoning fields EXPECT_EQ(restored.int_properties.at(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_REASONING_INT), 1); @@ -172,6 +182,9 @@ TEST(ModelInfoRoundTrip, AllMetadataFieldsSurviveRoundTrip) { // Int properties EXPECT_EQ(restored.int_properties.at(FOUNDRY_LOCAL_MODEL_PROP_FILESIZE_MB_INT), 4096); EXPECT_EQ(restored.int_properties.at(FOUNDRY_LOCAL_MODEL_PROP_MAX_OUTPUT_TOKENS_INT), 8192); + EXPECT_EQ(restored.int_properties.at(FOUNDRY_LOCAL_MODEL_PROP_CONTEXT_LENGTH_INT), 32768); + EXPECT_EQ(restored.int_properties.at("custom_count"), 42); + EXPECT_EQ(restored.int_properties.at("shared_property"), 7); // Prompt templates ASSERT_FALSE(restored.prompt_templates.empty()); diff --git a/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc b/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc index c17817b9c..9a525b863 100644 --- a/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc +++ b/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc @@ -41,19 +41,31 @@ class CpuOnlyDetector : public fl::IEpDetector { std::map> GetAvailableDevicesToEPs() const override { return {{"CPU", {"CPUExecutionProvider"}}}; } + + bool PrepareForModelLoad(std::string_view ep_name) override { + prepared_ep = ep_name; + return true; + } + + std::string prepared_ep; }; /// Creates a minimal model directory with a dummy genai_config.json. /// Cleans up on destruction. class TempModelDir { public: - TempModelDir(const std::string& model_name) { + explicit TempModelDir(const std::string& model_name, const std::string& provider = "") { path_ = (std::filesystem::temp_directory_path() / ("fl_test_" + model_name)).string(); std::filesystem::create_directories(path_); // Write a minimal genai_config.json std::ofstream config(std::filesystem::path(path_) / "genai_config.json"); - config << R"({"model": {"type": "phi3"}})"; + if (provider.empty()) { + config << R"({"model": {"type": "phi3"}})"; + } else { + config << R"({"model":{"type":"phi3","decoder":{"session_options":{"provider_options":[{")" + << provider << R"(":{}}]}}}})"; + } } ~TempModelDir() { @@ -132,6 +144,117 @@ TEST(ModelLoadManagerTest, LoadCudaGpuModel_CudaNotAvailable_ErrorMessage) { } } +TEST(ModelLoadManagerTest, LoadRuntimeIdWithCudaConfig_CudaNotAvailable_Throws) { + CpuOnlyDetector ep; + fl::StderrLogger logger; + fl::ModelLoadManager mgr(ep, logger); + + TempModelDir dir("alias-cuda-config", "cuda"); + + try { + mgr.LoadModel(dir.path(), "local/test-registration"); + FAIL() << "Expected exception"; + } catch (const fl::Exception& e) { + EXPECT_EQ(e.code(), FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + EXPECT_NE(std::string(e.what()).find("CUDAExecutionProvider"), std::string::npos); + } +} + +TEST(ModelLoadManagerTest, LoadRuntimeIdWithWebGpuConfig_WebGpuNotAvailable_Throws) { + CpuOnlyDetector ep; + fl::StderrLogger logger; + fl::ModelLoadManager mgr(ep, logger); + + TempModelDir dir("alias-webgpu-config", "WebGPU"); + + try { + mgr.LoadModel(dir.path(), "local/test-registration"); + FAIL() << "Expected exception"; + } catch (const fl::Exception& e) { + EXPECT_EQ(e.code(), FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + EXPECT_NE(std::string(e.what()).find("WebGpuExecutionProvider"), std::string::npos); + } +} + +TEST(ModelLoadManagerTest, LoadRuntimeIdWithCudaConfig_CudaAvailable_PreparesCuda) { + GpuEpDetector ep; + fl::StderrLogger logger; + fl::ModelLoadManager mgr(ep, logger); + + TempModelDir dir("alias-cuda-available", "cuda"); + + try { + mgr.LoadModel(dir.path(), "local/test-registration"); + } catch (const fl::Exception& e) { + EXPECT_NE(e.code(), FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + } + + EXPECT_EQ(ep.prepared_ep, "CUDAExecutionProvider"); +} + +TEST(ModelLoadManagerTest, LoadRuntimeIdWithCanonicalWinMlProvider_NotAvailable_Throws) { + CpuOnlyDetector ep; + fl::StderrLogger logger; + fl::ModelLoadManager mgr(ep, logger); + + TempModelDir dir("alias-migraphx-config", "MIGraphXExecutionProvider"); + + try { + mgr.LoadModel(dir.path(), "local/test-registration"); + FAIL() << "Expected exception"; + } catch (const fl::Exception& e) { + EXPECT_EQ(e.code(), FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + EXPECT_NE(std::string(e.what()).find("MIGraphXExecutionProvider"), std::string::npos); + } +} + +TEST(ModelLoadManagerTest, LoadRuntimeIdWithFullDmlProviderName_DoesNotRequireDownloadableEp) { + CpuOnlyDetector ep; + fl::StderrLogger logger; + fl::ModelLoadManager mgr(ep, logger); + + TempModelDir dir("alias-dml-config", "DmlExecutionProvider"); + + try { + mgr.LoadModel(dir.path(), "local/test-registration"); + } catch (const fl::Exception& e) { + EXPECT_NE(e.code(), FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + } + + EXPECT_TRUE(ep.prepared_ep.empty()); +} + +TEST(ModelLoadManagerTest, LoadWithUnknownOverride_ThrowsInvalidArgument) { + CpuOnlyDetector ep; + fl::StderrLogger logger; + fl::ModelLoadManager mgr(ep, logger); + + TempModelDir dir("unknown-override"); + + try { + mgr.LoadModel(dir.path(), "local/test-registration", fl::ExecutionProvider::kUnknown); + FAIL() << "Expected exception"; + } catch (const fl::Exception& e) { + EXPECT_EQ(e.code(), FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT); + } +} + +TEST(ModelLoadManagerTest, LoadWithCpuOverride_IgnoresCudaConfigAndModelId) { + CpuOnlyDetector ep; + fl::StderrLogger logger; + fl::ModelLoadManager mgr(ep, logger); + + TempModelDir dir("explicit-cpu", "cuda"); + + try { + mgr.LoadModel(dir.path(), "local/alias-cuda-gpu:0", fl::ExecutionProvider::kCPU); + } catch (const fl::Exception& e) { + EXPECT_NE(e.code(), FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + } + + EXPECT_TRUE(ep.prepared_ep.empty()); +} + TEST(ModelLoadManagerTest, LoadOpenVinoNpuModel_NotAvailable_Throws) { CpuOnlyDetector ep; fl::StderrLogger logger; diff --git a/sdk_v2/cpp/test/internal_api/time_utils_test.cc b/sdk_v2/cpp/test/internal_api/time_utils_test.cc new file mode 100644 index 000000000..c80b72356 --- /dev/null +++ b/sdk_v2/cpp/test/internal_api/time_utils_test.cc @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "util/time_utils.h" + +#include + +#include + +namespace fl { +namespace { + +TEST(TimeUtilsTest, FormatsUnixEpochInUtc) { + EXPECT_EQ(FormatUtcTimestamp(std::time_t{0}), "1970-01-01T00:00:00Z"); +} + +TEST(TimeUtilsTest, FormatsNonzeroTimestampInUtc) { + EXPECT_EQ(FormatUtcTimestamp(static_cast(946684800)), "2000-01-01T00:00:00Z"); +} + +} // namespace +} // namespace fl \ No newline at end of file diff --git a/sdk_v2/cpp/test/sdk_api/bring_your_own_model_e2e_test.cc b/sdk_v2/cpp/test/sdk_api/bring_your_own_model_e2e_test.cc new file mode 100644 index 000000000..94d2e0911 --- /dev/null +++ b/sdk_v2/cpp/test/sdk_api/bring_your_own_model_e2e_test.cc @@ -0,0 +1,178 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// End-to-end coverage for registering existing local model assets and then running inference. + +#include "model_fixture.h" + +#include "internal_api/test_model_cache.h" +#include "utils/temp_path.h" + +#include +#include +#include +#include +#include + +namespace { + +constexpr const char* kByomSourceModelEnvironmentVariable = "FOUNDRY_LOCAL_BYOM_TEST_MODEL_PATH"; +constexpr const char* kByomChatModelAlias = "qwen2.5-coder-0.5b-instruct-generic-cpu-3"; + +std::optional FindGenAiModelDirectory(const fs::path& model_path) { + if (fs::exists(model_path / "genai_config.json")) { + return fs::canonical(model_path); + } + + if (!fs::is_directory(model_path)) { + return std::nullopt; + } + + for (const auto& entry : fs::directory_iterator(model_path)) { + if (entry.is_directory() && fs::exists(entry.path() / "genai_config.json")) { + return fs::canonical(entry.path()); + } + } + + // Shared test data uses /Microsoft//. Also accept / as the override so the + // documented path remains useful even when that directory is an empty catalog placeholder. + const auto publisher_model_path = model_path.parent_path() / "Microsoft" / model_path.filename(); + if (publisher_model_path != model_path && fs::is_directory(publisher_model_path)) { + return FindGenAiModelDirectory(publisher_model_path); + } + + return std::nullopt; +} + +std::optional GetByomSourceModelPath() { + const auto override_path = fl::test::SafeGetEnv(kByomSourceModelEnvironmentVariable); + if (!override_path.empty()) { + return FindGenAiModelDirectory(override_path); + } + + try { + return fl::test::GetTestModelPath(kByomChatModelAlias); + } catch (const std::exception&) { + return std::nullopt; + } +} + +void StageModelAssets(const fs::path& source, const fs::path& destination) { + for (const auto& entry : fs::recursive_directory_iterator(source)) { + const auto relative_path = fs::relative(entry.path(), source); + if (relative_path.filename() == "inference_model.json") { + continue; + } + + const auto destination_path = destination / relative_path; + if (entry.is_directory()) { + fs::create_directories(destination_path); + continue; + } + if (!entry.is_regular_file()) { + continue; + } + + fs::create_directories(destination_path.parent_path()); + std::error_code hard_link_error; + fs::create_hard_link(entry.path(), destination_path, hard_link_error); + if (hard_link_error) { + fs::copy_file(entry.path(), destination_path, fs::copy_options::overwrite_existing); + } + } +} + +class LocalRegistrationGuard { + public: + LocalRegistrationGuard(foundry_local::ICatalog& catalog, + std::unique_ptr model, + std::string model_id) + : catalog_(catalog), model_(std::move(model)), model_id_(std::move(model_id)) {} + + ~LocalRegistrationGuard() { + try { + if (model_ && model_->IsLoaded()) { + model_->Unload(); + } + } catch (...) { + } + + model_.reset(); + try { + catalog_.UnregisterModel(model_id_); + } catch (...) { + } + } + + foundry_local::IModel& model() { return *model_; } + + private: + foundry_local::ICatalog& catalog_; + std::unique_ptr model_; + std::string model_id_; +}; + +} // namespace + +TEST(ByomE2eTest, RegisterModelPreservesAssetsAndRunsChatInference) { + using namespace foundry_local; + + const auto source_model_path = GetByomSourceModelPath(); + if (!source_model_path) { + GTEST_SKIP() << "BYOM source model not found. Set " << kByomSourceModelEnvironmentVariable + << " or stage " << kByomChatModelAlias << " under FOUNDRY_TEST_DATA_DIR/Microsoft."; + } + + auto temp_root = fl::test::TempPath::CreateTempDir("fl_byom_e2e_"); + const auto staged_model_path = temp_root.path() / "model"; + fs::create_directories(staged_model_path); + StageModelAssets(*source_model_path, staged_model_path); + + ASSERT_TRUE(fs::exists(staged_model_path / "genai_config.json")); + ASSERT_FALSE(fs::exists(staged_model_path / "inference_model.json")); + + auto& local_catalog = SharedTestEnv::Get().manager()->GetCatalog(CatalogType::Local); + const auto registration_alias = temp_root.path().filename().string(); + const auto registration_id = registration_alias + ":1"; + ModelInfo metadata; + metadata.SetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_TASK_STR, "chat-completion"); + + LocalRegistrationGuard registered( + local_catalog, local_catalog.RegisterModel(staged_model_path.string(), registration_id, metadata), + registration_id); + const auto sibling_id = registration_alias + ":2"; + ModelInfo sibling_metadata; + sibling_metadata.SetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_TASK_STR, "chat-completion"); + LocalRegistrationGuard registered_sibling( + local_catalog, local_catalog.RegisterModel(staged_model_path.string(), sibling_id, sibling_metadata), sibling_id); + auto& model = registered.model(); + + EXPECT_EQ(model.GetInfo().Id(), registration_id); + EXPECT_EQ(model.GetInfo().Alias(), registration_alias); + EXPECT_EQ(model.GetInfo().Version(), 1); + EXPECT_EQ(model.GetInfo().Task(), "chat-completion"); + EXPECT_TRUE(model.IsCached()); + EXPECT_FALSE(model.IsLoaded()); + EXPECT_TRUE(fs::is_regular_file(staged_model_path / "genai_config.json")); + EXPECT_FALSE(fs::exists(staged_model_path / "inference_model.json")); + + model.Load(); + ASSERT_TRUE(model.IsLoaded()); + EXPECT_THROW(local_catalog.UnregisterModel(registration_alias), Error); + + ChatSession session(model); + Request request{ + SystemMessage("You are a concise math assistant."), + UserMessage("What is 2+2? Answer with only the number."), + }; + RequestOptions options; + options.search.temperature = 0.0f; + options.search.max_output_tokens = 16; + request.SetOptions(options); + + Response response = session.ProcessRequest(request); + + EXPECT_EQ(response.GetFinishReason(), FOUNDRY_LOCAL_FINISH_STOP); + const auto response_text = CollectResponseText(response); + EXPECT_FALSE(response_text.empty()); + EXPECT_NE(response_text.find('4'), std::string::npos) << "Unexpected response: " << response_text; +} \ No newline at end of file diff --git a/sdk_v2/cpp/test/sdk_api/cache_only_test.cc b/sdk_v2/cpp/test/sdk_api/cache_only_test.cc index 364b6a4ba..a175b124e 100644 --- a/sdk_v2/cpp/test/sdk_api/cache_only_test.cc +++ b/sdk_v2/cpp/test/sdk_api/cache_only_test.cc @@ -257,11 +257,11 @@ TEST_F(CacheOnlyTest, MissingCacheFileReturnsEmptyModelList) { } } -TEST_F(CacheOnlyTest, SnapshotAndByomModelsUseScannedLocalPaths) { +TEST_F(CacheOnlyTest, SnapshotModelsUseScannedPathsAndUnknownDirectoriesAreIgnored) { WriteTwoModelCacheFile(); const auto snapshot_model_path = CreateLocalModel("phi-4-mini-instruct-generic-cpu:2", "phi-snapshot-model"); - const auto byom_model_path = CreateLocalModel("contoso-custom-model:7", "contoso-byom-model"); + CreateLocalModel("contoso-custom-model:7", "contoso-byom-model"); { foundry_local::Manager manager(MakeCacheOnlyConfig()); @@ -269,8 +269,8 @@ TEST_F(CacheOnlyTest, SnapshotAndByomModelsUseScannedLocalPaths) { auto models = catalog.GetModels(); auto cached_models = catalog.GetCachedModels(); - ASSERT_EQ(models.size(), 3u); - ASSERT_EQ(cached_models.size(), 2u); + ASSERT_EQ(models.size(), 2u); + ASSERT_EQ(cached_models.size(), 1u); auto snapshot_model = catalog.GetModelVariant("phi-4-mini-instruct-generic-cpu:2"); ASSERT_NE(snapshot_model, nullptr); @@ -278,15 +278,6 @@ TEST_F(CacheOnlyTest, SnapshotAndByomModelsUseScannedLocalPaths) { EXPECT_EQ(std::string(snapshot_model->GetPath()), snapshot_model_path); EXPECT_EQ(snapshot_model->GetInfo().Publisher().value_or(""), "Microsoft"); - auto byom_model = catalog.GetModelVariant("contoso-custom-model:7"); - ASSERT_NE(byom_model, nullptr); - EXPECT_TRUE(byom_model->IsCached()); - EXPECT_EQ(std::string(byom_model->GetPath()), byom_model_path); - EXPECT_EQ(byom_model->GetInfo().Name(), "contoso-custom-model"); - EXPECT_EQ(byom_model->GetInfo().Alias(), "contoso-custom-model"); - EXPECT_EQ(byom_model->GetInfo().Version(), 7); - EXPECT_EQ(byom_model->GetInfo().Uri(), "local://contoso-custom-model"); - EXPECT_EQ(byom_model->GetInfo().ModelProvider().value_or(""), "Local"); - EXPECT_EQ(byom_model->GetInfo().ModelType().value_or(""), "ONNX"); + EXPECT_EQ(catalog.GetModelVariant("contoso-custom-model:7"), nullptr); } } diff --git a/sdk_v2/python/src/foundry_local_sdk/_native/build_cffi.py b/sdk_v2/python/src/foundry_local_sdk/_native/build_cffi.py index c48ec92ff..0dd85b837 100644 --- a/sdk_v2/python/src/foundry_local_sdk/_native/build_cffi.py +++ b/sdk_v2/python/src/foundry_local_sdk/_native/build_cffi.py @@ -79,6 +79,11 @@ FOUNDRY_LOCAL_DEVICE_NPU = 3, } flDeviceType; +typedef enum flCatalogType { + FOUNDRY_LOCAL_CATALOG_PUBLIC = 0, + FOUNDRY_LOCAL_CATALOG_LOCAL = 1, +} flCatalogType; + typedef enum flTensorDataType { FOUNDRY_LOCAL_TENSOR_UNDEFINED = 0, FOUNDRY_LOCAL_TENSOR_FLOAT = 1, @@ -389,6 +394,7 @@ _Bool (*Manager_IsEpDownloadInProgress)(const flManager* manager); flStatusPtr (*Manager_Shutdown)(flManager* manager); _Bool (*Manager_IsShutdownRequested)(const flManager* manager); + flStatusPtr (*Manager_GetCatalogByType)(const flManager* manager, flCatalogType catalog_type, flCatalog** out_catalog); } flApi; /* ----------------------------------------------------------------------- @@ -489,6 +495,8 @@ flStatusPtr (*GetCachedModels)(const flCatalog* catalog, flModelList** out_models); flStatusPtr (*GetLoadedModels)(const flCatalog* catalog, flModelList** out_models); flStatusPtr (*GetModelVersions)(const flCatalog* catalog, const char* model_alias, const char* model_name, int32_t max_versions, flModelList** out_models); + flStatusPtr (*RegisterModel)(flCatalog* catalog, const char* model_path, const char* model_id, const flModelInfo* metadata, flModel** out_model); + flStatusPtr (*UnregisterModel)(flCatalog* catalog, const char* alias_or_model_id); } flCatalogApi; /* ----------------------------------------------------------------------- @@ -519,6 +527,10 @@ const flKeyValuePairs* (*Info_GetModelSettings)(const flModelInfo* info); const char* (*Info_GetStringProperty)(const flModelInfo* info, const char* key); int64_t (*Info_GetIntProperty)(const flModelInfo* info, const char* key, int64_t default_value); + flStatusPtr (*CreateModelInfo)(flModelInfo** out_info); + void (*ReleaseModelInfo)(flModelInfo* info); + flStatusPtr (*Info_SetStringProperty)(flModelInfo* info, const char* key, const char* value); + flStatusPtr (*Info_SetIntProperty)(flModelInfo* info, const char* key, int64_t value); } flModelApi; /* -----------------------------------------------------------------------