HotFix bug schema version

cmake/Version.cmake

@@ -1,45 +1,45 @@
 
 
 set(DATAFED_RELEASE_YEAR 2026)
-set(DATAFED_RELEASE_MONTH 2)
-set(DATAFED_RELEASE_DAY 24)
+set(DATAFED_RELEASE_MONTH 3)
+set(DATAFED_RELEASE_DAY 31)
 set(DATAFED_RELEASE_HOUR 10)
 set(DATAFED_RELEASE_MINUTE 0)
 
 set(DATAFED_COMMON_LIB_MAJOR 3)
-set(DATAFED_COMMON_LIB_MINOR 0)
+set(DATAFED_COMMON_LIB_MINOR 1)
 set(DATAFED_COMMON_LIB_PATCH 0)
 
 set(DATAFED_COMMON_PROTOCOL_API_MAJOR 3)
-set(DATAFED_COMMON_PROTOCOL_API_MINOR 0)
+set(DATAFED_COMMON_PROTOCOL_API_MINOR 1)
 set(DATAFED_COMMON_PROTOCOL_API_PATCH 0)
 
 set(DATAFED_CORE_MAJOR 3)
-set(DATAFED_CORE_MINOR 0)
+set(DATAFED_CORE_MINOR 1)
 set(DATAFED_CORE_PATCH 0)
 
 set(DATAFED_FOXX_MAJOR 1)
-set(DATAFED_FOXX_MINOR 3)
+set(DATAFED_FOXX_MINOR 4)
 set(DATAFED_FOXX_PATCH 0)
 
 set(DATAFED_FOXX_API_MAJOR 1)
-set(DATAFED_FOXX_API_MINOR 3)
+set(DATAFED_FOXX_API_MINOR 4)
 set(DATAFED_FOXX_API_PATCH 0)
 
 set(DATAFED_WEB_MAJOR 3)
-set(DATAFED_WEB_MINOR 0)
+set(DATAFED_WEB_MINOR 1)
 set(DATAFED_WEB_PATCH 0)
 
 set(DATAFED_REPO_MAJOR 3)
-set(DATAFED_REPO_MINOR 0)
+set(DATAFED_REPO_MINOR 1)
 set(DATAFED_REPO_PATCH 0)
 
 set(DATAFED_AUTHZ_MAJOR 3)
 set(DATAFED_AUTHZ_MINOR 0)
 set(DATAFED_AUTHZ_PATCH 0)
 
 set(DATAFED_PYTHON_CLIENT_MAJOR 5)
-set(DATAFED_PYTHON_CLIENT_MINOR 0)
+set(DATAFED_PYTHON_CLIENT_MINOR 1)
 set(DATAFED_PYTHON_CLIENT_PATCH 0)
 set(DATAFED_PYTHON_CLIENT_RELEASE_TYPE "")
 set(DATAFED_PYTHON_CLIENT_PRE_RELEASE_IDENTIFER "")

core/server/ClientWorker.cpp

@@ -841,7 +841,7 @@ ClientWorker::procRecordCreateRequest(const std::string &a_uid,
 
   if (request->has_metadata() && request->has_sch_id()) {
     validator_err = m_schema_handler->validateMetadataContent(
-        request->sch_id(), request->metadata(), log_context);
+        a_uid, request->sch_id(), request->metadata(), log_context);
 
     if (!validator_err.empty()) {
       DL_ERROR(log_context, "Metadata validation error: " << validator_err);
@@ -942,7 +942,7 @@ ClientWorker::procRecordUpdateRequest(const std::string &a_uid,
 
       if (validator_err.empty()) {
         validator_err = m_schema_handler->validateMetadataContent(
-            sch_id, effective_metadata, log_context);
+            a_uid, sch_id, effective_metadata, log_context);
 
         if (!validator_err.empty()) {
           DL_WARNING(log_context,

core/server/Config.hpp

@@ -4,6 +4,7 @@
 
 // Core local private includes
 #include "AuthenticationManager.hpp"
+#include "SchemaAPIConfig.hpp"
 
 // DataFed Common public includes
 #include "common/DynaLog.hpp"
@@ -16,6 +17,7 @@
 #include <mutex>
 #include <stdint.h>
 #include <string>
+#include <unordered_map>
 
 namespace SDMS {
 namespace Core {
@@ -79,6 +81,8 @@ class Config {
   uint32_t metrics_purge_period;
   uint32_t metrics_purge_age;
 
+  std::unordered_map<std::string, SchemaAPIConfig> schemas;
+
   // MsgComm::SecurityContext            sec_ctx;
   std::unique_ptr<ICredentials> sec_ctx;
 

core/server/DatabaseAPI.cpp

@@ -2955,6 +2955,12 @@ void DatabaseAPI::schemaCreate(const SDMS::SchemaCreateRequest &a_request,
   payload["pub"] = a_request.pub();
   payload["sys"] = a_request.sys();
   payload["def"] = a_request.def();
+  if (!a_request.format().empty()) {
+    payload["format"] = a_request.format();
+  }
+  if (!a_request.type().empty()) {
+    payload["type"] = a_request.type();
+  }
   string body = payload.dump(-1, ' ', true);
 
   dbPost("schema/create", {}, &body, result, log_context);
@@ -3058,29 +3064,45 @@ void DatabaseAPI::setSchemaData(SchemaData *a_schema,
   a_schema->set_id(a_obj.getString("id"));
   a_schema->set_ver(a_obj.getNumber("ver"));
 
-  if (a_obj.has("cnt"))
+  if (a_obj.has("cnt")) {
     a_schema->set_cnt(a_obj.asNumber());
+  }
 
-  if (a_obj.has("own_id") && !a_obj.value().isNull())
+  if (a_obj.has("own_id") && !a_obj.value().isNull()) {
     a_schema->set_own_id(a_obj.asString());
+  }
 
-  if (a_obj.has("own_nm") && !a_obj.value().isNull())
+  if (a_obj.has("own_nm") && !a_obj.value().isNull()) {
     a_schema->set_own_nm(a_obj.asString());
+  }
 
-  if (a_obj.has("desc"))
+  if (a_obj.has("desc")) {
     a_schema->set_desc(a_obj.asString());
+  }
 
-  if (a_obj.has("pub"))
+  if (a_obj.has("pub")) {
     a_schema->set_pub(a_obj.asBool());
+  }
 
-  if (a_obj.has("depr"))
+  if (a_obj.has("depr")) {
     a_schema->set_depr(a_obj.asBool());
+  }
 
-  if (a_obj.has("ref"))
+  if (a_obj.has("ref")) {
     a_schema->set_ref(a_obj.asBool());
+  }
 
-  if (a_obj.has("def"))
+  if (a_obj.has("def")) {
     a_schema->set_def(a_obj.value().toString());
+  }
+
+  if (a_obj.has("type")) {
+    a_schema->set_type(a_obj.asString());
+  }
+
+  if (a_obj.has("format")) {
+    a_schema->set_format(a_obj.asString());
+  }
 
   Value::ArrayConstIter j;
   SchemaData *dep;

core/server/SchemaAPIClient.cpp

@@ -315,13 +315,34 @@ bool SchemaAPIClient::validateMetadata(const std::string &a_schema_id,
     a_warnings = result.value("warnings", "");
     return true;
   }
+
   if (code == 422) {
-    a_errors = result.value("message", "metadata validation failed");
+    if (result.contains("details") && result["details"].contains("errors")) {
+        std::string raw = result["details"]["errors"].get<std::string>();
+        if (a_engine == "linkml") {
+            // Normalize semicolon-separated errors to newline-separated
+            // for consistency with json-schema validator output
+            std::string normalized;
+            size_t pos = 0, found;
+            while ((found = raw.find("; ", pos)) != std::string::npos) {
+                normalized += raw.substr(pos, found - pos) + "\n";
+                pos = found + 2;
+            }
+            normalized += raw.substr(pos);
+            a_errors = normalized;
+        } else {
+            a_errors = raw;
+        }
+    } else {
+        a_errors = result.value("message", "metadata validation failed");
+    }
     return false;
   }
-  if (code == 404)
+
+  if (code == 404) {
     EXCEPT_PARAM(BAD_REQUEST,
                  "SchemaAPI: schema " << a_schema_id << " not found");
+  }
   EXCEPT_PARAM(SERVICE_ERROR,
                "SchemaAPI validateMetadata failed, HTTP " << code);
 }

core/server/client_handlers/SchemaHandler.cpp

@@ -3,7 +3,10 @@
 #include "LocalJsonErrorHandler.hpp"
 #include "common/TraceException.hpp"
 #include "schema_storage/ArangoSchemaStorage.hpp"
+#include "schema_storage/ExternalSchemaStorage.hpp"
 #include "schema_validators/JsonSchemaValidator.hpp"
+#include "schema_validators/ExternalSchemaValidator.hpp"
+#include "Config.hpp"
 
 // Standard includes
 #include <functional>
@@ -13,8 +16,19 @@ using namespace std;
 namespace SDMS {
 namespace Core {
 
-SchemaHandler::SchemaHandler(DatabaseAPI &a_db_client)
+  SchemaHandler::SchemaHandler(DatabaseAPI &a_db_client)
     : m_db_client(a_db_client) {
+
+    if ( Config::getInstance().schemas.count("linkml") ){
+      SchemaAPIConfig linkml_config = Config::getInstance().schemas.at("linkml");
+      auto schema_api_client_storage = make_unique<SchemaAPIClient>(linkml_config);
+      auto linkml_storage = std::make_shared<ExternalSchemaStorage>(std::move(schema_api_client_storage));
+      m_schema_factory.registerStorage("linkml", std::move(linkml_storage));
+      auto schema_api_client_validator = make_unique<SchemaAPIClient>(linkml_config);
+      auto linkml_schema_validator = std::make_shared<ExternalSchemaValidator>(std::move(schema_api_client_validator), "linkml");
+      m_schema_factory.registerValidator("linkml",
+          std::move(linkml_schema_validator));
+    }
     // Assumes that we have already placed the schema in the database, arango
     // storage is a shell to be consistent with the interface.
     auto arango_storage = std::make_shared<ArangoSchemaStorage>();
@@ -32,7 +46,7 @@ SchemaHandler::SchemaHandler(DatabaseAPI &a_db_client)
     m_schema_factory.registerValidator("json-schema",
                                        std::move(json_schema_validator));
     m_schema_factory.setDefaultSchemaType("json-schema");
-}
+  }
 
 // ── Schema Definition Handlers ──────────────────────────────────────────────
 
@@ -62,7 +76,13 @@ void SchemaHandler::handleCreate(const std::string &a_uid,
   }
 
   // Persist to Arango — exception propagates naturally on failure
-  m_db_client.schemaCreate(a_request, a_reply, log_context);
+  // For external schema types, store a stub in Arango —
+  // the real content lives in external storage.
+  SchemaCreateRequest arango_req(a_request);
+  if (!a_request.type().empty() && a_request.type() != "json-schema") {
+    arango_req.set_def("{}");
+  }
+  m_db_client.schemaCreate(arango_req, a_reply, log_context);
 
   // Store content through factory (no-op for Arango, meaningful for external)
   try {
@@ -140,7 +160,13 @@ void SchemaHandler::handleRevise(const std::string &a_uid,
   }
 
   // Create new revision in Arango
-  m_db_client.schemaRevise(a_request, a_reply, log_context);
+  // For external schema types, store a stub in Arango —
+  // the real content lives in external storage.
+  SchemaReviseRequest arango_req(a_request);
+  if (!schema_type.empty() && schema_type != "json-schema") {
+    arango_req.set_def("{}");
+  }
+  m_db_client.schemaRevise(arango_req, a_reply, log_context);
 
   // Store content for new revision only when def was provided
   if (a_request.has_def()) {
@@ -220,7 +246,13 @@ void SchemaHandler::handleUpdate(const std::string &a_uid,
   }
 
   // Update in-place in Arango
-  m_db_client.schemaUpdate(a_request, log_context);
+  // For external schema types, store a stub in Arango —
+  // the real content lives in external storage.
+  SchemaUpdateRequest arango_req(a_request);
+  if (!schema_type.empty() && schema_type != "json-schema") {
+    arango_req.set_def("{}");
+  }
+  m_db_client.schemaUpdate(arango_req, log_context);
 
   // Update content in external storage only when def was provided.
   // Uses updateContent (in-place) rather than storeContent (new entry).
@@ -250,7 +282,9 @@ void SchemaHandler::handleUpdate(const std::string &a_uid,
       rollback_request.set_id(sch_doc.getString("id"));
       rollback_request.set_desc(sch_doc.getString("desc"));
       rollback_request.set_pub(sch_doc.getBool("pub"));
-      rollback_request.set_sys(sch_doc.getBool("sys"));
+      if (sch_doc.has("sys")) {
+        rollback_request.set_sys(sch_doc.getBool("sys"));
+      }
 
       DL_ERROR(log_context,
                "Schema storage update failed attempting rollback: " << e.what());
@@ -269,14 +303,17 @@ void SchemaHandler::handleUpdate(const std::string &a_uid,
 // ── Metadata Validation ─────────────────────────────────────────────────────
 
 std::string SchemaHandler::validateMetadataContent(
+    const std::string &a_uid,
     const std::string &a_sch_id,
     const std::string &a_metadata,
     LogContext log_context) {
 
+  m_db_client.setClient(a_uid);
   DL_DEBUG(log_context, "validateMetadataContent schema=" << a_sch_id);
 
   std::string schema_type = "json-schema";
   std::string schema_format = "json";
+  std::string metadata_format = "json";
   std::string schema_def;
 
   // Load schema record from DB
@@ -323,7 +360,7 @@ std::string SchemaHandler::validateMetadataContent(
     }
 
     auto result = validator.validateMetadata(
-        a_sch_id, schema_format, a_metadata, log_context);
+        a_sch_id, metadata_format, a_metadata, log_context);
 
     if (!result.valid) {
       return result.errors;
@@ -342,10 +379,9 @@ void SchemaHandler::handleMetadataValidate(
     LogContext log_context) {
 
   DL_DEBUG(log_context, "Metadata validate");
-  m_db_client.setClient(a_uid);
 
   std::string errors = validateMetadataContent(
-      a_request.sch_id(), a_request.metadata(), log_context);
+      a_uid, a_request.sch_id(), a_request.metadata(), log_context);
 
   if (!errors.empty()) {
     a_reply.set_errors(errors);

core/server/client_handlers/SchemaHandler.hpp

@@ -118,9 +118,10 @@ class SchemaHandler {
                     AckReply &a_reply,
                     LogContext log_context);
 
-  std::string validateMetadataContent(const std::string &a_sch_id,
-                                    const std::string &a_metadata,
-                                    LogContext log_context);
+  std::string validateMetadataContent(const std::string &a_uid,
+                                      const std::string &a_sch_id,
+                                      const std::string &a_metadata,
+                                      LogContext log_context);
 
 private:
   /**

core/server/main.cpp

@@ -14,6 +14,7 @@
 
 // Standard includes
 #include "Config.hpp"
+#include "SchemaAPIConfig.hpp"
 #include <climits>
 #include <fstream>
 #include <iostream>
@@ -34,6 +35,7 @@ int main(int a_argc, char **a_argv) {
   LogContext log_context;
   log_context.thread_name = "core_server";
   log_context.thread_id = 0;
+  Core::SchemaAPIConfig linkml_schema_config;
 
   try {
 
@@ -64,6 +66,12 @@ int main(int a_argc, char **a_argv) {
         "Globus authorization API base URL")(
         "glob-xfr-url", po::value<string>(&config.glob_xfr_url),
         "Globus transfer API base URL")(
+        "linkml-schema-api-url",
+         po::value<string>(&linkml_schema_config.base_url),
+         "External LinkML schema API base URL (enables LinkML support)")(
+        "linkml-schema-api-token",
+         po::value<string>(&linkml_schema_config.bearer_token),
+         "Bearer token for LinkML schema API authentication")(
         "client-id", po::value<string>(&config.client_id), "Client ID")(
         "client-secret", po::value<string>(&config.client_secret),
         "Client secret")("task-purge-age",
@@ -152,6 +160,16 @@ int main(int a_argc, char **a_argv) {
 
         return 0;
       }
+
+      if (linkml_schema_config.isConfigured()) {
+        config.schemas["linkml"] = linkml_schema_config;
+        DL_INFO(log_context, "LinkML schema engine enabled, API: "
+                                 << linkml_schema_config.base_url);
+      } else {
+        DL_INFO(log_context,
+                "LinkML schema engine disabled (no linkml-schema-api-url configured)");
+      }
+
       if (cfg_log_level != UINT_MAX) {
         if (cfg_log_level >=
             static_cast<unsigned int>(LogLevel::LAST_SENTINEL)) {