Merge branch 'master' into wip/wal_sync_client

Author: umegane

CMakeLists.txt

@@ -73,6 +73,7 @@ else()
     message(FATAL_ERROR "unsupported RECOVERY_SORTER_KVSLIB value: ${RECOVERY_SORTER_KVSLIB_UPPERCASE}")
 endif()
 find_package(nlohmann_json 3.7.0 REQUIRED)
+find_package(OpenSSL REQUIRED)
 if (ENABLE_ALTIMETER)
     find_package(altimeter REQUIRED)
     find_package(fmt REQUIRED)

include/limestone/api/blob_pool.h

@@ -23,6 +23,9 @@ namespace limestone::api {
 /// @brief BLOB reference type.
 using blob_id_type = std::uint64_t;
 
+/// @brief BLOB reference tag type.
+using blob_reference_tag_type = std::uint64_t;
+
 /**
  * @brief represents a pool for provisional registration of BLOB data.
  */
@@ -87,6 +90,22 @@ class blob_pool {
      * @throws limestone_blob_exception if an I/O error occurs during the operation
      */
     [[nodiscard]] virtual blob_id_type duplicate_data(blob_id_type reference) = 0;
+
+    /**
+     * @brief generates a BLOB reference tag for access control.
+     * @param blob_id the BLOB reference
+     * @param transaction_id the transaction ID
+     * @return the generated BLOB reference tag
+     * @throws limestone_blob_exception if an internal error occurs during tag generation.
+     *         Possible reasons include:
+     *         - Environment issues (e.g., cryptographic library not initialized or misconfigured)
+     *         - Resource exhaustion (e.g., out of memory)
+     *         - Other unexpected internal errors
+     * @note No validation is performed for blob_id or transaction_id values; any value is accepted.
+     */
+    [[nodiscard]] virtual blob_reference_tag_type generate_reference_tag(
+            blob_id_type blob_id,
+            std::uint64_t transaction_id) = 0;
 };
 
 } // namespace limestone::api

src/CMakeLists.txt

@@ -82,6 +82,8 @@ target_link_libraries(${package_name}
         PRIVATE grpc++
         PRIVATE grpc
         PRIVATE protobuf
+        PRIVATE OpenSSL::SSL
+        PRIVATE OpenSSL::Crypto
 )
 
 if (ENABLE_ALTIMETER)
@@ -115,6 +117,8 @@ target_link_libraries(limestone-impl
         INTERFACE protobuf
         INTERFACE gpr
         INTERFACE absl_synchronization
+        INTERFACE OpenSSL::SSL
+        INTERFACE OpenSSL::Crypto
 )
 
 # utils

src/limestone/blob_pool_impl.cpp

@@ -17,17 +17,46 @@
 #include "blob_pool_impl.h"
 #include "limestone_exception_helper.h"
 #include "limestone/api/datastore.h"
+#include "datastore_impl.h"
 #include <filesystem>
 #include <boost/filesystem.hpp>
 #include <iostream>
 #include <fstream>
 #include <cstdio>
 #include <memory>
+#include <cstring>
+
+#include <openssl/hmac.h>
+#include <openssl/evp.h>
+#include <openssl/err.h>
 
 
 
 namespace limestone::internal {
 
+
+void blob_pool_impl::handle_hmac_result(unsigned char const* result) {
+    if (result == nullptr) {
+        // Retrieve all OpenSSL error codes and error strings
+        std::string msg = "Failed to calculate reference tag: ";
+        // NOLINTNEXTLINE(google-runtime-int) : OpenSSL API requires unsigned long
+        unsigned long openssl_err = 0;
+        bool has_error = false;
+        while ((openssl_err = ERR_get_error()) != 0) {
+            has_error = true;
+            std::array<char, 256> err_msg_buf{};
+            ERR_error_string_n(openssl_err,
+                              err_msg_buf.data(),
+                              err_msg_buf.size());
+            msg += "[" + std::to_string(openssl_err) + ": " + err_msg_buf.data() + "] ";
+        }
+        if (! has_error) {
+            msg += "No OpenSSL error code available.";
+        }
+        LOG_AND_THROW_BLOB_EXCEPTION_NO_ERRNO(msg);
+    }
+}
+
 blob_pool_impl::blob_pool_impl(std::function<blob_id_type()> id_generator,
                                limestone::internal::blob_file_resolver& resolver,
                                limestone::api::datastore& datastore)
@@ -312,4 +341,40 @@ blob_id_type blob_pool_impl::register_data(std::string_view data) {
     return id;
 }
 
+blob_reference_tag_type blob_pool_impl::generate_reference_tag(
+    blob_id_type blob_id,
+    std::uint64_t transaction_id) {
+    
+    // Prepare input data: concatenate blob_id and transaction_id using portable approach
+    std::array<unsigned char, sizeof(blob_id_type) + sizeof(std::uint64_t)> input_bytes{};
+    std::memcpy(input_bytes.data(), &blob_id, sizeof(blob_id_type));
+    std::memcpy(input_bytes.data() + sizeof(blob_id_type), &transaction_id, sizeof(std::uint64_t));
+
+    // Get the secret key from datastore_impl
+    const auto& secret_key = datastore_.get_impl()->get_hmac_secret_key();
+
+    // Clear OpenSSL error queue to avoid noise from previous API calls
+    ERR_clear_error();
+
+    // Calculate HMAC-SHA256
+    std::array<unsigned char, EVP_MAX_MD_SIZE> md{};
+    unsigned int md_len = 0;
+
+    unsigned char* result = HMAC(EVP_sha256(), 
+                                secret_key.data(), 
+                                static_cast<int>(secret_key.size()),
+                                input_bytes.data(),
+                                input_bytes.size(),
+                                md.data(), 
+                                &md_len);
+
+    this->handle_hmac_result(result);
+
+    // Use the first 8 bytes of the HMAC result as the tag
+    blob_reference_tag_type tag = 0;
+    std::memcpy(&tag, md.data(), sizeof(blob_reference_tag_type));
+    
+    return tag;
+}
+
 } // namespace limestone::internal

src/limestone/blob_pool_impl.h

@@ -33,6 +33,10 @@ class blob_pool_impl : public blob_pool {
 
     [[nodiscard]] blob_id_type duplicate_data(blob_id_type reference) override;
 
+    [[nodiscard]] blob_reference_tag_type generate_reference_tag(
+        blob_id_type blob_id,
+        std::uint64_t transaction_id) override;
+
 protected:
     // These protected fields and methods include:
     // - Test-specific methods
@@ -94,6 +98,13 @@ class blob_pool_impl : public blob_pool {
         std::lock_guard<std::mutex> lock(mutex_);
         return blob_ids_;
     }
+
+    /**
+     * @brief Checks HMAC result and throws exception if failed.
+     * @param result HMAC result pointer (nullptr if failed)
+     */
+    void handle_hmac_result(unsigned char const* result);
+
 private:
     /**
      * @brief Generates a unique ID for a BLOB.

src/limestone/datastore_impl.cpp

@@ -17,6 +17,11 @@
 
 #include <cstdlib>
 #include <iostream>
+#include <cstring>
+#include <stdexcept>
+#include <openssl/hmac.h>
+#include <openssl/rand.h>
+#include <openssl/evp.h>
 
 #include "blob_file_resolver.h"
 #include "blob_file_scanner.h"
@@ -60,6 +65,9 @@ datastore_impl::datastore_impl(datastore& ds)
     replica_exists_.store(has_replica, std::memory_order_release);
     LOG_LP(INFO) << "Replica " << (has_replica ? "enabled" : "disabled")
                     << "; endpoint valid: " << replication_endpoint_.is_valid();
+    
+    // Generate HMAC secret key for BLOB reference tag generation
+    generate_hmac_secret_key();
 }
 
 // Default destructor.
@@ -351,4 +359,17 @@ std::set<boost::filesystem::path> datastore_impl::get_files() const {
     return datastore_.get_files();
 }
 
+void datastore_impl::generate_hmac_secret_key() {
+    // Generate 16 random bytes using OpenSSL RAND_bytes()
+    // TODO: Future improvement - throw exception instead of abort when public API allows it
+    if (RAND_bytes(hmac_secret_key_.data(), static_cast<int>(hmac_secret_key_.size())) != 1) {
+        LOG_LP(ERROR) << "Failed to generate random bytes for BLOB access control secret key";
+        std::abort(); // Current: abort due to noexcept constraint in public API
+    }
+}
+
+const std::array<std::uint8_t, 16>& datastore_impl::get_hmac_secret_key() const noexcept {
+    return hmac_secret_key_;
+}
+
 }  // namespace limestone::api

src/limestone/datastore_impl.h

@@ -16,10 +16,12 @@
 #pragma once
 
 #include <limestone/api/datastore.h>
+#include <limestone/api/blob_pool.h>
 
 #include <atomic>
 #include <memory>
 #include <optional>
+#include <array>
 
 #include "manifest.h"
 #include "replication/replica_connector.h"
@@ -131,6 +133,11 @@ class datastore_impl {
      * @return A set of file paths.
      */
     [[nodiscard]] std::set<boost::filesystem::path> get_files() const;
+    /**
+     * @brief gets the HMAC secret key for BLOB reference tag generation.
+     * @return reference to the HMAC secret key
+     */
+    [[nodiscard]] const std::array<std::uint8_t, 16>& get_hmac_secret_key() const noexcept;
 
 private:
     datastore& datastore_;
@@ -158,6 +165,14 @@ class datastore_impl {
 
     // Durable epoch ID at boot time
     std::atomic<epoch_id_type> boot_durable_epoch_id_{0};
+
+    // HMAC secret key for BLOB reference tag generation (16 bytes)
+    std::array<std::uint8_t, 16> hmac_secret_key_{};
+
+    /**
+     * @brief generates HMAC secret key for BLOB reference tag generation.
+     */
+    void generate_hmac_secret_key();
 };
 
 }  // namespace limestone::api

src/limestone/datastore_restore.cpp

@@ -139,18 +139,24 @@ status copy_backup_files(const boost::filesystem::path& from_dir, const std::vec
         }
         try {
             if (!resolver.is_blob_file(src)) {
-                // overwrite if destination exists
-                boost::filesystem::copy_file(src, location / dst, boost::filesystem::copy_options::overwrite_existing);
+                std::filesystem::copy_file(
+                    std::filesystem::path{src.string()},
+                    std::filesystem::path{(location / dst).string()},
+                    std::filesystem::copy_options::overwrite_existing
+                );
             } else {
                 boost::filesystem::path full_dst = resolver.resolve_path(resolver.extract_blob_id(src));
                 boost::filesystem::path dst_dir = full_dst.parent_path();
                 if (!boost::filesystem::exists(dst_dir)) {
                     boost::filesystem::create_directories(dst_dir);
                 }
-                // overwrite blob file if destination exists
-                boost::filesystem::copy_file(src, full_dst, boost::filesystem::copy_options::overwrite_existing);
+                std::filesystem::copy_file(
+                    std::filesystem::path{src.string()},
+                    std::filesystem::path{full_dst.string()},
+                    std::filesystem::copy_options::overwrite_existing
+                );
             }
-        } catch (boost::filesystem::filesystem_error& ex) {
+        } catch (std::filesystem::filesystem_error& ex) {
             LOG_LP(ERROR) << ex.what() << " file = " << src.string();
             return status::err_permission_error;
         }
@@ -209,18 +215,24 @@ status datastore::restore(std::string_view from, bool keep_backup, bool purge_de
                 }
 
                 if (!resolver.is_blob_file(p)) {
-                    // overwrite destination file if exists
-                    boost::filesystem::copy_file(p, location_ / p.filename(), boost::filesystem::copy_options::overwrite_existing);
+                    std::filesystem::copy_file(
+                        std::filesystem::path{p.string()},
+                        std::filesystem::path{(location_ / p.filename()).string()},
+                        std::filesystem::copy_options::overwrite_existing
+                    );
                 } else {
                     boost::filesystem::path full_dst = resolver.resolve_path(resolver.extract_blob_id(p));
                     boost::filesystem::path dst_dir = full_dst.parent_path();
                     if (!boost::filesystem::exists(dst_dir)) {
                         boost::filesystem::create_directories(dst_dir);
                     }
-                    // overwrite blob file if exists
-                    boost::filesystem::copy_file(p, full_dst, boost::filesystem::copy_options::overwrite_existing);
+                    std::filesystem::copy_file(
+                        std::filesystem::path{p.string()},
+                        std::filesystem::path{full_dst.string()},
+                        std::filesystem::copy_options::overwrite_existing
+                    );
                 }
-            } catch (boost::filesystem::filesystem_error& ex) {
+            } catch (std::filesystem::filesystem_error& ex) {
                 LOG_LP(ERROR) << ex.what() << " file = " << p.string();
                 return status::err_permission_error;
             }