Refactor WebKit URL validation to use libcurl

src/CMakeLists.txt

@@ -56,6 +56,7 @@ set(SOURCE_FILES
     engine/plugin_manager.cc
     engine/plugin_webkit_store.cc
     engine/plugin_webkit_world_mgr.cc
+    engine/target_url.cc
     engine/thread_pool.cc
     util/cmdline_parser.cc
     util/file_watcher.cc

src/engine/http_hooks.cc

@@ -38,10 +38,11 @@
 #include "millennium/url_parser.h"
 #include "millennium/virtfs.h"
 #include "millennium/types.h"
+#include "millennium/target_url.h"
+#include "millennium/http.h"
 
 #include <nlohmann/json_fwd.hpp>
 #include <thread>
-#include <unordered_set>
 
 std::atomic<unsigned long long> g_hookedModuleId{ 0 };
 
@@ -195,11 +196,10 @@ void network_hook_ctl::mime_doc_request_handler(const nlohmann::basic_json<>& me
     };
 
     /** check if the request URL is a do-not-hook URL. */
-    for (const auto& doNotHook : g_js_and_css_hook_blacklist) {
-        if (std::regex_match(requestUrl, std::regex(doNotHook))) {
-            m_cdp->send_host("Fetch.continueResponse", params);
-            return;
-        }
+    target_url target(requestUrl);
+    if (!target.is_safe_for_network_hooks()) {
+        m_cdp->send_host("Fetch.continueResponse", params);
+        return;
     }
 
     const auto redirect_codes = { http_code::SEE_OTHER, http_code::MOVED_PERMANENTLY, http_code::FOUND, http_code::TEMPORARY_REDIRECT, http_code::PERMANENT_REDIRECT };
@@ -211,7 +211,7 @@ void network_hook_ctl::mime_doc_request_handler(const nlohmann::basic_json<>& me
         }
     }
 
-    const processed_hooks hooks = apply_user_webkit_hooks(requestUrl);
+    const processed_hooks hooks = apply_user_webkit_hooks(target);
     if (hooks.empty()) {
         m_cdp->send_host("Fetch.continueResponse", params);
         return;
@@ -243,14 +243,20 @@ void network_hook_ctl::mime_doc_request_handler(const nlohmann::basic_json<>& me
     m_cdp->send_host("Fetch.fulfillRequest", fullfillParams);
 }
 
-network_hook_ctl::processed_hooks network_hook_ctl::apply_user_webkit_hooks(const std::string& requestUrl) const
+network_hook_ctl::processed_hooks network_hook_ctl::apply_user_webkit_hooks(const target_url& target) const
 {
     processed_hooks result;
     auto hookList = get_hook_list();
     bool anyHookMatched = false;
+    bool safe_for_js = target.is_safe_for_js();
 
     for (const auto& hook : hookList) {
-        if (!std::regex_match(requestUrl, hook.hook.url_pattern)) continue;
+        if (!std::regex_match(target.raw_url(), hook.hook.url_pattern)) continue;
+
+        if (hook.hook.type == TagTypes::JAVASCRIPT && !safe_for_js) {
+            continue;
+        }
+
         anyHookMatched = true;
 
         if (hook.hook.type == TagTypes::STYLESHEET)

src/engine/plugin_webkit_world_mgr.cc

@@ -36,6 +36,7 @@
 #include "millennium/logger.h"
 #include "millennium/auth.h"
 #include "millennium/url_parser.h"
+#include "millennium/target_url.h"
 #include <format>
 
 webkit_world_mgr::webkit_world_mgr(std::shared_ptr<cdp_client> client, std::shared_ptr<plugin_manager> plugin_manager, std::shared_ptr<network_hook_ctl> network_hook_ctl,
@@ -73,49 +74,10 @@ void webkit_world_mgr::initialize()
     }
 }
 
-static bool is_steam_owned_url(const std::string& url)
-{
-    if (url.find(std::string("https://") + k_steam_loopback + "/") == 0) return true;
-    for (const auto* tld : k_steam_tlds) {
-        if (url.find(std::string(".") + tld + "/") != std::string::npos) return true;
-    }
-    return false;
-}
 
 bool webkit_world_mgr::is_valid_target_url(const std::string& url) const
 {
-    if (url.empty()) {
-        return false;
-    }
-
-    // only web protocols are supported
-    if (url.find("http://") != 0 && url.find("https://") != 0) {
-        return false;
-    }
-
-    // steamloopback is the main UI, handled natively by SharedJSContext
-    if (url.find("https://steamloopback.host/") == 0) {
-        return false;
-    }
-
-    // forbid URLs matching the global blacklist (e.g. checkout pages)
-    for (const auto& pattern : g_js_hook_blacklist) {
-        if (std::regex_match(url, std::regex(pattern))) {
-            return false;
-        }
-    }
-
-    // allow all steam-owned popups/frames (e.g. store, community)
-    if (is_steam_owned_url(url)) {
-        return true;
-    }
-
-    // allow external URLs if explicitly hooked by a plugin or theme (e.g. via add_browser_js / add_browser_css in the backend)
-    const auto hookList = m_network_hook_ctl->get_hook_list();
-    return std::any_of(hookList.begin(), hookList.end(), [&url](const auto& hook)
-    {
-        return std::regex_match(url, hook.hook.url_pattern);
-    });
+    return target_url(url).allows_webkit_injection(m_network_hook_ctl->get_hook_list());
 }
 
 void webkit_world_mgr::attach_to_target(const std::string& target_id, const std::string& url)
@@ -181,7 +143,7 @@ void webkit_world_mgr::attach_to_target(const std::string& target_id, const std:
          * it seems this reload interferes with some versions of CF turnstile.
          */
         bool is_top_level = !frame_tree_result["frameTree"]["frame"].contains("parentId");
-        bool can_reload = is_top_level && is_steam_owned_url(url);
+        bool can_reload = is_top_level && target_url(url).is_steam_owned();
 
         {
             std::lock_guard<std::mutex> lock(m_targets_mutex);

src/engine/target_url.cc

@@ -0,0 +1,106 @@
+#include "millennium/target_url.h"
+#include <curl/curl.h>
+#include <algorithm>
+#include <cctype>
+
+bool target_url::is_domain_match(std::string_view host, std::string_view domain) {
+    if (host == domain) return true;
+    if (host.length() > domain.length() && 
+        host[host.length() - domain.length() - 1] == '.' &&
+        host.ends_with(domain)) {
+        return true;
+    }
+    return false;
+}
+
+target_url::target_url(const std::string& url) : m_raw_url(url) {
+    CURLU* h = curl_url();
+    if (!h) return;
+
+    if (curl_url_set(h, CURLUPART_URL, url.c_str(), 0) == CURLUE_OK) {
+        char* host_ptr = nullptr;
+        char* scheme_ptr = nullptr;
+
+        if (curl_url_get(h, CURLUPART_HOST, &host_ptr, 0) == CURLUE_OK &&
+            curl_url_get(h, CURLUPART_SCHEME, &scheme_ptr, 0) == CURLUE_OK) {
+
+            m_host = host_ptr;
+            m_scheme = scheme_ptr;
+
+            curl_free(host_ptr);
+            curl_free(scheme_ptr);
+
+            while (!m_host.empty() && m_host.back() == '.') {
+                m_host.pop_back();
+            }
+
+            std::transform(m_host.begin(), m_host.end(), m_host.begin(),
+                           [](unsigned char c){ return std::tolower(c); });
+            std::transform(m_scheme.begin(), m_scheme.end(), m_scheme.begin(),
+                           [](unsigned char c){ return std::tolower(c); });
+
+            m_valid = true;
+        } else {
+            if (host_ptr) curl_free(host_ptr);
+            if (scheme_ptr) curl_free(scheme_ptr);
+        }
+    }
+    curl_url_cleanup(h);
+}
+
+target_url::~target_url() = default;
+
+bool target_url::is_valid() const {
+    return m_valid;
+}
+
+bool target_url::is_safe_for_network_hooks() const {
+    if (!m_valid) return false;
+    if (m_scheme != "http" && m_scheme != "https") return false;
+
+    static constexpr std::string_view forbidden_domains[] = {
+        "paypal.com", "paypalobjects.com", "recaptcha.net",
+        "youtube.com", "youtube-nocookie.com", "youtu.be", "ytimg.com", 
+        "googlevideo.com", "googleusercontent.com", "studioyoutube.com",
+        "chromewebstore.google.com"
+    };
+
+    for (const auto& domain : forbidden_domains) {
+        if (is_domain_match(m_host, domain)) return false;
+    }
+    return true;
+}
+
+bool target_url::is_safe_for_js() const {
+    if (!m_valid) return false;
+
+    if (is_domain_match(m_host, "checkout.steampowered.com")) {
+        return false;
+    }
+
+    return is_safe_for_network_hooks();
+}
+
+bool target_url::is_steam_owned() const {
+    if (!m_valid) return false;
+
+    if (m_host == k_steam_loopback) return true;
+
+    for (const auto* domain : k_steam_tlds) {
+        if (is_domain_match(m_host, domain)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+bool target_url::allows_webkit_injection(const std::vector<network_hook_ctl::hook_item>& hooks) const {
+    if (!m_valid) return false;
+    if (!is_safe_for_js()) return false;
+    if (m_host == k_steam_loopback) return false;
+    if (is_steam_owned()) return true;
+
+    return std::any_of(hooks.begin(), hooks.end(), [this](const auto& hook_item) {
+        return std::regex_match(m_raw_url, hook_item.hook.url_pattern);
+    });
+}

src/include/millennium/http_hooks.h

@@ -46,40 +46,20 @@
 #include <shared_mutex>
 #include <string>
 #include <vector>
-#include <unordered_set>
+
 
 extern std::atomic<unsigned long long> g_hookedModuleId;
 std::string get_cdp_isolated_ctx_script();
 
-/**
- * Millennium will not load JavaScript into the following URLs to favor user safety.
- * This is a list of URLs that may have sensitive information or are not safe to load JavaScript into.
- */
-static const std::vector<std::string> g_js_hook_blacklist = { "https://checkout\\.steampowered\\.com/.*" };
+
 
 /** Canonical list of Steam-owned TLDs. Both the CDP network interceptor and the webkit world manager derive their domain checks from this. */
 static constexpr const char* k_steam_tlds[] = {
     "steampowered.com", "steamcommunity.com", "steamgames.com", "steam-chat.com", "steamstatic.com",
 };
 static constexpr const char* k_steam_loopback = "steamloopback.host";
 
-// clang-format off
-/** Millennium will not hook the following URLs to favor user safety. (Neither JavaScript nor CSS will be injected into these URLs.) */
-static const std::unordered_set<std::string> g_js_and_css_hook_blacklist = {
-    /** Ignore paypal related content */
-    R"(https?:\/\/(?:[\w-]+\.)*paypal\.com\/[^\s"']*)",
-    R"(https?:\/\/(?:[\w-]+\.)*paypalobjects\.com\/[^\s"']*)",
-    R"(https?:\/\/(?:[\w-]+\.)*recaptcha\.net\/[^\s"']*)",
-
-    /** Ignore youtube related content */
-    R"(https?://(?:[\w-]+\.)*(?:youtube(?:-nocookie)?|youtu|ytimg|googlevideo|googleusercontent|studioyoutube)\.com/[^\s"']*)",
-    R"(https?://(?:[\w-]+\.)*youtu\.be/[^\s"']*)",
-
-    /** Ignore Chrome Web Store (causes a webhelper crash on Fetch.fulfillRequest) */
-    R"(https?:\/\/(?:[\w-]+\.)*chromewebstore\.google\.com\/[^\s"']*)",
-};
-// clang-format on
-
+class target_url;
 class network_hook_ctl
 {
   public:
@@ -185,6 +165,6 @@ class network_hook_ctl
     void vfs_request_handler(const nlohmann::basic_json<>& message);
     void mime_doc_request_handler(const nlohmann::basic_json<>& message);
     std::filesystem::path path_from_url(const std::string& requestUrl);
-    processed_hooks apply_user_webkit_hooks(const std::string& requestUrl) const;
+    processed_hooks apply_user_webkit_hooks(const target_url& target) const;
     std::string inject_into_document_head(const std::string& original, const std::string& content) const;
 };

src/include/millennium/target_url.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include "millennium/http_hooks.h"
+
+class target_url {
+public:
+    explicit target_url(const std::string& url);
+    ~target_url();
+
+    // Was parsed successfully?
+    bool is_valid() const;
+
+    // Policy queries
+    bool is_safe_for_network_hooks() const;
+    bool is_safe_for_js() const;
+    bool is_steam_owned() const;
+    bool allows_webkit_injection(const std::vector<network_hook_ctl::hook_item>& hooks) const;
+
+    static bool is_domain_match(std::string_view host, std::string_view domain);
+
+    const std::string& scheme() const { return m_scheme; }
+    const std::string& host() const { return m_host; }
+    const std::string& raw_url() const { return m_raw_url; }
+
+private:
+    bool m_valid = false;
+    std::string m_scheme;
+    std::string m_host;
+    std::string m_raw_url;
+};

src/instrumentation/internal/steam_hooks.cc

@@ -43,9 +43,9 @@
 #include <sys/stat.h>
 #include <unistd.h>
 #elif _WIN32
-#include "millennium/filesystem.h"
 #endif
 
+#include "millennium/filesystem.h"
 #include "millennium/logger.h"
 #include "millennium/steam_hooks.h"
 #include "millennium/cmdline_api.h"

tests/cpp/CMakeLists.txt

@@ -1,6 +1,8 @@
 set(TEST_SOURCES
   ffi_recorder_test.cc
+  test_target_url.cc
   ${CMAKE_SOURCE_DIR}/src/mep/ffi_recorder.cc
+  ${CMAKE_SOURCE_DIR}/src/engine/target_url.cc
 )
 
 add_executable(millennium_cpp_tests ${TEST_SOURCES})
@@ -11,7 +13,7 @@ target_include_directories(millennium_cpp_tests PRIVATE
   ${CMAKE_SOURCE_DIR}/src/include
 )
 
-target_link_libraries(millennium_cpp_tests PRIVATE Catch2::Catch2WithMain)
+target_link_libraries(millennium_cpp_tests PRIVATE Catch2::Catch2WithMain libcurl nlohmann_json::nlohmann_json)
 
 include(Catch)
 catch_discover_tests(millennium_cpp_tests)