feat: bundle Firecrawl as managed Docker service for web search

Firecrawl (open source, ghcr.io/mendableai/firecrawl) is now a managed
service like Neo4j/Qdrant — starts automatically with `prism node up`.
Web tools auto-detect local instance at localhost:3002, no API key needed.

- Add FirecrawlConfig to ServiceConfig (port 3002, enabled by default)
- Add start_firecrawl() to DockerOrchestrator
- web.py: auto-detect local Firecrawl → cloud API → DDG fallback chain
- pyproject.toml: firecrawl-py + duckduckgo-search + beautifulsoup4

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: Darth-Hidious

app/tools/web.py

@@ -13,9 +13,42 @@
 
 logger = logging.getLogger(__name__)
 
-# Firecrawl config — works with local self-hosted instance OR the API
+# Firecrawl config — prefers local self-hosted instance (bundled with PRISM),
+# falls back to cloud API if key is set.
 FIRECRAWL_KEY = os.environ.get("FIRECRAWL_API_KEY", "")
-FIRECRAWL_URL = os.environ.get("FIRECRAWL_API_URL", "https://api.firecrawl.dev/v1")
+FIRECRAWL_LOCAL_URL = os.environ.get("FIRECRAWL_LOCAL_URL", "http://localhost:3002")
+FIRECRAWL_API_URL = os.environ.get("FIRECRAWL_API_URL", "https://api.firecrawl.dev/v1")
+
+
+def _firecrawl_available() -> bool:
+    """Check if local Firecrawl instance is running."""
+    try:
+        import httpx
+
+        r = httpx.get(f"{FIRECRAWL_LOCAL_URL}/", timeout=2)
+        return r.status_code < 500
+    except Exception:
+        return False
+
+
+# Cache the check at import time so we don't hit it on every call
+_LOCAL_FIRECRAWL = _firecrawl_available()
+
+# Resolve which Firecrawl URL and key to use
+if _LOCAL_FIRECRAWL:
+    # Local instance — no API key needed
+    FIRECRAWL_URL = FIRECRAWL_LOCAL_URL
+    FIRECRAWL_ACTIVE = True
+    logger.info(f"Firecrawl: using local instance at {FIRECRAWL_URL}")
+elif FIRECRAWL_KEY:
+    # Cloud API with key
+    FIRECRAWL_URL = FIRECRAWL_API_URL
+    FIRECRAWL_ACTIVE = True
+    logger.info("Firecrawl: using cloud API")
+else:
+    FIRECRAWL_URL = ""
+    FIRECRAWL_ACTIVE = False
+    logger.info("Firecrawl: not available, using DuckDuckGo fallback")
 
 
 def _web_read(**kwargs) -> dict:
@@ -29,11 +62,14 @@ def _web_read(**kwargs) -> dict:
         return {"error": "url is required"}
 
     # Try Firecrawl first (best quality — handles JS, returns markdown)
-    if FIRECRAWL_KEY:
+    if FIRECRAWL_ACTIVE:
         try:
             from firecrawl import FirecrawlApp
 
-            app = FirecrawlApp(api_key=FIRECRAWL_KEY, api_url=FIRECRAWL_URL)
+            app = FirecrawlApp(
+                api_key=FIRECRAWL_KEY or "local",
+                api_url=FIRECRAWL_URL,
+            )
             result = app.scrape_url(url, params={"formats": ["markdown"]})
             content = result.get("markdown", "") if isinstance(result, dict) else str(result)
             title = ""
@@ -106,12 +142,15 @@ def _web_search(**kwargs) -> dict:
     if not query:
         return {"error": "query is required"}
 
-    # Try Firecrawl search first (if key is set)
-    if FIRECRAWL_KEY:
+    # Try Firecrawl search first (local or cloud)
+    if FIRECRAWL_ACTIVE:
         try:
             from firecrawl import FirecrawlApp
 
-            app = FirecrawlApp(api_key=FIRECRAWL_KEY, api_url=FIRECRAWL_URL)
+            app = FirecrawlApp(
+                api_key=FIRECRAWL_KEY or "local",
+                api_url=FIRECRAWL_URL,
+            )
             results = app.search(query, params={"limit": limit})
             items = results if isinstance(results, list) else results.get("data", [])
             return {

crates/cli/src/main.rs

@@ -1156,7 +1156,8 @@ async fn main() -> Result<()> {
                     let wants_managed_services = svc_config.neo4j.is_some()
                         || svc_config.vector_db.is_some()
                         || svc_config.kafka.is_some()
-                        || svc_config.spark.is_some();
+                        || svc_config.spark.is_some()
+                        || svc_config.firecrawl.is_some();
 
                     if wants_managed_services {
                         println!("\n  PRISM v{}", env!("CARGO_PKG_VERSION"));

crates/orch/src/docker.rs

@@ -350,6 +350,49 @@ impl DockerOrchestrator {
         })
     }
 
+    /// Start Firecrawl (open-source web scraper) and return a ServiceHandle.
+    async fn start_firecrawl(
+        &self,
+        config: &crate::services::FirecrawlConfig,
+    ) -> Result<ServiceHandle> {
+        self.ensure_image(&config.image).await?;
+
+        let env = vec![
+            // Firecrawl minimal config — no external dependencies needed
+            format!("PORT={}", config.port),
+            "HOST=0.0.0.0".to_string(),
+            "NUM_WORKERS_PER_QUEUE=2".to_string(),
+        ];
+
+        let mut port_bindings = HashMap::new();
+        port_bindings.insert(
+            format!("{}/tcp", config.port),
+            Some(vec![PortBinding {
+                host_ip: Some("127.0.0.1".to_string()),
+                host_port: Some(config.port.to_string()),
+            }]),
+        );
+
+        let container_id = self
+            .run_container("firecrawl", &config.image, env, None, port_bindings, None)
+            .await?;
+
+        Ok(ServiceHandle {
+            name: "firecrawl".to_string(),
+            container_id: Some(container_id),
+            port: config.port,
+            healthy: false,
+        })
+    }
+
+    /// Restart Firecrawl (public, used by health monitor).
+    pub async fn start_firecrawl_public(
+        &self,
+        config: &crate::services::FirecrawlConfig,
+    ) -> Result<ServiceHandle> {
+        self.start_firecrawl(config).await
+    }
+
     /// Restart Spark (public, used by health monitor).
     pub async fn start_spark_public(
         &self,
@@ -479,6 +522,13 @@ impl ServiceOrchestrator for DockerOrchestrator {
             services.push(spark);
         }
 
+        // Firecrawl — open-source web scraping (enabled by default)
+        if let Some(ref firecrawl_cfg) = config.firecrawl {
+            info!("Starting Firecrawl...");
+            let firecrawl = self.start_firecrawl(firecrawl_cfg).await?;
+            services.push(firecrawl);
+        }
+
         // Wait for services to become healthy
         let checker = HealthChecker::new();
         for handle in &mut services {

crates/orch/src/services.rs

@@ -6,6 +6,7 @@ pub struct ServiceConfig {
     pub vector_db: Option<VectorDbConfig>,
     pub kafka: Option<KafkaConfig>,
     pub spark: Option<SparkConfig>,
+    pub firecrawl: Option<FirecrawlConfig>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -73,13 +74,32 @@ impl Default for SparkConfig {
     }
 }
 
+/// Firecrawl — open-source web scraping & search engine.
+/// Docker image: `ghcr.io/mendableai/firecrawl`
+/// Default port: 3002 (API)
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct FirecrawlConfig {
+    pub image: String,
+    pub port: u16,
+}
+
+impl Default for FirecrawlConfig {
+    fn default() -> Self {
+        Self {
+            image: "ghcr.io/mendableai/firecrawl:latest".to_string(),
+            port: 3002,
+        }
+    }
+}
+
 impl Default for ServiceConfig {
     fn default() -> Self {
         Self {
             neo4j: Some(Neo4jConfig::default()),
             vector_db: Some(VectorDbConfig::default()),
             kafka: None,
             spark: None,
+            firecrawl: Some(FirecrawlConfig::default()),
         }
     }
 }