inference.yaml

inference:
  # ═══════════════════════════════════════════════════════════════════════════
  # Local Ollama Configuration
  # ═══════════════════════════════════════════════════════════════════════════
  # Uses presets defined in config/ollama.yaml for easy switching between
  # CPU/GPU configurations and different models.
  #
  # Available presets (defined in ollama.yaml):
  #   - granite-cpu: Granite 4.0-1B optimized for CPU (16GB RAM)
  #   - granite-gpu: Granite 4.0-1B with full GPU acceleration
  #   - qwen3-4b-gpu: Qwen3 4B with GPU (larger, more capable)
  #   - llama3-3b-cpu: Llama 3.2 3B optimized for CPU
  #
  # To switch configurations, simply change the use_preset value below.
  # ═══════════════════════════════════════════════════════════════════════════
  ollama:
    enabled: true
    use_preset: "smollm2-1.7b-cpu"  # Reference to preset in config/ollama.yaml
  ollama_cloud:
    enabled: true
    api_key: ${OLLAMA_CLOUD_API_KEY}
    model: "gpt-oss:120b" # Supported models: https://docs.ollama.com/cloud
    stream: true
    think: false  # Enable/disable think mode (shows reasoning in <think> tags)
    # Generation parameters - Optimized for large SQL result sets
    temperature: 0.3  # Lower for more factual, deterministic outputs
    top_p: 0.8  # More focused outputs for data analysis
    top_k: 40
    num_predict: 4096  # Increased to handle comprehensive analysis of hundreds of records
    repeat_penalty: 1.1
    repeat_last_n: 32
    # Context window - large enough for system prompt + hundreds of SQL rows + response
    # Calculation: ~1K system prompt + ~10-20K SQL results + ~4K response = ~25K minimum
    num_ctx: 32768  # 32K context for large SQL result sets
    # Stop sequences
    stop: []
    # Cloud service - these settings are managed by Ollama Cloud infrastructure
    # num_threads, num_batch, use_mmap, use_mlock, vocab_only, numa are not applicable
    # Random seed for reproducible outputs
    seed: null
  ollama_remote:
    enabled: false
    # Remote Ollama server URL (e.g., AWS EC2, self-hosted server)
    base_url: "http://your-remote-server:11434"
    # Optional API key for secured remote servers (leave null if not required)
    api_key: null
    model: "granite4:1b"  # Model available on your remote Ollama server
    stream: true
    think: false  # Enable/disable think mode (shows reasoning in <think> tags)
    # Generation parameters - Adjust based on your use case and model
    temperature: 0.1  # Balanced for creative yet factual outputs
    top_p: 0.9  # Good diversity while maintaining coherence
    top_k: 30
    min_p: 0.0
    typical_p: 0.9
    num_predict: 4096  # Maximum tokens to generate
    repeat_penalty: 1.15  # Moderate penalty to prevent repetition
    repeat_last_n: 64  # Check tokens for repetition
    presence_penalty: 0.0
    frequency_penalty: 0.0
    # Mirostat sampling (0=disabled, 1=Mirostat, 2=Mirostat v2)
    mirostat: 0
    mirostat_tau: 4.0
    mirostat_eta: 0.1
    # Context settings - Adjust based on your remote server's capabilities
    num_ctx: 8192  # Context window size (depends on server's model and hardware)
    num_keep: 2048  # Keep prompt context and chat history
    penalize_newline: false
    # Stop sequences
    stop: []
    # Remote server - these settings are managed by the remote Ollama server
    # num_threads, num_batch, num_gpu, use_mmap, use_mlock, vocab_only, numa are not applicable
    # Random seed for reproducible outputs
    seed: null
    # Retry configuration for handling network issues and cold starts
    retry:
      enabled: true
      max_retries: 5
      initial_wait_ms: 2000  # Start with 2 seconds
      max_wait_ms: 30000     # Max 30 seconds per retry
      exponential_base: 2    # Exponential backoff multiplier
    # Timeout configuration
    timeout:
      connect: 10000         # 10 seconds for connection
      total: 120000          # 2 minutes total timeout
      warmup: 120000         # 2 minutes for initial warmup (increased for cold starts)
    # Keep alive - how long to keep model loaded in memory on remote server
    # Options: "5m", "10m", "1h", "-1" (indefinite), "0" (unload immediately)
    # Longer values reduce cold start latency but use more server memory
    keep_alive: "10m"
  vllm:
    enabled: false
    # Mode: "api" (connect to vLLM server) or "direct" (load model in-process with GPU)
    mode: "api"

    # ═══════════════════════════════════════════════════════════════════════════
    # API Mode Configuration
    # ═══════════════════════════════════════════════════════════════════════════
    # Connect to a running vLLM server (started with `vllm serve` or `python -m vllm.entrypoints.openai.api_server`)
    host: "localhost"
    port: 8000
    base_url: null  # Override full URL (e.g., "http://localhost:8000/v1"), takes precedence over host/port
    api_key: null   # Optional API key for secured servers

    # ═══════════════════════════════════════════════════════════════════════════
    # Direct Mode Configuration
    # ═══════════════════════════════════════════════════════════════════════════
    # Load model directly in-process (requires GPU and vLLM package installed)
    # model_path: "meta-llama/Llama-3.1-8B-Instruct"  # HuggingFace model ID or local path

    # GPU and Parallelism
    tensor_parallel_size: 1       # Number of GPUs for tensor parallelism (split model across GPUs)
    pipeline_parallel_size: 1     # Number of GPUs for pipeline parallelism
    gpu_memory_utilization: 0.90  # Fraction of GPU memory to use (0.0-1.0)
    max_model_len: 4096           # Maximum sequence length (context window)

    # Quantization (optional, for reduced memory usage)
    quantization: null            # Options: "awq", "gptq", "squeezellm", "fp8", null (none)
    dtype: "auto"                 # Data type: "auto", "half", "float16", "bfloat16", "float32"

    # Advanced settings
    trust_remote_code: false      # Trust remote code in HuggingFace models (security risk)
    enforce_eager: false          # Disable CUDA graphs (useful for debugging)
    swap_space: 4                 # CPU swap space in GB for KV cache offloading
    kv_cache_dtype: "auto"        # KV cache data type
    seed: null                    # Random seed for reproducibility

    # ═══════════════════════════════════════════════════════════════════════════
    # Shared Generation Parameters (used in both modes)
    # ═══════════════════════════════════════════════════════════════════════════
    model: "Qwen/Qwen2.5-1.5B-Instruct"  # Model to use (HuggingFace ID for direct, model name for API)
    temperature: 0.1
    top_p: 0.8
    top_k: 20                     # Set to -1 to disable
    max_tokens: 2048
    repetition_penalty: 1.0       # Penalty for repeating tokens (1.0 = no penalty)
    presence_penalty: 0.0         # Penalty for tokens already in context
    frequency_penalty: 0.0        # Penalty based on token frequency
    stream: true

    # Stop sequences
    stop: ["<|eot_id|>"]          # Stop generation when these tokens are encountered

    # Timeout configuration (API mode)
    timeout:
      connect: 10000              # 10 seconds
      total: 120000               # 2 minutes

    # Retry configuration (API mode)
    retry:
      enabled: true
      max_retries: 3
      initial_wait_ms: 1000
      max_wait_ms: 30000
      exponential_base: 2
  tensorrt:
    enabled: true
    # Mode: "direct" (load model locally with TensorRT-LLM) or "api" (connect to trtllm-serve)
    mode: "direct"

    # ═══════════════════════════════════════════════════════════════════════════
    # API Mode Configuration
    # ═══════════════════════════════════════════════════════════════════════════
    # Connect to a running trtllm-serve server (started with `trtllm-serve model_name`)
    host: "localhost"
    port: 8000
    base_url: null  # Override full URL (e.g., "http://localhost:8000/v1"), takes precedence over host/port
    api_key: null   # Optional API key for secured servers

    # ═══════════════════════════════════════════════════════════════════════════
    # Direct Mode Configuration
    # ═══════════════════════════════════════════════════════════════════════════
    # Load model directly in-process (requires NVIDIA GPU and tensorrt-llm package)
    model: "TinyLlama/TinyLlama-1.1B-Chat-v1.0"  # HuggingFace model ID or local path
    # Pre-quantized NVIDIA models: nvidia/Llama-3.1-8B-Instruct-FP8, nvidia/Qwen3-8B-FP8

    # GPU and Parallelism
    tensor_parallel_size: 1       # Number of GPUs for tensor parallelism (split model across GPUs)
    pipeline_parallel_size: 1     # Number of GPUs for pipeline parallelism
    gpu_memory_utilization: 0.90  # Fraction of GPU memory to use (0.0-1.0)
    max_model_len: 4096           # Maximum sequence length (context window)

    # Quantization (for optimized inference)
    # Options: "fp8" (FP8 quantization), "int8" (INT8 weight-only),
    #          "int4_awq" (AWQ 4-bit), "int4_gptq" (GPTQ 4-bit), null (none)
    # Note: Pre-quantized models from NVIDIA (e.g., nvidia/Llama-3.1-8B-Instruct-FP8)
    #       don't need this setting - they are already quantized
    quantization: null
    dtype: "auto"                 # Data type: "auto", "float16", "bfloat16", "float32"

    # KV cache configuration
    kv_cache_dtype: "auto"        # KV cache data type

    # ═══════════════════════════════════════════════════════════════════════════
    # Generation Parameters (used in both modes)
    # ═══════════════════════════════════════════════════════════════════════════
    temperature: 0.1
    top_p: 0.8
    top_k: 20
    max_tokens: 2048
    repetition_penalty: 1.0       # Penalty for repeating tokens (1.0 = no penalty)
    presence_penalty: 0.0         # Penalty for tokens already in context
    frequency_penalty: 0.0        # Penalty based on token frequency
    stream: true

    # Stop sequences
    stop: []                      # Stop generation when these tokens are encountered

    # Timeout configuration (API mode)
    timeout:
      connect: 10000              # 10 seconds
      total: 120000               # 2 minutes

    # Retry configuration (API mode)
    retry:
      enabled: true
      max_retries: 3
      initial_wait_ms: 1000
      max_wait_ms: 30000
      exponential_base: 2
  shimmy:
    enabled: false
    base_url: "http://localhost:11435"  # Shimmy server URL (OpenAI-compatible API)
    model: "gemma-3-1b-it-q4-0"  # Model name to request from the server (smallest available - good for quick testing)
    api_key: null  # Optional - Shimmy doesn't require authentication
    
    # Generation parameters
    temperature: 0.4  # Balanced for creative yet factual outputs
    top_p: 0.9  # Good diversity while maintaining coherence
    max_tokens: 4096  # Maximum tokens to generate
    
    # Context configuration
    context_window: 65536  # Shimmy supports large context windows
    stream: true
    
    # Stop sequences (optional)
    stop_tokens: []
    
    # Timeout configuration
    timeout:
      connect: 10000   # 10 seconds
      total: 120000    # 2 minutes
    
    # Retry configuration
    retry:
      enabled: true
      max_retries: 5
      initial_wait_ms: 2000
      max_wait_ms: 30000
      exponential_base: 2
  llama_cpp:
    enabled: true
    use_preset: "gemma4-e2b-direct-cpu"  # Reference to preset in config/llama_cpp.yaml
  gemini:
    enabled: true
    api_key: ${GOOGLE_API_KEY}
    model: "gemini-3.6-flash"
    context_window: 1000000  # Gemini supports up to 1M tokens (input context)
    temperature: 0.1
    top_p: 0.8
    top_k: 20
    max_tokens: 8192         # Max output tokens; 1024 was causing cut-off responses
    stream: true
    transport: "rest"
    # Safety settings - set to true to disable all safety filters (useful for reducing false positives)
    disable_safety: true
  groq:
    enabled: true
    api_key: ${GROQ_API_KEY}
    model: "llama-3.1-8b-instant"
    context_window: 131072  # Llama 3.1 supports 128K context on Groq
    temperature: 0.1
    top_p: 0.8
    max_tokens: 32000
    stream: true
  deepseek:
    enabled: true
    api_key: ${DEEPSEEK_API_KEY}
    model: "deepseek-chat"
    context_window: 65536  # DeepSeek Chat supports 64K context
    temperature: 0.1
    top_p: 0.8
    max_tokens: 1024
    stream: true
  vertex:
    enabled: false
    project_id: "your-project-id"
    location: "us-central1"
    model: "gemini-1.5-pro"
    context_window: 1000000  # Gemini 1.5 Pro on Vertex supports up to 1M tokens
    temperature: 0.1
    top_p: 0.8
    top_k: 20
    max_tokens: 1024
    credentials_path: ""
    stream: true
  aws:
    enabled: false
    access_key: ${AWS_BEDROCK_ACCESS_KEY}
    secret_access_key: ${AWS_SECRET_ACCESS_KEY}
    region: "ca-central-1"
    model: "claude-sonnet-4-5-20250929"
    context_window: 200000  # Claude on Bedrock supports 200K context
    content_type: "application/json"
    accept: "application/json"
    max_tokens: 1024
  azure:
    enabled: false
    endpoint: "https://<your-resource-name>.openai.azure.com/"
    deployment: "your-deployment-name"
    api_key: ${AZURE_ACCESS_KEY}
    api_version: "2024-06-01"
    context_window: 128000  # Azure OpenAI GPT-4 Turbo supports 128K context
    temperature: 0.1
    top_p: 0.8
    max_tokens: 1024
    stream: true
    verbose: true
  openai:
    enabled: true
    api_key: ${OPENAI_API_KEY}
    model: "gpt-5.4-mini"
    context_window: 128000  # GPT-4 Turbo/GPT-4o supports 128K context
    temperature: 1
    top_p: 0.8
    max_tokens: 16000
    stream: true
  mistral:
    enabled: true
    api_key: ${MISTRAL_API_KEY}
    api_base: "https://api.mistral.ai/v1"
    model: "mistral-small-2603"
    context_window: 32768  # Mistral Small supports 32K context
    temperature: 0.1
    top_p: 0.8
    max_tokens: 1024
    stream: true
  anthropic:
    enabled: true
    api_key: ${ANTHROPIC_API_KEY}
    api_base: "https://api.anthropic.com/v1"
    model: "claude-sonnet-4-6"
    context_window: 200000  # Claude 3/4 supports 200K context
    max_tokens: 1024
    stream: true
  together:
    enabled: false
    api_key: ${TOGETHER_API_KEY}
    api_base: "https://api.together.xyz/v1"
    model: "Qwen/Qwen3-235B-A22B-fp8-tput"
    context_window: 32768  # Varies by model, Qwen supports 32K+
    temperature: 0.1
    top_p: 0.8
    max_tokens: 1024
    stream: true
    show_thinking: false
  xai:
    enabled: true
    api_key: ${XAI_API_KEY}
    api_base: "https://api.x.ai/v1"
    model: "grok-4.3"
    context_window: 131072  # Grok supports 128K context
    temperature: 0.1
    top_p: 0.8
    max_tokens: 16000
    stream: true
    show_thinking: false
  # Hugging Face Inference API (serverless or dedicated Inference Endpoints).
  # Use any model ID from https://huggingface.co/models
  huggingface:
    enabled: false
    model: "HuggingFaceTB/SmolLM2-1.7B-Instruct"
    api_key: ${HUGGINGFACE_API_KEY}
    endpoint_url: null  # Optional: custom URL for Inference Endpoints
    # Inference provider routing. Controls which backend runs the model.
    # Options: hf-inference, cerebras, together, fireworks-ai, replicate,
    #          sambanova, nebius, novita, scaleway, hyperbolic, auto
    # Default "hf-inference" uses HF's own serverless API.
    # Use "auto" to let the client pick the best available provider.
    provider: "hf-inference"
    temperature: 0.7
    max_tokens: 1024
    stream: true
    # Retry and timeout (optional)
    retry:
      enabled: true
      max_retries: 3
      initial_wait_ms: 1000
      max_wait_ms: 30000
      exponential_base: 2
    timeout:
      connect: 10000
      total: 120000
  transformers:
    enabled: false
    model: "RISys-Lab/RedSage-Qwen3-8B-Ins"
    device: "auto"              # auto, cuda, mps, cpu
    device_map: "auto"          # auto (accelerate distributes), cuda:0, null
    dtype: "auto"               # auto, float16, bfloat16, float32
    trust_remote_code: false
    max_memory: null            # e.g. {"0": "20GiB", "cpu": "30GiB"}
    attn_implementation: null   # flash_attention_2, sdpa, or null
    temperature: 0.7
    top_p: 0.9
    top_k: -1
    max_tokens: 2048
    repetition_penalty: 1.0
    do_sample: true
    stream: true
    stop: []
  airllm:
    enabled: false
    # Loads models locally via AirLLM (https://github.com/lyogavin/airllm), which
    # streams model layers from disk so multi-billion-parameter models can run
    # on a single GPU at the cost of generation speed. No streaming token
    # callback is available, so streaming is simulated by chunking.
    # NOTE: requires Linux + a CUDA GPU. On macOS, airllm uses a different
    # MLX-only backend (Llama models only) that this integration does not support.
    model: "garage-bAInd/Platypus2-70B-instruct"  # HuggingFace model ID or local path
    device: "cuda:0"               # AirLLM currently targets a single CUDA device
    compression: null             # Options: "4bit", "8bit", null (none)
    max_seq_len: 4096             # Maximum sequence length (context window)
    hf_token: null                # Optional HuggingFace token for gated models
    layer_shards_saving_path: null  # Optional custom path for converted layer shards
    delete_original: false        # Delete original safetensors after converting to layer shards
    profiling_mode: false         # Enable AirLLM's internal profiling/timing logs
    temperature: 0.1
    top_p: 0.8
    top_k: -1                     # Set to -1 to disable
    max_tokens: 512
    repetition_penalty: 1.0       # Penalty for repeating tokens (1.0 = no penalty)
    do_sample: true
    stream: true
    stop: []
  openrouter:
    enabled: true
    api_key: ${OPENROUTER_API_KEY}
    base_url: "https://openrouter.ai/api/v1"
    model: "meta-llama/llama-3.1-8b-instruct"
    context_window: 131072  # Llama 3.1 supports 128K context
    temperature: 0.1
    top_p: 0.8
    max_tokens: 1024
    stream: true
    verbose: false
  cohere:
    enabled: true
    api_key: ${COHERE_API_KEY}
    api_base: "https://api.cohere.ai/compatibility/v1"
    model: "command-r7b-12-2024"
    context_window: 128000  # Command A API limit (input + output); reserve space for max_tokens
    temperature: 0.3
    top_p: 0.8
    max_tokens: 4000 # Max output tokens (on-demand may cap at 4K; dedicated has no cap)
    stream: true
  watson:
    enabled: false
    api_key: ${WATSON_API_KEY}
    api_base: "https://domain.region.cloud.ibm.com"
    project_id: "your-project_id"
    instance_id: "openshift"
    model: "ibm/granite-3-8b-instruct"
    context_window: 8192  # Granite 3 8B supports 8K context
    temperature: 0.1
    top_k: 20
    top_p: 0.8
    max_tokens: 1024
    stream: true
    show_thinking: false
    space_id: ""
    region: "your-region"
    auth_type: "iam"
    time_limit: 10000
    verify: false
  perplexity:
    enabled: false
    api_key: ${PERPLEXITY_API_KEY}
    api_base: "https://api.perplexity.ai"
    model: "llama-3-sonar-small-32k-online"
    context_window: 32768  # Sonar 32K model
    temperature: 0.1
    top_p: 0.8
    max_tokens: 1024
    stream: true
  fireworks:
    enabled: false
    api_key: ${FIREWORKS_API_KEY}
    api_base: "https://api.fireworks.ai/inference/v1"
    model: "accounts/fireworks/models/firellava-13b"
    context_window: 4096  # FireLLaVA 13B context (varies by model)
    temperature: 0.1
    top_p: 0.8
    max_tokens: 1024
    stream: true
  replicate:
    enabled: false
    api_key: ${REPLICATE_API_KEY}
    model: "meta/llama-2-70b-chat:02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3"
    context_window: 4096  # Llama 2 70B supports 4K context
    temperature: 0.1
    top_p: 0.8
    max_tokens: 1024
    stream: true
  nvidia:
    enabled: false
    api_key: ${NVIDIA_API_KEY}
    api_base: "http://localhost:8000/v1"
    model: "meta/llama3-8b-instruct"
    context_window: 8192  # Llama 3 8B supports 8K context
    temperature: 0.1
    top_p: 0.8
    max_tokens: 1024
    stream: true
  bitnet:
    enabled: false
    # Mode: "direct" (load model locally) or "api" (connect to BitNet server)
    # NOTE: "direct" mode requires a `bitnet` Python package exposing BitNetInference,
    # which upstream microsoft/BitNet does not provide (see docs/bitnet-setup.md).
    # Use "api" mode with BitNet's run_inference_server.py instead.
    mode: "api"
    
    # Direct mode configuration
    # To get GGUF file run: ./install/setup.sh --download-gguf bitnet-2b-4t
    model_path: "models/ggml-model-i2_s.gguf"
    quant_type: "i2_s"  # Quantization type: i2_s or tl1
    use_pretuned: true  # Use pretuned kernel parameters
    quant_embd: false   # Quantize embeddings to f16
    
    # API mode configuration
    base_url: "http://localhost:8080"
    api_key: null  # Optional API key for secured servers
    
    # Generation parameters
    temperature: 0.7
    top_p: 0.9
    top_k: 40
    max_tokens: 1024
    
    # Context and threading
    n_ctx: 2048
    n_threads: 8
    n_batch: 2
    
    # GPU settings
    n_gpu_layers: 0  # Number of layers to offload to GPU (0 = CPU only, -1 = all)
    main_gpu: 0      # Main GPU device
    low_vram: false  # Enable for systems with limited VRAM
    
    # BitNet-specific optimizations
    kernel_params:
      enable_custom: false  # Use custom kernel parameters
      # Custom kernel parameters can be added here
    
    # Memory management
    use_mmap: true
    use_mlock: false
    
    # Streaming
    stream: true
    
    # Stop sequences
    stop: []
    
    # Timeout configuration
    timeout:
      connect: 10000   # 10 seconds
      total: 120000    # 2 minutes
    
    # Retry configuration
    retry:
      enabled: true
      max_retries: 3
      initial_wait_ms: 1000
      max_wait_ms: 30000
      exponential_base: 2
  zai:
    enabled: false
    api_key: ${ZAI_API_KEY}
    base_url: "https://api.z.ai/api/paas/v4/"
    model: "glm-4.6"
    context_window: 128000  # GLM-4 supports 128K context
    temperature: 0.1
    top_p: 0.8
    max_tokens: 2000
    stream: true
    # Timeout configuration
    timeout:
      connect: 10000   # 10 seconds
      total: 120000    # 2 minutes
    # Retry configuration
    retry:
      enabled: true
      max_retries: 3
      initial_wait_ms: 1000
      max_wait_ms: 30000
      exponential_base: 2
  cerebras:
    enabled: false
    api_key: ${CEREBRAS_API_KEY}
    model: "llama-3.3-70b"
    context_window: 131072  # Cerebras supports 128K context
    temperature: 0.1
    top_p: 0.8
    max_tokens: 8192
    stream: true
  deepinfra:
    enabled: false
    api_key: ${DEEPINFRA_API_KEY}
    model: "meta-llama/Meta-Llama-3.1-70B-Instruct"
    context_window: 131072  # Llama 3.1 supports 128K context
    temperature: 0.1
    top_p: 0.8
    max_tokens: 4096
    stream: true
  lmstudio:
    enabled: false
    api_key: "lm-studio"  # Placeholder — LM Studio local server ignores auth
    base_url: "http://localhost:1234/v1"
    model: "lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF"  # Update to match loaded model
    context_window: 32768
    temperature: 0.7
    top_p: 0.9
    max_tokens: 4096
    stream: true
  moonshot:
    enabled: false
    api_key: ${MOONSHOT_API_KEY}
    model: "moonshot-v1-128k"
    context_window: 131072  # Kimi supports 128K context
    temperature: 0.1
    top_p: 0.8
    max_tokens: 4096
    stream: true
  minimax:
    enabled: false
    api_key: ${MINIMAX_API_KEY}
    model: "MiniMax-Text-01"
    context_window: 1000000  # MiniMax-Text-01 supports 1M context
    temperature: 0.1
    top_p: 0.8
    max_tokens: 4096
    stream: true
  nearai:
    enabled: false
    api_key: ${NEARAI_API_KEY}
    model: "zai-org/GLM-5.1-FP8"
    context_window: 202752  # NEAR AI Cloud TEE model catalog, fetched 2026-05-21
    temperature: 0.1
    top_p: 0.8
    max_tokens: 4096
    stream: true
  nebius:
    enabled: false
    api_key: ${NEBIUS_API_KEY}
    model: "meta-llama/Meta-Llama-3.1-70B-Instruct"
    context_window: 131072  # Llama 3.1 supports 128K context
    temperature: 0.1
    top_p: 0.8
    max_tokens: 4096
    stream: true
  venice:
    enabled: false
    api_key: ${VENICE_API_KEY}
    model: "llama-3.3-70b"
    context_window: 131072  # Llama 3.3 supports 128K context
    temperature: 0.1
    top_p: 0.8
    max_tokens: 4096
    stream: true
  scaleway:
    enabled: false
    api_key: ${SCALEWAY_API_KEY}
    model: "llama-3.3-70b-instruct"
    context_window: 131072  # Llama 3.3 supports 128K context
    temperature: 0.1
    top_p: 0.8
    max_tokens: 4096
    stream: true
  fugu:
    enabled: true
    api_key: ${FUGU_API_KEY}
    base_url: "https://api.sakana.ai/v1"
    model: "fugu"  # Options: fugu, fugu-ultra, fugu-ultra-20260615
    temperature: 0.1
    top_p: 0.8
    max_tokens: 4096
    stream: true
    reasoning_effort: "high"  # Options: high, xhigh, max