config.yaml

# Master configuration file for Synthetic Data Kit

# Global paths configuration
paths:
  # Input data location (directory containing files to process)
  input: "data/input"           # Directory containing PDF, HTML, DOCX, PPT, TXT files
  
  # Output locations (4-stage pipeline directories)
  output:
    parsed: "data/parsed"       # Stage 1: Where parsed text files are saved (ingest output)
    generated: "data/generated" # Stage 2: Where generated QA pairs are saved (create output)
    curated: "data/curated"     # Stage 3: Where curated QA pairs are saved (curate output)
    final: "data/final"         # Stage 4: Where final training formats are saved (save-as output)

# LLM Provider configuration
llm:
  # Provider selection: "vllm" or "api-endpoint"
  provider: "api-endpoint"

# VLLM server configuration
vllm:
  api_base: "http://localhost:8000/v1" # Base URL for VLLM API
  port: 8000                           # Port for VLLM server
  model: "meta-llama/Llama-3.3-70B-Instruct" # Default model to use
  max_retries: 3                       # Number of retries for API calls
  retry_delay: 1.0                     # Initial delay between retries (seconds)
  sleep_time: 0.1                      # Small delay in seconds between batches to avoid rate limits
  http_request_timeout: 180            # Http Request timeout in seconds (3 minutes)
  
# API endpoint configuration
api-endpoint:
  api_base: "https://api.llama.com/v1" # Optional base URL for API endpoint (null for default API)
  api_key: "llama_api_key"               # API key for API endpoint or compatible service (can use env var instead)
  model: "Llama-4-Maverick-17B-128E-Instruct-FP8" # Default model to use
  max_retries: 3                       # Number of retries for API calls
  retry_delay: 1.0                     # Initial delay between retries (seconds)
  sleep_time: 0.5                      # Small delay in seconds between batches to avoid rate limits

# Ingest configuration
ingest:
  default_format: "txt"  # Default output format for parsed files
  youtube_captions: "auto"  # Options: "auto", "manual" - caption preference

# LLM generation parameters
generation:
  temperature: 0.7   # Higher = more creative, lower = more deterministic
  top_p: 0.95        # Nucleus sampling parameter
  chunk_size: 4000   # Size of text chunks for processing
  overlap: 200       # Overlap between chunks to maintain context
  max_tokens: 4096   # Maximum tokens in LLM responses
  num_pairs: 25      # Default number of QA pairs to generate
  num_cot_examples: 5  # Default number of Chain of Thought examples to generate
  num_cot_enhance_examples: null  # Maximum number of conversations to enhance (null = enhance all)
  batch_size: 32     # Number of requests to batch together (for create)
  max_context_length: 8000       # Context Length of the MODEL. Useful while Generating Summary
  summary_overlap: 0       # Overlap between chunks to maintain context. Useful while Generating Summary
  
# Content curation parameters
curate:
  threshold: 7.0     # Default quality threshold (1-10)
  batch_size: 5      # Number of items per batch for rating (smaller batches for API stability)
  inference_batch: 5 # Number of batches to process at once with VLLM
  temperature: 0.1   # Temperature for rating (lower = more consistent)

# Format conversion parameters
format:
  default: "jsonl"   # Default output format
  include_metadata: true  # Include metadata in output files
  pretty_json: true  # Use indentation in JSON output

# Prompts for different tasks
prompts:
  # Summary generation prompt
  summary: |
    Summarize this document in 3-5 sentences, focusing on the main topic and key concepts.
  
  # QA pair generation prompt
  qa_generation: |
    Create question-answer pairs from this text for LLM training.
    
    Rules:
    1. Questions must be about important facts in the text
    2. Answers must be directly supported by the text
    3. Return JSON format only:
    
    [
      {{
        "question": "Question 1?",
        "answer": "Answer 1."
      }},
      {{
        "question": "Question 2?",
        "answer": "Answer 2."
      }}
    ]
    
    Text:
    {text}
  
  # QA pair rating prompt
  qa_rating: |
    Rate each question-answer pair on a scale from 1-10, based on:
    - Accuracy (0-3): factual correctness
    - Relevance (0-2): relevance to content
    - Clarity (0-2): clear language
    - Usefulness (0-3): value for model learning
    
    YOU MUST RETURN A VALID JSON OBJECT OR ARRAY WITH THIS EXACT SCHEMA:
    {{
      "question": "Exact question text",
      "answer": "Exact answer text",
      "rating": 8
    }}
    
    OR FOR MULTIPLE PAIRS:
    [
      {{"question": "Q1", "answer": "A1", "rating": 8}},
      {{"question": "Q2", "answer": "A2", "rating": 9}}
    ]
    
    *** YOUR RESPONSE MUST BE VALID JSON AND NOTHING ELSE - NO EXPLANATION, NO MARKDOWN ***
    
    QA pairs to rate:
    {pairs}
    
  # Chain of Thought generation prompt
  cot_generation: |
    Create complex reasoning examples from this text that demonstrate chain-of-thought thinking.
    
    Each example should have:
    1. A challenging question that requires step-by-step reasoning
    2. Detailed reasoning steps that break down the problem
    3. A concise final answer
    
    Return JSON format only:
    
    [
      {{
        "question": "Complex question about the text?",
        "reasoning": "Step 1: First, I need to consider...\nStep 2: Then, I analyze...\nStep 3: Finally, I can conclude...",
        "answer": "Final answer based on the reasoning."
      }},
      {{
        "question": "Another complex question?",
        "reasoning": "Step 1: First, I'll analyze...\nStep 2: Next, I need to determine...\nStep 3: Based on this analysis...",
        "answer": "Final answer drawn from the reasoning."
      }}
    ]
    
    Text:
    {text}
  
  # Chain of Thought enhancement prompt
  cot_enhancement: |
    You are an expert reasoning assistant. Your task is to enhance the given conversations by adding chain-of-thought reasoning.
    
    For each conversation, add detailed step-by-step reasoning to the assistant's responses while preserving the original answer.
    
    {include_simple_steps} = Whether to add reasoning to simple responses too. If false, only add reasoning to complex responses.
    
    Return the enhanced conversations as a JSON array matching this format:
    [
      [
        {{"role": "system", "content": "System message"}},
        {{"role": "user", "content": "User question"}},
        {{"role": "assistant", "content": "Let me think through this step by step:\n\n1. First, I need to consider...\n2. Then...\n\nTherefore, [original answer]"}}
      ],
      [
        {{"role": "system", "content": "System message"}},
        {{"role": "user", "content": "Another user question"}},
        {{"role": "assistant", "content": "Let me work through this:\n\n1. I'll start by...\n2. Next...\n\nIn conclusion, [original answer]"}}
      ]
    ]
    
    Original conversations:
    {conversations}