- Apple Silicon (M1/M2/M3) with 16GB+ RAM (24GB+ recommended for 7B models)
- macOS Sonoma 14.0+ with Xcode 15.4+
# Install Homebrew (if not already installed)
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
# Install Python and system tools
brew install python@3.10 git cmake
# Create virtual environment
python3.10 -m venv ~/grpo-env
source ~/grpo-env/bin/activate
# Install PyTorch with MPS support
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
# Install vLLM from source (experimental Mac support)
git clone https://github.com/vllm-project/vllm.git
cd vllm && pip install -r requirements-cpu.txt && pip install -e .from transformers import AutoModelForCausalLM, AutoTokenizer
model_name = "Qwen/Qwen2.5-7B"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(
model_name,
torch_dtype=torch.float16,
device_map="auto",
trust_remote_code=True
)import os
import torch
import wandb
from datasets import load_dataset
from transformers import (
AutoTokenizer,
AutoModelForCausalLM,
TrainingArguments,
set_seed
)
from trl import GRPOConfig, GRPOTrainer
from peft import LoraConfig, get_peft_model
# Initialize W&B
wandb.init(project="qwen-first-principles", name="grpo-mac-run")
class FirstPrinciplesTrainer:
def __init__(self):
self.config = {
"model_name": "Qwen/Qwen2.5-7B",
"lora_config": {
"r": 32,
"lora_alpha": 64,
"target_modules": ["q_proj", "v_proj"],
"lora_dropout": 0.1,
"bias": "none",
"task_type": "CAUSAL_LM"
},
"grpo_config": {
"num_generations": 4,
"temperature": 0.7,
"max_prompt_length": 512,
"max_completion_length": 1024,
"kl_penalty": 0.1
},
"training_args": {
"output_dir": "./grpo-output",
"num_train_epochs": 1,
"per_device_train_batch_size": 2,
"gradient_accumulation_steps": 8,
"learning_rate": 1e-6,
"fp16": True,
"logging_steps": 10,
"optim": "adamw_torch",
"report_to": "wandb"
}
}
def load_model(self):
self.tokenizer = AutoTokenizer.from_pretrained(self.config["model_name"])
self.model = AutoModelForCausalLM.from_pretrained(
self.config["model_name"],
torch_dtype=torch.float16,
device_map="auto",
trust_remote_code=True
)
# Apply LoRA for memory efficiency
peft_config = LoraConfig(**self.config["lora_config"])
self.model = get_peft_model(self.model, peft_config)
self.model.print_trainable_parameters()
def create_reward_function(self, response):
"""Verifiable first principles reward function"""
score = 0.0
# Structure validation
if re.search(r'<reasoning>.*?</reasoning>\s*<answer>.*?</answer>', response):
score += 0.3
# First principles indicators
fp_keywords = ['fundamental', 'core concept', 'building block', 'basic principle']
score += 0.1 * sum(1 for kw in fp_keywords if kw in response.lower())
# Feynman technique check
feynman_indicators = ['analogy', 'simple terms', 'imagine that', 'for example']
score += 0.1 * sum(1 for fi in feynman_indicators if fi in response.lower())
# Answer correctness (simple placeholder)
if "42" in response: # Replace with actual validation
score += 0.5
return min(score, 1.0)
def prepare_dataset(self):
"""Create first principles prompts dataset"""
return load_dataset("json", data_files={"train": "first_principles_prompts.json"})
def train(self):
grpo_config = GRPOConfig(
**self.config["grpo_config"],
use_vllm=True,
vllm_mode="colocate",
vllm_gpu_memory_utilization=0.7
)
training_args = TrainingArguments(
**self.config["training_args"],
remove_unused_columns=False,
dataloader_num_workers=0 # Required for Mac compatibility
)
trainer = GRPOTrainer(
model=self.model,
args=training_args,
train_dataset=self.prepare_dataset()["train"],
tokenizer=self.tokenizer,
reward_func=self.create_reward_function
)
trainer.train()
if __name__ == "__main__":
trainer = FirstPrinciplesTrainer()
trainer.load_model()
trainer.train()[
{
"prompt": "Using first principles, explain why objects fall at the same rate in a vacuum. Format your answer with <reasoning> and <answer> blocks.",
"category": "physics"
},
{
"prompt": "Break down the concept of supply and demand from fundamental economic principles. Use the Feynman technique in your explanation.",
"category": "economics"
}
]# Activate environment
source ~/grpo-env/bin/activate
# Login to W&B
wandb login
# Start training with MPS acceleration
MPS_DEVICE=1 CUDA_VISIBLE_DEVICES=0 python grpo_train.py# Add to training arguments
training_args = TrainingArguments(
...
gradient_checkpointing=True,
optim="adafactor",
torch_compile=True # Use Metal Shader Graph
)GRPOConfig(
...
vllm_max_model_len=2048,
vllm_enable_chunked_prefill=True,
vllm_gpu_memory_utilization=0.8
)
GRPO Training Workflow for First Principles Reasoning on Mac
)
def evaluate_response(prompt, response):
wandb.log({
"reasoning_score": float('<reasoning>' in response),
"answer_score": float('<answer>' in response),
"fp_keywords": sum(1 for kw in fp_keywords if kw in response),
"feynman_score": sum(1 for fi in feynman_indicators if fi in response)
})from peft import PeftModel
# Load fine-tuned model
model = PeftModel.from_pretrained(
AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-7B"),
"./grpo-output"
)
# Merge LoRA weights
model = model.merge_and_unload()- MPS Memory Errors:
- Reduce batch size:
per_device_train_batch_size=1 - Enable gradient checkpointing
- Use
torch.mps.empty_cache()periodically
- Reduce batch size:
- vLLM Compatibility:
# Reinstall with specific flags
CMAKE_ARGS="-DLLAMA_METAL=on" pip install -e .- Performance Optimization:
# Enable Metal kernels
torch.set_float32_matmul_precision('high')This guide provides an end-to-end solution for implementing verifiable first principles reasoning with Qwen on Apple Silicon, leveraging GRPO's efficiency and Mac's native Metal acceleration.
⁂