validate.py

"""
Pre-submission validation script for AdaptiveTutor-Env.

Checks all items on the Pre-Submission Checklist:
  1. openenv.yaml is valid and parseable
  2. All typed models are importable
  3. reset() returns a valid Observation
  4. step() returns valid (Observation, Reward, done, info)
  5. state() returns a dict with required keys
  6. 3+ tasks exist
  7. Each grader produces a score in [0.0, 1.0]
  8. Graders are deterministic

Run with:  python validate.py
"""

from __future__ import annotations
import sys
import yaml
import traceback
from pathlib import Path
from typing import List, Tuple

PASS = "✅"
FAIL = "❌"
results: List[Tuple[str, bool, str]] = []


def check(name: str, fn) -> bool:
    try:
        msg = fn()
        results.append((name, True, msg or "OK"))
        return True
    except Exception as e:
        results.append((name, False, str(e)))
        return False


# ─── Checks ───────────────────────────────────────────────────────────────────


def check_yaml():
    with open("openenv.yaml") as f:
        data = yaml.safe_load(f)
    required = [
        "name",
        "version",
        "description",
        "tasks",
        "action_space",
        "observation_space",
    ]
    missing = [k for k in required if k not in data]
    if missing:
        raise ValueError(f"Missing keys in openenv.yaml: {missing}")
    n_tasks = len(data["tasks"])
    return f"YAML valid. {n_tasks} tasks defined."


def check_imports():
    from server.models import Action, Observation, Reward, StepResponse
    from server.environment import AdaptiveTutorEnv
    from server.tasks import TASK_CATALOGUE
    from server.reward import compute_reward

    return "All modules imported successfully."


def check_reset():
    from server.environment import AdaptiveTutorEnv

    env = AdaptiveTutorEnv()
    obs = env.reset(task_id="single_subject_mastery", seed=42)
    assert obs.step == 0
    assert obs.task_id == "single_subject_mastery"
    assert 0.0 <= obs.overall_mastery <= 1.0
    assert obs.last_action is None
    return f"reset() OK. Initial mastery={obs.overall_mastery:.3f}"


def check_step():
    from server.environment import AdaptiveTutorEnv
    from server.models import Action

    env = AdaptiveTutorEnv()
    env.reset(seed=0)
    result = env.step(
        Action(
            subject="mathematics",
            topic="algebra",
            activity_type=1,
            difficulty=0,
            strategy=0,
        )
    )
    assert hasattr(result, "observation")
    assert hasattr(result, "reward")
    assert isinstance(result.done, bool)
    assert -1.0 <= result.reward.total <= 1.0
    return f"step() OK. reward={result.reward.total:.4f} done={result.done}"


def check_state():
    from server.environment import AdaptiveTutorEnv

    env = AdaptiveTutorEnv()
    env.reset(seed=0)
    s = env.state()
    required = ["initialized", "task_id", "step", "masteries", "engagement", "fatigue"]
    missing = [k for k in required if k not in s]
    if missing:
        raise ValueError(f"state() missing keys: {missing}")
    return f"state() OK. Keys present: {list(s.keys())[:6]}..."


def check_tasks():
    from server.tasks import TASK_CATALOGUE

    assert len(TASK_CATALOGUE) >= 3, f"Only {len(TASK_CATALOGUE)} tasks found"
    diffs = [t.difficulty for t in TASK_CATALOGUE.values()]
    assert "easy" in diffs, "No easy task"
    assert "medium" in diffs, "No medium task"
    assert "hard" in diffs, "No hard task"
    return f"{len(TASK_CATALOGUE)} tasks: {list(TASK_CATALOGUE.keys())}"


def check_graders():
    from server.environment import AdaptiveTutorEnv
    from server.models import Action, SUBJECTS, TOPICS_PER_SUBJECT
    from server.tasks import TASK_CATALOGUE

    scores = {}
    for task_id in TASK_CATALOGUE:
        env = AdaptiveTutorEnv()
        env.reset(task_id=task_id, seed=3)
        done = False
        step = 0
        while not done and step < TASK_CATALOGUE[task_id].max_steps + 5:
            subj = SUBJECTS[step % len(SUBJECTS)]
            topic = TOPICS_PER_SUBJECT[subj][step % 5]
            result = env.step(
                Action(
                    subject=subj,
                    topic=topic,
                    activity_type=1,
                    difficulty=1,
                    strategy=1,
                )
            )
            done = result.done
            step += 1
        grade_result = env.grade_episode()
        assert 0.002 <= grade_result.score <= 0.998, (
            f"Score {grade_result.score} out of range [0.002, 0.998] for task {task_id}"
        )
        scores[task_id] = round(grade_result.score, 4)

    return f"Grader scores: {scores}"


def check_grader_determinism():
    from server.environment import AdaptiveTutorEnv
    from server.models import Action

    env = AdaptiveTutorEnv()
    env.reset(task_id="multi_subject_balancing", seed=7)
    done = False
    while not done:
        result = env.step(
            Action(
                subject="mathematics",
                topic="algebra",
                activity_type=0,
                difficulty=1,
                strategy=1,
            )
        )
        done = result.done

    r1 = env.grade_episode()
    r2 = env.grade_episode()
    assert r1.score == r2.score, f"Non-deterministic grader: {r1.score} vs {r2.score}"
    return f"Grader deterministic (score={r1.score:.4f})."


def check_reward_density():
    from server.environment import AdaptiveTutorEnv
    from server.models import Action, SUBJECTS, TOPICS_PER_SUBJECT

    env = AdaptiveTutorEnv()
    env.reset(task_id="multi_subject_balancing", seed=1)
    rewards = []
    for i in range(10):
        subj = SUBJECTS[i % len(SUBJECTS)]
        topic = TOPICS_PER_SUBJECT[subj][i % 5]
        result = env.step(
            Action(
                subject=subj,
                topic=topic,
                activity_type=1,
                difficulty=1,
                strategy=1,
            )
        )
        rewards.append(result.reward.total)
        if result.done:
            break

    all_zero = all(r == 0.0 for r in rewards)
    assert not all_zero, "All rewards are zero — reward function not working"
    unique_vals = len(set(round(r, 4) for r in rewards))
    return f"Dense rewards over {len(rewards)} steps. Unique values: {unique_vals}."


# ─── Run All Checks ───────────────────────────────────────────────────────────

CHECKS = [
    ("openenv.yaml valid", check_yaml),
    ("Module imports", check_imports),
    ("reset() API", check_reset),
    ("step() API", check_step),
    ("state() API", check_state),
    ("3+ tasks with difficulty range", check_tasks),
    ("Graders score in (0.002, 0.998)", check_graders),
    ("Graders deterministic", check_grader_determinism),
    ("Dense reward signal", check_reward_density),
]


def main():
    print("=" * 60)
    print("AdaptiveTutor-Env — Pre-Submission Validation")
    print("=" * 60)

    passed = 0
    for name, fn in CHECKS:
        ok = check(name, fn)
        if ok:
            passed += 1

    print()
    for name, ok, msg in results:
        icon = PASS if ok else FAIL
        print(f"{icon} {name}")
        if not ok:
            print(f"   ↳ {msg}")
        else:
            print(f"   ↳ {msg}")

    print()
    print(f"{'=' * 60}")
    print(f"Result: {passed}/{len(CHECKS)} checks passed.")
    if passed == len(CHECKS):
        print("✅ All checks passed. Ready to submit!")
    else:
        print("❌ Some checks failed. Please fix before submitting.")
    print("=" * 60)

    sys.exit(0 if passed == len(CHECKS) else 1)


if __name__ == "__main__":
    main()