test(v2): cover provider-owned client architecture

Author: jxnl-oai

.github/workflows/ty.yml

@@ -27,6 +27,8 @@ jobs:
         run: uv run ty check --error-on-warning --output-format github instructor/
       - name: Run type check with ty (tests)
         run: uv run ty check --config-file ty-tests.toml --error-on-warning --output-format github tests
+      - name: Run public surface inference check with Pyright
+        run: uv run --with 'pyright==1.1.408' pyright tests/typing/test_public_surface.py
       - name: Check installed-package public typing
         run: tests/typing/check_installed_package.sh
       - name: Check supported Python versions and platforms

tests/llm/test_openai/test_multimodal.py

@@ -8,6 +8,8 @@
 import base64
 import os
 
+from instructor.core.exceptions import InstructorRetryException
+
 audio_url = "https://raw.githubusercontent.com/instructor-ai/instructor/main/tests/assets/gettysburg.wav"
 image_url = "https://raw.githubusercontent.com/instructor-ai/instructor/main/tests/assets/image.jpg"
 
@@ -60,21 +62,32 @@ def test_multimodal_audio_description(audio_file, mode, client):
     class AudioDescription(BaseModel):
         source: str
 
-    response = client.chat.completions.create(
-        model="gpt-audio-1.5",
-        response_model=AudioDescription,
-        modalities=["text"],
-        messages=[
-            {
-                "role": "user",
-                "content": [
-                    "Where's this excerpt from?",
-                    audio_file,
-                ],  # type: ignore
-            },
-        ],
-        audio={"voice": "alloy", "format": "wav"},  # type: ignore
-    )
+    try:
+        response = client.chat.completions.create(
+            model="gpt-audio-1.5",
+            response_model=AudioDescription,
+            modalities=["text"],
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        "Where's this excerpt from?",
+                        audio_file,
+                    ],  # type: ignore
+                },
+            ],
+            audio={"voice": "alloy", "format": "wav"},  # type: ignore
+        )
+    except InstructorRetryException as exc:
+        message = str(exc).lower()
+        if (
+            "model_not_found" in message
+            or "does not exist or you do not have access" in message
+        ):
+            pytest.skip(f"Audio model unavailable in this environment: {exc}")
+        raise
+
+    assert isinstance(response, AudioDescription)
 
 
 class ImageDescription(BaseModel):

tests/test_lazy_imports.py

@@ -1,9 +1,13 @@
 import importlib
 import json
+import os
 import subprocess
 import sys
+from pathlib import Path
 from typing import TypedDict
 
+_PROJECT_ROOT = Path(__file__).resolve().parents[1]
+
 
 class ColdImportState(TypedDict):
     modules: list[str]
@@ -21,11 +25,17 @@ def _cold_import_state(code: str) -> ColdImportState:
 )
 print(json.dumps(mods))
 """
+    env = dict(os.environ)
+    env["PYTHONPATH"] = os.pathsep.join(
+        path for path in (str(_PROJECT_ROOT), env.get("PYTHONPATH")) if path
+    )
     result = subprocess.run(
         [sys.executable, "-c", probe, code],
         check=True,
         capture_output=True,
         text=True,
+        cwd=_PROJECT_ROOT,
+        env=env,
     )
     modules = json.loads(result.stdout)
     return {"modules": modules, "count": len(modules)}

tests/v2/provider_matrix.py

@@ -4,7 +4,6 @@
 
 import importlib.util
 from pathlib import Path
-from typing import Any
 
 import pytest
 
@@ -23,25 +22,31 @@
 PROVIDER_HANDLER_MODES = {
     provider: spec.supported_modes for provider, spec in PROVIDER_SPECS.items()
 }
-
-
-def legacy_config_dicts() -> dict[Provider, dict[str, Any]]:
-    """Expose the old dict shape while the baseline tests migrate."""
-    return {
-        provider: {
-            "provider_string": spec.provider_string,
-            "supported_modes": list(spec.supported_modes),
-            "unsupported_modes": list(spec.unsupported_modes),
-            "legacy_modes": spec.legacy_modes,
-            "from_function": spec.from_function,
-            "sdk_module": spec.sdk_module,
-            "basic_modes": list(spec.basic_modes),
-            "async_modes": list(spec.async_modes),
-            "missing_sdk_message": spec.missing_sdk_message,
-        }
-        for provider, spec in TEST_PROVIDER_SPECS.items()
-    }
-
+PARTIAL_STREAM_CASES = tuple(
+    (provider, mode)
+    for provider, spec in TEST_PROVIDER_SPECS.items()
+    for mode in spec.capabilities.partial_stream_modes
+)
+ITERABLE_STREAM_CASES = tuple(
+    (provider, mode)
+    for provider, spec in TEST_PROVIDER_SPECS.items()
+    for mode in spec.capabilities.iterable_stream_modes
+)
+TYPED_MULTIMODAL_PROVIDERS = tuple(
+    provider
+    for provider, spec in TEST_PROVIDER_SPECS.items()
+    if spec.capabilities.multimodal_inputs
+)
+TYPED_MULTIMODAL_CASES = tuple(
+    (provider, media_type)
+    for provider, spec in TEST_PROVIDER_SPECS.items()
+    for media_type in spec.capabilities.multimodal_inputs
+)
+EXPLICIT_PARALLEL_PROVIDERS = tuple(
+    provider
+    for provider, spec in TEST_PROVIDER_SPECS.items()
+    if spec.capabilities.explicit_parallel_tools
+)
 
 _PROJECT_ROOT = Path(__file__).resolve().parents[2]
 _HANDLERS_LOADED: set[Provider] = set()

tests/v2/test_anthropic_parallel_runtime.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+import pytest
+from pydantic import BaseModel
+
+from instructor.v2.providers.anthropic import parallel
+
+
+class Weather(BaseModel):
+    city: str
+
+
+class Score(BaseModel):
+    value: int
+
+
+def test_parallel_schema_generation_is_owned_by_anthropic(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    calls: list[type[BaseModel]] = []
+
+    def fake_schema(model: type[BaseModel]) -> dict[str, str]:
+        calls.append(model)
+        return {"name": model.__name__}
+
+    monkeypatch.setattr(parallel, "generate_anthropic_schema", fake_schema)
+
+    schemas = parallel.handle_parallel_model(Iterable[Weather | Score])
+
+    assert schemas == [{"name": "Weather"}, {"name": "Score"}]
+    assert calls == [Weather, Score]