fix: skip prompt cache hints for Bedrock Nova and add per-model override

holmes/core/llm.py

@@ -177,6 +177,27 @@ def _is_gemini_route(litellm_model_name: str) -> bool:
     return False
 
 
+def _supports_prompt_caching(litellm_model_name: str) -> bool:
+    """True if the provider/model accepts cache-control hints.
+
+    Gemini routes (Google AI Studio and Vertex-AI hosted Gemini) reject
+    GenerateContent requests combining CachedContent with system_instruction /
+    tools / tool_config, which is exactly what cache_control_injection_points
+    produces. Amazon Bedrock Nova models likewise reject the cachePoint field
+    that LiteLLM translates the hint into on Bedrock. Other providers -
+    including non-Gemini models on Vertex and Bedrock Claude - keep their
+    cache benefit. Add new incompatible routes here rather than special-casing
+    the call site.
+    """
+    if _is_gemini_route(litellm_model_name):
+        return False
+    if litellm_model_name.startswith("bedrock/"):
+        # Bedrock routes carry a regional alias (e.g. bedrock/us.amazon.nova-pro-v1:0);
+        # only the Nova family rejects cachePoint.
+        return "amazon.nova" not in litellm_model_name.split("/", 1)[1].lower()
+    return True
+
+
 class ContextWindowUsage(BaseModel):
     total_tokens: int
     tools_tokens: int
@@ -202,6 +223,13 @@ class ModelEntry(BaseModel):
     api_base: Optional[str] = None
     api_version: Optional[str] = None
 
+    # Explicit prompt-caching override. True forces cache_control hints, False
+    # suppresses them, None (default) falls back to the automatic per-route
+    # default (_supports_prompt_caching). Set this for models whose provider
+    # rejects cache hints and isn't covered by the default yet, e.g.
+    # cache_control: false for a Bedrock model.
+    cache_control: Optional[bool] = None
+
     model_config = ConfigDict(
         extra="allow",
     )
@@ -358,6 +386,7 @@ class DefaultLLM(LLM):
     api_version: Optional[str]
     args: Dict
     is_robusta_model: bool
+    cache_control: Optional[bool] = None
 
     def __init__(
         self,
@@ -386,6 +415,9 @@ def __init__(
     def update_custom_args(self):
         self.max_context_size = self.args.get("custom_args", {}).get("max_context_size")
         self.args.pop("custom_args", None)
+        # ModelEntry.cache_control explicit override flows in via args; pop it
+        # so it never leaks into the litellm.completion call.
+        self.cache_control = self.args.pop("cache_control", None)
 
     def check_llm(
         self,
@@ -737,14 +769,18 @@ def completion(
             # Leave api_key as None in completion call when AZURE_AD_TOKEN_AUTH is enabled
             self.api_key = None
 
-        # Gemini rejects GenerateContent requests that combine CachedContent with
-        # system_instruction / tools / tool_config, which is exactly what
-        # cache_control_injection_points produces for us. Skip the cache hint for
-        # Gemini routes (both Google AI Studio and Vertex-AI hosted Gemini); other
-        # providers - including non-Gemini models on Vertex like Claude - keep
-        # their cache benefit.
+        # Some providers reject cache-control hints: Gemini rejects
+        # GenerateContent requests that combine CachedContent with
+        # system_instruction / tools / tool_config, and Bedrock Nova rejects
+        # the cachePoint field - both of which cache_control_injection_points
+        # produces. Skip the cache hint for those routes (see
+        # _supports_prompt_caching); a per-model `cache_control` override in
+        # model_list.yaml wins over the automatic default.
         cache_kwargs: Dict[str, Any] = {}
-        if not _is_gemini_route(litellm_model_name):
+        cache_control = self.cache_control
+        if cache_control is None:
+            cache_control = _supports_prompt_caching(litellm_model_name)
+        if cache_control:
             cache_kwargs["cache_control_injection_points"] = [
                 {
                     "location": "message",

tests/core/test_llm_completion_cache_control.py

@@ -33,6 +33,7 @@ def _make_llm(model: str) -> DefaultLLM:
     llm.name = None
     llm.is_robusta_model = False
     llm.max_context_size = None
+    llm.cache_control = None
     return llm
 
 
@@ -70,6 +71,24 @@ def test_gemini_models_skip_cache_control(self, mock_completion, model):
             "Gemini rejects CachedContent + system_instruction/tools/tool_config."
         )
 
+    @pytest.mark.parametrize(
+        "model",
+        [
+            "bedrock/us.amazon.nova-pro-v1:0",
+            "bedrock/us.amazon.nova-lite-v1:0",
+            "bedrock/us.amazon.nova-micro-v1:0",
+            "bedrock/eu-west-1.amazon.nova-pro-v1:0",
+        ],
+    )
+    def test_bedrock_nova_models_skip_cache_control(self, mock_completion, model):
+        llm = _make_llm(model)
+        llm.completion(messages=[{"role": "user", "content": "hi"}])
+        kwargs = mock_completion.call_args.kwargs
+        assert "cache_control_injection_points" not in kwargs, (
+            f"cache_control_injection_points must not be sent to {model}; "
+            "Bedrock Nova rejects the cachePoint field it translates into."
+        )
+
     @pytest.mark.parametrize(
         "model",
         [
@@ -98,3 +117,54 @@ def test_gemini_completion_still_forwards_messages_and_model(self, mock_completi
         assert kwargs["model"] == "gemini/gemini-3.1-pro-preview"
         assert kwargs["messages"] == messages
         assert kwargs["temperature"] == 0.3
+
+
+class TestCacheControlOverride:
+    """A per-model `cache_control` field in model_list.yaml must override the
+    automatic per-route default: False suppresses the cache hint even for
+    models that normally support it, True forces it for models that normally
+    skip it (e.g. a Bedrock Nova model not covered by the default yet).
+    """
+
+    @pytest.mark.parametrize(
+        "model",
+        [
+            "gemini/gemini-3.1-pro-preview",
+            "bedrock/us.amazon.nova-pro-v1:0",
+        ],
+    )
+    def test_cache_control_override_force_on(self, mock_completion, model):
+        llm = _make_llm(model)
+        llm.cache_control = True
+        llm.completion(messages=[{"role": "user", "content": "hi"}])
+        kwargs = mock_completion.call_args.kwargs
+        assert kwargs.get("cache_control_injection_points") == [
+            {"location": "message", "index": -1}
+        ], f"cache_control: true must force the cache hint for {model}"
+
+    @pytest.mark.parametrize(
+        "model",
+        [
+            "openai/gpt-4o",
+            "anthropic/claude-sonnet-4-5",
+            "bedrock/anthropic.claude-sonnet-4-20250514-v1:0",
+        ],
+    )
+    def test_cache_control_override_force_off(self, mock_completion, model):
+        llm = _make_llm(model)
+        llm.cache_control = False
+        llm.completion(messages=[{"role": "user", "content": "hi"}])
+        kwargs = mock_completion.call_args.kwargs
+        assert "cache_control_injection_points" not in kwargs, (
+            f"cache_control: false must suppress the cache hint for {model}"
+        )
+
+    def test_cache_control_popped_from_args(self):
+        """The override flows into DefaultLLM.args from model_list.yaml and must
+        be consumed by update_custom_args, never leaking into the litellm call."""
+        llm = DefaultLLM.__new__(DefaultLLM)
+        llm.args = {"cache_control": False, "temperature": 0.1}
+        llm.update_custom_args()
+        assert llm.cache_control is False
+        assert "cache_control" not in llm.args
+        assert llm.args == {"temperature": 0.1}