feat(index): support raw-query ivf rq search

docs/src/format/index/vector/index.md

@@ -254,6 +254,7 @@
 | `num_bits`            | u8   | Number of bits per dimension, in the range 1..=9     |
 | `code_dim`            | u32  | Rotated vector dimension for the 1-bit binary code   |
 | `packed`              | bool | Whether codes are packed for optimized computation   |
+| `query_estimator`     | string | Distance estimator layout: `residual_query` or `raw_query`. Missing values are read as `residual_query` for compatibility with released 1-bit IVF_RQ indexes. |
 
 #### Lance File Global Buffer
 
@@ -279,8 +280,9 @@
 The rotation matrix has shape `[code_dim, code_dim]` where `code_dim` is the rotated vector
 dimension. IVF_RQ always stores the 1-bit binary sign code in `_rabit_codes`; for `num_bits > 1`,
 the remaining `num_bits - 1` ex-code bits are stored in `__ex_codes` instead of widening the
-binary code path. `num_bits=1` indexes only store the binary-code factor columns; multi-bit indexes
-also store separate ex-code additive and scale factors.
+binary code path. New IVF_RQ indexes store raw-query estimator factors. `num_bits=1` indexes only
+store the binary-code factor columns; multi-bit indexes also store separate ex-code additive and
+scale factors.
 
 ## Appendices
 
@@ -342,10 +344,10 @@
 
 #### Auxiliary File
 
 - Arrow Schema Metadata:
   - `"distance_type"` → `"l2"`
   - `"lance:ivf"` → tracks per-partition `offsets` and `lengths` (no centroids here)
-  - `"lance:rabit"` → `"{"rotate_mat_position":1,"num_bits":1,"packed":true}"`
+  - `"lance:rabit"` → `"{"rotate_mat_position":1,"num_bits":1,"packed":true,"query_estimator":"raw_query"}"`
 - Lance File Global buffer:
   - `Tensor` rotation matrix with shape `[code_dim, code_dim]` = `[128, 128]` (float32)
 - Rows with Arrow schema:

python/python/tests/compat/compat_decorator.py

@@ -153,6 +153,10 @@ def skip_read_after_current_write(self, version: str) -> bool:
         """Return True to skip the old-version read after current-version writes."""
         return False
 
+    def skip_write_after_current_write(self, version: str) -> bool:
+        """Return True to skip the old-version write after current-version writes."""
+        return False
+
     def skip_downgrade(self, version: str) -> bool:
         """Return True to skip the current-write -> old-read downgrade test."""
         return False
@@ -333,8 +337,10 @@ def test_func({sig_params}):
         obj.create()
     # Old version: verify can read
     venv = venv_factory.get_venv(version)
-    venv.execute_method(obj, "check_read", obj.compat_env(version, "check_read"))
-    venv.execute_method(obj, "check_write", obj.compat_env(version, "check_write"))
+    if not obj.skip_read_after_current_write(version):
+        venv.execute_method(obj, "check_read", obj.compat_env(version, "check_read"))
+    if not obj.skip_write_after_current_write(version):
+        venv.execute_method(obj, "check_write", obj.compat_env(version, "check_write"))
 '''
     else:  # upgrade_downgrade
         func_body = f'''

python/python/tests/compat/test_vector_indices.py

@@ -274,6 +274,13 @@ def current_env(self, method_name: str):
             return {"LANCE_COMPAT_CURRENT_RUNTIME": "1"}
         return {}
 
+    def skip_write_after_current_write(self, version: str) -> bool:
+        # Newly written IVF_RQ indexes carry raw-query estimator metadata and
+        # split-code schema that older runtimes can query but cannot optimize.
+        # The upgrade_downgrade variant still covers old 1-bit residual-query
+        # indexes being read and rewritten by the current runtime.
+        return True
+
     def create(self):
         """Create dataset with IVF_RQ vector index."""
         shutil.rmtree(self.path, ignore_errors=True)

python/python/tests/test_vector_index.py

@@ -1067,13 +1067,7 @@
     assert stats["indices"][0]["sub_index"]["packed"] is False
 
 
-@pytest.mark.skip(
-    reason=(
-        "IVF_RQ num_bits>1 creation is gated until split-code search support "
-        "is implemented"
-    )
-)
-def test_create_ivf_rq_multi_bit_gates_search():
+def test_create_ivf_rq_multi_bit_searches_l2_and_cosine():
     ds = lance.write_dataset(create_table(), "memory://")
 
     ds = ds.create_index(
@@ -1084,15 +1078,38 @@
     )
     stats = ds.stats.index_stats("vector_idx")
     assert stats["indices"][0]["sub_index"]["num_bits"] == 9
+    assert stats["indices"][0]["sub_index"]["query_estimator"] == "raw_query"
 
-    with pytest.raises(pa.ArrowInvalid, match="num_bits>1 search is not supported"):
-        ds.to_table(
-            nearest={
-                "column": "vector",
-                "q": np.random.randn(128).astype(np.float32),
-                "k": 10,
-            }
-        )
+    result = ds.to_table(
+        nearest={
+            "column": "vector",
+            "q": np.random.randn(128).astype(np.float32),
+            "k": 10,
+        }
+    )
+    assert result.num_rows == 10
+
+    cosine_ds = lance.write_dataset(create_table(), "memory://")
+    cosine_ds = cosine_ds.create_index(
+        "vector",
+        index_type="IVF_RQ",
+        metric="cosine",
+        num_partitions=4,
+        num_bits=9,
+    )
+    cosine_stats = cosine_ds.stats.index_stats("vector_idx")
+    assert cosine_stats["indices"][0]["sub_index"]["num_bits"] == 9
+    assert cosine_stats["indices"][0]["sub_index"]["query_estimator"] == "raw_query"
+
+    cosine_result = cosine_ds.to_table(
+        nearest={
+            "column": "vector",
+            "q": np.random.randn(128).astype(np.float32),
+            "k": 10,
+            "metric": "cosine",
+        }
+    )
+    assert cosine_result.num_rows == 10
 
 
 def test_create_ivf_rq_requires_dim_divisible_by_8():

rust/lance-index/src/vector/bq.rs

@@ -128,14 +128,7 @@ pub fn validate_rq_num_bits(num_bits: u8) -> Result<()> {
 }
 
 pub fn validate_supported_rq_num_bits(num_bits: u8) -> Result<()> {
-    validate_rq_num_bits(num_bits)?;
-    if num_bits != RABIT_BINARY_NUM_BITS {
-        return Err(Error::not_supported(format!(
-            "IVF_RQ num_bits={} index creation is not supported until split-code search support is implemented",
-            num_bits
-        )));
-    }
-    Ok(())
+    validate_rq_num_bits(num_bits)
 }
 
 pub fn rabit_ex_bits(num_bits: u8) -> Result<u8> {
@@ -261,13 +254,7 @@ mod tests {
         );
 
         validate_supported_rq_num_bits(1).unwrap();
-        let err = validate_supported_rq_num_bits(9).unwrap_err();
-        assert!(
-            err.to_string()
-                .contains("num_bits=9 index creation is not supported"),
-            "{}",
-            err
-        );
+        validate_supported_rq_num_bits(9).unwrap();
     }
 
     #[test]