Align ParadeDB scripts to 5443 and close DSL parity gaps

ankitml · ankitml · commit 33845e02d109 · 2026-02-19T21:29:45.000-05:00
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -14,7 +14,7 @@ pip install -e .[test,dev]
 ruff check .
 mypy paradedb
 python -m pytest tests/unit
-PARADEDB_TEST_DSN=postgres://postgres:postgres@localhost:5432/postgres python -m pytest -m integration
+PARADEDB_TEST_DSN=postgres://postgres:postgres@localhost:5443/postgres python -m pytest -m integration
 ```
 
 ## Guidelines
diff --git a/README.md b/README.md
@@ -26,7 +26,7 @@ pip install -e .[test,dev]
 - `paradedb.sqlalchemy.search`: ParadeDB predicates (`match_all`, `fuzzy`, `parse`, `more_like_this`, etc.).
 - `paradedb.sqlalchemy.pdb`: function wrappers (`score`, `snippet`, `snippets`, `agg`).
 - `paradedb.sqlalchemy.facets`: aggregate/facet JSON builders and rows+facets helper.
-- `paradedb.sqlalchemy.select_with`: select decorators for score/snippet columns.
+- `paradedb.sqlalchemy.select_with`: select decorators for score/snippet/snippet_positions columns.
 - `paradedb.sqlalchemy.alembic`: Alembic operations for BM25 index lifecycle.
 
 ## Quickstart
@@ -49,11 +49,32 @@ products_bm25_idx.create(engine)
 stmt = select(Product.id, Product.description).where(search.match_any(Product.description, "running", "shoes"))
 ```
 
+Index JSON keys using BM25Field expressions:
+
+```python
+from paradedb.sqlalchemy import expr
+
+products_bm25_idx = Index(
+    "products_bm25_idx",
+    indexing.BM25Field(Product.id),
+    indexing.BM25Field(
+        expr.json_text(Product.metadata, "color"),
+        tokenizer=indexing.tokenize.literal(alias="metadata_color"),
+    ),
+    indexing.BM25Field(
+        expr.json_text(Product.metadata, "location"),
+        tokenizer=indexing.tokenize.literal(alias="metadata_location"),
+    ),
+    postgresql_using="bm25",
+    postgresql_with={"key_field": "id"},
+)
+```
+
 ## Query APIs
 
 - Basic predicates: `match_all`, `match_any`, `term`, `phrase`, `fuzzy`, `regex`, `all`
 - Advanced predicates: `parse`, `phrase_prefix`, `regex_phrase`, `near`, `proximity`, `more_like_this`
-- Scoring/snippets: `pdb.score`, `pdb.snippet`, `pdb.snippets`, `select_with.score`, `select_with.snippet`
+- Scoring/snippets: `pdb.score`, `pdb.snippet`, `pdb.snippets`, `pdb.snippet_positions`, `select_with.score`, `select_with.snippet`, `select_with.snippet_positions`
 - Aggregations/facets: `facets.*` builders + `pdb.agg(...)`
 - Rows + facets: `facets.with_rows(...)`
 
@@ -67,6 +88,8 @@ stmt = (
     select(
         pdb.agg(facets.value_count(field="id")).label("count"),
         pdb.agg(facets.avg(field="rating")).label("avg_rating"),
+        pdb.agg(facets.percentiles(field="rating", percents=[50, 95])).label("rating_percentiles"),
+        pdb.agg(facets.top_hits(size=2, sort=[{"rating": "desc"}], docvalue_fields=["id", "rating"])).label("top_hits"),
     )
     .select_from(Product)
     .where(search.match_all(Product.description, "running"))
@@ -117,7 +140,7 @@ python -m pytest tests/unit
 Integration tests (requires running ParadeDB):
 
 ```bash
-PARADEDB_TEST_DSN=postgres://postgres:postgres@localhost:5432/postgres python -m pytest -m integration
+PARADEDB_TEST_DSN=postgres://postgres:postgres@localhost:5443/postgres python -m pytest -m integration
 ```
 
 ## CI
diff --git a/examples/common.py b/examples/common.py
@@ -23,7 +23,7 @@
 
 
 def engine_from_env() -> Engine:
-    dsn = os.getenv("DATABASE_URL", "postgresql+psycopg://postgres:postgres@localhost:5432/postgres")
+    dsn = os.getenv("DATABASE_URL", "postgresql+psycopg://postgres:postgres@localhost:5443/postgres")
     return create_engine(dsn)
 
 
diff --git a/examples/faceted_search.py b/examples/faceted_search.py
@@ -28,6 +28,14 @@ def main() -> None:
         select(
             pdb.agg(facets.value_count(field="id")).label("count"),
             pdb.agg(facets.avg(field="rating")).label("avg_rating"),
+            pdb.agg(facets.percentiles(field="rating", percents=[50, 95])).label("rating_percentiles"),
+            pdb.agg(
+                facets.top_hits(
+                    size=2,
+                    sort=[{"rating": "desc"}],
+                    docvalue_fields=["id", "rating"],
+                )
+            ).label("top_hits"),
         )
         .select_from(Product)
         .where(search.match_all(Product.description, "running"))
diff --git a/paradedb/sqlalchemy/indexing.py b/paradedb/sqlalchemy/indexing.py
@@ -123,7 +123,7 @@ def _compile_bm25_field(element: BM25Field, compiler, **kw: Any) -> str:
     expr_sql = compiler.process(element.expr, **kw)
     if element.tokenizer is None:
         return expr_sql
-    return f"({expr_sql}::{element.tokenizer.render()})"
+    return f"(({expr_sql})::{element.tokenizer.render()})"
 
 
 @compiles(BM25Field)
diff --git a/paradedb/sqlalchemy/search.py b/paradedb/sqlalchemy/search.py
@@ -102,8 +102,17 @@ class ProximityExpr:
     def __init__(self, expr: ClauseElement) -> None:
         self.expr = expr
 
-    def near(self, other: str | ClauseElement | ProximityExpr, *, distance: int, ordered: bool = False) -> ProximityExpr:
-        return ProximityExpr(_near_chain(self.expr, other, distance=distance, ordered=ordered))
+    def near(
+        self,
+        other: str | ClauseElement | ProximityExpr | None = None,
+        *,
+        distance: int,
+        ordered: bool = False,
+        right_pattern: str | None = None,
+        max_expansions: int = 100,
+    ) -> ProximityExpr:
+        right = _resolve_near_operand(other, right_pattern=right_pattern, max_expansions=max_expansions)
+        return ProximityExpr(_near_chain(self.expr, right, distance=distance, ordered=ordered))
 
 
 def _to_proximity_operand(value: str | ClauseElement | ProximityExpr) -> ClauseElement:
@@ -122,12 +131,29 @@ def _to_proximity_clause(value: str | ClauseElement | ProximityExpr) -> ClauseEl
 
 
 def _near_chain(left: str | ClauseElement | ProximityExpr, right: str | ClauseElement | ProximityExpr, *, distance: int, ordered: bool = False) -> ClauseElement:
+    require_non_negative(distance, field_name="distance")
     left_expr = _to_proximity_clause(left)
     right_expr = _to_proximity_clause(right)
     op = _NEAR_ORDERED if ordered else _NEAR
     return left_expr.operate(op, literal(distance)).operate(op, right_expr)
 
 
+def _resolve_near_operand(
+    right: str | ClauseElement | ProximityExpr | None,
+    *,
+    right_pattern: str | None,
+    max_expansions: int,
+) -> str | ClauseElement | ProximityExpr:
+    if right_pattern is not None:
+        if right is not None:
+            raise InvalidArgumentError("right and right_pattern cannot be used together")
+        require_non_negative(max_expansions, field_name="max_expansions")
+        return prox_regex(right_pattern, max_expansions)
+    if right is None:
+        raise InvalidArgumentError("right is required unless right_pattern is provided")
+    return right
+
+
 def parse(field: ColumnElement, query: str, *, lenient: bool = False, conjunction_mode: bool = False) -> ColumnElement[bool]:
     return field.operate(_QUERY, func.pdb.parse(query, lenient, conjunction_mode))
 
@@ -153,11 +179,22 @@ def regex_phrase(
     return field.operate(_QUERY, func.pdb.regex_phrase(array(terms, type_=Text()), slop, max_expansions))
 
 
-def near(field: ColumnElement, left: str | ClauseElement, right: str | ClauseElement, *, distance: int, ordered: bool = False) -> ColumnElement[bool]:
-    return field.operate(_QUERY, _near_chain(left, right, distance=distance, ordered=ordered))
+def near(
+    field: ColumnElement,
+    left: str | ClauseElement | ProximityExpr,
+    right: str | ClauseElement | ProximityExpr | None = None,
+    *,
+    distance: int,
+    ordered: bool = False,
+    right_pattern: str | None = None,
+    max_expansions: int = 100,
+) -> ColumnElement[bool]:
+    right_operand = _resolve_near_operand(right, right_pattern=right_pattern, max_expansions=max_expansions)
+    return field.operate(_QUERY, _near_chain(left, right_operand, distance=distance, ordered=ordered))
 
 
 def prox_regex(pattern: str, max_expansions: int = 100) -> ProximityExpr:
+    require_non_negative(max_expansions, field_name="max_expansions")
     return ProximityExpr(func.pdb.prox_regex(pattern, max_expansions))
 
 
diff --git a/paradedb/sqlalchemy/select_with.py b/paradedb/sqlalchemy/select_with.py
@@ -57,3 +57,13 @@ def snippets(
             sort_by=sort_by,
         ).label(label)
     )
+
+
+def snippet_positions(
+    stmt: Select,
+    field: ColumnElement,
+    *,
+    label: str = "snippet_positions",
+) -> Select:
+    _assert_snippet_supported(stmt)
+    return stmt.add_columns(pdb.snippet_positions(field).label(label))
diff --git a/scripts/run_examples.sh b/scripts/run_examples.sh
@@ -8,7 +8,12 @@ if [[ "${SKIP_PARADEDB_START:-0}" != "1" ]]; then
   source "${SCRIPT_DIR}/run_paradedb.sh"
 fi
 
-export DATABASE_URL="${DATABASE_URL:-postgresql+psycopg://postgres:postgres@localhost:${PARADEDB_PORT:-5443}/postgres}"
+PORT="${PARADEDB_PORT:-5443}"
+USER="${PARADEDB_USER:-postgres}"
+PASSWORD="${PARADEDB_PASSWORD:-postgres}"
+DB="${PARADEDB_DB:-postgres}"
+
+export DATABASE_URL="${DATABASE_URL:-postgresql+psycopg://${USER}:${PASSWORD}@localhost:${PORT}/${DB}}"
 
 ./.venv/bin/python examples/quickstart.py
 ./.venv/bin/python examples/autocomplete.py
diff --git a/scripts/run_integration_tests.sh b/scripts/run_integration_tests.sh
@@ -6,10 +6,10 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 source "${SCRIPT_DIR}/run_paradedb.sh"
 
-PORT="${PARADEDB_PORT:-5443}"
-USER="${PARADEDB_USER:-postgres}"
-PASSWORD="${PARADEDB_PASSWORD:-postgres}"
-DB="${PARADEDB_DB:-postgres}"
+PORT="${PARADEDB_PORT}"
+USER="${PARADEDB_USER}"
+PASSWORD="${PARADEDB_PASSWORD}"
+DB="${PARADEDB_DB}"
 
 export PARADEDB_INTEGRATION=1
 export PARADEDB_TEST_DSN="postgresql+psycopg://${USER}:${PASSWORD}@localhost:${PORT}/${DB}"
diff --git a/scripts/run_paradedb.sh b/scripts/run_paradedb.sh
@@ -8,13 +8,26 @@ else
   RUNNING=0
 fi
 
-IMAGE="${PARADEDB_IMAGE:-paradedb/paradedb:0.21.4-pg18}"
-CONTAINER_NAME="${PARADEDB_CONTAINER_NAME:-paradedb-sqlalchemy-integration}"
+PARADEDB_IMAGE="${PARADEDB_IMAGE:-paradedb/paradedb:0.21.8-pg18}"
+PARADEDB_CONTAINER_NAME="${PARADEDB_CONTAINER_NAME:-paradedb-sqlalchemy-integration}"
+PARADEDB_PORT="${PARADEDB_PORT:-5443}"
+PARADEDB_USER="${PARADEDB_USER:-postgres}"
+PARADEDB_PASSWORD="${PARADEDB_PASSWORD:-postgres}"
+PARADEDB_DB="${PARADEDB_DB:-postgres}"
 
-PORT="${PARADEDB_PORT:-5443}"
-USER="${PARADEDB_USER:-postgres}"
-PASSWORD="${PARADEDB_PASSWORD:-postgres}"
-DB="${PARADEDB_DB:-postgres}"
+export PARADEDB_IMAGE
+export PARADEDB_CONTAINER_NAME
+export PARADEDB_PORT
+export PARADEDB_USER
+export PARADEDB_PASSWORD
+export PARADEDB_DB
+
+IMAGE="${PARADEDB_IMAGE}"
+CONTAINER_NAME="${PARADEDB_CONTAINER_NAME}"
+PORT="${PARADEDB_PORT}"
+USER="${PARADEDB_USER}"
+PASSWORD="${PARADEDB_PASSWORD}"
+DB="${PARADEDB_DB}"
 
 DATABASE_URL="${DATABASE_URL:-postgresql+psycopg://${USER}:${PASSWORD}@localhost:${PORT}/${DB}}"
 export DATABASE_URL
diff --git a/tests/integration/test_facets_integration.py b/tests/integration/test_facets_integration.py
@@ -41,6 +41,36 @@ def test_multiple_agg_columns_with_search_all(session):
     assert row._mapping["count"] is not None
 
 
+def test_percentiles_agg_with_search_all(session):
+    stmt = (
+        select(pdb.agg(facets.percentiles(field="rating", percents=[50, 95])).label("pct"))
+        .select_from(Product)
+        .where(search.all(Product.id))
+    )
+    assert_uses_paradedb_scan(session, stmt)
+    row = session.execute(stmt).one()
+    assert row._mapping["pct"] is not None
+
+
+def test_top_hits_agg_with_search_all(session):
+    stmt = (
+        select(
+            pdb.agg(
+                facets.top_hits(
+                    size=2,
+                    sort=[{"rating": "desc"}],
+                    docvalue_fields=["id", "rating"],
+                )
+            ).label("hits")
+        )
+        .select_from(Product)
+        .where(search.all(Product.id))
+    )
+    assert_uses_paradedb_scan(session, stmt)
+    row = session.execute(stmt).one()
+    assert row._mapping["hits"] is not None
+
+
 def test_with_rows_adds_window_agg_and_extracts_payload(session):
     base = (
         select(Product.id, Product.description, Product.rating)
diff --git a/tests/integration/test_indexing_integration.py b/tests/integration/test_indexing_integration.py
@@ -2,7 +2,9 @@
 
 import pytest
 from sqlalchemy import Column, Index, Integer, MetaData, String, Table, Text, text
+from sqlalchemy.dialects.postgresql import JSONB
 
+from paradedb.sqlalchemy.expr import json_text
 from paradedb.sqlalchemy.indexing import BM25Field, assert_indexed, describe, tokenize
 from paradedb.sqlalchemy.errors import FieldNotIndexedError
 
@@ -133,6 +135,59 @@ def test_bm25_index_with_tokenizers_when_supported(engine):
     _drop_table_and_index(engine, table_name, index_name)
 
 
+def test_bm25_index_json_keys_when_supported(engine):
+    if not _tokenizer_cast_supported(engine):
+        pytest.skip("ParadeDB instance does not support tokenizer cast index syntax yet")
+
+    table_name = "indexing_products_json"
+    index_name = "indexing_products_json_bm25_idx"
+    _drop_table_and_index(engine, table_name, index_name)
+
+    metadata = MetaData()
+    products = Table(
+        table_name,
+        metadata,
+        Column("id", Integer, primary_key=True),
+        Column("metadata", JSONB, nullable=False),
+    )
+    metadata.create_all(engine)
+
+    idx = Index(
+        index_name,
+        BM25Field(products.c.id),
+        BM25Field(
+            json_text(products.c.metadata, "color"),
+            tokenizer=tokenize.literal(alias="metadata_color"),
+        ),
+        BM25Field(
+            json_text(products.c.metadata, "location"),
+            tokenizer=tokenize.literal(alias="metadata_location"),
+        ),
+        postgresql_using="bm25",
+        postgresql_with={"key_field": "id"},
+    )
+    idx.create(engine)
+
+    with engine.begin() as conn:
+        row = conn.execute(
+            text(
+                """
+                SELECT indexdef
+                FROM pg_indexes
+                WHERE tablename = :table_name
+                  AND indexname = :index_name
+                """
+            ),
+            {"table_name": table_name, "index_name": index_name},
+        ).one()
+
+    assert "->>" in row.indexdef
+    assert "alias=metadata_color" in row.indexdef
+    assert "alias=metadata_location" in row.indexdef
+
+    _drop_table_and_index(engine, table_name, index_name)
+
+
 def test_create_all_with_attached_bm25_index(engine):
     table_name = "attached_index_items"
     index_name = "attached_index_items_bm25_idx"
@@ -335,5 +390,3 @@ def test_assert_indexed_passes_and_raises(engine):
         assert_indexed(engine, tbl.c.extra)
 
     _drop_table_and_index(engine, table_name, index_name)
-
-
diff --git a/tests/integration/test_query_interface_integration.py b/tests/integration/test_query_interface_integration.py
@@ -73,13 +73,15 @@ def test_snippet_projection(session):
 
 
 def test_select_with_helpers(session):
-    base = select(Product.id).where(search.match_any(Product.description, "running"))
+    base = select(Product.id, Product.description).where(search.match_any(Product.description, "running"))
     stmt = select_with.score(base, Product.id, label="search_score")
+    stmt = select_with.snippet_positions(stmt, Product.description, label="positions")
     assert_uses_paradedb_scan(session, stmt)
 
     rows = session.execute(stmt.order_by(Product.id)).all()
     assert len(rows) == 2
     assert rows[0][1] is not None
+    assert rows[0][2] is not None
 
 
 def test_select_with_snippet_rejects_fuzzy_predicate():
@@ -94,6 +96,12 @@ def test_select_with_snippets_rejects_fuzzy_predicate():
         select_with.snippets(base, Product.description)
 
 
+def test_select_with_snippet_positions_rejects_fuzzy_predicate():
+    base = select(Product.id, Product.description).where(search.fuzzy(Product.description, "wirless", distance=1))
+    with pytest.raises(SnippetWithFuzzyPredicateError):
+        select_with.snippet_positions(base, Product.description)
+
+
 def test_snippets_and_positions_projection(session):
     stmt = (
         select(
diff --git a/tests/integration/test_snippet_functions_integration.py b/tests/integration/test_snippet_functions_integration.py
diff --git a/tests/unit/test_indexing_unit.py b/tests/unit/test_indexing_unit.py
diff --git a/tests/unit/test_phase0_modules_unit.py b/tests/unit/test_phase0_modules_unit.py
diff --git a/tests/unit/test_sql_compilation_unit.py b/tests/unit/test_sql_compilation_unit.py
diff --git a/tests/unit/test_validation_cache_unit.py b/tests/unit/test_validation_cache_unit.py
