Harden BM25 field parsing and container startup

Author: philippemnoel

paradedb/sqlalchemy/indexing.py

@@ -452,9 +452,8 @@ class IndexMeta:
 
 _KEY_FIELD_RE = re.compile(r"key_field\s*=\s*'?\"?([^'\",)\s]+)\"?'?", re.IGNORECASE)
 _ALIAS_RE = re.compile(r"alias\s*=\s*([A-Za-z_][A-Za-z0-9_]*)", re.IGNORECASE)
-_CAST_FIELD_RE = re.compile(r"^\(*\"?([A-Za-z_][A-Za-z0-9_]*)\"?\)*\s*::\s*pdb\.", re.IGNORECASE)
-_PLAIN_FIELD_RE = re.compile(r'^\(*"?([A-Za-z_][A-Za-z0-9_]*)"?\)*$')
 _TOKENIZER_NAME_RE = re.compile(r"::pdb\.([A-Za-z_][A-Za-z0-9_]*)", re.IGNORECASE)
+_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
 
 
 def _split_top_level_csv(expr: str) -> list[str]:
@@ -520,16 +519,58 @@ def _extract_bm25_field_list(indexdef: str) -> list[str]:
 
 
 def _extract_field_name(field_expr: str) -> str | None:
-    expr = field_expr.strip()
-    cast_match = _CAST_FIELD_RE.match(expr)
-    if cast_match:
-        return cast_match.group(1)
-    plain_match = _PLAIN_FIELD_RE.match(expr)
-    if plain_match:
-        return plain_match.group(1)
+    expr = _strip_outer_parens(field_expr.strip())
+    cast_marker = re.search(r"::\s*pdb\.", expr, re.IGNORECASE)
+    if cast_marker is not None:
+        expr = _strip_outer_parens(expr[: cast_marker.start()].strip())
+
+    if "->" in expr:
+        expr = _strip_outer_parens(expr.split("->", 1)[0].strip())
+
+    # Strip schema/table qualifiers and keep the terminal identifier.
+    if "." in expr:
+        expr = _strip_outer_parens(expr.rsplit(".", 1)[1].strip())
+
+    if expr.startswith('"') and expr.endswith('"') and len(expr) >= 2:
+        return expr[1:-1].replace('""', '"')
+    if _IDENT_RE.match(expr):
+        return expr
     return None
 
 
+def _strip_outer_parens(value: str) -> str:
+    expr = value
+    while expr.startswith("(") and expr.endswith(")") and _has_balanced_outer_parens(expr):
+        expr = expr[1:-1].strip()
+    return expr
+
+
+def _has_balanced_outer_parens(value: str) -> bool:
+    depth = 0
+    in_single = False
+    in_double = False
+
+    for i, ch in enumerate(value):
+        if ch == "'" and not in_double:
+            in_single = not in_single
+            continue
+        if ch == '"' and not in_single:
+            in_double = not in_double
+            continue
+        if in_single or in_double:
+            continue
+
+        if ch == "(":
+            depth += 1
+        elif ch == ")":
+            depth -= 1
+            if depth == 0 and i != len(value) - 1:
+                return False
+            if depth < 0:
+                return False
+    return depth == 0
+
+
 def _extract_key_field(indexdef: str) -> str | None:
     match = _KEY_FIELD_RE.search(indexdef)
     if match:

scripts/run_paradedb.sh

@@ -51,20 +51,22 @@ run_container() {
     "${IMAGE}" >/dev/null
 }
 
+created_or_recreated=0
 if [[ "$container_exists" == "0" ]]; then
   echo "Starting ParadeDB container ${CONTAINER_NAME} from ${IMAGE}..."
   run_container
+  created_or_recreated=1
 else
   current_image="$(docker inspect -f '{{.Config.Image}}' "${CONTAINER_NAME}" 2>/dev/null || true)"
   if [[ -n "${current_image}" && "${current_image}" != "${IMAGE}" ]]; then
     echo "Container ${CONTAINER_NAME} uses image ${current_image}; recreating with ${IMAGE}..."
     docker rm -f "${CONTAINER_NAME}" >/dev/null
     echo "Starting ParadeDB container ${CONTAINER_NAME} from ${IMAGE}..."
     run_container
-    container_exists=0
+    created_or_recreated=1
   fi
 
-  if [[ "$container_exists" == "1" ]]; then
+  if [[ "${created_or_recreated}" == "0" ]]; then
     mapped_port="$(docker port "${CONTAINER_NAME}" 5432/tcp 2>/dev/null | head -n1 | awk -F: '{print $NF}')"
     if [[ -n "${mapped_port}" && "${mapped_port}" != "${PORT}" ]]; then
       echo "Container ${CONTAINER_NAME} is already mapped to host port ${mapped_port}; using that port."
@@ -73,9 +75,18 @@ else
       echo "Container ${CONTAINER_NAME} has no published 5432 port; recreating with ${PORT}:5432..."
       docker rm -f "${CONTAINER_NAME}" >/dev/null
       run_container
+      created_or_recreated=1
+    fi
+  fi
+
+  if [[ "${created_or_recreated}" == "0" ]]; then
+    is_running="$(docker inspect -f '{{.State.Running}}' "${CONTAINER_NAME}" 2>/dev/null || echo false)"
+    if [[ "${is_running}" == "true" ]]; then
+      echo "Container ${CONTAINER_NAME} is already running."
+    else
+      echo "Container ${CONTAINER_NAME} already exists; starting it..."
+      docker start "${CONTAINER_NAME}" >/dev/null
     fi
-    echo "Container ${CONTAINER_NAME} already exists; starting it..."
-    docker start "${CONTAINER_NAME}" >/dev/null
   fi
 fi
 

tests/unit/test_indexing_unit.py

@@ -294,6 +294,21 @@ def test_extract_bm25_field_list_parses_tokenizer_casts():
     assert _extract_alias(parts[2]) == "category_exact"
 
 
+def test_extract_field_name_from_json_key_tokenizer_cast():
+    expr = "((metadata ->> 'color')::pdb.literal('alias=metadata_color'))"
+    assert _extract_field_name(expr) == "metadata"
+
+
+def test_extract_field_name_from_qualified_tokenizer_cast():
+    expr = "((public.products.description)::pdb.unicode_words('lowercase=true'))"
+    assert _extract_field_name(expr) == "description"
+
+
+def test_extract_field_name_from_quoted_identifier():
+    expr = '(("Display Name")::pdb.literal(\'alias=display_name\'))'
+    assert _extract_field_name(expr) == "Display Name"
+
+
 # ---------------------------------------------------------------------------
 # _extract_tokenizer_name
 # ---------------------------------------------------------------------------