wip:RAG pipeline:general structure #6 - test(pipeline): cover chunker dispatch and the digest stamped on every chunk of a file

Author: ByteVoyageur

test/unit/test_indexed_file_state.py

@@ -0,0 +1,103 @@
+from ingestion import indexed_file_state
+
+
+# --- compute_file_sha1: stable whole-file digest --------------------------
+
+def test_compute_file_sha1_is_the_hex_sha1_of_the_utf8_bytes():
+    # Pinned digest: it is stored in Qdrant and compared against the next
+    # run, so changing the algorithm or the encoding would mark every
+    # indexed file as changed and re-embed the whole corpus. The accented
+    # character makes the utf-8 choice observable.
+    assert (indexed_file_state.compute_file_sha1("héllo")
+            == "35b5ea45c5e41f78b46a937cc74d41dfea920890")
+
+
+def test_compute_file_sha1_is_deterministic():
+    assert (indexed_file_state.compute_file_sha1("const a = 1")
+            == indexed_file_state.compute_file_sha1("const a = 1"))
+
+
+def test_compute_file_sha1_changes_with_content():
+    assert (indexed_file_state.compute_file_sha1("const a = 1")
+            != indexed_file_state.compute_file_sha1("const a = 2"))
+
+
+# --- load_indexed_file_hashes: rebuild the state from stored payloads -----
+
+def test_load_indexed_file_hashes_keys_on_repository_and_source_path(
+        monkeypatch):
+    monkeypatch.setattr(indexed_file_state.vectordb, "iter_payloads", lambda: [
+        {"repository": "kdk", "source_path": "map/x.js", "file_sha1": "aaa"},
+        {"repository": "kano", "source_path": "map/x.js", "file_sha1": "bbb"},
+    ])
+    assert indexed_file_state.load_indexed_file_hashes() == {
+        ("kdk", "map/x.js"): "aaa",
+        ("kano", "map/x.js"): "bbb",
+    }
+
+
+def test_load_indexed_file_hashes_shares_one_entry_per_file(monkeypatch):
+    # Every chunk of a file carries the same file_sha1, so many payloads
+    # collapse into a single entry.
+    monkeypatch.setattr(indexed_file_state.vectordb, "iter_payloads", lambda: [
+        {"repository": "kdk", "source_path": "map/x.js", "file_sha1": "aaa"},
+        {"repository": "kdk", "source_path": "map/x.js", "file_sha1": "aaa"},
+    ])
+    assert indexed_file_state.load_indexed_file_hashes() == {
+        ("kdk", "map/x.js"): "aaa"}
+
+
+def test_load_indexed_file_hashes_skips_payloads_without_a_digest(monkeypatch):
+    monkeypatch.setattr(indexed_file_state.vectordb, "iter_payloads", lambda: [
+        {"repository": "kdk", "source_path": "map/x.js", "file_sha1": ""},
+        {"repository": "kdk", "source_path": "map/y.js"},
+    ])
+    assert indexed_file_state.load_indexed_file_hashes() == {}
+
+
+def test_load_indexed_file_hashes_is_empty_on_the_first_run(monkeypatch):
+    monkeypatch.setattr(
+        indexed_file_state.vectordb, "iter_payloads", lambda: [])
+    assert indexed_file_state.load_indexed_file_hashes() == {}
+
+
+# --- select_changed_chunks: drop chunks whose file is unchanged -----------
+
+def test_select_changed_chunks_keeps_a_file_that_is_not_indexed_yet():
+    chunk = {"metadata": {
+        "repository": "kdk", "source_path": "map/x.js", "file_sha1": "aaa"}}
+    assert indexed_file_state.select_changed_chunks([chunk], {}) == [chunk]
+
+
+def test_select_changed_chunks_drops_a_file_indexed_at_the_same_digest():
+    chunk = {"metadata": {
+        "repository": "kdk", "source_path": "map/x.js", "file_sha1": "aaa"}}
+    indexed = {("kdk", "map/x.js"): "aaa"}
+    assert indexed_file_state.select_changed_chunks([chunk], indexed) == []
+
+
+def test_select_changed_chunks_keeps_a_file_whose_content_changed():
+    chunk = {"metadata": {
+        "repository": "kdk", "source_path": "map/x.js", "file_sha1": "bbb"}}
+    indexed = {("kdk", "map/x.js"): "aaa"}
+    assert indexed_file_state.select_changed_chunks([chunk], indexed) == [chunk]
+
+
+def test_select_changed_chunks_distinguishes_repositories():
+    # source_path is repo-relative, so the same path in another repo must not
+    # be mistaken for an already indexed file.
+    chunk = {"metadata": {
+        "repository": "kano", "source_path": "map/x.js", "file_sha1": "aaa"}}
+    indexed = {("kdk", "map/x.js"): "aaa"}
+    assert indexed_file_state.select_changed_chunks([chunk], indexed) == [chunk]
+
+
+def test_select_changed_chunks_keeps_every_chunk_of_a_changed_file():
+    chunks = [
+        {"metadata": {"repository": "kdk", "source_path": "map/x.js",
+                      "file_sha1": "bbb", "chunk_index": 0}},
+        {"metadata": {"repository": "kdk", "source_path": "map/x.js",
+                      "file_sha1": "bbb", "chunk_index": 1}},
+    ]
+    indexed = {("kdk", "map/x.js"): "aaa"}
+    assert indexed_file_state.select_changed_chunks(chunks, indexed) == chunks

test/unit/test_pipeline.py

@@ -0,0 +1,123 @@
+from ingestion import pipeline
+from ingestion.indexed_file_state import compute_file_sha1
+
+JAVASCRIPT = "export function center () {\n  return [0, 0]\n}\n"
+MARKDOWN = "# Title\n\nSome prose about the map.\n"
+
+
+# --- dispatch by extension ------------------------------------------------
+
+def test_chunk_files_dispatches_each_extension_to_a_chunker(tmp_path):
+    _write(tmp_path, "kdk/map/base.js", JAVASCRIPT)
+    _write(tmp_path, "kdk/docs/guide.md", MARKDOWN)
+
+    chunks = pipeline.chunk_files(_files(tmp_path), tmp_path)
+
+    assert {chunk["metadata"]["source_path"] for chunk in chunks} == {
+        "map/base.js", "docs/guide.md"}
+
+
+def test_chunk_files_skips_extensions_without_a_chunker(tmp_path):
+    # The incremental path feeds this function straight from git, which knows
+    # nothing about the scanner's filters, so unsupported files arrive here.
+    _write(tmp_path, "kdk/scripts/deploy.py", "print('hi')")
+    _write(tmp_path, "kdk/map/base.js", JAVASCRIPT)
+
+    chunks = pipeline.chunk_files(_files(tmp_path), tmp_path)
+
+    assert all(chunk["metadata"]["source_path"] == "map/base.js"
+               for chunk in chunks)
+
+
+def test_chunk_files_dispatches_on_a_lowercased_extension(tmp_path):
+    _write(tmp_path, "kdk/docs/GUIDE.MD", MARKDOWN)
+
+    chunks = pipeline.chunk_files(_files(tmp_path), tmp_path)
+
+    assert chunks
+
+
+def test_chunk_files_returns_nothing_for_no_files(tmp_path):
+    assert pipeline.chunk_files([], tmp_path) == []
+
+
+# --- repository and source_path: the key everything else uses -------------
+
+def test_chunk_files_splits_the_repository_from_the_source_path(tmp_path):
+    # delete_file() and the indexed state both key on this pair, so a wrong
+    # split silently orphans a file's chunks instead of replacing them.
+    _write(tmp_path, "kdk/map/mixin.base-map.js", JAVASCRIPT)
+
+    chunk = pipeline.chunk_files(_files(tmp_path), tmp_path)[0]
+
+    assert chunk["metadata"]["repository"] == "kdk"
+    assert chunk["metadata"]["source_path"] == "map/mixin.base-map.js"
+
+
+def test_chunk_files_keeps_the_source_path_repository_relative(tmp_path):
+    _write(tmp_path, "kdk/packages/core/src/deep.js", JAVASCRIPT)
+
+    chunk = pipeline.chunk_files(_files(tmp_path), tmp_path)[0]
+
+    assert chunk["metadata"]["repository"] == "kdk"
+    assert chunk["metadata"]["source_path"] == "packages/core/src/deep.js"
+
+
+def test_chunk_files_distinguishes_the_same_path_in_two_repositories(tmp_path):
+    _write(tmp_path, "kdk/map/base.js", JAVASCRIPT)
+    _write(tmp_path, "kano/map/base.js", JAVASCRIPT)
+
+    chunks = pipeline.chunk_files(_files(tmp_path), tmp_path)
+
+    assert {chunk["metadata"]["repository"] for chunk in chunks} == {
+        "kdk", "kano"}
+
+
+# --- file_sha1: the gate on the next run ----------------------------------
+
+def test_chunk_files_stamps_the_digest_of_the_whole_file(tmp_path):
+    _write(tmp_path, "kdk/map/base.js", JAVASCRIPT)
+
+    chunk = pipeline.chunk_files(_files(tmp_path), tmp_path)[0]
+
+    assert chunk["metadata"]["file_sha1"] == compute_file_sha1(JAVASCRIPT)
+
+
+def test_chunk_files_stamps_every_chunk_of_a_file_with_the_same_digest(
+        tmp_path):
+    # select_changed_chunks() compares per file, not per chunk: one chunk
+    # carrying a different digest would re-embed a file that never changed.
+    _write(tmp_path, "kdk/map/base.js", JAVASCRIPT * 40)
+
+    chunks = pipeline.chunk_files(_files(tmp_path), tmp_path)
+
+    assert len(chunks) > 1
+    assert len({chunk["metadata"]["file_sha1"] for chunk in chunks}) == 1
+
+
+def test_chunk_files_changes_the_digest_when_the_content_changes(tmp_path):
+    path = _write(tmp_path, "kdk/map/base.js", JAVASCRIPT)
+    before = pipeline.chunk_files([path], tmp_path)[0]
+
+    path.write_text(JAVASCRIPT.replace("[0, 0]", "[1, 1]"))
+    after = pipeline.chunk_files([path], tmp_path)[0]
+
+    assert before["metadata"]["file_sha1"] != after["metadata"]["file_sha1"]
+
+
+# ---------------------------------------------------------------------------
+# UTILS
+# ---------------------------------------------------------------------------
+
+
+# Write a workspace-relative file, creating parent directories as needed.
+def _write(root, relative, text):
+    path = root / relative
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(text)
+    return path
+
+
+# Every file under the workspace, as the callers of chunk_files pass them.
+def _files(root):
+    return sorted(path for path in root.rglob("*") if path.is_file())