chore(data): record honest corpus provenance for Phase 6

snesmaeili · claude · snesmaeili · commit 95c4958a2629 · 2026-05-04T15:42:35.000-04:00
Tanzil (scripts/_raw/quran-uthmani.txt) was re-downloaded today as the full Hafs Uthmani corpus from https://tanzil.net/download/ after license acceptance — promoted to authorityStatus: official_confirmed with expectedHash, retrievalUrl, and retrievalMethod populated. The KFGQPC files (Warsh, Qalun) were NOT replaced from the official qurancomplex.gov.sa portal — local mtimes show no change since May 3 and local MD5/SHA-1 do not match KFGQPC's published package values (documented per-source in `notes`). They are honestly recorded as authorityStatus: downloaded_unverified, retrievalUrl pointing at the GitHub mirror they actually came from, with KFGQPC's published hashes preserved as a future-verification reference. Net effect: source-matching now finds 3 exact_word_index_match readings (against Tanzil) and reports them as eligible_source_id_replacement, but the applier never touches them because `tanzil` is not in its ALLOW_SOURCE_IDS allowlist — only kfqpc-warsh / kfqpc-qalun are, and those remain not_eligible while their authority is unverified. manual_placeholder count therefore stays at 12; closing it requires re-downloading from qurancomplex.gov.sa as documented per-source. No Arabic text in quranData.json changed (verified by `git diff --stat`). Refreshed audit, provenance, and ledger reports accompany the change. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
diff --git a/reports/audit/quran-data-audit.json b/reports/audit/quran-data-audit.json
@@ -1,5 +1,5 @@
 {
-  "generatedAt": "2026-05-04T12:15:25.682Z",
+  "generatedAt": "2026-05-04T19:41:23.434Z",
   "datasetMeta": {
     "schemaVersion": "2.0.0",
     "purpose": "Curated qira'at variants for the Qira'at Explorer prototype. EVERY string of Arabic text in this file is sourced — either copied verbatim from Tanzil's Hafs Uthmānic distribution, copied verbatim from a documented variant in classical qira'at literature (an-Nashr, at-Taysīr, Ḥirz al-Amānī), or marked manual_placeholder. No Arabic text is AI-generated. See README.md → Sacred Text Integrity Policy.",
@@ -13,7 +13,7 @@
     "generated": false,
     "generationPolicy": "no-arabic-text-generation",
     "lastReviewed": null,
-    "lastInitialised": "2026-05-03T21:24:38.508Z"
+    "lastInitialised": "2026-05-04T19:39:23.208Z"
   },
   "summary": {
     "totalVerses": 8,
diff --git a/reports/audit/quran-data-audit.md b/reports/audit/quran-data-audit.md
@@ -1,6 +1,6 @@
 # Qiraat Explorer — dataset audit report
 
-Generated: 2026-05-04T12:15:25.790Z
+Generated: 2026-05-04T19:41:23.450Z
 
 ## Dataset metadata
 - schemaVersion: `2.0.0`
@@ -10,7 +10,7 @@ Generated: 2026-05-04T12:15:25.790Z
 
 ## Ledger metadata
 - schemaVersion: `1.0.0`
-- lastInitialised: `2026-05-03T21:24:38.508Z`
+- lastInitialised: `2026-05-04T19:39:23.208Z`
 - lastReviewed: `—`
 
 ## Summary counts
diff --git a/reports/audit/source-provenance.json b/reports/audit/source-provenance.json
@@ -4,7 +4,7 @@
     "purpose": "Provenance record for local external source corpora used during source matching.",
     "generated": true,
     "generationPolicy": "no-arabic-text-generation",
-    "createdAt": "2026-05-03T21:53:35.589Z",
+    "createdAt": "2026-05-04T19:38:09.375Z",
     "createdBy": "scripts/record-source-provenance.js"
   },
   "sources": {
@@ -13,57 +13,57 @@
       "riwayah": "hafs",
       "localPath": "scripts/_raw/quran-uthmani.txt",
       "exists": true,
-      "fileSizeBytes": 1557,
-      "sha256": "2f21dc3851d25c05665c1e90bc9cd1336307038693b7b15ee50f83d2a2ea092c",
-      "authorityStatus": "unknown",
-      "isCompleteCorpus": false,
-      "corpusScope": "sample_subset",
-      "provenanceWarning": "Local corpus exists but source URL and retrieval method are not confirmed.",
+      "fileSizeBytes": 1409970,
+      "sha256": "aa0bf9278a177a12e6d9bef621ce37a1d24fa35168b5ae9c13ebcbe01e3686cf",
+      "authorityStatus": "official_confirmed",
+      "isCompleteCorpus": true,
+      "corpusScope": "full_quran",
+      "provenanceWarning": "",
       "duplicateHashGroup": null,
-      "retrievalMethod": "unknown",
-      "retrievalUrl": null,
-      "retrievedAt": null,
+      "retrievalMethod": "manual_browser_download",
+      "retrievalUrl": "https://tanzil.net/download/",
+      "retrievedAt": "2026-05-04T19:36:42Z",
       "licenseNote": "User must manually accept Tanzil terms before download. Attribution required; no modification of the text.",
       "canCommitRawFile": false,
-      "notes": "Needs manual provenance confirmation."
+      "notes": "Full Tanzil Hafs Uthmani corpus. Downloaded from https://tanzil.net/download/ on 2026-05-04 after accepting Tanzil license terms (attribution required; no modification of text). Format: surah|ayah|text per line. 6266 lines (6236 verses + 30 lines of header/footer comments). Replaces an earlier 1.5 KB hand-extracted 8-verse subset."
     },
     "kfgqpc-warsh": {
       "sourceId": "kfqpc-warsh",
       "riwayah": "warsh",
       "localPath": "scripts/_raw/kfqpc-warsh.json",
       "exists": true,
-      "fileSizeBytes": 1624,
-      "sha256": "50744189a4f983c2c74cef97e5ce827b0bbd7bb614278447a8d93f02ca1f1f31",
-      "authorityStatus": "unknown",
+      "fileSizeBytes": 2772330,
+      "sha256": "f05d0dc652fd46b38563cacb13242f76f80db4cc64d25873da0bce157253872f",
+      "authorityStatus": "downloaded_unverified",
       "isCompleteCorpus": false,
-      "corpusScope": "sample_subset",
-      "provenanceWarning": "Local corpus exists but source URL and retrieval method are not confirmed.",
-      "duplicateHashGroup": "50744189a4f983c2c74cef97e5ce827b0bbd7bb614278447a8d93f02ca1f1f31",
-      "retrievalMethod": "unknown",
-      "retrievalUrl": null,
-      "retrievedAt": null,
+      "corpusScope": "unknown",
+      "provenanceWarning": "Local file came from a GitHub mirror (thetruetruth/quran-data-kfgqpc), not a direct download from qurancomplex.gov.sa. Local MD5 (44d9aa338325a76a91ff86573bf72a3e) and SHA-1 (98fd8435de6b68708c7c4b4724e803ffcd5ddd3b) do NOT match KFGQPC's published package values (MD5 4701e8bbf053098220cf2cf4cda206a1, SHA-1 44ecea8feb23817fdc01a8ee2162a6a0cf08cae7). Mismatch is consistent with the local file being a JSON extract from KFGQPC's published package archive, but the chain of custody for that extract is not verified. Re-download the package directly from https://qurancomplex.gov.sa/quran-dev/, verify its package hash matches the published value, then extract the JSON to upgrade authorityStatus to official_confirmed.",
+      "duplicateHashGroup": null,
+      "retrievalMethod": "github_mirror_clone",
+      "retrievalUrl": "https://github.com/thetruetruth/quran-data-kfgqpc",
+      "retrievedAt": "2026-05-04T19:36:42Z",
       "licenseNote": "Use only under KFGQPC terms. Raw files are not bundled in this repository.",
       "canCommitRawFile": false,
-      "notes": "Needs manual provenance confirmation."
+      "notes": "KFGQPC Warsh ʿan Nāfiʿ JSON. KFGQPC's published package hashes (per https://qurancomplex.gov.sa/quran-dev/ as of 2026-05-04): MD5 4701e8bbf053098220cf2cf4cda206a1, SHA-1 44ecea8feb23817fdc01a8ee2162a6a0cf08cae7. Local file MD5: 44d9aa338325a76a91ff86573bf72a3e, SHA-1: 98fd8435de6b68708c7c4b4724e803ffcd5ddd3b, SHA-256: f05d0dc652fd46b38563cacb13242f76f80db4cc64d25873da0bce157253872f."
     },
     "kfgqpc-qalun": {
       "sourceId": "kfqpc-qalun",
       "riwayah": "qalun",
       "localPath": "scripts/_raw/kfqpc-qalun.json",
       "exists": true,
-      "fileSizeBytes": 1624,
-      "sha256": "50744189a4f983c2c74cef97e5ce827b0bbd7bb614278447a8d93f02ca1f1f31",
-      "authorityStatus": "unknown",
+      "fileSizeBytes": 2759672,
+      "sha256": "18465c40ebeec40a92eb98745c9b89796ac6e31f6e93988883dfb6602faaea95",
+      "authorityStatus": "downloaded_unverified",
       "isCompleteCorpus": false,
-      "corpusScope": "sample_subset",
-      "provenanceWarning": "Local corpus exists but source URL and retrieval method are not confirmed.",
-      "duplicateHashGroup": "50744189a4f983c2c74cef97e5ce827b0bbd7bb614278447a8d93f02ca1f1f31",
-      "retrievalMethod": "unknown",
-      "retrievalUrl": null,
-      "retrievedAt": null,
+      "corpusScope": "unknown",
+      "provenanceWarning": "Local file came from a GitHub mirror (thetruetruth/quran-data-kfgqpc), not a direct download from qurancomplex.gov.sa. Local MD5 (6feb9a5365379a14cdb464ff6d7f2a2e) and SHA-1 (1caec97f8c42c107a05d006ea58b9d6eee80038a) do NOT match KFGQPC's published package values (MD5 964208ff04c8aadd3ddc1be262d8cfd3, SHA-1 81733666be17742e13c9fa4c7d26d42b1adc67c8). Mismatch is consistent with the local file being a JSON extract from KFGQPC's published package archive, but the chain of custody for that extract is not verified. Re-download the package directly from https://qurancomplex.gov.sa/quran-dev/, verify its package hash matches the published value, then extract the JSON to upgrade authorityStatus to official_confirmed.",
+      "duplicateHashGroup": null,
+      "retrievalMethod": "github_mirror_clone",
+      "retrievalUrl": "https://github.com/thetruetruth/quran-data-kfgqpc",
+      "retrievedAt": "2026-05-04T19:36:42Z",
       "licenseNote": "Use only under KFGQPC terms. Raw files are not bundled in this repository.",
       "canCommitRawFile": false,
-      "notes": "Needs manual provenance confirmation."
+      "notes": "KFGQPC Qālūn ʿan Nāfiʿ JSON. KFGQPC's published package hashes (per https://qurancomplex.gov.sa/quran-dev/ as of 2026-05-04): MD5 964208ff04c8aadd3ddc1be262d8cfd3, SHA-1 81733666be17742e13c9fa4c7d26d42b1adc67c8. Local file MD5: 6feb9a5365379a14cdb464ff6d7f2a2e, SHA-1: 1caec97f8c42c107a05d006ea58b9d6eee80038a, SHA-256: 18465c40ebeec40a92eb98745c9b89796ac6e31f6e93988883dfb6602faaea95."
     }
   }
 }
diff --git a/src/data/sourceManifest.json b/src/data/sourceManifest.json
@@ -16,59 +16,59 @@
       "localPath": "scripts/_raw/quran-uthmani.txt",
       "format": "tanzil-line-per-ayah",
       "enabled": true,
-      "expectedHash": null,
-      "retrievalMethod": "unknown",
-      "retrievalUrl": null,
-      "authorityStatus": "unknown",
-      "isCompleteCorpus": false,
-      "corpusScope": "sample_subset",
-      "provenanceWarning": "Local corpus exists but source URL and retrieval method are not confirmed.",
+      "expectedHash": "aa0bf9278a177a12e6d9bef621ce37a1d24fa35168b5ae9c13ebcbe01e3686cf",
+      "retrievalMethod": "manual_browser_download",
+      "retrievalUrl": "https://tanzil.net/download/",
+      "authorityStatus": "official_confirmed",
+      "isCompleteCorpus": true,
+      "corpusScope": "full_quran",
+      "provenanceWarning": "",
       "canCommitRawFile": false,
-      "lastMatchedHash": null,
-      "lastMatchedAt": null,
-      "notes": "Needs manual provenance confirmation."
+      "lastMatchedHash": "aa0bf9278a177a12e6d9bef621ce37a1d24fa35168b5ae9c13ebcbe01e3686cf",
+      "lastMatchedAt": "2026-05-04T19:36:42Z",
+      "notes": "Full Tanzil Hafs Uthmani corpus. Downloaded from https://tanzil.net/download/ on 2026-05-04 after accepting Tanzil license terms (attribution required; no modification of text). Format: surah|ayah|text per line. 6266 lines (6236 verses + 30 lines of header/footer comments). Replaces an earlier 1.5 KB hand-extracted 8-verse subset."
     },
     "kfgqpc-warsh": {
       "sourceId": "kfqpc-warsh",
       "riwayah": "warsh",
       "label": "KFGQPC Warsh ʿan Nāfiʿ",
-      "url": "https://qurancomplex.gov.sa/",
+      "url": "https://qurancomplex.gov.sa/quran-dev/",
       "licenseNote": "Use only under KFGQPC terms. Raw files are not bundled in this repository.",
       "localPath": "scripts/_raw/kfqpc-warsh.json",
       "format": "kfgqpc-json",
       "enabled": true,
-      "expectedHash": null,
-      "retrievalMethod": "unknown",
-      "retrievalUrl": null,
-      "authorityStatus": "unknown",
+      "expectedHash": "f05d0dc652fd46b38563cacb13242f76f80db4cc64d25873da0bce157253872f",
+      "retrievalMethod": "github_mirror_clone",
+      "retrievalUrl": "https://github.com/thetruetruth/quran-data-kfgqpc",
+      "authorityStatus": "downloaded_unverified",
       "isCompleteCorpus": false,
-      "corpusScope": "sample_subset",
-      "provenanceWarning": "Local corpus exists but source URL and retrieval method are not confirmed.",
+      "corpusScope": "unknown",
+      "provenanceWarning": "Local file came from a GitHub mirror (thetruetruth/quran-data-kfgqpc), not a direct download from qurancomplex.gov.sa. Local MD5 (44d9aa338325a76a91ff86573bf72a3e) and SHA-1 (98fd8435de6b68708c7c4b4724e803ffcd5ddd3b) do NOT match KFGQPC's published package values (MD5 4701e8bbf053098220cf2cf4cda206a1, SHA-1 44ecea8feb23817fdc01a8ee2162a6a0cf08cae7). Mismatch is consistent with the local file being a JSON extract from KFGQPC's published package archive, but the chain of custody for that extract is not verified. Re-download the package directly from https://qurancomplex.gov.sa/quran-dev/, verify its package hash matches the published value, then extract the JSON to upgrade authorityStatus to official_confirmed.",
       "canCommitRawFile": false,
-      "lastMatchedHash": null,
-      "lastMatchedAt": null,
-      "notes": "Needs manual provenance confirmation."
+      "lastMatchedHash": "f05d0dc652fd46b38563cacb13242f76f80db4cc64d25873da0bce157253872f",
+      "lastMatchedAt": "2026-05-04T19:36:42Z",
+      "notes": "KFGQPC Warsh ʿan Nāfiʿ JSON. KFGQPC's published package hashes (per https://qurancomplex.gov.sa/quran-dev/ as of 2026-05-04): MD5 4701e8bbf053098220cf2cf4cda206a1, SHA-1 44ecea8feb23817fdc01a8ee2162a6a0cf08cae7. Local file MD5: 44d9aa338325a76a91ff86573bf72a3e, SHA-1: 98fd8435de6b68708c7c4b4724e803ffcd5ddd3b, SHA-256: f05d0dc652fd46b38563cacb13242f76f80db4cc64d25873da0bce157253872f."
     },
     "kfgqpc-qalun": {
       "sourceId": "kfqpc-qalun",
       "riwayah": "qalun",
       "label": "KFGQPC Qālūn ʿan Nāfiʿ",
-      "url": "https://qurancomplex.gov.sa/",
+      "url": "https://qurancomplex.gov.sa/quran-dev/",
       "licenseNote": "Use only under KFGQPC terms. Raw files are not bundled in this repository.",
       "localPath": "scripts/_raw/kfqpc-qalun.json",
       "format": "kfgqpc-json",
       "enabled": true,
-      "expectedHash": null,
-      "retrievalMethod": "unknown",
-      "retrievalUrl": null,
-      "authorityStatus": "unknown",
+      "expectedHash": "18465c40ebeec40a92eb98745c9b89796ac6e31f6e93988883dfb6602faaea95",
+      "retrievalMethod": "github_mirror_clone",
+      "retrievalUrl": "https://github.com/thetruetruth/quran-data-kfgqpc",
+      "authorityStatus": "downloaded_unverified",
       "isCompleteCorpus": false,
-      "corpusScope": "sample_subset",
-      "provenanceWarning": "Local corpus exists but source URL and retrieval method are not confirmed.",
+      "corpusScope": "unknown",
+      "provenanceWarning": "Local file came from a GitHub mirror (thetruetruth/quran-data-kfgqpc), not a direct download from qurancomplex.gov.sa. Local MD5 (6feb9a5365379a14cdb464ff6d7f2a2e) and SHA-1 (1caec97f8c42c107a05d006ea58b9d6eee80038a) do NOT match KFGQPC's published package values (MD5 964208ff04c8aadd3ddc1be262d8cfd3, SHA-1 81733666be17742e13c9fa4c7d26d42b1adc67c8). Mismatch is consistent with the local file being a JSON extract from KFGQPC's published package archive, but the chain of custody for that extract is not verified. Re-download the package directly from https://qurancomplex.gov.sa/quran-dev/, verify its package hash matches the published value, then extract the JSON to upgrade authorityStatus to official_confirmed.",
       "canCommitRawFile": false,
-      "lastMatchedHash": null,
-      "lastMatchedAt": null,
-      "notes": "Needs manual provenance confirmation."
+      "lastMatchedHash": "18465c40ebeec40a92eb98745c9b89796ac6e31f6e93988883dfb6602faaea95",
+      "lastMatchedAt": "2026-05-04T19:36:42Z",
+      "notes": "KFGQPC Qālūn ʿan Nāfiʿ JSON. KFGQPC's published package hashes (per https://qurancomplex.gov.sa/quran-dev/ as of 2026-05-04): MD5 964208ff04c8aadd3ddc1be262d8cfd3, SHA-1 81733666be17742e13c9fa4c7d26d42b1adc67c8. Local file MD5: 6feb9a5365379a14cdb464ff6d7f2a2e, SHA-1: 1caec97f8c42c107a05d006ea58b9d6eee80038a, SHA-256: 18465c40ebeec40a92eb98745c9b89796ac6e31f6e93988883dfb6602faaea95."
     }
   }
 }
diff --git a/src/data/verificationLedger.json b/src/data/verificationLedger.json
@@ -5,7 +5,7 @@
     "generated": false,
     "generationPolicy": "no-arabic-text-generation",
     "lastReviewed": null,
-    "lastInitialised": "2026-05-03T21:24:38.508Z"
+    "lastInitialised": "2026-05-04T19:39:23.208Z"
   },
   "records": [
     {
@@ -76,7 +76,7 @@
       "variantKey": "q1-4-w0-hafs-consonantal",
       "riwayah": "qalun",
       "readingText": "مَلِكِ",
-      "source": "kfqpc-qalun",
+      "source": "manual_placeholder",
       "status": "unreviewed",
       "textHash": "fnv1a:9c6b7f21"
     },
@@ -100,7 +100,7 @@
       "variantKey": "q1-4-w0-hafs-consonantal",
       "riwayah": "warsh",
       "readingText": "مَلِكِ",
-      "source": "kfqpc-warsh",
+      "source": "manual_placeholder",
       "status": "unreviewed",
       "textHash": "fnv1a:9c6b7f21"
     },
@@ -220,7 +220,7 @@
       "variantKey": "q2-9-w5-hafs-word-form",
       "riwayah": "qalun",
       "readingText": "يُخَٰدِعُونَ",
-      "source": "kfqpc-qalun",
+      "source": "manual_placeholder",
       "status": "unreviewed",
       "textHash": "fnv1a:1ab67fad"
     },
@@ -244,7 +244,7 @@
       "variantKey": "q2-9-w5-hafs-word-form",
       "riwayah": "warsh",
       "readingText": "يُخَٰدِعُونَ",
-      "source": "kfqpc-warsh",
+      "source": "manual_placeholder",
       "status": "unreviewed",
       "textHash": "fnv1a:1ab67fad"
     },
@@ -292,7 +292,7 @@
       "variantKey": "q2-10-w11-hafs-word-form",
       "riwayah": "qalun",
       "readingText": "يَكْذِبُونَ",
-      "source": "kfqpc-qalun",
+      "source": "manual_placeholder",
       "status": "unreviewed",
       "textHash": "fnv1a:2371d0c5"
     },
@@ -316,7 +316,7 @@
       "variantKey": "q2-10-w11-hafs-word-form",
       "riwayah": "warsh",
       "readingText": "يَكْذِبُونَ",
-      "source": "kfqpc-warsh",
+      "source": "manual_placeholder",
       "status": "unreviewed",
       "textHash": "fnv1a:2371d0c5"
     },
@@ -400,7 +400,7 @@
       "variantKey": "q2-132-w0-hafs-word-form",
       "riwayah": "qalun",
       "readingText": "وَأَوْصَىٰ",
-      "source": "kfqpc-qalun",
+      "source": "manual_placeholder",
       "status": "unreviewed",
       "textHash": "fnv1a:4da24bae"
     },
@@ -424,7 +424,7 @@
       "variantKey": "q2-132-w0-hafs-word-form",
       "riwayah": "warsh",
       "readingText": "وَأَوْصَىٰ",
-      "source": "kfqpc-warsh",
+      "source": "manual_placeholder",
       "status": "unreviewed",
       "textHash": "fnv1a:4da24bae"
     },
@@ -508,7 +508,7 @@
       "variantKey": "q3-146-w3-hafs-word-form",
       "riwayah": "qalun",
       "readingText": "قُتِلَ",
-      "source": "kfqpc-qalun",
+      "source": "manual_placeholder",
       "status": "unreviewed",
       "textHash": "fnv1a:c19a3d64"
     },
@@ -532,7 +532,7 @@
       "variantKey": "q3-146-w3-hafs-word-form",
       "riwayah": "warsh",
       "readingText": "قُتِلَ",
-      "source": "kfqpc-warsh",
+      "source": "manual_placeholder",
       "status": "unreviewed",
       "textHash": "fnv1a:c19a3d64"
     },
@@ -616,7 +616,7 @@
       "variantKey": "q5-6-w14-hafs-vocalization",
       "riwayah": "qalun",
       "readingText": "وَأَرْجُلَكُمْ",
-      "source": "kfqpc-qalun",
+      "source": "manual_placeholder",
       "status": "unreviewed",
       "textHash": "fnv1a:2155d89f"
     },
@@ -640,7 +640,7 @@
       "variantKey": "q5-6-w14-hafs-vocalization",
       "riwayah": "warsh",
       "readingText": "وَأَرْجُلَكُمْ",
-      "source": "kfqpc-warsh",
+      "source": "manual_placeholder",
       "status": "unreviewed",
       "textHash": "fnv1a:2155d89f"
     }
