From 997f1df1013b87df8e168719a4d6df00e799a86c Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Thu, 2 Jul 2026 18:30:40 +0800 Subject: [PATCH 01/37] refactor(knowledge): promote pipeline stage dirs out of utils/ Pure moves + import path rewrites, zero logic change. The utils/ grab-bag hid the entire ingestion pipeline; stage directories now sit at the feature root in pipeline order: - utils/sources/ -> sources/ (input: directory expansion, url fetch, snapshots) - utils/indexing/ -> indexing/ (chunk/splitter/embed/rerank/materialFields) - utils/storage/ -> storage/ (raw/ path allocation) - utils/cleanup/{subtreePurge,vectorCleanup} -> cleanup/ - utils/cleanup/statusCleanup -> ingestion/ (write-side only consumer) - utils/addConflicts -> ingestion/ - utils/search -> query/ (read-side ranking helpers) - utils/items -> items.ts (shared predicates/source probing) External deep imports updated in KnowledgeMigrator/KnowledgeVectorMigrator (+ test). vi.mock specifiers in task handler tests updated to match. Co-Authored-By: Claude Fable 5 Signed-off-by: eeee0717 --- .../migration/v2/migrators/KnowledgeMigrator.ts | 2 +- .../v2/migrators/KnowledgeVectorMigrator.ts | 14 +++++++------- .../__tests__/KnowledgeVectorMigrator.test.ts | 4 ++-- src/main/features/knowledge/KnowledgeService.ts | 12 ++++++------ .../knowledge/KnowledgeWorkflowService.ts | 12 ++++++------ .../knowledge/__tests__/KnowledgeService.test.ts | 8 ++++---- .../{utils => }/__tests__/items.test.ts | 0 .../cleanup/__tests__/vectorCleanup.test.ts | 0 .../{utils => }/cleanup/subtreePurge.ts | 6 +++--- .../{utils => }/cleanup/vectorCleanup.ts | 0 .../{utils => }/indexing/__tests__/chunk.test.ts | 0 .../{utils => }/indexing/__tests__/embed.test.ts | 0 .../indexing/__tests__/materialFields.test.ts | 0 .../indexing/__tests__/rerank.test.ts | 0 .../indexing/__tests__/splitter.test.ts | 0 .../knowledge/{utils => }/indexing/chunk.ts | 0 .../knowledge/{utils => }/indexing/embed.ts | 0 .../{utils => }/indexing/materialFields.ts | 0 .../knowledge/{utils => }/indexing/rerank.ts | 0 .../knowledge/{utils => }/indexing/splitter.ts | 0 .../__tests__/addConflicts.test.ts | 0 .../__tests__/statusCleanup.integration.test.ts | 0 .../{utils => ingestion}/addConflicts.ts | 0 .../cleanup => ingestion}/statusCleanup.ts | 0 src/main/features/knowledge/{utils => }/items.ts | 2 +- .../{utils => query}/__tests__/search.test.ts | 0 .../knowledge/{utils => query}/search.ts | 0 .../knowledge/readers/KnowledgeFileReader.ts | 2 +- .../knowledge/readers/KnowledgeSnapshotReader.ts | 4 ++-- .../sources/__tests__/directory.test.ts | 0 .../sources/__tests__/noteSnapshot.test.ts | 0 .../sources/__tests__/okfFrontmatter.test.ts | 0 .../sources/__tests__/prepare.test.ts | 0 .../sources/__tests__/sourcePlanning.test.ts | 0 .../{utils => }/sources/__tests__/url.test.ts | 0 .../sources/__tests__/urlSnapshot.test.ts | 0 .../knowledge/{utils => }/sources/directory.ts | 0 .../{utils => }/sources/noteSnapshot.ts | 0 .../{utils => }/sources/okfFrontmatter.ts | 0 .../knowledge/{utils => }/sources/prepare.ts | 2 +- .../{utils => }/sources/sourcePlanning.ts | 0 .../knowledge/{utils => }/sources/url.ts | 0 .../knowledge/{utils => }/sources/urlSnapshot.ts | 0 .../storage/__tests__/pathStorage.test.ts | 0 .../knowledge/{utils => }/storage/pathStorage.ts | 0 .../tasks/__tests__/jobHandlerTestUtils.ts | 16 ++++++++-------- .../tasks/checkFileProcessingResultJobHandler.ts | 2 +- .../knowledge/tasks/deleteSubtreeJobHandler.ts | 4 ++-- .../knowledge/tasks/indexDocumentsJobHandler.ts | 16 ++++++++-------- .../knowledge/tasks/prepareRootJobHandler.ts | 10 +++++----- .../knowledge/tasks/reindexSubtreeJobHandler.ts | 6 +++--- .../vectorstore/KnowledgeVectorStoreService.ts | 4 ++-- .../KnowledgeVectorStoreService.test.ts | 2 +- .../KnowledgeIndexStore.integration.test.ts | 2 +- 54 files changed, 65 insertions(+), 65 deletions(-) rename src/main/features/knowledge/{utils => }/__tests__/items.test.ts (100%) rename src/main/features/knowledge/{utils => }/cleanup/__tests__/vectorCleanup.test.ts (100%) rename src/main/features/knowledge/{utils => }/cleanup/subtreePurge.ts (96%) rename src/main/features/knowledge/{utils => }/cleanup/vectorCleanup.ts (100%) rename src/main/features/knowledge/{utils => }/indexing/__tests__/chunk.test.ts (100%) rename src/main/features/knowledge/{utils => }/indexing/__tests__/embed.test.ts (100%) rename src/main/features/knowledge/{utils => }/indexing/__tests__/materialFields.test.ts (100%) rename src/main/features/knowledge/{utils => }/indexing/__tests__/rerank.test.ts (100%) rename src/main/features/knowledge/{utils => }/indexing/__tests__/splitter.test.ts (100%) rename src/main/features/knowledge/{utils => }/indexing/chunk.ts (100%) rename src/main/features/knowledge/{utils => }/indexing/embed.ts (100%) rename src/main/features/knowledge/{utils => }/indexing/materialFields.ts (100%) rename src/main/features/knowledge/{utils => }/indexing/rerank.ts (100%) rename src/main/features/knowledge/{utils => }/indexing/splitter.ts (100%) rename src/main/features/knowledge/{utils => ingestion}/__tests__/addConflicts.test.ts (100%) rename src/main/features/knowledge/{utils/cleanup => ingestion}/__tests__/statusCleanup.integration.test.ts (100%) rename src/main/features/knowledge/{utils => ingestion}/addConflicts.ts (100%) rename src/main/features/knowledge/{utils/cleanup => ingestion}/statusCleanup.ts (100%) rename src/main/features/knowledge/{utils => }/items.ts (99%) rename src/main/features/knowledge/{utils => query}/__tests__/search.test.ts (100%) rename src/main/features/knowledge/{utils => query}/search.ts (100%) rename src/main/features/knowledge/{utils => }/sources/__tests__/directory.test.ts (100%) rename src/main/features/knowledge/{utils => }/sources/__tests__/noteSnapshot.test.ts (100%) rename src/main/features/knowledge/{utils => }/sources/__tests__/okfFrontmatter.test.ts (100%) rename src/main/features/knowledge/{utils => }/sources/__tests__/prepare.test.ts (100%) rename src/main/features/knowledge/{utils => }/sources/__tests__/sourcePlanning.test.ts (100%) rename src/main/features/knowledge/{utils => }/sources/__tests__/url.test.ts (100%) rename src/main/features/knowledge/{utils => }/sources/__tests__/urlSnapshot.test.ts (100%) rename src/main/features/knowledge/{utils => }/sources/directory.ts (100%) rename src/main/features/knowledge/{utils => }/sources/noteSnapshot.ts (100%) rename src/main/features/knowledge/{utils => }/sources/okfFrontmatter.ts (100%) rename src/main/features/knowledge/{utils => }/sources/prepare.ts (98%) rename src/main/features/knowledge/{utils => }/sources/sourcePlanning.ts (100%) rename src/main/features/knowledge/{utils => }/sources/url.ts (100%) rename src/main/features/knowledge/{utils => }/sources/urlSnapshot.ts (100%) rename src/main/features/knowledge/{utils => }/storage/__tests__/pathStorage.test.ts (100%) rename src/main/features/knowledge/{utils => }/storage/pathStorage.ts (100%) diff --git a/src/main/data/migration/v2/migrators/KnowledgeMigrator.ts b/src/main/data/migration/v2/migrators/KnowledgeMigrator.ts index 1f763f012f3..b6366529512 100644 --- a/src/main/data/migration/v2/migrators/KnowledgeMigrator.ts +++ b/src/main/data/migration/v2/migrators/KnowledgeMigrator.ts @@ -12,7 +12,7 @@ import { loggerService } from '@logger' import { needsProcessedArtifactReservation, reserveImportedFileRelativePath -} from '@main/features/knowledge/utils/storage/pathStorage' +} from '@main/features/knowledge/storage/pathStorage' import { sanitizeFilename } from '@main/utils/file' import { copy, ensureDir } from '@main/utils/file/fs' import type { ExecuteResult, PrepareResult, ValidateResult, ValidationError } from '@shared/data/migration/v2/types' diff --git a/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts b/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts index 0ce8306a726..5688dea9098 100644 --- a/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts +++ b/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts @@ -3,19 +3,19 @@ import path from 'node:path' import { knowledgeBaseTable, knowledgeItemTable } from '@data/db/schemas/knowledge' import { loggerService } from '@logger' -import { DOCUMENT_SEPARATOR } from '@main/features/knowledge/utils/indexing/chunk' +import { DOCUMENT_SEPARATOR } from '@main/features/knowledge/indexing/chunk' import { type MaterialFieldSource, toMaterialRelativePath -} from '@main/features/knowledge/utils/indexing/materialFields' -import { deriveNoteSnapshotSlug } from '@main/features/knowledge/utils/sources/noteSnapshot' -import { serializeOkfFrontmatter } from '@main/features/knowledge/utils/sources/okfFrontmatter' -import { deriveUrlSnapshotSlug, deriveUrlSnapshotTitle } from '@main/features/knowledge/utils/sources/urlSnapshot' +} from '@main/features/knowledge/indexing/materialFields' +import { deriveNoteSnapshotSlug } from '@main/features/knowledge/sources/noteSnapshot' +import { serializeOkfFrontmatter } from '@main/features/knowledge/sources/okfFrontmatter' +import { deriveUrlSnapshotSlug, deriveUrlSnapshotTitle } from '@main/features/knowledge/sources/urlSnapshot' import { assertSafeKnowledgeRelativePath, collectKnowledgeReservedRelativePaths, reserveImportedFileRelativePath -} from '@main/features/knowledge/utils/storage/pathStorage' +} from '@main/features/knowledge/storage/pathStorage' import { type BetterSqlite3Driver, openBetterSqlite3IndexDriver @@ -48,7 +48,7 @@ import { const logger = loggerService.withContext('KnowledgeVectorMigrator') // Runtime vector store + material layout — source of truth: -// src/main/features/knowledge/utils/storage/pathStorage.ts +// src/main/features/knowledge/storage/pathStorage.ts // (CHERRY_META_DIR / VECTOR_STORE_FILE / MATERIAL_ROOT_DIR). Runtime opens // {knowledgeBaseDir}/{baseId}/.cherry/index.sqlite by the migrated (new) base id, and resolves // every material's bytes at {knowledgeBaseDir}/{baseId}/raw/{relativePath}, so the migrator must diff --git a/src/main/data/migration/v2/migrators/__tests__/KnowledgeVectorMigrator.test.ts b/src/main/data/migration/v2/migrators/__tests__/KnowledgeVectorMigrator.test.ts index 5212862cfe8..90772577201 100644 --- a/src/main/data/migration/v2/migrators/__tests__/KnowledgeVectorMigrator.test.ts +++ b/src/main/data/migration/v2/migrators/__tests__/KnowledgeVectorMigrator.test.ts @@ -2,7 +2,7 @@ import * as fs from 'node:fs' import * as os from 'node:os' import path from 'node:path' -import { stripOkfFrontmatter } from '@main/features/knowledge/utils/sources/okfFrontmatter' +import { stripOkfFrontmatter } from '@main/features/knowledge/sources/okfFrontmatter' import { hashEmbeddingText } from '@main/features/knowledge/vectorstore/indexStore/hashing' import { KnowledgeIndexStore } from '@main/features/knowledge/vectorstore/indexStore/KnowledgeIndexStore' import { encodeVectorBlob } from '@main/features/knowledge/vectorstore/indexStore/vectorBlob' @@ -73,7 +73,7 @@ function createTempRoot() { } // Mirrors the runtime vector store layout in -// src/main/features/knowledge/utils/storage/pathStorage.ts: {root}/{baseId}/.cherry/index.sqlite. +// src/main/features/knowledge/storage/pathStorage.ts: {root}/{baseId}/.cherry/index.sqlite. // Read-back assertions use this so they fail if the migrator ever writes to a path the runtime // would not open — the exact bug this regression guards against. function runtimeVectorStorePath(baseId: string): string { diff --git a/src/main/features/knowledge/KnowledgeService.ts b/src/main/features/knowledge/KnowledgeService.ts index b7c997848a2..717f3a29d04 100644 --- a/src/main/features/knowledge/KnowledgeService.ts +++ b/src/main/features/knowledge/KnowledgeService.ts @@ -44,12 +44,12 @@ import { toKnowledgeItemId, toKnowledgeItemIds } from './types' -import { embedKnowledgeQuery } from './utils/indexing/embed' -import { toMaterialRelativePath } from './utils/indexing/materialFields' -import { rerankKnowledgeSearchResults } from './utils/indexing/rerank' -import { classifyKnowledgeItemSource } from './utils/items' -import { applyRelevanceThreshold, getInitialSearchScoreKind, withSearchRanks } from './utils/search' -import { getKnowledgeBaseFilePath } from './utils/storage/pathStorage' +import { embedKnowledgeQuery } from './indexing/embed' +import { toMaterialRelativePath } from './indexing/materialFields' +import { rerankKnowledgeSearchResults } from './indexing/rerank' +import { classifyKnowledgeItemSource } from './items' +import { applyRelevanceThreshold, getInitialSearchScoreKind, withSearchRanks } from './query/search' +import { getKnowledgeBaseFilePath } from './storage/pathStorage' import type { KnowledgeIndexStore } from './vectorstore/indexStore/KnowledgeIndexStore' import type { KnowledgeIndexSearchMatch } from './vectorstore/indexStore/model' diff --git a/src/main/features/knowledge/KnowledgeWorkflowService.ts b/src/main/features/knowledge/KnowledgeWorkflowService.ts index 3f13b22ff4e..844f9159e8a 100644 --- a/src/main/features/knowledge/KnowledgeWorkflowService.ts +++ b/src/main/features/knowledge/KnowledgeWorkflowService.ts @@ -32,11 +32,11 @@ import { toKnowledgeItemId, toKnowledgeItemIds } from './types' -import { resolveKnowledgeAddConflicts } from './utils/addConflicts' -import { markUnscheduledKnowledgeItemsFailed } from './utils/cleanup/statusCleanup' -import { cancelActiveKnowledgeSubtreeJobs, purgeKnowledgeSubtreeWithinLock } from './utils/cleanup/subtreePurge' -import { isContainerKnowledgeItem } from './utils/items' -import { planKnowledgeItemSource } from './utils/sources/sourcePlanning' +import { resolveKnowledgeAddConflicts } from './ingestion/addConflicts' +import { markUnscheduledKnowledgeItemsFailed } from './ingestion/statusCleanup' +import { cancelActiveKnowledgeSubtreeJobs, purgeKnowledgeSubtreeWithinLock } from './cleanup/subtreePurge' +import { isContainerKnowledgeItem } from './items' +import { planKnowledgeItemSource } from './sources/sourcePlanning' import { assertKnowledgeFileTargetAvailable, collectKnowledgeReservedRelativePaths, @@ -47,7 +47,7 @@ import { getProcessedMarkdownRelativePath, needsProcessedArtifactReservation, reserveImportedFileRelativePath -} from './utils/storage/pathStorage' +} from './storage/pathStorage' const logger = loggerService.withContext('Knowledge:WorkflowService') // Keep poll jobs delayed enough to avoid hot-looping while remote processors are still working. diff --git a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts index 4b256dd0a49..35c10f4d37d 100644 --- a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts +++ b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts @@ -10,7 +10,7 @@ import { } from '@shared/data/types/knowledge' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import type * as PathStorage from '../utils/storage/pathStorage' +import type * as PathStorage from '../storage/pathStorage' const { cancelManyMock, @@ -170,12 +170,12 @@ vi.mock('@data/services/KnowledgeItemService', () => ({ } })) -vi.mock('../utils/indexing/rerank', () => ({ +vi.mock('../indexing/rerank', () => ({ rerankKnowledgeSearchResults: rerankKnowledgeSearchResultsMock })) -vi.mock('../utils/storage/pathStorage', async () => { - const actual = await vi.importActual('../utils/storage/pathStorage') +vi.mock('../storage/pathStorage', async () => { + const actual = await vi.importActual('../storage/pathStorage') return { ...actual, copyFileIntoKnowledgeBaseAt: copyFileIntoKnowledgeBaseAtMock, diff --git a/src/main/features/knowledge/utils/__tests__/items.test.ts b/src/main/features/knowledge/__tests__/items.test.ts similarity index 100% rename from src/main/features/knowledge/utils/__tests__/items.test.ts rename to src/main/features/knowledge/__tests__/items.test.ts diff --git a/src/main/features/knowledge/utils/cleanup/__tests__/vectorCleanup.test.ts b/src/main/features/knowledge/cleanup/__tests__/vectorCleanup.test.ts similarity index 100% rename from src/main/features/knowledge/utils/cleanup/__tests__/vectorCleanup.test.ts rename to src/main/features/knowledge/cleanup/__tests__/vectorCleanup.test.ts diff --git a/src/main/features/knowledge/utils/cleanup/subtreePurge.ts b/src/main/features/knowledge/cleanup/subtreePurge.ts similarity index 96% rename from src/main/features/knowledge/utils/cleanup/subtreePurge.ts rename to src/main/features/knowledge/cleanup/subtreePurge.ts index 665c62d2983..c69aa1e0150 100644 --- a/src/main/features/knowledge/utils/cleanup/subtreePurge.ts +++ b/src/main/features/knowledge/cleanup/subtreePurge.ts @@ -2,14 +2,14 @@ import { application } from '@application' import { knowledgeItemService } from '@data/services/KnowledgeItemService' import type { KnowledgeBase, KnowledgeItem } from '@shared/data/types/knowledge' -import { cancelJobOrThrow } from '../../tasks/utils/cancel' -import { narrowKnowledgeJobInput } from '../../tasks/utils/jobInput' +import { cancelJobOrThrow } from '../tasks/utils/cancel' +import { narrowKnowledgeJobInput } from '../tasks/utils/jobInput' import { KNOWLEDGE_ACTIVE_JOB_LIMIT, KNOWLEDGE_ACTIVE_JOB_STATUSES, knowledgeQueueName, toKnowledgeBaseId -} from '../../types' +} from '../types' import { isIndexableKnowledgeItem } from '../items' import { deleteKnowledgeItemFilesBestEffort } from '../storage/pathStorage' import { deleteKnowledgeItemVectors } from './vectorCleanup' diff --git a/src/main/features/knowledge/utils/cleanup/vectorCleanup.ts b/src/main/features/knowledge/cleanup/vectorCleanup.ts similarity index 100% rename from src/main/features/knowledge/utils/cleanup/vectorCleanup.ts rename to src/main/features/knowledge/cleanup/vectorCleanup.ts diff --git a/src/main/features/knowledge/utils/indexing/__tests__/chunk.test.ts b/src/main/features/knowledge/indexing/__tests__/chunk.test.ts similarity index 100% rename from src/main/features/knowledge/utils/indexing/__tests__/chunk.test.ts rename to src/main/features/knowledge/indexing/__tests__/chunk.test.ts diff --git a/src/main/features/knowledge/utils/indexing/__tests__/embed.test.ts b/src/main/features/knowledge/indexing/__tests__/embed.test.ts similarity index 100% rename from src/main/features/knowledge/utils/indexing/__tests__/embed.test.ts rename to src/main/features/knowledge/indexing/__tests__/embed.test.ts diff --git a/src/main/features/knowledge/utils/indexing/__tests__/materialFields.test.ts b/src/main/features/knowledge/indexing/__tests__/materialFields.test.ts similarity index 100% rename from src/main/features/knowledge/utils/indexing/__tests__/materialFields.test.ts rename to src/main/features/knowledge/indexing/__tests__/materialFields.test.ts diff --git a/src/main/features/knowledge/utils/indexing/__tests__/rerank.test.ts b/src/main/features/knowledge/indexing/__tests__/rerank.test.ts similarity index 100% rename from src/main/features/knowledge/utils/indexing/__tests__/rerank.test.ts rename to src/main/features/knowledge/indexing/__tests__/rerank.test.ts diff --git a/src/main/features/knowledge/utils/indexing/__tests__/splitter.test.ts b/src/main/features/knowledge/indexing/__tests__/splitter.test.ts similarity index 100% rename from src/main/features/knowledge/utils/indexing/__tests__/splitter.test.ts rename to src/main/features/knowledge/indexing/__tests__/splitter.test.ts diff --git a/src/main/features/knowledge/utils/indexing/chunk.ts b/src/main/features/knowledge/indexing/chunk.ts similarity index 100% rename from src/main/features/knowledge/utils/indexing/chunk.ts rename to src/main/features/knowledge/indexing/chunk.ts diff --git a/src/main/features/knowledge/utils/indexing/embed.ts b/src/main/features/knowledge/indexing/embed.ts similarity index 100% rename from src/main/features/knowledge/utils/indexing/embed.ts rename to src/main/features/knowledge/indexing/embed.ts diff --git a/src/main/features/knowledge/utils/indexing/materialFields.ts b/src/main/features/knowledge/indexing/materialFields.ts similarity index 100% rename from src/main/features/knowledge/utils/indexing/materialFields.ts rename to src/main/features/knowledge/indexing/materialFields.ts diff --git a/src/main/features/knowledge/utils/indexing/rerank.ts b/src/main/features/knowledge/indexing/rerank.ts similarity index 100% rename from src/main/features/knowledge/utils/indexing/rerank.ts rename to src/main/features/knowledge/indexing/rerank.ts diff --git a/src/main/features/knowledge/utils/indexing/splitter.ts b/src/main/features/knowledge/indexing/splitter.ts similarity index 100% rename from src/main/features/knowledge/utils/indexing/splitter.ts rename to src/main/features/knowledge/indexing/splitter.ts diff --git a/src/main/features/knowledge/utils/__tests__/addConflicts.test.ts b/src/main/features/knowledge/ingestion/__tests__/addConflicts.test.ts similarity index 100% rename from src/main/features/knowledge/utils/__tests__/addConflicts.test.ts rename to src/main/features/knowledge/ingestion/__tests__/addConflicts.test.ts diff --git a/src/main/features/knowledge/utils/cleanup/__tests__/statusCleanup.integration.test.ts b/src/main/features/knowledge/ingestion/__tests__/statusCleanup.integration.test.ts similarity index 100% rename from src/main/features/knowledge/utils/cleanup/__tests__/statusCleanup.integration.test.ts rename to src/main/features/knowledge/ingestion/__tests__/statusCleanup.integration.test.ts diff --git a/src/main/features/knowledge/utils/addConflicts.ts b/src/main/features/knowledge/ingestion/addConflicts.ts similarity index 100% rename from src/main/features/knowledge/utils/addConflicts.ts rename to src/main/features/knowledge/ingestion/addConflicts.ts diff --git a/src/main/features/knowledge/utils/cleanup/statusCleanup.ts b/src/main/features/knowledge/ingestion/statusCleanup.ts similarity index 100% rename from src/main/features/knowledge/utils/cleanup/statusCleanup.ts rename to src/main/features/knowledge/ingestion/statusCleanup.ts diff --git a/src/main/features/knowledge/utils/items.ts b/src/main/features/knowledge/items.ts similarity index 99% rename from src/main/features/knowledge/utils/items.ts rename to src/main/features/knowledge/items.ts index 426feb694d3..4fdbcc8aeec 100644 --- a/src/main/features/knowledge/utils/items.ts +++ b/src/main/features/knowledge/items.ts @@ -1,7 +1,7 @@ import type { PathReadability } from '@main/utils/file/fs' import type { KnowledgeItem } from '@shared/data/types/knowledge' -import type { ContainerKnowledgeItem, IndexableKnowledgeItem } from '../types/items' +import type { ContainerKnowledgeItem, IndexableKnowledgeItem } from './types/items' import { probeKnowledgeFile, probeKnowledgeSourcePath } from './storage/pathStorage' export function isIndexableKnowledgeItem(item: KnowledgeItem): item is IndexableKnowledgeItem { diff --git a/src/main/features/knowledge/utils/__tests__/search.test.ts b/src/main/features/knowledge/query/__tests__/search.test.ts similarity index 100% rename from src/main/features/knowledge/utils/__tests__/search.test.ts rename to src/main/features/knowledge/query/__tests__/search.test.ts diff --git a/src/main/features/knowledge/utils/search.ts b/src/main/features/knowledge/query/search.ts similarity index 100% rename from src/main/features/knowledge/utils/search.ts rename to src/main/features/knowledge/query/search.ts diff --git a/src/main/features/knowledge/readers/KnowledgeFileReader.ts b/src/main/features/knowledge/readers/KnowledgeFileReader.ts index 7f183edd3bf..bb431faaa4a 100644 --- a/src/main/features/knowledge/readers/KnowledgeFileReader.ts +++ b/src/main/features/knowledge/readers/KnowledgeFileReader.ts @@ -10,7 +10,7 @@ import { MarkdownReader } from '@vectorstores/readers/markdown' import { PDFReader } from '@vectorstores/readers/pdf' import { TextFileReader } from '@vectorstores/readers/text' -import { getKnowledgeBaseFilePath } from '../utils/storage/pathStorage' +import { getKnowledgeBaseFilePath } from '../storage/pathStorage' import { DraftsExportReader } from './files/DraftsExportReader' import { EpubReader } from './files/EpubReader' diff --git a/src/main/features/knowledge/readers/KnowledgeSnapshotReader.ts b/src/main/features/knowledge/readers/KnowledgeSnapshotReader.ts index 53b7da2cfe3..8472c1acafa 100644 --- a/src/main/features/knowledge/readers/KnowledgeSnapshotReader.ts +++ b/src/main/features/knowledge/readers/KnowledgeSnapshotReader.ts @@ -2,8 +2,8 @@ import { read } from '@main/utils/file/fs' import type { KnowledgeItemOf, KnowledgeSourceMetadata } from '@shared/data/types/knowledge' import { Document } from '@vectorstores/core' -import { stripOkfFrontmatter } from '../utils/sources/okfFrontmatter' -import { getKnowledgeBaseFilePath } from '../utils/storage/pathStorage' +import { stripOkfFrontmatter } from '../sources/okfFrontmatter' +import { getKnowledgeBaseFilePath } from '../storage/pathStorage' /** * Read a url or note item from its captured on-disk snapshot — never the network diff --git a/src/main/features/knowledge/utils/sources/__tests__/directory.test.ts b/src/main/features/knowledge/sources/__tests__/directory.test.ts similarity index 100% rename from src/main/features/knowledge/utils/sources/__tests__/directory.test.ts rename to src/main/features/knowledge/sources/__tests__/directory.test.ts diff --git a/src/main/features/knowledge/utils/sources/__tests__/noteSnapshot.test.ts b/src/main/features/knowledge/sources/__tests__/noteSnapshot.test.ts similarity index 100% rename from src/main/features/knowledge/utils/sources/__tests__/noteSnapshot.test.ts rename to src/main/features/knowledge/sources/__tests__/noteSnapshot.test.ts diff --git a/src/main/features/knowledge/utils/sources/__tests__/okfFrontmatter.test.ts b/src/main/features/knowledge/sources/__tests__/okfFrontmatter.test.ts similarity index 100% rename from src/main/features/knowledge/utils/sources/__tests__/okfFrontmatter.test.ts rename to src/main/features/knowledge/sources/__tests__/okfFrontmatter.test.ts diff --git a/src/main/features/knowledge/utils/sources/__tests__/prepare.test.ts b/src/main/features/knowledge/sources/__tests__/prepare.test.ts similarity index 100% rename from src/main/features/knowledge/utils/sources/__tests__/prepare.test.ts rename to src/main/features/knowledge/sources/__tests__/prepare.test.ts diff --git a/src/main/features/knowledge/utils/sources/__tests__/sourcePlanning.test.ts b/src/main/features/knowledge/sources/__tests__/sourcePlanning.test.ts similarity index 100% rename from src/main/features/knowledge/utils/sources/__tests__/sourcePlanning.test.ts rename to src/main/features/knowledge/sources/__tests__/sourcePlanning.test.ts diff --git a/src/main/features/knowledge/utils/sources/__tests__/url.test.ts b/src/main/features/knowledge/sources/__tests__/url.test.ts similarity index 100% rename from src/main/features/knowledge/utils/sources/__tests__/url.test.ts rename to src/main/features/knowledge/sources/__tests__/url.test.ts diff --git a/src/main/features/knowledge/utils/sources/__tests__/urlSnapshot.test.ts b/src/main/features/knowledge/sources/__tests__/urlSnapshot.test.ts similarity index 100% rename from src/main/features/knowledge/utils/sources/__tests__/urlSnapshot.test.ts rename to src/main/features/knowledge/sources/__tests__/urlSnapshot.test.ts diff --git a/src/main/features/knowledge/utils/sources/directory.ts b/src/main/features/knowledge/sources/directory.ts similarity index 100% rename from src/main/features/knowledge/utils/sources/directory.ts rename to src/main/features/knowledge/sources/directory.ts diff --git a/src/main/features/knowledge/utils/sources/noteSnapshot.ts b/src/main/features/knowledge/sources/noteSnapshot.ts similarity index 100% rename from src/main/features/knowledge/utils/sources/noteSnapshot.ts rename to src/main/features/knowledge/sources/noteSnapshot.ts diff --git a/src/main/features/knowledge/utils/sources/okfFrontmatter.ts b/src/main/features/knowledge/sources/okfFrontmatter.ts similarity index 100% rename from src/main/features/knowledge/utils/sources/okfFrontmatter.ts rename to src/main/features/knowledge/sources/okfFrontmatter.ts diff --git a/src/main/features/knowledge/utils/sources/prepare.ts b/src/main/features/knowledge/sources/prepare.ts similarity index 98% rename from src/main/features/knowledge/utils/sources/prepare.ts rename to src/main/features/knowledge/sources/prepare.ts index 19e0891379f..2adf04ca1fb 100644 --- a/src/main/features/knowledge/utils/sources/prepare.ts +++ b/src/main/features/knowledge/sources/prepare.ts @@ -7,7 +7,7 @@ import { type KnowledgeItemType } from '@shared/data/types/knowledge' -import type { IndexableKnowledgeItem } from '../../types/items' +import type { IndexableKnowledgeItem } from '../types/items' import { isContainerKnowledgeItem, isIndexableKnowledgeItem } from '../items' import { collectKnowledgeReservedRelativePaths } from '../storage/pathStorage' import { expandDirectoryOwnerToTree, type ExpandedDirectoryNode } from './directory' diff --git a/src/main/features/knowledge/utils/sources/sourcePlanning.ts b/src/main/features/knowledge/sources/sourcePlanning.ts similarity index 100% rename from src/main/features/knowledge/utils/sources/sourcePlanning.ts rename to src/main/features/knowledge/sources/sourcePlanning.ts diff --git a/src/main/features/knowledge/utils/sources/url.ts b/src/main/features/knowledge/sources/url.ts similarity index 100% rename from src/main/features/knowledge/utils/sources/url.ts rename to src/main/features/knowledge/sources/url.ts diff --git a/src/main/features/knowledge/utils/sources/urlSnapshot.ts b/src/main/features/knowledge/sources/urlSnapshot.ts similarity index 100% rename from src/main/features/knowledge/utils/sources/urlSnapshot.ts rename to src/main/features/knowledge/sources/urlSnapshot.ts diff --git a/src/main/features/knowledge/utils/storage/__tests__/pathStorage.test.ts b/src/main/features/knowledge/storage/__tests__/pathStorage.test.ts similarity index 100% rename from src/main/features/knowledge/utils/storage/__tests__/pathStorage.test.ts rename to src/main/features/knowledge/storage/__tests__/pathStorage.test.ts diff --git a/src/main/features/knowledge/utils/storage/pathStorage.ts b/src/main/features/knowledge/storage/pathStorage.ts similarity index 100% rename from src/main/features/knowledge/utils/storage/pathStorage.ts rename to src/main/features/knowledge/storage/pathStorage.ts diff --git a/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts b/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts index 506ac4e473d..51deaa76acc 100644 --- a/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts +++ b/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts @@ -4,7 +4,7 @@ import type { KnowledgeBase, KnowledgeItemOf } from '@shared/data/types/knowledg import { MockMainCacheServiceUtils } from '@test-mocks/main/CacheService' import { beforeEach, vi } from 'vitest' -import type * as PathStorage from '../../utils/storage/pathStorage' +import type * as PathStorage from '../../storage/pathStorage' const mocks = vi.hoisted(() => ({ cancelMock: vi.fn(), @@ -135,24 +135,24 @@ vi.mock('../../readers/KnowledgeReader', () => ({ loadKnowledgeItemDocuments: loadKnowledgeItemDocumentsMock })) -vi.mock('../../utils/sources/prepare', () => ({ +vi.mock('../../sources/prepare', () => ({ prepareKnowledgeItem: prepareKnowledgeItemMock })) -vi.mock('../../utils/sources/url', () => ({ +vi.mock('../../sources/url', () => ({ fetchKnowledgeWebPage: fetchKnowledgeWebPageMock })) -vi.mock('../../utils/sources/urlSnapshot', () => ({ +vi.mock('../../sources/urlSnapshot', () => ({ captureUrlSnapshotFile: captureUrlSnapshotFileMock })) -vi.mock('../../utils/sources/noteSnapshot', () => ({ +vi.mock('../../sources/noteSnapshot', () => ({ captureNoteSnapshotFile: captureNoteSnapshotFileMock })) -vi.mock('../../utils/storage/pathStorage', async () => { - const actual = await vi.importActual('../../utils/storage/pathStorage') +vi.mock('../../storage/pathStorage', async () => { + const actual = await vi.importActual('../../storage/pathStorage') return { ...actual, // Stub the best-effort cleanup the handlers call. Its swallow-on-failure @@ -167,7 +167,7 @@ vi.mock('../../utils/storage/pathStorage', async () => { } }) -vi.mock('../../utils/indexing/embed', () => ({ +vi.mock('../../indexing/embed', () => ({ embedKnowledgeTexts: embedKnowledgeTextsMock })) diff --git a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts index ae8a4b8d98f..046c8b27c94 100644 --- a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts +++ b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts @@ -15,7 +15,7 @@ import type { FileProcessingJobPayload } from '../../fileProcessing/tasks/shared import type { KnowledgeLockManager } from '../KnowledgeLockManager' import type { KnowledgeWorkflowService } from '../KnowledgeWorkflowService' import { knowledgeQueueName, toKnowledgeBaseId, toKnowledgeItemId } from '../types' -import { toKnowledgeRelativePath } from '../utils/storage/pathStorage' +import { toKnowledgeRelativePath } from '../storage/pathStorage' import type { KnowledgeCheckFileProcessingResultPayload } from './jobTypes' import { cancelJobOrThrow } from './utils/cancel' import { isDataApiNotFoundError, markKnowledgeItemFailedOnSettled } from './utils/settled' diff --git a/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts b/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts index b1dc54a85f4..d48964da3bf 100644 --- a/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts +++ b/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts @@ -7,8 +7,8 @@ import type { JobHandler } from '@main/core/job/types' import type { KnowledgeLockManager } from '../KnowledgeLockManager' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId } from '../types' -import { cancelActiveKnowledgeSubtreeJobs, purgeKnowledgeSubtreeWithinLock } from '../utils/cleanup/subtreePurge' -import { reclaimKnowledgeIndexSpace } from '../utils/cleanup/vectorCleanup' +import { cancelActiveKnowledgeSubtreeJobs, purgeKnowledgeSubtreeWithinLock } from '../cleanup/subtreePurge' +import { reclaimKnowledgeIndexSpace } from '../cleanup/vectorCleanup' import type { KnowledgeDeleteSubtreePayload } from './jobTypes' const logger = loggerService.withContext('Knowledge:DeleteSubtreeJobHandler') diff --git a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts index ade892c4562..b66a328bce6 100644 --- a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts +++ b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts @@ -12,14 +12,14 @@ import type { KnowledgeLockManager } from '../KnowledgeLockManager' import { loadKnowledgeItemDocuments } from '../readers/KnowledgeReader' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId } from '../types' import type { IndexableKnowledgeItem } from '../types/items' -import { type ChunkedKnowledgeContent, chunkKnowledgeDocuments } from '../utils/indexing/chunk' -import { embedKnowledgeTexts } from '../utils/indexing/embed' -import { toMaterialRelativePath } from '../utils/indexing/materialFields' -import { isIndexableKnowledgeItem } from '../utils/items' -import { captureNoteSnapshotFile } from '../utils/sources/noteSnapshot' -import { fetchKnowledgeWebPage } from '../utils/sources/url' -import { captureUrlSnapshotFile } from '../utils/sources/urlSnapshot' -import { collectKnowledgeReservedRelativePaths } from '../utils/storage/pathStorage' +import { type ChunkedKnowledgeContent, chunkKnowledgeDocuments } from '../indexing/chunk' +import { embedKnowledgeTexts } from '../indexing/embed' +import { toMaterialRelativePath } from '../indexing/materialFields' +import { isIndexableKnowledgeItem } from '../items' +import { captureNoteSnapshotFile } from '../sources/noteSnapshot' +import { fetchKnowledgeWebPage } from '../sources/url' +import { captureUrlSnapshotFile } from '../sources/urlSnapshot' +import { collectKnowledgeReservedRelativePaths } from '../storage/pathStorage' import { hashEmbeddingText } from '../vectorstore/indexStore/hashing' import type { RebuildMaterialEmbeddingInput, RebuildMaterialInput } from '../vectorstore/indexStore/model' import type { KnowledgeIndexDocumentsPayload } from './jobTypes' diff --git a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts index c41f9495b73..5f1b4ed930f 100644 --- a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts +++ b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts @@ -9,11 +9,11 @@ import type { KnowledgeItem } from '@shared/data/types/knowledge' import type { KnowledgeLockManager } from '../KnowledgeLockManager' import type { KnowledgeWorkflowService } from '../KnowledgeWorkflowService' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId, toKnowledgeItemId } from '../types' -import { markUnscheduledKnowledgeItemsFailed } from '../utils/cleanup/statusCleanup' -import { deleteKnowledgeItemVectors } from '../utils/cleanup/vectorCleanup' -import { isIndexableKnowledgeItem } from '../utils/items' -import { prepareKnowledgeItem } from '../utils/sources/prepare' -import { deleteKnowledgeItemFilesBestEffort } from '../utils/storage/pathStorage' +import { markUnscheduledKnowledgeItemsFailed } from '../ingestion/statusCleanup' +import { deleteKnowledgeItemVectors } from '../cleanup/vectorCleanup' +import { isIndexableKnowledgeItem } from '../items' +import { prepareKnowledgeItem } from '../sources/prepare' +import { deleteKnowledgeItemFilesBestEffort } from '../storage/pathStorage' import type { KnowledgePrepareRootPayload } from './jobTypes' import { isDataApiNotFoundError, markKnowledgeItemFailedOnSettled } from './utils/settled' diff --git a/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts b/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts index ef571f05825..bea0788bd49 100644 --- a/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts +++ b/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts @@ -18,9 +18,9 @@ import { toKnowledgeBaseId, toKnowledgeItemId } from '../types' -import { deleteKnowledgeItemVectors } from '../utils/cleanup/vectorCleanup' -import { canKnowledgeItemRebuildSource, isContainerKnowledgeItem } from '../utils/items' -import { deleteKnowledgeItemFilesBestEffort } from '../utils/storage/pathStorage' +import { deleteKnowledgeItemVectors } from '../cleanup/vectorCleanup' +import { canKnowledgeItemRebuildSource, isContainerKnowledgeItem } from '../items' +import { deleteKnowledgeItemFilesBestEffort } from '../storage/pathStorage' import type { KnowledgeReindexSubtreePayload } from './jobTypes' import { narrowKnowledgeJobInput } from './utils/jobInput' diff --git a/src/main/features/knowledge/vectorstore/KnowledgeVectorStoreService.ts b/src/main/features/knowledge/vectorstore/KnowledgeVectorStoreService.ts index d1dec47f056..df4f78ce959 100644 --- a/src/main/features/knowledge/vectorstore/KnowledgeVectorStoreService.ts +++ b/src/main/features/knowledge/vectorstore/KnowledgeVectorStoreService.ts @@ -7,8 +7,8 @@ import { DataApiErrorFactory } from '@shared/data/api' import type { CompletedKnowledgeBase, KnowledgeBase } from '@shared/data/types/knowledge' import { isCompletedKnowledgeBase } from '@shared/data/types/knowledge' -import { isIndexableKnowledgeItem } from '../utils/items' -import { deleteKnowledgeBaseDir, getKnowledgeVectorStoreFilePathSync } from '../utils/storage/pathStorage' +import { isIndexableKnowledgeItem } from '../items' +import { deleteKnowledgeBaseDir, getKnowledgeVectorStoreFilePathSync } from '../storage/pathStorage' import { openBetterSqlite3IndexDriver } from './indexStore/BetterSqlite3Driver' import { betterSqlite3VectorIndex } from './indexStore/BetterSqlite3VectorIndex' import { ensureIndexMeta, hasAnyMaterial, readIndexSchemaVersion } from './indexStore/indexMeta' diff --git a/src/main/features/knowledge/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts b/src/main/features/knowledge/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts index a2eeaf81861..15c7c28b149 100644 --- a/src/main/features/knowledge/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts +++ b/src/main/features/knowledge/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts @@ -98,7 +98,7 @@ vi.mock('@data/services/KnowledgeItemService', () => ({ knowledgeItemService: { getItemsByBaseId: getItemsByBaseIdMock } })) -vi.mock('../../utils/storage/pathStorage', () => ({ +vi.mock('../../storage/pathStorage', () => ({ getKnowledgeVectorStoreFilePathSync: getPathSyncMock, deleteKnowledgeBaseDir: deleteDirMock })) diff --git a/src/main/features/knowledge/vectorstore/indexStore/__tests__/KnowledgeIndexStore.integration.test.ts b/src/main/features/knowledge/vectorstore/indexStore/__tests__/KnowledgeIndexStore.integration.test.ts index d3a61212352..25a14ea760d 100644 --- a/src/main/features/knowledge/vectorstore/indexStore/__tests__/KnowledgeIndexStore.integration.test.ts +++ b/src/main/features/knowledge/vectorstore/indexStore/__tests__/KnowledgeIndexStore.integration.test.ts @@ -4,7 +4,7 @@ import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { splitTextWithOffsets } from '../../../utils/indexing/splitter' +import { splitTextWithOffsets } from '../../../indexing/splitter' import { openBetterSqlite3IndexDriver } from '../BetterSqlite3Driver' import { betterSqlite3VectorIndex } from '../BetterSqlite3VectorIndex' import { hashEmbeddingText } from '../hashing' From 688d5b79f70f24d993536606531ef113b6c9cdf6 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Thu, 2 Jul 2026 18:35:06 +0800 Subject: [PATCH 02/37] refactor(knowledge): extract base admin service from KnowledgeService Move the base-lifecycle block (~250 lines) out of the facade into base/KnowledgeBaseAdminService: createBase (with rollback), deleteBase, restoreBase (+toRestoreRuntimeInput), listBases, hasAnyBase, and the deleteBase-only cancelAllJobsForBase. Code is moved verbatim; the facade methods become one-line delegates, so all external call sites and tests are untouched. assertBaseCanRunRuntimeOperation becomes a free function in base/guards so the read side (search/concepts) and the write side can share the same base-status guard without going through the facade. Co-Authored-By: Claude Fable 5 Signed-off-by: eeee0717 --- .../features/knowledge/KnowledgeService.ts | 273 ++--------------- .../base/KnowledgeBaseAdminService.ts | 279 ++++++++++++++++++ src/main/features/knowledge/base/guards.ts | 18 ++ 3 files changed, 317 insertions(+), 253 deletions(-) create mode 100644 src/main/features/knowledge/base/KnowledgeBaseAdminService.ts create mode 100644 src/main/features/knowledge/base/guards.ts diff --git a/src/main/features/knowledge/KnowledgeService.ts b/src/main/features/knowledge/KnowledgeService.ts index 717f3a29d04..7b1034a919b 100644 --- a/src/main/features/knowledge/KnowledgeService.ts +++ b/src/main/features/knowledge/KnowledgeService.ts @@ -5,14 +5,12 @@ import { loggerService } from '@logger' import { BaseService, DependsOn, Injectable, Phase, ServicePhase } from '@main/core/lifecycle' import { TraceMethod } from '@mcp-trace/trace-core' import { DataApiErrorFactory, ErrorCode, isDataApiError } from '@shared/data/api' -import { KNOWLEDGE_BASES_MAX_LIMIT } from '@shared/data/api/schemas/knowledges' import { type CreateKnowledgeBaseDto, getKnowledgeItemDisplayTitle, KNOWLEDGE_ITEM_ERROR_INDEXING_INTERRUPTED, type KnowledgeAddConflictStrategy, type KnowledgeAddItemInput, - KnowledgeAddItemInputSchema, type KnowledgeAddItemsResult, type KnowledgeBase, type KnowledgeItem, @@ -26,30 +24,27 @@ import { import { isCompletedVectorKnowledgeBase } from '@shared/data/types/knowledge' import { estimateTokenCount } from 'tokenx' +import { assertBaseCanRunRuntimeOperation } from './base/guards' +import { KnowledgeBaseAdminService } from './base/KnowledgeBaseAdminService' +import { embedKnowledgeQuery } from './indexing/embed' +import { toMaterialRelativePath } from './indexing/materialFields' +import { rerankKnowledgeSearchResults } from './indexing/rerank' +import { classifyKnowledgeItemSource } from './items' import { KnowledgeLockManager } from './KnowledgeLockManager' import { KnowledgeWorkflowService } from './KnowledgeWorkflowService' +import { applyRelevanceThreshold, getInitialSearchScoreKind, withSearchRanks } from './query/search' import { createCheckFileProcessingResultJobHandler } from './tasks/checkFileProcessingResultJobHandler' import { createDeleteSubtreeJobHandler } from './tasks/deleteSubtreeJobHandler' import { createIndexDocumentsJobHandler } from './tasks/indexDocumentsJobHandler' import { createPrepareRootJobHandler } from './tasks/prepareRootJobHandler' import { createReindexSubtreeJobHandler } from './tasks/reindexSubtreeJobHandler' -import { narrowKnowledgeJobInput } from './tasks/utils/jobInput' import { - KNOWLEDGE_ACTIVE_JOB_LIMIT, - KNOWLEDGE_ACTIVE_JOB_STATUSES, - KNOWLEDGE_JOB_TYPES, knowledgeDeleteSubtreeIdempotencyKey, knowledgeQueueName, toKnowledgeBaseId, toKnowledgeItemId, toKnowledgeItemIds } from './types' -import { embedKnowledgeQuery } from './indexing/embed' -import { toMaterialRelativePath } from './indexing/materialFields' -import { rerankKnowledgeSearchResults } from './indexing/rerank' -import { classifyKnowledgeItemSource } from './items' -import { applyRelevanceThreshold, getInitialSearchScoreKind, withSearchRanks } from './query/search' -import { getKnowledgeBaseFilePath } from './storage/pathStorage' import type { KnowledgeIndexStore } from './vectorstore/indexStore/KnowledgeIndexStore' import type { KnowledgeIndexSearchMatch } from './vectorstore/indexStore/model' @@ -66,7 +61,6 @@ const KNOWLEDGE_SEARCH_OVERFETCH_FACTOR = 5 /** Hard ceiling on fetched candidates, bounding the brute-force vector scan and rerank cost regardless of topK. */ const KNOWLEDGE_SEARCH_CANDIDATE_CAP = 200 const REINDEX_ALLOWED_STATUSES = new Set(['completed', 'failed']) -const KNOWLEDGE_JOB_TYPE_SET = new Set(KNOWLEDGE_JOB_TYPES) /** Max characters {@link KnowledgeService.readConcept} returns in one slice, so a large document can't flood the agent's context. */ const CONCEPT_READ_MAX_CHARS = 20_000 @@ -271,6 +265,9 @@ function scanConceptMatches( export class KnowledgeService extends BaseService { private readonly knowledgeLockManager = new KnowledgeLockManager() private readonly workflowService = new KnowledgeWorkflowService(this.knowledgeLockManager) + private readonly baseAdmin = new KnowledgeBaseAdminService(this.knowledgeLockManager, (baseId, items) => + this.addItems(baseId, items) + ) protected onInit(): void { const jobManager = application.get('JobManager') @@ -296,153 +293,15 @@ export class KnowledgeService extends BaseService { } async createBase(dto: CreateKnowledgeBaseDto): Promise { - const base = knowledgeBaseService.create(dto) - const vectorStoreService = application.get('KnowledgeVectorStoreService') - - try { - await vectorStoreService.getIndexStore(base) - } catch (error) { - await this.rollbackFailedBaseCreation(base.id) - throw error - } - - return base - } - - /** - * Undo a half-created base after its index store failed to open: remove the - * orphaned `.cherry/` directory `getIndexStore` left on disk and drop the DB - * row. Both steps are best-effort and logged — a cleanup failure must never - * mask the original open error the caller needs to see. - */ - private async rollbackFailedBaseCreation(baseId: string): Promise { - const vectorStoreService = application.get('KnowledgeVectorStoreService') - try { - await vectorStoreService.deleteStore(baseId) - } catch (cleanupError) { - logger.warn('Failed to remove index store dir during createBase rollback', cleanupError as Error, { baseId }) - } - try { - knowledgeBaseService.delete(baseId) - } catch (cleanupError) { - logger.warn('Failed to delete knowledge base row during createBase rollback', cleanupError as Error, { baseId }) - } + return await this.baseAdmin.createBase(dto) } async deleteBase(baseId: string): Promise { - await this.cancelAllJobsForBase(baseId) - - await this.knowledgeLockManager.withBaseMutationLock(baseId, async () => { - try { - const vectorStoreService = application.get('KnowledgeVectorStoreService') - await vectorStoreService.deleteStore(baseId) - } catch (error) { - const normalizedError = error instanceof Error ? error : new Error(String(error)) - logger.error('Failed to delete knowledge base vector artifacts', normalizedError, { baseId }) - throw error - } - - try { - knowledgeBaseService.delete(baseId) - } catch (error) { - const normalizedError = error instanceof Error ? error : new Error(String(error)) - logger.error('Failed to delete knowledge base SQLite row after artifact cleanup', normalizedError, { - baseId - }) - throw DataApiErrorFactory.invalidOperation( - 'deleteBase', - `Vector artifacts were deleted, but SQLite knowledge base cleanup failed: ${normalizedError.message}` - ) - } - }) + await this.baseAdmin.deleteBase(baseId) } async restoreBase(dto: RestoreKnowledgeBaseDto): Promise { - const sourceBase = knowledgeBaseService.getById(dto.sourceBaseId) - - // Restore always supplies an embedding model (RestoreKnowledgeBaseSchema requires - // it). A BM25-only source's searchMode is pinned to 'bm25' by the no-model - // invariant, so carrying it over would leave the restored base's semantic search - // silently disabled despite it now paying for the full embedding backfill. Drop - // it (and the always-null hybridAlpha) so create() applies its own hybrid - // default; a source that already had a model keeps its existing preference. - const gainsEmbeddings = sourceBase.embeddingModelId === null - - const createDto: CreateKnowledgeBaseDto = { - name: dto.name?.trim() ?? sourceBase.name, - dimensions: dto.dimensions, - embeddingModelId: dto.embeddingModelId, - rerankModelId: sourceBase.rerankModelId, - fileProcessorId: sourceBase.fileProcessorId, - chunkSize: sourceBase.chunkSize, - chunkOverlap: sourceBase.chunkOverlap, - threshold: sourceBase.threshold, - documentCount: sourceBase.documentCount, - searchMode: gainsEmbeddings ? undefined : sourceBase.searchMode, - hybridAlpha: gainsEmbeddings ? undefined : sourceBase.hybridAlpha, - groupId: sourceBase.groupId ?? undefined - } - - const rootItems = knowledgeItemService.getRootItemsByBaseId(sourceBase.id) - - // Partial restore: probe each root's source and skip the ones whose source is genuinely gone, so - // a single missing source no longer aborts the entire restore. This is the common case for a - // failed base — a v1-migrated directory child has a virtual path with no raw/ file, and a file - // whose original was deleted has no material to copy; addItems would throw on the first such - // item and roll back the whole batch. An 'unverifiable' source (transient/permission error) is - // kept, not skipped — like reindex, we never drop a source we could not confirm is gone. - const restorableRootItems: KnowledgeItem[] = [] - for (const item of rootItems) { - if ((await classifyKnowledgeItemSource(sourceBase.id, item)) === 'missing') { - logger.warn('Skipping knowledge item with a missing source during restore', { - sourceBaseId: sourceBase.id, - itemId: item.id, - type: item.type - }) - continue - } - restorableRootItems.push(item) - } - const skippedMissingSourceCount = rootItems.length - restorableRootItems.length - if (skippedMissingSourceCount > 0) { - logger.info('Restore skipped knowledge items whose source no longer exists', { - sourceBaseId: sourceBase.id, - skippedMissingSourceCount, - restorableCount: restorableRootItems.length - }) - } - - const inputs = restorableRootItems.map((item) => this.toRestoreRuntimeInput(sourceBase.id, item)) - const restoredBase = await this.createBase(createDto) - try { - await this.addItems(restoredBase.id, inputs) - } catch (error) { - try { - await this.deleteBase(restoredBase.id) - } catch (cleanupError) { - const cleanupMessage = cleanupError instanceof Error ? cleanupError.message : String(cleanupError) - logger.error( - 'Failed to delete restored knowledge base after item restoration failed', - cleanupError instanceof Error ? cleanupError : new Error(cleanupMessage), - { - sourceBaseId: sourceBase.id, - restoredBaseId: restoredBase.id - } - ) - throw DataApiErrorFactory.invalidOperation( - 'restoreBase', - `Failed to restore knowledge items: ${ - error instanceof Error ? error.message : String(error) - }. Restored knowledge base '${restoredBase.id}' could not be cleaned up automatically: ${cleanupMessage}. Please delete it manually.` - ) - } - throw DataApiErrorFactory.invalidOperation( - 'restoreBase', - `Failed to restore knowledge items: ${error instanceof Error ? error.message : String(error)}` - ) - } - - return { base: restoredBase, skippedMissingSourceCount } + return await this.baseAdmin.restoreBase(dto) } async addItems( @@ -450,7 +309,7 @@ export class KnowledgeService extends BaseService { items: KnowledgeAddItemInput[], conflictStrategy?: KnowledgeAddConflictStrategy ): Promise { - this.assertBaseCanRunRuntimeOperation(baseId, 'addItems') + assertBaseCanRunRuntimeOperation(baseId, 'addItems') return await this.workflowService.addItems(baseId, items, conflictStrategy) } @@ -464,7 +323,7 @@ export class KnowledgeService extends BaseService { } async reindexItems(baseId: string, itemIds: string[]): Promise { - this.assertBaseCanRunRuntimeOperation(baseId, 'reindexItems') + assertBaseCanRunRuntimeOperation(baseId, 'reindexItems') const rootItemIds = knowledgeItemService.getOutermostSelectedItemIds(baseId, itemIds) if (rootItemIds.length === 0) { return @@ -476,14 +335,12 @@ export class KnowledgeService extends BaseService { } listBases(): KnowledgeBase[] { - const { items } = knowledgeBaseService.list({ page: 1, limit: KNOWLEDGE_BASES_MAX_LIMIT }) - return items + return this.baseAdmin.listBases() } /** Whether the user has any knowledge base at all — a cheap count (not a full list) for tool-availability gating. */ hasAnyBase(): boolean { - const { total } = knowledgeBaseService.list({ page: 1, limit: 1 }) - return total > 0 + return this.baseAdmin.hasAnyBase() } listRootItems(baseId: string): KnowledgeItem[] { @@ -492,7 +349,7 @@ export class KnowledgeService extends BaseService { @TraceMethod({ spanName: 'Knowledge.search', tag: 'Knowledge' }) async search(baseId: string, query: string): Promise { - this.assertBaseCanRunRuntimeOperation(baseId, 'search') + assertBaseCanRunRuntimeOperation(baseId, 'search') if (!SEARCH_TOKEN_PATTERN.test(query)) { throw DataApiErrorFactory.validation( @@ -539,7 +396,7 @@ export class KnowledgeService extends BaseService { async listItemChunks(baseId: string, itemId: string): Promise { const knowledgeBaseId = toKnowledgeBaseId(baseId) const knowledgeItemId = toKnowledgeItemId(itemId) - this.assertBaseCanRunRuntimeOperation(knowledgeBaseId, 'listItemChunks') + assertBaseCanRunRuntimeOperation(knowledgeBaseId, 'listItemChunks') const item = await this.assertItemCanRunChunkOperation(knowledgeBaseId, knowledgeItemId, 'list chunks') this.assertCompletedContainerHasNoDeletingChildren(knowledgeBaseId, item) @@ -732,7 +589,7 @@ export class KnowledgeService extends BaseService { conceptId: string, operation: string ): Promise<{ item: KnowledgeItem; text: string }> { - this.assertBaseCanRunRuntimeOperation(baseId, operation) + assertBaseCanRunRuntimeOperation(baseId, operation) const base = knowledgeBaseService.getById(baseId) const vectorStoreService = application.get('KnowledgeVectorStoreService') @@ -928,25 +785,6 @@ export class KnowledgeService extends BaseService { } } - private async cancelAllJobsForBase(baseId: string): Promise { - const jobManager = application.get('JobManager') - const activeJobs = await jobManager.list({ - queue: knowledgeQueueName(toKnowledgeBaseId(baseId)), - status: [...KNOWLEDGE_ACTIVE_JOB_STATUSES], - limit: KNOWLEDGE_ACTIVE_JOB_LIMIT - }) - const jobsToCancel = activeJobs.filter((job) => KNOWLEDGE_JOB_TYPE_SET.has(job.type)) - const linkedFileProcessingJobIds = activeJobs.flatMap((job) => { - const narrowed = narrowKnowledgeJobInput(job) - return narrowed?.type === 'knowledge.check-file-processing-result' ? [narrowed.input.fileProcessingJobId] : [] - }) - - await Promise.all([ - ...jobsToCancel.map((job) => jobManager.cancel(job.id, 'delete-base')), - ...linkedFileProcessingJobIds.map((jobId) => jobManager.cancel(jobId, 'delete-base')) - ]) - } - /** * Park items stranded mid-indexing by an app quit / restart at `failed`. * @@ -1010,21 +848,6 @@ export class KnowledgeService extends BaseService { } } - private assertBaseCanRunRuntimeOperation(baseId: string, operation: string): void { - const base = knowledgeBaseService.getById(baseId) - - if (base.status !== 'failed') { - return - } - - throw DataApiErrorFactory.validation( - { - base: [`Knowledge base '${baseId}' is in failed state; restore it before ${operation}.`] - }, - `Cannot ${operation} failed knowledge base` - ) - } - private async getRootItemsInBase(baseId: string, itemIds: string[]): Promise { const rootIds = [...new Set(itemIds)] const items = await Promise.all(rootIds.map((itemId) => knowledgeItemService.getById(itemId))) @@ -1138,60 +961,4 @@ export class KnowledgeService extends BaseService { ) } - private toRestoreRuntimeInput(sourceBaseId: string, item: KnowledgeItem): KnowledgeAddItemInput { - try { - if (item.type === 'file') { - return KnowledgeAddItemInputSchema.parse({ - type: 'file', - data: { - source: item.data.source, - path: getKnowledgeBaseFilePath(sourceBaseId, item.data.relativePath), - // Carry the processed artifact across so the new base indexes from it - // instead of re-running the (slow, paid) file processor. - ...(item.data.indexedRelativePath - ? { indexedPath: getKnowledgeBaseFilePath(sourceBaseId, item.data.indexedRelativePath) } - : {}) - } - }) - } - - if (item.type === 'url') { - return KnowledgeAddItemInputSchema.parse({ - type: 'url', - data: { - source: item.data.source, - url: item.data.url, - // Carry the captured snapshot across so the restored URL indexes offline - // instead of re-fetching the live page (which may have changed or died). - // If the source never captured one, omit it and let the first index capture. - ...(item.data.relativePath - ? { snapshotPath: getKnowledgeBaseFilePath(sourceBaseId, item.data.relativePath) } - : {}) - } - }) - } - - if (item.type === 'note') { - return KnowledgeAddItemInputSchema.parse({ - type: 'note', - // The snapshot relativePath is intentionally dropped: the content is the - // source of truth and re-capturing it into the new base on first index is - // free and deterministic, so there is no snapshot file to carry across. - data: { source: item.data.source, content: item.data.content } - }) - } - - return KnowledgeAddItemInputSchema.parse({ - type: item.type, - data: item.data - }) - } catch (error) { - throw DataApiErrorFactory.invalidOperation( - 'restoreBase', - `Cannot restore knowledge item '${item.id}' (${item.type}): ${ - error instanceof Error ? error.message : String(error) - }` - ) - } - } } diff --git a/src/main/features/knowledge/base/KnowledgeBaseAdminService.ts b/src/main/features/knowledge/base/KnowledgeBaseAdminService.ts new file mode 100644 index 00000000000..4612d73746d --- /dev/null +++ b/src/main/features/knowledge/base/KnowledgeBaseAdminService.ts @@ -0,0 +1,279 @@ +import { application } from '@application' +import { knowledgeBaseService } from '@data/services/KnowledgeBaseService' +import { knowledgeItemService } from '@data/services/KnowledgeItemService' +import { loggerService } from '@logger' +import { DataApiErrorFactory } from '@shared/data/api' +import { KNOWLEDGE_BASES_MAX_LIMIT } from '@shared/data/api/schemas/knowledges' +import { + type CreateKnowledgeBaseDto, + type KnowledgeAddItemInput, + KnowledgeAddItemInputSchema, + type KnowledgeAddItemsResult, + type KnowledgeBase, + type KnowledgeItem, + type RestoreKnowledgeBaseDto, + type RestoreKnowledgeBaseResult +} from '@shared/data/types/knowledge' + +import { classifyKnowledgeItemSource } from '../items' +import type { KnowledgeLockManager } from '../KnowledgeLockManager' +import { getKnowledgeBaseFilePath } from '../storage/pathStorage' +import { narrowKnowledgeJobInput } from '../tasks/utils/jobInput' +import { + KNOWLEDGE_ACTIVE_JOB_LIMIT, + KNOWLEDGE_ACTIVE_JOB_STATUSES, + KNOWLEDGE_JOB_TYPES, + knowledgeQueueName, + toKnowledgeBaseId +} from '../types' + +const logger = loggerService.withContext('Knowledge:BaseAdmin') +const KNOWLEDGE_JOB_TYPE_SET = new Set(KNOWLEDGE_JOB_TYPES) + +/** The add-items entry `restoreBase` feeds the restored items through (the facade's guarded add flow). */ +type AddKnowledgeItemsFn = (baseId: string, items: KnowledgeAddItemInput[]) => Promise + +/** Knowledge base lifecycle: create (with rollback), delete, restore, and list — everything about the base row + its on-disk artifacts, not about items. */ +export class KnowledgeBaseAdminService { + constructor( + private readonly knowledgeLockManager: KnowledgeLockManager, + private readonly addItems: AddKnowledgeItemsFn + ) {} + + async createBase(dto: CreateKnowledgeBaseDto): Promise { + const base = knowledgeBaseService.create(dto) + const vectorStoreService = application.get('KnowledgeVectorStoreService') + + try { + await vectorStoreService.getIndexStore(base) + } catch (error) { + await this.rollbackFailedBaseCreation(base.id) + throw error + } + + return base + } + + /** + * Undo a half-created base after its index store failed to open: remove the + * orphaned `.cherry/` directory `getIndexStore` left on disk and drop the DB + * row. Both steps are best-effort and logged — a cleanup failure must never + * mask the original open error the caller needs to see. + */ + private async rollbackFailedBaseCreation(baseId: string): Promise { + const vectorStoreService = application.get('KnowledgeVectorStoreService') + try { + await vectorStoreService.deleteStore(baseId) + } catch (cleanupError) { + logger.warn('Failed to remove index store dir during createBase rollback', cleanupError as Error, { baseId }) + } + try { + knowledgeBaseService.delete(baseId) + } catch (cleanupError) { + logger.warn('Failed to delete knowledge base row during createBase rollback', cleanupError as Error, { baseId }) + } + } + + async deleteBase(baseId: string): Promise { + await this.cancelAllJobsForBase(baseId) + + await this.knowledgeLockManager.withBaseMutationLock(baseId, async () => { + try { + const vectorStoreService = application.get('KnowledgeVectorStoreService') + await vectorStoreService.deleteStore(baseId) + } catch (error) { + const normalizedError = error instanceof Error ? error : new Error(String(error)) + logger.error('Failed to delete knowledge base vector artifacts', normalizedError, { baseId }) + throw error + } + + try { + knowledgeBaseService.delete(baseId) + } catch (error) { + const normalizedError = error instanceof Error ? error : new Error(String(error)) + logger.error('Failed to delete knowledge base SQLite row after artifact cleanup', normalizedError, { + baseId + }) + throw DataApiErrorFactory.invalidOperation( + 'deleteBase', + `Vector artifacts were deleted, but SQLite knowledge base cleanup failed: ${normalizedError.message}` + ) + } + }) + } + + async restoreBase(dto: RestoreKnowledgeBaseDto): Promise { + const sourceBase = knowledgeBaseService.getById(dto.sourceBaseId) + + // Restore always supplies an embedding model (RestoreKnowledgeBaseSchema requires + // it). A BM25-only source's searchMode is pinned to 'bm25' by the no-model + // invariant, so carrying it over would leave the restored base's semantic search + // silently disabled despite it now paying for the full embedding backfill. Drop + // it (and the always-null hybridAlpha) so create() applies its own hybrid + // default; a source that already had a model keeps its existing preference. + const gainsEmbeddings = sourceBase.embeddingModelId === null + + const createDto: CreateKnowledgeBaseDto = { + name: dto.name?.trim() ?? sourceBase.name, + dimensions: dto.dimensions, + embeddingModelId: dto.embeddingModelId, + rerankModelId: sourceBase.rerankModelId, + fileProcessorId: sourceBase.fileProcessorId, + chunkSize: sourceBase.chunkSize, + chunkOverlap: sourceBase.chunkOverlap, + threshold: sourceBase.threshold, + documentCount: sourceBase.documentCount, + searchMode: gainsEmbeddings ? undefined : sourceBase.searchMode, + hybridAlpha: gainsEmbeddings ? undefined : sourceBase.hybridAlpha, + groupId: sourceBase.groupId ?? undefined + } + + const rootItems = knowledgeItemService.getRootItemsByBaseId(sourceBase.id) + + // Partial restore: probe each root's source and skip the ones whose source is genuinely gone, so + // a single missing source no longer aborts the entire restore. This is the common case for a + // failed base — a v1-migrated directory child has a virtual path with no raw/ file, and a file + // whose original was deleted has no material to copy; addItems would throw on the first such + // item and roll back the whole batch. An 'unverifiable' source (transient/permission error) is + // kept, not skipped — like reindex, we never drop a source we could not confirm is gone. + const restorableRootItems: KnowledgeItem[] = [] + for (const item of rootItems) { + if ((await classifyKnowledgeItemSource(sourceBase.id, item)) === 'missing') { + logger.warn('Skipping knowledge item with a missing source during restore', { + sourceBaseId: sourceBase.id, + itemId: item.id, + type: item.type + }) + continue + } + restorableRootItems.push(item) + } + const skippedMissingSourceCount = rootItems.length - restorableRootItems.length + if (skippedMissingSourceCount > 0) { + logger.info('Restore skipped knowledge items whose source no longer exists', { + sourceBaseId: sourceBase.id, + skippedMissingSourceCount, + restorableCount: restorableRootItems.length + }) + } + + const inputs = restorableRootItems.map((item) => this.toRestoreRuntimeInput(sourceBase.id, item)) + const restoredBase = await this.createBase(createDto) + try { + await this.addItems(restoredBase.id, inputs) + } catch (error) { + try { + await this.deleteBase(restoredBase.id) + } catch (cleanupError) { + const cleanupMessage = cleanupError instanceof Error ? cleanupError.message : String(cleanupError) + logger.error( + 'Failed to delete restored knowledge base after item restoration failed', + cleanupError instanceof Error ? cleanupError : new Error(cleanupMessage), + { + sourceBaseId: sourceBase.id, + restoredBaseId: restoredBase.id + } + ) + throw DataApiErrorFactory.invalidOperation( + 'restoreBase', + `Failed to restore knowledge items: ${ + error instanceof Error ? error.message : String(error) + }. Restored knowledge base '${restoredBase.id}' could not be cleaned up automatically: ${cleanupMessage}. Please delete it manually.` + ) + } + throw DataApiErrorFactory.invalidOperation( + 'restoreBase', + `Failed to restore knowledge items: ${error instanceof Error ? error.message : String(error)}` + ) + } + + return { base: restoredBase, skippedMissingSourceCount } + } + + listBases(): KnowledgeBase[] { + const { items } = knowledgeBaseService.list({ page: 1, limit: KNOWLEDGE_BASES_MAX_LIMIT }) + return items + } + + /** Whether the user has any knowledge base at all — a cheap count (not a full list) for tool-availability gating. */ + hasAnyBase(): boolean { + const { total } = knowledgeBaseService.list({ page: 1, limit: 1 }) + return total > 0 + } + + private async cancelAllJobsForBase(baseId: string): Promise { + const jobManager = application.get('JobManager') + const activeJobs = await jobManager.list({ + queue: knowledgeQueueName(toKnowledgeBaseId(baseId)), + status: [...KNOWLEDGE_ACTIVE_JOB_STATUSES], + limit: KNOWLEDGE_ACTIVE_JOB_LIMIT + }) + const jobsToCancel = activeJobs.filter((job) => KNOWLEDGE_JOB_TYPE_SET.has(job.type)) + const linkedFileProcessingJobIds = activeJobs.flatMap((job) => { + const narrowed = narrowKnowledgeJobInput(job) + return narrowed?.type === 'knowledge.check-file-processing-result' ? [narrowed.input.fileProcessingJobId] : [] + }) + + await Promise.all([ + ...jobsToCancel.map((job) => jobManager.cancel(job.id, 'delete-base')), + ...linkedFileProcessingJobIds.map((jobId) => jobManager.cancel(jobId, 'delete-base')) + ]) + } + + private toRestoreRuntimeInput(sourceBaseId: string, item: KnowledgeItem): KnowledgeAddItemInput { + try { + if (item.type === 'file') { + return KnowledgeAddItemInputSchema.parse({ + type: 'file', + data: { + source: item.data.source, + path: getKnowledgeBaseFilePath(sourceBaseId, item.data.relativePath), + // Carry the processed artifact across so the new base indexes from it + // instead of re-running the (slow, paid) file processor. + ...(item.data.indexedRelativePath + ? { indexedPath: getKnowledgeBaseFilePath(sourceBaseId, item.data.indexedRelativePath) } + : {}) + } + }) + } + + if (item.type === 'url') { + return KnowledgeAddItemInputSchema.parse({ + type: 'url', + data: { + source: item.data.source, + url: item.data.url, + // Carry the captured snapshot across so the restored URL indexes offline + // instead of re-fetching the live page (which may have changed or died). + // If the source never captured one, omit it and let the first index capture. + ...(item.data.relativePath + ? { snapshotPath: getKnowledgeBaseFilePath(sourceBaseId, item.data.relativePath) } + : {}) + } + }) + } + + if (item.type === 'note') { + return KnowledgeAddItemInputSchema.parse({ + type: 'note', + // The snapshot relativePath is intentionally dropped: the content is the + // source of truth and re-capturing it into the new base on first index is + // free and deterministic, so there is no snapshot file to carry across. + data: { source: item.data.source, content: item.data.content } + }) + } + + return KnowledgeAddItemInputSchema.parse({ + type: item.type, + data: item.data + }) + } catch (error) { + throw DataApiErrorFactory.invalidOperation( + 'restoreBase', + `Cannot restore knowledge item '${item.id}' (${item.type}): ${ + error instanceof Error ? error.message : String(error) + }` + ) + } + } +} diff --git a/src/main/features/knowledge/base/guards.ts b/src/main/features/knowledge/base/guards.ts new file mode 100644 index 00000000000..feeeb75c321 --- /dev/null +++ b/src/main/features/knowledge/base/guards.ts @@ -0,0 +1,18 @@ +import { knowledgeBaseService } from '@data/services/KnowledgeBaseService' +import { DataApiErrorFactory } from '@shared/data/api' + +/** A base in `failed` state must be restored before any runtime operation (search, add, reindex, deep-read). */ +export function assertBaseCanRunRuntimeOperation(baseId: string, operation: string): void { + const base = knowledgeBaseService.getById(baseId) + + if (base.status !== 'failed') { + return + } + + throw DataApiErrorFactory.validation( + { + base: [`Knowledge base '${baseId}' is in failed state; restore it before ${operation}.`] + }, + `Cannot ${operation} failed knowledge base` + ) +} From 91fc95197669644c9467e9b3838b0378cf94a243 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Thu, 2 Jul 2026 18:43:10 +0800 Subject: [PATCH 03/37] refactor(knowledge): rename workflow service to ingestion service, absorb admission checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git mv KnowledgeWorkflowService.ts -> ingestion/KnowledgeIngestionService.ts and align the class, handler params, and test mocks with the new name. Admission checks sink from the facade into the ingestion service so one flow's validation + orchestration live in one place: - addItems absorbs the failed-base guard (base/guards) - deleteItems absorbs the outermost-root expansion + empty early-return - reindexItems absorbs the guard, root expansion, and the whole assertSubtreesCanReindex source/status precheck Boot-time recovery (recoverDeletingItems / recoverInterruptedItems) moves in as well — it re-enqueues write-side jobs. Facade methods become pure delegates; behavior and call order are unchanged. The barrel now exports only KnowledgeService (the other exports had no consumers). Co-Authored-By: Claude Fable 5 Signed-off-by: eeee0717 --- .../tasks/__tests__/jobExecution.test.ts | 2 +- .../features/knowledge/KnowledgeService.ts | 184 ++-------------- .../__tests__/KnowledgeService.test.ts | 48 ++--- .../base/KnowledgeBaseAdminService.ts | 9 +- src/main/features/knowledge/index.ts | 2 - .../KnowledgeIngestionService.ts} | 203 +++++++++++++++--- ...heckFileProcessingResultJobHandler.test.ts | 64 +++--- .../tasks/__tests__/jobHandlerTestUtils.ts | 6 +- .../__tests__/prepareRootJobHandler.test.ts | 22 +- .../reindexSubtreeJobHandler.test.ts | 30 +-- .../checkFileProcessingResultJobHandler.ts | 8 +- .../knowledge/tasks/prepareRootJobHandler.ts | 10 +- .../tasks/reindexSubtreeJobHandler.ts | 6 +- 13 files changed, 291 insertions(+), 303 deletions(-) rename src/main/features/knowledge/{KnowledgeWorkflowService.ts => ingestion/KnowledgeIngestionService.ts} (71%) diff --git a/src/main/features/fileProcessing/tasks/__tests__/jobExecution.test.ts b/src/main/features/fileProcessing/tasks/__tests__/jobExecution.test.ts index ef773c11c11..a0201a5fc56 100644 --- a/src/main/features/fileProcessing/tasks/__tests__/jobExecution.test.ts +++ b/src/main/features/fileProcessing/tasks/__tests__/jobExecution.test.ts @@ -163,7 +163,7 @@ describe('prepareFileProcessingJob', () => { }) // The `{kind:'path'}` branch is what the knowledge workflow actually uses -// (KnowledgeWorkflowService passes `{kind:'path', path}`), bypassing FileManager. +// (KnowledgeIngestionService passes `{kind:'path', path}`), bypassing FileManager. describe('resolveFileProcessingFileInfo — kind:path', () => { beforeEach(() => { fsStatMock.mockReset() diff --git a/src/main/features/knowledge/KnowledgeService.ts b/src/main/features/knowledge/KnowledgeService.ts index 7b1034a919b..9c12ef5bfaa 100644 --- a/src/main/features/knowledge/KnowledgeService.ts +++ b/src/main/features/knowledge/KnowledgeService.ts @@ -8,7 +8,6 @@ import { DataApiErrorFactory, ErrorCode, isDataApiError } from '@shared/data/api import { type CreateKnowledgeBaseDto, getKnowledgeItemDisplayTitle, - KNOWLEDGE_ITEM_ERROR_INDEXING_INTERRUPTED, type KnowledgeAddConflictStrategy, type KnowledgeAddItemInput, type KnowledgeAddItemsResult, @@ -29,28 +28,20 @@ import { KnowledgeBaseAdminService } from './base/KnowledgeBaseAdminService' import { embedKnowledgeQuery } from './indexing/embed' import { toMaterialRelativePath } from './indexing/materialFields' import { rerankKnowledgeSearchResults } from './indexing/rerank' -import { classifyKnowledgeItemSource } from './items' +import { KnowledgeIngestionService } from './ingestion/KnowledgeIngestionService' import { KnowledgeLockManager } from './KnowledgeLockManager' -import { KnowledgeWorkflowService } from './KnowledgeWorkflowService' import { applyRelevanceThreshold, getInitialSearchScoreKind, withSearchRanks } from './query/search' import { createCheckFileProcessingResultJobHandler } from './tasks/checkFileProcessingResultJobHandler' import { createDeleteSubtreeJobHandler } from './tasks/deleteSubtreeJobHandler' import { createIndexDocumentsJobHandler } from './tasks/indexDocumentsJobHandler' import { createPrepareRootJobHandler } from './tasks/prepareRootJobHandler' import { createReindexSubtreeJobHandler } from './tasks/reindexSubtreeJobHandler' -import { - knowledgeDeleteSubtreeIdempotencyKey, - knowledgeQueueName, - toKnowledgeBaseId, - toKnowledgeItemId, - toKnowledgeItemIds -} from './types' +import { toKnowledgeBaseId, toKnowledgeItemId } from './types' import type { KnowledgeIndexStore } from './vectorstore/indexStore/KnowledgeIndexStore' import type { KnowledgeIndexSearchMatch } from './vectorstore/indexStore/model' const logger = loggerService.withContext('KnowledgeService') const SEARCH_TOKEN_PATTERN = /[\p{L}\p{N}_]+/u -const DELETE_RECOVERY_ROOT_CHUNK_SIZE = 500 /** * Fetch this many × the requested result count as index candidates. The index * store only filters by material state; the item-visibility filter (missing / @@ -60,7 +51,6 @@ const DELETE_RECOVERY_ROOT_CHUNK_SIZE = 500 const KNOWLEDGE_SEARCH_OVERFETCH_FACTOR = 5 /** Hard ceiling on fetched candidates, bounding the brute-force vector scan and rerank cost regardless of topK. */ const KNOWLEDGE_SEARCH_CANDIDATE_CAP = 200 -const REINDEX_ALLOWED_STATUSES = new Set(['completed', 'failed']) /** Max characters {@link KnowledgeService.readConcept} returns in one slice, so a large document can't flood the agent's context. */ const CONCEPT_READ_MAX_CHARS = 20_000 @@ -264,32 +254,30 @@ function scanConceptMatches( @DependsOn(['KnowledgeVectorStoreService', 'JobManager', 'FileProcessingService']) export class KnowledgeService extends BaseService { private readonly knowledgeLockManager = new KnowledgeLockManager() - private readonly workflowService = new KnowledgeWorkflowService(this.knowledgeLockManager) - private readonly baseAdmin = new KnowledgeBaseAdminService(this.knowledgeLockManager, (baseId, items) => - this.addItems(baseId, items) - ) + private readonly ingestionService = new KnowledgeIngestionService(this.knowledgeLockManager) + private readonly baseAdmin = new KnowledgeBaseAdminService(this.knowledgeLockManager, this.ingestionService) protected onInit(): void { const jobManager = application.get('JobManager') jobManager.registerHandler( 'knowledge.prepare-root', - createPrepareRootJobHandler(this.knowledgeLockManager, this.workflowService) + createPrepareRootJobHandler(this.knowledgeLockManager, this.ingestionService) ) jobManager.registerHandler('knowledge.index-documents', createIndexDocumentsJobHandler(this.knowledgeLockManager)) jobManager.registerHandler( 'knowledge.check-file-processing-result', - createCheckFileProcessingResultJobHandler(this.knowledgeLockManager, this.workflowService) + createCheckFileProcessingResultJobHandler(this.knowledgeLockManager, this.ingestionService) ) jobManager.registerHandler('knowledge.delete-subtree', createDeleteSubtreeJobHandler(this.knowledgeLockManager)) jobManager.registerHandler( 'knowledge.reindex-subtree', - createReindexSubtreeJobHandler(this.knowledgeLockManager, this.workflowService) + createReindexSubtreeJobHandler(this.knowledgeLockManager, this.ingestionService) ) } protected async onAllReady(): Promise { - this.recoverDeletingItems() - this.recoverInterruptedItems() + this.ingestionService.recoverDeletingItems() + this.ingestionService.recoverInterruptedItems() } async createBase(dto: CreateKnowledgeBaseDto): Promise { @@ -309,29 +297,15 @@ export class KnowledgeService extends BaseService { items: KnowledgeAddItemInput[], conflictStrategy?: KnowledgeAddConflictStrategy ): Promise { - assertBaseCanRunRuntimeOperation(baseId, 'addItems') - return await this.workflowService.addItems(baseId, items, conflictStrategy) + return await this.ingestionService.addItems(baseId, items, conflictStrategy) } async deleteItems(baseId: string, itemIds: string[]): Promise { - const rootItemIds = knowledgeItemService.getOutermostSelectedItemIds(baseId, itemIds) - if (rootItemIds.length === 0) { - return - } - - await this.workflowService.deleteItems(baseId, rootItemIds) + await this.ingestionService.deleteItems(baseId, itemIds) } async reindexItems(baseId: string, itemIds: string[]): Promise { - assertBaseCanRunRuntimeOperation(baseId, 'reindexItems') - const rootItemIds = knowledgeItemService.getOutermostSelectedItemIds(baseId, itemIds) - if (rootItemIds.length === 0) { - return - } - - await this.assertSubtreesCanReindex(baseId, rootItemIds) - - await this.workflowService.reindexItems(baseId, rootItemIds) + await this.ingestionService.reindexItems(baseId, itemIds) } listBases(): KnowledgeBase[] { @@ -785,69 +759,6 @@ export class KnowledgeService extends BaseService { } } - /** - * Park items stranded mid-indexing by an app quit / restart at `failed`. - * - * Indexing handlers declare `recovery: 'abandon'`, so an interrupted job is - * cancelled rather than silently resumed on the next launch — a deliberate - * quit must not auto-spend the (paid) embedding API. The job side is handled - * by JobManager's startup recovery; this closes the item side. The common case - * (handler settled the abort as cancelled) is already flipped to `failed` by - * the job's onSettled; this is the boot-time safety net for the stragglers - * onSettled lost the race to write (process exited first) or never ran (hard - * kill / crash). Marking them `failed` clears the perpetual spinner and makes - * them reindexable so the user can finish them on demand. - */ - private recoverInterruptedItems(): void { - try { - const failedCount = knowledgeItemService.failInterruptedItems(KNOWLEDGE_ITEM_ERROR_INDEXING_INTERRUPTED) - if (failedCount > 0) { - logger.info('Recovered interrupted knowledge items', { count: failedCount }) - } - } catch (error) { - logger.error('Failed to recover interrupted knowledge items', error as Error) - } - } - - private recoverDeletingItems(): void { - let deletingRootGroups: Awaited> - try { - deletingRootGroups = knowledgeItemService.getDeletingRootGroups() - } catch (error) { - logger.error('Failed to scan deleting knowledge items for recovery', error as Error) - return - } - - if (deletingRootGroups.length === 0) { - return - } - - const jobManager = application.get('JobManager') - for (const { baseId, rootItemIds } of deletingRootGroups) { - for (let i = 0; i < rootItemIds.length; i += DELETE_RECOVERY_ROOT_CHUNK_SIZE) { - const rootItemIdChunk = rootItemIds.slice(i, i + DELETE_RECOVERY_ROOT_CHUNK_SIZE) - try { - jobManager.enqueue( - 'knowledge.delete-subtree', - { baseId, rootItemIds: rootItemIdChunk }, - { - idempotencyKey: knowledgeDeleteSubtreeIdempotencyKey( - toKnowledgeBaseId(baseId), - toKnowledgeItemIds(rootItemIdChunk) - ), - queue: knowledgeQueueName(toKnowledgeBaseId(baseId)) - } - ) - } catch (error) { - logger.error('Failed to enqueue recovered knowledge delete cleanup', error as Error, { - baseId, - rootItemIds: rootItemIdChunk - }) - } - } - } - } - private async getRootItemsInBase(baseId: string, itemIds: string[]): Promise { const rootIds = [...new Set(itemIds)] const items = await Promise.all(rootIds.map((itemId) => knowledgeItemService.getById(itemId))) @@ -890,75 +801,4 @@ export class KnowledgeService extends BaseService { ) } } - - private async assertSubtreesCanReindex(baseId: string, rootItemIds: string[]): Promise { - const blockingStatusCounts = new Map() - const missingSourceItemIds: string[] = [] - const unverifiableSourceItemIds: string[] = [] - - for (const rootItemId of rootItemIds) { - const subtreeItems = knowledgeItemService.getSubtreeItems(baseId, [rootItemId], { includeRoots: true }) - - // Reindex deletes the subtree's vectors before re-reading the source (reindexSubtreeJobHandler), - // so a root whose source is gone would lose its vectors with nothing to rebuild from — reject up - // front. Only the root's own source matters: a directory is rescanned from data.source and its - // children recreated (never read from their raw/ files), a file leaf reads its own raw/ file, and - // note/url always rebuild from the DB / network. A v1-migrated folder child reindexed on its own - // is a file root whose raw/ file never existed, so this rejects it too. Distinguish a genuinely - // missing source (delete-and-re-add) from one we could not verify (transient/permission error, - // which should retry rather than be destroyed). - const root = subtreeItems.find((item) => item.id === rootItemId) - if (root) { - const sourceState = await classifyKnowledgeItemSource(baseId, root) - if (sourceState === 'missing') { - missingSourceItemIds.push(rootItemId) - } else if (sourceState === 'unverifiable') { - unverifiableSourceItemIds.push(rootItemId) - } - } - - for (const item of subtreeItems) { - if (REINDEX_ALLOWED_STATUSES.has(item.status)) { - continue - } - - blockingStatusCounts.set(item.status, (blockingStatusCounts.get(item.status) ?? 0) + 1) - } - } - - if (missingSourceItemIds.length > 0) { - throw DataApiErrorFactory.validation( - { - item: [`Knowledge item source no longer exists on disk for ${missingSourceItemIds.length} item(s)`] - }, - 'Cannot reindex a knowledge item whose source file or folder no longer exists; delete it and add it again to rebuild' - ) - } - - if (unverifiableSourceItemIds.length > 0) { - throw DataApiErrorFactory.validation( - { - item: [`Could not verify the knowledge item source on disk for ${unverifiableSourceItemIds.length} item(s)`] - }, - 'Could not verify the knowledge item source (it may be temporarily unavailable); please try again' - ) - } - - if (blockingStatusCounts.size === 0) { - return - } - - const statusSummary = [...blockingStatusCounts.entries()] - .sort(([left], [right]) => left.localeCompare(right)) - .map(([status, count]) => `${status}=${count}`) - .join(', ') - - throw DataApiErrorFactory.validation( - { - item: [`Knowledge item subtree is still running or being deleted: ${statusSummary}`] - }, - 'Cannot reindex knowledge item until the entire subtree is completed or failed' - ) - } - } diff --git a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts index 35c10f4d37d..eddc9ff23eb 100644 --- a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts +++ b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts @@ -1220,17 +1220,17 @@ describe('KnowledgeService', () => { } ]) - const workflowService = ( + const ingestionService = ( service as unknown as { - workflowService: { + ingestionService: { scheduleItem(baseId: string, itemId: string, parentJobId?: string | null): Promise } } - ).workflowService + ).ingestionService // The processed-artifact reservation guard must treat the note snapshot as occupied (it lives // under `raw/` too), so it refuses the colliding `.md` output instead of overwriting it on disk. - await expect(workflowService.scheduleItem('kb-1', 'file-1')).rejects.toThrow( + await expect(ingestionService.scheduleItem('kb-1', 'file-1')).rejects.toThrow( 'Knowledge file already exists: source.md' ) expect(fileProcessingStartJobMock).not.toHaveBeenCalled() @@ -1285,14 +1285,14 @@ describe('KnowledgeService', () => { knowledgeBaseGetByIdMock.mockReturnValue(createBase({ fileProcessorId: 'doc2x' })) knowledgeItemGetByIdMock.mockReturnValueOnce(processingFile) - const workflowService = ( + const ingestionService = ( service as unknown as { - workflowService: { + ingestionService: { scheduleItem(baseId: string, itemId: string, parentJobId?: string | null): Promise } } - ).workflowService - await workflowService.scheduleItem('kb-1', 'file-1', 'reindex-job') + ).ingestionService + await ingestionService.scheduleItem('kb-1', 'file-1', 'reindex-job') expect(fileProcessingStartJobMock).toHaveBeenCalledWith( { @@ -1334,15 +1334,15 @@ describe('KnowledgeService', () => { throw new Error('check enqueue failed') }) - const workflowService = ( + const ingestionService = ( service as unknown as { - workflowService: { + ingestionService: { scheduleItem(baseId: string, itemId: string, parentJobId?: string | null): Promise } } - ).workflowService + ).ingestionService - await expect(workflowService.scheduleItem('kb-1', 'file-1')).rejects.toThrow('check enqueue failed') + await expect(ingestionService.scheduleItem('kb-1', 'file-1')).rejects.toThrow('check enqueue failed') expect(fileProcessingStartJobMock).toHaveBeenCalled() expect(cancelMock).toHaveBeenCalledWith('fp-job-1', 'knowledge-file-processing-check-enqueue-failed') @@ -1358,15 +1358,15 @@ describe('KnowledgeService', () => { }) cancelMock.mockRejectedValueOnce(new Error('cancel failed')) - const workflowService = ( + const ingestionService = ( service as unknown as { - workflowService: { + ingestionService: { scheduleItem(baseId: string, itemId: string, parentJobId?: string | null): Promise } } - ).workflowService + ).ingestionService - await expect(workflowService.scheduleItem('kb-1', 'file-1')).rejects.toThrow('check enqueue failed') + await expect(ingestionService.scheduleItem('kb-1', 'file-1')).rejects.toThrow('check enqueue failed') expect(cancelMock).toHaveBeenCalledWith('fp-job-1', 'knowledge-file-processing-check-enqueue-failed') }) @@ -1376,14 +1376,14 @@ describe('KnowledgeService', () => { knowledgeBaseGetByIdMock.mockReturnValue(createBase({ fileProcessorId: 'doc2x' })) knowledgeItemGetByIdMock.mockReturnValueOnce(processingFile) - const workflowService = ( + const ingestionService = ( service as unknown as { - workflowService: { + ingestionService: { scheduleItem(baseId: string, itemId: string, parentJobId?: string | null): Promise } } - ).workflowService - await workflowService.scheduleItem('kb-1', 'file-1', 'reindex-job') + ).ingestionService + await ingestionService.scheduleItem('kb-1', 'file-1', 'reindex-job') expect(enqueueMock).toHaveBeenCalledWith( 'knowledge.index-documents', @@ -1400,9 +1400,9 @@ describe('KnowledgeService', () => { vi.useFakeTimers() vi.setSystemTime(new Date('2026-04-08T00:00:00.000Z')) const service = new KnowledgeService() - const workflowService = ( + const ingestionService = ( service as unknown as { - workflowService: { + ingestionService: { scheduleFileProcessingCheck( baseId: string, itemId: string, @@ -1411,9 +1411,9 @@ describe('KnowledgeService', () => { ): Promise } } - ).workflowService + ).ingestionService - await workflowService.scheduleFileProcessingCheck('kb-1', 'file-1', 'fp-job-1', { + await ingestionService.scheduleFileProcessingCheck('kb-1', 'file-1', 'fp-job-1', { pollRound: 1, firstScheduledAt: Date.parse('2026-04-08T00:00:00.000Z'), parentJobId: 'check-job-0' diff --git a/src/main/features/knowledge/base/KnowledgeBaseAdminService.ts b/src/main/features/knowledge/base/KnowledgeBaseAdminService.ts index 4612d73746d..0e30fd50c62 100644 --- a/src/main/features/knowledge/base/KnowledgeBaseAdminService.ts +++ b/src/main/features/knowledge/base/KnowledgeBaseAdminService.ts @@ -8,13 +8,13 @@ import { type CreateKnowledgeBaseDto, type KnowledgeAddItemInput, KnowledgeAddItemInputSchema, - type KnowledgeAddItemsResult, type KnowledgeBase, type KnowledgeItem, type RestoreKnowledgeBaseDto, type RestoreKnowledgeBaseResult } from '@shared/data/types/knowledge' +import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' import { classifyKnowledgeItemSource } from '../items' import type { KnowledgeLockManager } from '../KnowledgeLockManager' import { getKnowledgeBaseFilePath } from '../storage/pathStorage' @@ -30,14 +30,11 @@ import { const logger = loggerService.withContext('Knowledge:BaseAdmin') const KNOWLEDGE_JOB_TYPE_SET = new Set(KNOWLEDGE_JOB_TYPES) -/** The add-items entry `restoreBase` feeds the restored items through (the facade's guarded add flow). */ -type AddKnowledgeItemsFn = (baseId: string, items: KnowledgeAddItemInput[]) => Promise - /** Knowledge base lifecycle: create (with rollback), delete, restore, and list — everything about the base row + its on-disk artifacts, not about items. */ export class KnowledgeBaseAdminService { constructor( private readonly knowledgeLockManager: KnowledgeLockManager, - private readonly addItems: AddKnowledgeItemsFn + private readonly ingestionService: KnowledgeIngestionService ) {} async createBase(dto: CreateKnowledgeBaseDto): Promise { @@ -160,7 +157,7 @@ export class KnowledgeBaseAdminService { const inputs = restorableRootItems.map((item) => this.toRestoreRuntimeInput(sourceBase.id, item)) const restoredBase = await this.createBase(createDto) try { - await this.addItems(restoredBase.id, inputs) + await this.ingestionService.addItems(restoredBase.id, inputs) } catch (error) { try { await this.deleteBase(restoredBase.id) diff --git a/src/main/features/knowledge/index.ts b/src/main/features/knowledge/index.ts index 29435836b83..aa6e0589a36 100644 --- a/src/main/features/knowledge/index.ts +++ b/src/main/features/knowledge/index.ts @@ -1,3 +1 @@ -export { KnowledgeLockManager } from './KnowledgeLockManager' export { KnowledgeService } from './KnowledgeService' -export { KnowledgeWorkflowService } from './KnowledgeWorkflowService' diff --git a/src/main/features/knowledge/KnowledgeWorkflowService.ts b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts similarity index 71% rename from src/main/features/knowledge/KnowledgeWorkflowService.ts rename to src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts index 844f9159e8a..64453bea7da 100644 --- a/src/main/features/knowledge/KnowledgeWorkflowService.ts +++ b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts @@ -1,24 +1,42 @@ -import './tasks/jobTypes' +import '../tasks/jobTypes' import { application } from '@application' import { knowledgeBaseService } from '@data/services/KnowledgeBaseService' import { knowledgeItemService } from '@data/services/KnowledgeItemService' import { loggerService } from '@logger' import { getFileExt } from '@main/utils/file' +import { DataApiErrorFactory } from '@shared/data/api' import { FileProcessorIdSchema } from '@shared/data/presets/fileProcessing' import { type CreateKnowledgeItemDto, DEFAULT_KNOWLEDGE_ADD_CONFLICT_STRATEGY, + KNOWLEDGE_ITEM_ERROR_INDEXING_INTERRUPTED, type KnowledgeAddConflictStrategy, type KnowledgeAddItemInput, type KnowledgeAddItemsResult, type KnowledgeBase, - type KnowledgeItem + type KnowledgeItem, + type KnowledgeItemStatus } from '@shared/data/types/knowledge' import { knowledgeSupportedFileExts } from '@shared/utils/file' -import type { KnowledgeLockManager } from './KnowledgeLockManager' -import { cancelJobOrThrow } from './tasks/utils/cancel' +import { assertBaseCanRunRuntimeOperation } from '../base/guards' +import { cancelActiveKnowledgeSubtreeJobs, purgeKnowledgeSubtreeWithinLock } from '../cleanup/subtreePurge' +import { classifyKnowledgeItemSource, isContainerKnowledgeItem } from '../items' +import type { KnowledgeLockManager } from '../KnowledgeLockManager' +import { planKnowledgeItemSource } from '../sources/sourcePlanning' +import { + assertKnowledgeFileTargetAvailable, + collectKnowledgeReservedRelativePaths, + copyFileIntoKnowledgeBaseAt, + deleteKnowledgeItemFilesBestEffort, + getKnowledgeBaseFilePath, + getKnowledgeSourceRelativePath, + getProcessedMarkdownRelativePath, + needsProcessedArtifactReservation, + reserveImportedFileRelativePath +} from '../storage/pathStorage' +import { cancelJobOrThrow } from '../tasks/utils/cancel' import { type KnowledgeBaseId, knowledgeDeleteSubtreeIdempotencyKey, @@ -31,30 +49,19 @@ import { toKnowledgeBaseId, toKnowledgeItemId, toKnowledgeItemIds -} from './types' -import { resolveKnowledgeAddConflicts } from './ingestion/addConflicts' -import { markUnscheduledKnowledgeItemsFailed } from './ingestion/statusCleanup' -import { cancelActiveKnowledgeSubtreeJobs, purgeKnowledgeSubtreeWithinLock } from './cleanup/subtreePurge' -import { isContainerKnowledgeItem } from './items' -import { planKnowledgeItemSource } from './sources/sourcePlanning' -import { - assertKnowledgeFileTargetAvailable, - collectKnowledgeReservedRelativePaths, - copyFileIntoKnowledgeBaseAt, - deleteKnowledgeItemFilesBestEffort, - getKnowledgeBaseFilePath, - getKnowledgeSourceRelativePath, - getProcessedMarkdownRelativePath, - needsProcessedArtifactReservation, - reserveImportedFileRelativePath -} from './storage/pathStorage' +} from '../types' +import { resolveKnowledgeAddConflicts } from './addConflicts' +import { markUnscheduledKnowledgeItemsFailed } from './statusCleanup' -const logger = loggerService.withContext('Knowledge:WorkflowService') +const logger = loggerService.withContext('Knowledge:IngestionService') // Keep poll jobs delayed enough to avoid hot-looping while remote processors are still working. const FILE_PROCESSING_CHECK_DELAY_MS = 5_000 const KNOWLEDGE_SUPPORTED_FILE_EXT_SET = new Set(knowledgeSupportedFileExts) +const REINDEX_ALLOWED_STATUSES = new Set(['completed', 'failed']) +const DELETE_RECOVERY_ROOT_CHUNK_SIZE = 500 -export class KnowledgeWorkflowService { +/** Write-side orchestration: admission checks, item creation, conflict handling, and job enqueueing for the add/delete/reindex flows. */ +export class KnowledgeIngestionService { constructor(private readonly knowledgeLockManager: KnowledgeLockManager) {} async addItems( @@ -62,6 +69,8 @@ export class KnowledgeWorkflowService { inputs: KnowledgeAddItemInput[], conflictStrategy: KnowledgeAddConflictStrategy = DEFAULT_KNOWLEDGE_ADD_CONFLICT_STRATEGY ): Promise { + assertBaseCanRunRuntimeOperation(baseId, 'addItems') + if (inputs.length === 0) { return { status: 'added' } } @@ -177,8 +186,12 @@ export class KnowledgeWorkflowService { } async deleteItems(baseId: string, itemIds: string[]): Promise { + const rootItemIds = knowledgeItemService.getOutermostSelectedItemIds(baseId, itemIds) + if (rootItemIds.length === 0) { + return + } + knowledgeBaseService.getById(baseId) - const rootItemIds = [...new Set(itemIds)] const knowledgeBaseId = toKnowledgeBaseId(baseId) const knowledgeRootItemIds = toKnowledgeItemIds(rootItemIds) const markedIds = await this.knowledgeLockManager.withBaseMutationLock(baseId, () => @@ -205,8 +218,15 @@ export class KnowledgeWorkflowService { } async reindexItems(baseId: string, itemIds: string[]): Promise { + assertBaseCanRunRuntimeOperation(baseId, 'reindexItems') + const rootItemIds = knowledgeItemService.getOutermostSelectedItemIds(baseId, itemIds) + if (rootItemIds.length === 0) { + return + } + + await this.assertSubtreesCanReindex(baseId, rootItemIds) + knowledgeBaseService.getById(baseId) - const rootItemIds = [...new Set(itemIds)] const knowledgeBaseId = toKnowledgeBaseId(baseId) const knowledgeRootItemIds = toKnowledgeItemIds(rootItemIds) const jobManager = application.get('JobManager') @@ -353,6 +373,139 @@ export class KnowledgeWorkflowService { ) } + /** + * Park items stranded mid-indexing by an app quit / restart at `failed`. + * + * Indexing handlers declare `recovery: 'abandon'`, so an interrupted job is + * cancelled rather than silently resumed on the next launch — a deliberate + * quit must not auto-spend the (paid) embedding API. The job side is handled + * by JobManager's startup recovery; this closes the item side. The common case + * (handler settled the abort as cancelled) is already flipped to `failed` by + * the job's onSettled; this is the boot-time safety net for the stragglers + * onSettled lost the race to write (process exited first) or never ran (hard + * kill / crash). Marking them `failed` clears the perpetual spinner and makes + * them reindexable so the user can finish them on demand. + */ + recoverInterruptedItems(): void { + try { + const failedCount = knowledgeItemService.failInterruptedItems(KNOWLEDGE_ITEM_ERROR_INDEXING_INTERRUPTED) + if (failedCount > 0) { + logger.info('Recovered interrupted knowledge items', { count: failedCount }) + } + } catch (error) { + logger.error('Failed to recover interrupted knowledge items', error as Error) + } + } + + recoverDeletingItems(): void { + let deletingRootGroups: Awaited> + try { + deletingRootGroups = knowledgeItemService.getDeletingRootGroups() + } catch (error) { + logger.error('Failed to scan deleting knowledge items for recovery', error as Error) + return + } + + if (deletingRootGroups.length === 0) { + return + } + + const jobManager = application.get('JobManager') + for (const { baseId, rootItemIds } of deletingRootGroups) { + for (let i = 0; i < rootItemIds.length; i += DELETE_RECOVERY_ROOT_CHUNK_SIZE) { + const rootItemIdChunk = rootItemIds.slice(i, i + DELETE_RECOVERY_ROOT_CHUNK_SIZE) + try { + jobManager.enqueue( + 'knowledge.delete-subtree', + { baseId, rootItemIds: rootItemIdChunk }, + { + idempotencyKey: knowledgeDeleteSubtreeIdempotencyKey( + toKnowledgeBaseId(baseId), + toKnowledgeItemIds(rootItemIdChunk) + ), + queue: knowledgeQueueName(toKnowledgeBaseId(baseId)) + } + ) + } catch (error) { + logger.error('Failed to enqueue recovered knowledge delete cleanup', error as Error, { + baseId, + rootItemIds: rootItemIdChunk + }) + } + } + } + } + + private async assertSubtreesCanReindex(baseId: string, rootItemIds: string[]): Promise { + const blockingStatusCounts = new Map() + const missingSourceItemIds: string[] = [] + const unverifiableSourceItemIds: string[] = [] + + for (const rootItemId of rootItemIds) { + const subtreeItems = knowledgeItemService.getSubtreeItems(baseId, [rootItemId], { includeRoots: true }) + + // Reindex deletes the subtree's vectors before re-reading the source (reindexSubtreeJobHandler), + // so a root whose source is gone would lose its vectors with nothing to rebuild from — reject up + // front. Only the root's own source matters: a directory is rescanned from data.source and its + // children recreated (never read from their raw/ files), a file leaf reads its own raw/ file, and + // note/url always rebuild from the DB / network. A v1-migrated folder child reindexed on its own + // is a file root whose raw/ file never existed, so this rejects it too. Distinguish a genuinely + // missing source (delete-and-re-add) from one we could not verify (transient/permission error, + // which should retry rather than be destroyed). + const root = subtreeItems.find((item) => item.id === rootItemId) + if (root) { + const sourceState = await classifyKnowledgeItemSource(baseId, root) + if (sourceState === 'missing') { + missingSourceItemIds.push(rootItemId) + } else if (sourceState === 'unverifiable') { + unverifiableSourceItemIds.push(rootItemId) + } + } + + for (const item of subtreeItems) { + if (REINDEX_ALLOWED_STATUSES.has(item.status)) { + continue + } + + blockingStatusCounts.set(item.status, (blockingStatusCounts.get(item.status) ?? 0) + 1) + } + } + + if (missingSourceItemIds.length > 0) { + throw DataApiErrorFactory.validation( + { + item: [`Knowledge item source no longer exists on disk for ${missingSourceItemIds.length} item(s)`] + }, + 'Cannot reindex a knowledge item whose source file or folder no longer exists; delete it and add it again to rebuild' + ) + } + + if (unverifiableSourceItemIds.length > 0) { + throw DataApiErrorFactory.validation( + { + item: [`Could not verify the knowledge item source on disk for ${unverifiableSourceItemIds.length} item(s)`] + }, + 'Could not verify the knowledge item source (it may be temporarily unavailable); please try again' + ) + } + + if (blockingStatusCounts.size === 0) { + return + } + + const statusSummary = [...blockingStatusCounts.entries()] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([status, count]) => `${status}=${count}`) + .join(', ') + + throw DataApiErrorFactory.validation( + { + item: [`Knowledge item subtree is still running or being deleted: ${statusSummary}`] + }, + 'Cannot reindex knowledge item until the entire subtree is completed or failed' + ) + } + private rollbackAcceptedItems(baseId: string, items: KnowledgeItem[], originalError: unknown): void { for (const item of items) { try { diff --git a/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts index 76989fea65d..4c869e94b3d 100644 --- a/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts @@ -18,7 +18,7 @@ import { knowledgeItemUpdateStatusMock, knowledgeLockManager, PROCESSED_RELATIVE_PATH, - workflowService + ingestionService } from './jobHandlerTestUtils' function createFileProcessingJobSnapshot(overrides: Partial> = {}) { @@ -55,7 +55,7 @@ function createCheckPayload( describe('check-file-processing-result job handler', () => { it('declares the knowledge check job contract', () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) expect(handler.recovery).toBe('abandon') expect(handler.defaultQueue?.(createCheckPayload())).toBe('base.kb-1') @@ -69,7 +69,7 @@ describe('check-file-processing-result job handler', () => { }) it('reschedules delayed polling while file processing is active', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) getJobMock.mockResolvedValue(createFileProcessingJobSnapshot({ status: 'running' })) @@ -82,17 +82,17 @@ describe('check-file-processing-result job handler', () => { ) await handler.execute(ctx) - expect(workflowService.scheduleFileProcessingCheck).toHaveBeenCalledWith('kb-1', FILE_ITEM_ID, 'fp-job-1', { + expect(ingestionService.scheduleFileProcessingCheck).toHaveBeenCalledWith('kb-1', FILE_ITEM_ID, 'fp-job-1', { pollRound: 3, firstScheduledAt, parentJobId: 'job-1' }) - expect(workflowService.scheduleIndexing).not.toHaveBeenCalled() + expect(ingestionService.scheduleIndexing).not.toHaveBeenCalled() expect(ctx.reportProgress).toHaveBeenCalledWith(0, { stage: 'waiting', pollRound: 3 }) }) it('keeps polling follow-ups attached to the original workflow parent', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) getJobMock.mockResolvedValue(createFileProcessingJobSnapshot({ status: 'running' })) @@ -108,7 +108,7 @@ describe('check-file-processing-result job handler', () => { ) ) - expect(workflowService.scheduleFileProcessingCheck).toHaveBeenCalledWith('kb-1', FILE_ITEM_ID, 'fp-job-1', { + expect(ingestionService.scheduleFileProcessingCheck).toHaveBeenCalledWith('kb-1', FILE_ITEM_ID, 'fp-job-1', { pollRound: 3, firstScheduledAt, parentJobId: 'reindex-job' @@ -116,7 +116,7 @@ describe('check-file-processing-result job handler', () => { }) it('mirrors file-processing progress while polling', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) getJobMock.mockResolvedValue(createFileProcessingJobSnapshot({ status: 'running' })) MockMainCacheServiceUtils.setSharedCacheValue(`${JOB_PROGRESS_KEY_PREFIX}fp-job-1`, { @@ -145,7 +145,7 @@ describe('check-file-processing-result job handler', () => { }) it('marks the item failed when file processing exceeds the wait limit', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) getJobMock.mockResolvedValue(createFileProcessingJobSnapshot({ status: 'running' })) @@ -161,12 +161,12 @@ describe('check-file-processing-result job handler', () => { error: 'File processing job fp-job-1 did not finish within 30 minutes' }) expect(cancelMock).toHaveBeenCalledWith('fp-job-1', 'knowledge-file-processing-timeout') - expect(workflowService.scheduleFileProcessingCheck).not.toHaveBeenCalled() + expect(ingestionService.scheduleFileProcessingCheck).not.toHaveBeenCalled() expect(ctx.reportProgress).toHaveBeenCalledWith(100, { stage: 'failed' }) }) it('stores the processed artifact relative path and schedules indexing on completion', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) getJobMock.mockResolvedValue( createFileProcessingJobSnapshot({ @@ -181,13 +181,13 @@ describe('check-file-processing-result job handler', () => { await handler.execute(ctx) expect(knowledgeItemUpdateIndexedRelativePathMock).toHaveBeenCalledWith(FILE_ITEM_ID, PROCESSED_RELATIVE_PATH) - expect(workflowService.scheduleIndexing).toHaveBeenCalledWith('kb-1', FILE_ITEM_ID, 'job-1') - expect(workflowService.scheduleFileProcessingCheck).not.toHaveBeenCalled() + expect(ingestionService.scheduleIndexing).toHaveBeenCalledWith('kb-1', FILE_ITEM_ID, 'job-1') + expect(ingestionService.scheduleFileProcessingCheck).not.toHaveBeenCalled() expect(ctx.reportProgress).toHaveBeenCalledWith(100, { stage: 'done' }) }) it('schedules indexing under the original workflow parent after polling completion', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) getJobMock.mockResolvedValue( createFileProcessingJobSnapshot({ @@ -206,11 +206,11 @@ describe('check-file-processing-result job handler', () => { ) ) - expect(workflowService.scheduleIndexing).toHaveBeenCalledWith('kb-1', FILE_ITEM_ID, 'reindex-job') + expect(ingestionService.scheduleIndexing).toHaveBeenCalledWith('kb-1', FILE_ITEM_ID, 'reindex-job') }) it('marks the item failed when the linked job is not the expected file-processing job', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) getJobMock.mockResolvedValue( createFileProcessingJobSnapshot({ @@ -230,12 +230,12 @@ describe('check-file-processing-result job handler', () => { expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith(FILE_ITEM_ID, 'failed', { error: 'Invalid file processing job for knowledge item: fp-job-1' }) - expect(workflowService.scheduleIndexing).not.toHaveBeenCalled() + expect(ingestionService.scheduleIndexing).not.toHaveBeenCalled() expect(ctx.reportProgress).toHaveBeenCalledWith(100, { stage: 'failed' }) }) it('skips attaching artifacts when the item becomes deleting before continuation side effects', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) knowledgeItemGetByIdMock .mockReturnValueOnce(createFileItem()) .mockReturnValueOnce(createFileItem(FILE_ITEM_ID, 'deleting')) @@ -252,12 +252,12 @@ describe('check-file-processing-result job handler', () => { await handler.execute(ctx) expect(knowledgeItemUpdateIndexedRelativePathMock).not.toHaveBeenCalled() - expect(workflowService.scheduleIndexing).not.toHaveBeenCalled() + expect(ingestionService.scheduleIndexing).not.toHaveBeenCalled() expect(ctx.reportProgress).not.toHaveBeenCalledWith(100, { stage: 'done' }) }) it('marks the item failed when file processing fails', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) getJobMock.mockResolvedValue( createFileProcessingJobSnapshot({ @@ -272,12 +272,12 @@ describe('check-file-processing-result job handler', () => { expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith(FILE_ITEM_ID, 'failed', { error: 'File processing job fp-job-1 failed: processor failed' }) - expect(workflowService.scheduleIndexing).not.toHaveBeenCalled() + expect(ingestionService.scheduleIndexing).not.toHaveBeenCalled() expect(ctx.reportProgress).toHaveBeenCalledWith(100, { stage: 'failed' }) }) it('marks the item failed when the completed output has no markdown artifact', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) getJobMock.mockResolvedValue( createFileProcessingJobSnapshot({ @@ -294,22 +294,22 @@ describe('check-file-processing-result job handler', () => { expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith(FILE_ITEM_ID, 'failed', { error: 'Invalid file processing result for job fp-job-1' }) - expect(workflowService.scheduleIndexing).not.toHaveBeenCalled() + expect(ingestionService.scheduleIndexing).not.toHaveBeenCalled() }) it('cancels linked file-processing work before skipping deleting items', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) knowledgeItemGetByIdMock.mockReturnValue(createFileItem(FILE_ITEM_ID, 'deleting')) await handler.execute(createCtx(createCheckPayload())) expect(cancelMock).toHaveBeenCalledWith('fp-job-1', 'knowledge-file-processing-item-unavailable') expect(knowledgeItemUpdateStatusMock).not.toHaveBeenCalled() - expect(workflowService.scheduleIndexing).not.toHaveBeenCalled() + expect(ingestionService.scheduleIndexing).not.toHaveBeenCalled() }) it('surfaces cancel failures before skipping deleting items', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) knowledgeItemGetByIdMock.mockReturnValue(createFileItem(FILE_ITEM_ID, 'deleting')) cancelMock.mockRejectedValueOnce(new Error('cancel failed')) @@ -320,7 +320,7 @@ describe('check-file-processing-result job handler', () => { }) it('surfaces cancel timeouts before skipping deleting items', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) knowledgeItemGetByIdMock.mockReturnValue(createFileItem(FILE_ITEM_ID, 'deleting')) cancelMock.mockResolvedValue({ outcome: 'timed-out' }) @@ -330,7 +330,7 @@ describe('check-file-processing-result job handler', () => { }) it('cancels linked file-processing work before skipping missing items', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) knowledgeItemGetByIdMock.mockImplementation(() => { throw DataApiErrorFactory.notFound('KnowledgeItem', FILE_ITEM_ID) }) @@ -339,11 +339,11 @@ describe('check-file-processing-result job handler', () => { expect(cancelMock).toHaveBeenCalledWith('fp-job-1', 'knowledge-file-processing-item-unavailable') expect(knowledgeItemUpdateStatusMock).not.toHaveBeenCalled() - expect(workflowService.scheduleIndexing).not.toHaveBeenCalled() + expect(ingestionService.scheduleIndexing).not.toHaveBeenCalled() }) it('onSettled marks active items failed when the check job fails', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) getJobMock.mockResolvedValue({ id: 'job-1', status: 'failed', @@ -385,7 +385,7 @@ describe('check-file-processing-result job handler', () => { }) it('onSettled falls back to the terminal status when a failed job has no error message', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) getJobMock.mockResolvedValue({ id: 'job-1', status: 'failed', @@ -427,7 +427,7 @@ describe('check-file-processing-result job handler', () => { }) it('onSettled marks a cancelled (app-quit) item failed with the reindex-to-finish message', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) getJobMock.mockResolvedValue({ id: 'job-1', status: 'cancelled', diff --git a/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts b/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts index 51deaa76acc..8a878e84f68 100644 --- a/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts +++ b/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts @@ -337,7 +337,7 @@ export const knowledgeLockManager = { withBaseMutationLock: vi.fn(async (_baseId: string, task: () => Promise) => await task()) } -export const workflowService = { +export const ingestionService = { scheduleFileProcessingCheck: vi.fn(), scheduleIndexing: vi.fn(), scheduleItem: scheduleItemMock @@ -394,7 +394,7 @@ beforeEach(() => { probeKnowledgeFileMock.mockResolvedValue('readable') probeKnowledgeSourcePathMock.mockResolvedValue('readable') cancelMock.mockResolvedValue({ outcome: 'cancelled' }) - workflowService.scheduleFileProcessingCheck.mockResolvedValue(undefined) - workflowService.scheduleIndexing.mockResolvedValue(undefined) + ingestionService.scheduleFileProcessingCheck.mockResolvedValue(undefined) + ingestionService.scheduleIndexing.mockResolvedValue(undefined) scheduleItemMock.mockResolvedValue({ id: 'scheduled-job' }) }) diff --git a/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts index 9b211cfa4a5..7b422ed8c00 100644 --- a/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts @@ -17,12 +17,12 @@ import { knowledgeLockManager, prepareKnowledgeItemMock, scheduleItemMock, - workflowService + ingestionService } from './jobHandlerTestUtils' describe('prepare-root job handler', () => { it('clears stale expansion and schedules recreated leaves', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) knowledgeItemGetByIdMock.mockReturnValue(createDirectoryItem()) await handler.execute(createCtx({ baseId: 'kb-1', itemId: 'dir-1' }, 'prepare-job')) @@ -35,7 +35,7 @@ describe('prepare-root job handler', () => { }) it('clears stale expansion vectors before deleting rows', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) const activeChild = createNoteItem('active-note', 'dir-1') knowledgeItemGetByIdMock.mockReturnValue(createDirectoryItem()) knowledgeItemGetSubtreeItemsMock.mockReturnValue([activeChild]) @@ -50,7 +50,7 @@ describe('prepare-root job handler', () => { }) it('routes stale-expansion cleanup through best-effort delete before deleting rows', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) const activeChild = createNoteItem('active-note', 'dir-1') knowledgeItemGetByIdMock.mockReturnValue(createDirectoryItem()) knowledgeItemGetSubtreeItemsMock.mockReturnValue([activeChild]) @@ -69,7 +69,7 @@ describe('prepare-root job handler', () => { }) it('leaves deleting descendants for delete-subtree cleanup', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) const activeChild = createNoteItem('active-note', 'dir-1') const deletingChild = createNoteItem('deleting-note', 'dir-1', 'deleting') knowledgeItemGetByIdMock.mockReturnValue(createDirectoryItem()) @@ -84,7 +84,7 @@ describe('prepare-root job handler', () => { }) it('skips expansion when the root becomes deleting inside the mutation lock', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) knowledgeItemGetByIdMock .mockReturnValueOnce(createDirectoryItem()) .mockReturnValueOnce(createDirectoryItem('dir-1', 'deleting')) @@ -99,7 +99,7 @@ describe('prepare-root job handler', () => { }) it('keeps terminal failure from an empty expansion', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) knowledgeItemGetByIdMock.mockReturnValue(createDirectoryItem()) prepareKnowledgeItemMock.mockResolvedValue([]) @@ -110,7 +110,7 @@ describe('prepare-root job handler', () => { }) it('marks unscheduled child leaves failed when enqueueing a child fails', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) const leaves = [ createNoteItem('leaf-1', 'dir-1'), createNoteItem('leaf-2', 'dir-1'), @@ -137,7 +137,7 @@ describe('prepare-root job handler', () => { }) it('falls back to subtree failed status when marking an unscheduled leaf fails', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) const leaves = [createNoteItem('leaf-1', 'dir-1'), createNoteItem('leaf-2', 'dir-1')] knowledgeItemGetByIdMock.mockReturnValue(createDirectoryItem()) prepareKnowledgeItemMock.mockResolvedValue(leaves) @@ -161,7 +161,7 @@ describe('prepare-root job handler', () => { }) it('reports unrecovered leaves when failed status cleanup also fails', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) const leaves = [createNoteItem('leaf-1', 'dir-1')] knowledgeItemGetByIdMock.mockReturnValue(createDirectoryItem()) prepareKnowledgeItemMock.mockResolvedValue(leaves) @@ -181,7 +181,7 @@ describe('prepare-root job handler', () => { }) it('onSettled skips failed status when the item is deleting', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) getJobMock.mockResolvedValue( createJobSnapshot({ id: 'prepare-job', diff --git a/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts index 1bf995f79a2..4e783a53549 100644 --- a/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts @@ -22,12 +22,12 @@ import { loggerWarnMock, probeKnowledgeSourcePathMock, scheduleItemMock, - workflowService + ingestionService } from './jobHandlerTestUtils' describe('reindex-subtree job handler', () => { it('clears old artifacts, resets selected roots, and schedules selected roots', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) const root = createDirectoryItem('dir-1') const child = createNoteItem('note-1', 'dir-1') knowledgeItemGetSubtreeItemsMock.mockImplementation( @@ -50,7 +50,7 @@ describe('reindex-subtree job handler', () => { }) it('routes container descendant cleanup through best-effort delete before deleting rows', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) const root = createDirectoryItem('dir-1') const child = createNoteItem('note-1', 'dir-1') knowledgeItemGetSubtreeItemsMock.mockImplementation( @@ -75,7 +75,7 @@ describe('reindex-subtree job handler', () => { }) it('skips deleting subtrees before reset', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) const root = createDirectoryItem('dir-1', 'deleting') const child = createNoteItem('note-1', 'dir-1', 'deleting') const ctx = createCtx({ baseId: 'kb-1', rootItemIds: ['dir-1'] }, 'reindex-job') @@ -93,7 +93,7 @@ describe('reindex-subtree job handler', () => { }) it('skips reset when the subtree becomes deleting inside the mutation lock', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) const root = createDirectoryItem('dir-1') const child = createNoteItem('note-1', 'dir-1') const deletingChild = createNoteItem('note-1', 'dir-1', 'deleting') @@ -110,7 +110,7 @@ describe('reindex-subtree job handler', () => { }) it('leaves a root untouched when its source vanished before the mutation lock', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) const root = createDirectoryItem('dir-1') const child = createNoteItem('note-1', 'dir-1') const ctx = createCtx({ baseId: 'kb-1', rootItemIds: ['dir-1'] }, 'reindex-job') @@ -140,7 +140,7 @@ describe('reindex-subtree job handler', () => { }) it('reindexes only the roots whose source still exists', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) const presentRoot = createDirectoryItem('dir-1') const presentChild = createNoteItem('note-1', 'dir-1') const missingRoot = createDirectoryItem('dir-2') @@ -176,7 +176,7 @@ describe('reindex-subtree job handler', () => { }) it('clears old artifacts for selected leaf roots', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) const root = createNoteItem('note-1') knowledgeItemGetSubtreeItemsMock.mockImplementation( (_baseId: string, _rootIds: string[], options: { includeRoots?: boolean; leafOnly?: boolean } = {}) => { @@ -194,7 +194,7 @@ describe('reindex-subtree job handler', () => { }) it('marks only unscheduled reset roots failed when rescheduling fails', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) const firstRoot = createDirectoryItem('dir-1') const secondRoot = createDirectoryItem('dir-2') const firstChild = createNoteItem('note-1', 'dir-1') @@ -220,7 +220,7 @@ describe('reindex-subtree job handler', () => { }) it('never fails a left-untouched missing-source root when rescheduling a rebuildable root fails', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) const presentRoot = createDirectoryItem('dir-1') const presentChild = createNoteItem('note-1', 'dir-1') const missingRoot = createDirectoryItem('dir-2') @@ -255,7 +255,7 @@ describe('reindex-subtree job handler', () => { }) it('stores the localized interruption code when a shutdown abort interrupts rescheduling', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) const root = createDirectoryItem('dir-1') const child = createNoteItem('note-1', 'dir-1') knowledgeItemGetSubtreeItemsMock.mockImplementation( @@ -285,7 +285,7 @@ describe('reindex-subtree job handler', () => { }) it('onSettled marks active roots without follow-up jobs failed', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) getJobMock.mockResolvedValue( createJobSnapshot({ id: 'reindex-job', @@ -317,7 +317,7 @@ describe('reindex-subtree job handler', () => { }) it('onSettled stores the localized interruption code when the reindex job was cancelled', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) getJobMock.mockResolvedValue( createJobSnapshot({ id: 'reindex-job', @@ -351,7 +351,7 @@ describe('reindex-subtree job handler', () => { }) it('onSettled skips deleting roots and roots with follow-up jobs', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) getJobMock.mockResolvedValue( createJobSnapshot({ id: 'reindex-job', @@ -388,7 +388,7 @@ describe('reindex-subtree job handler', () => { }) it('onSettled treats file-processing check jobs as follow-up jobs', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, workflowService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) getJobMock.mockResolvedValue( createJobSnapshot({ id: 'reindex-job', diff --git a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts index 046c8b27c94..75bc22df1a9 100644 --- a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts +++ b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts @@ -13,7 +13,7 @@ import { } from '../../fileProcessing/persistence/artifacts' import type { FileProcessingJobPayload } from '../../fileProcessing/tasks/shared' import type { KnowledgeLockManager } from '../KnowledgeLockManager' -import type { KnowledgeWorkflowService } from '../KnowledgeWorkflowService' +import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' import { knowledgeQueueName, toKnowledgeBaseId, toKnowledgeItemId } from '../types' import { toKnowledgeRelativePath } from '../storage/pathStorage' import type { KnowledgeCheckFileProcessingResultPayload } from './jobTypes' @@ -27,7 +27,7 @@ const FILE_PROCESSING_ITEM_UNAVAILABLE_CANCEL_REASON = 'knowledge-file-processin export function createCheckFileProcessingResultJobHandler( knowledgeLockManager: KnowledgeLockManager, - workflowService: KnowledgeWorkflowService + ingestionService: KnowledgeIngestionService ): JobHandler { return { // Don't auto-resume on restart — a deliberate app quit must not re-spend the @@ -78,7 +78,7 @@ export function createCheckFileProcessingResultJobHandler( } const nextPollRound = ctx.input.pollRound + 1 - await workflowService.scheduleFileProcessingCheck( + await ingestionService.scheduleFileProcessingCheck( toKnowledgeBaseId(baseId), toKnowledgeItemId(itemId), fileProcessingJobId, @@ -114,7 +114,7 @@ export function createCheckFileProcessingResultJobHandler( } knowledgeItemService.updateIndexedRelativePath(itemId, indexedRelativePath) - await workflowService.scheduleIndexing( + await ingestionService.scheduleIndexing( toKnowledgeBaseId(baseId), toKnowledgeItemId(itemId), workflowParentJobId diff --git a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts index 5f1b4ed930f..0456263bc82 100644 --- a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts +++ b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts @@ -7,7 +7,7 @@ import type { JobContext, JobHandler } from '@main/core/job/types' import type { KnowledgeItem } from '@shared/data/types/knowledge' import type { KnowledgeLockManager } from '../KnowledgeLockManager' -import type { KnowledgeWorkflowService } from '../KnowledgeWorkflowService' +import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId, toKnowledgeItemId } from '../types' import { markUnscheduledKnowledgeItemsFailed } from '../ingestion/statusCleanup' import { deleteKnowledgeItemVectors } from '../cleanup/vectorCleanup' @@ -21,7 +21,7 @@ const logger = loggerService.withContext('Knowledge:PrepareRootJobHandler') export function createPrepareRootJobHandler( knowledgeLockManager: KnowledgeLockManager, - workflowService: KnowledgeWorkflowService + ingestionService: KnowledgeIngestionService ): JobHandler { return { // Don't auto-resume on restart — a deliberate app quit must not re-spend the @@ -56,7 +56,7 @@ export function createPrepareRootJobHandler( // Source expansion creates child items, so it runs under the base mutation lock. const leafItems = await scanRootItem(ctx, knowledgeLockManager) // Child indexing is scheduled after expansion succeeds so partial scans do not enqueue stale leaves. - await enqueueLeafItems(ctx, leafItems, workflowService) + await enqueueLeafItems(ctx, leafItems, ingestionService) }, async onSettled(event) { @@ -150,7 +150,7 @@ async function scanRootItem( async function enqueueLeafItems( ctx: JobContext, leafItems: KnowledgeItem[], - workflowService: KnowledgeWorkflowService + ingestionService: KnowledgeIngestionService ): Promise { const { baseId } = ctx.input @@ -160,7 +160,7 @@ async function enqueueLeafItems( for (const [index, leaf] of leafItems.entries()) { ctx.signal.throwIfAborted() try { - await workflowService.scheduleItem(baseIdInput, toKnowledgeItemId(leaf.id), ctx.jobId) + await ingestionService.scheduleItem(baseIdInput, toKnowledgeItemId(leaf.id), ctx.jobId) completedSchedulingLeafIds.add(leaf.id) } catch (error) { markUnscheduledLeafItemsFailed(baseId, leafItems, completedSchedulingLeafIds, error) diff --git a/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts b/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts index bea0788bd49..7d34cd5f0cb 100644 --- a/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts +++ b/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts @@ -9,7 +9,7 @@ import type { JobSnapshot } from '@shared/data/api/schemas/jobs' import { KNOWLEDGE_ITEM_ERROR_INDEXING_INTERRUPTED, type KnowledgeItemStatus } from '@shared/data/types/knowledge' import type { KnowledgeLockManager } from '../KnowledgeLockManager' -import type { KnowledgeWorkflowService } from '../KnowledgeWorkflowService' +import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' import { KNOWLEDGE_ACTIVE_JOB_LIMIT, KNOWLEDGE_ACTIVE_JOB_STATUSES, @@ -29,7 +29,7 @@ const REINDEX_RECOVERY_ACTIVE_STATUSES = new Set(['preparin export function createReindexSubtreeJobHandler( knowledgeLockManager: KnowledgeLockManager, - workflowService: KnowledgeWorkflowService + ingestionService: KnowledgeIngestionService ): JobHandler { return { // Don't auto-resume on restart — a deliberate app quit must not re-spend the @@ -133,7 +133,7 @@ export function createReindexSubtreeJobHandler( try { for (const item of resetResult.roots) { ctx.signal.throwIfAborted() - await workflowService.scheduleItem(toKnowledgeBaseId(baseId), toKnowledgeItemId(item.id), ctx.jobId) + await ingestionService.scheduleItem(toKnowledgeBaseId(baseId), toKnowledgeItemId(item.id), ctx.jobId) completedSchedulingRootIds.add(item.id) } } catch (error) { From 52d19914708485a0ce2c4b925d183a10851fba5e Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Thu, 2 Jul 2026 18:47:46 +0800 Subject: [PATCH 04/37] refactor(knowledge): extract query and concept services from KnowledgeService Move the read side (~700 lines) out of the facade into query/: - KnowledgeQueryService: search (with visibility filtering, over-fetch trim, rerank) + listItemChunks + listRootItems and their private helpers, code moved verbatim - KnowledgeConceptService: the Concept ID-addressed agent tool surface (readConcept/grepConcept/deleteConcepts/refreshConcepts/ getOrganizationTree) together with its types, constants, and the grep scanner pure functions; delete/refresh delegate to the ingestion service's guarded deleteItems/reindexItems - visibility.ts / storeOperation.ts: loadVisibleItems, deriveConceptId, and runStoreOperation become free functions shared by both classes KnowledgeService is now a pure facade (~150 lines): lifecycle hooks, module assembly, and one-line delegates. All public signatures are unchanged, so external call sites are untouched. Co-Authored-By: Claude Fable 5 Signed-off-by: eeee0717 --- .../features/knowledge/KnowledgeService.ts | 740 +----------------- .../__tests__/KnowledgeService.test.ts | 3 +- .../query/KnowledgeConceptService.ts | 433 ++++++++++ .../knowledge/query/KnowledgeQueryService.ts | 226 ++++++ .../knowledge/query/storeOperation.ts | 33 + .../features/knowledge/query/visibility.ts | 55 ++ 6 files changed, 791 insertions(+), 699 deletions(-) create mode 100644 src/main/features/knowledge/query/KnowledgeConceptService.ts create mode 100644 src/main/features/knowledge/query/KnowledgeQueryService.ts create mode 100644 src/main/features/knowledge/query/storeOperation.ts create mode 100644 src/main/features/knowledge/query/visibility.ts diff --git a/src/main/features/knowledge/KnowledgeService.ts b/src/main/features/knowledge/KnowledgeService.ts index 9c12ef5bfaa..97e9b738a44 100644 --- a/src/main/features/knowledge/KnowledgeService.ts +++ b/src/main/features/knowledge/KnowledgeService.ts @@ -1,254 +1,41 @@ import { application } from '@application' -import { knowledgeBaseService } from '@data/services/KnowledgeBaseService' -import { knowledgeItemService } from '@data/services/KnowledgeItemService' -import { loggerService } from '@logger' import { BaseService, DependsOn, Injectable, Phase, ServicePhase } from '@main/core/lifecycle' -import { TraceMethod } from '@mcp-trace/trace-core' -import { DataApiErrorFactory, ErrorCode, isDataApiError } from '@shared/data/api' -import { - type CreateKnowledgeBaseDto, - getKnowledgeItemDisplayTitle, - type KnowledgeAddConflictStrategy, - type KnowledgeAddItemInput, - type KnowledgeAddItemsResult, - type KnowledgeBase, - type KnowledgeItem, - type KnowledgeItemChunk, - type KnowledgeItemStatus, - type KnowledgeItemType, - type KnowledgeSearchResult, - type RestoreKnowledgeBaseDto, - type RestoreKnowledgeBaseResult +import type { + CreateKnowledgeBaseDto, + KnowledgeAddConflictStrategy, + KnowledgeAddItemInput, + KnowledgeAddItemsResult, + KnowledgeBase, + KnowledgeItem, + KnowledgeItemChunk, + KnowledgeSearchResult, + RestoreKnowledgeBaseDto, + RestoreKnowledgeBaseResult } from '@shared/data/types/knowledge' -import { isCompletedVectorKnowledgeBase } from '@shared/data/types/knowledge' -import { estimateTokenCount } from 'tokenx' -import { assertBaseCanRunRuntimeOperation } from './base/guards' import { KnowledgeBaseAdminService } from './base/KnowledgeBaseAdminService' -import { embedKnowledgeQuery } from './indexing/embed' -import { toMaterialRelativePath } from './indexing/materialFields' -import { rerankKnowledgeSearchResults } from './indexing/rerank' import { KnowledgeIngestionService } from './ingestion/KnowledgeIngestionService' import { KnowledgeLockManager } from './KnowledgeLockManager' -import { applyRelevanceThreshold, getInitialSearchScoreKind, withSearchRanks } from './query/search' +import type { + KnowledgeConceptContent, + KnowledgeConceptGrep, + KnowledgeConceptMutationResult, + KnowledgeOrganizationTree +} from './query/KnowledgeConceptService' +import { KnowledgeConceptService } from './query/KnowledgeConceptService' +import { KnowledgeQueryService } from './query/KnowledgeQueryService' import { createCheckFileProcessingResultJobHandler } from './tasks/checkFileProcessingResultJobHandler' import { createDeleteSubtreeJobHandler } from './tasks/deleteSubtreeJobHandler' import { createIndexDocumentsJobHandler } from './tasks/indexDocumentsJobHandler' import { createPrepareRootJobHandler } from './tasks/prepareRootJobHandler' import { createReindexSubtreeJobHandler } from './tasks/reindexSubtreeJobHandler' -import { toKnowledgeBaseId, toKnowledgeItemId } from './types' -import type { KnowledgeIndexStore } from './vectorstore/indexStore/KnowledgeIndexStore' -import type { KnowledgeIndexSearchMatch } from './vectorstore/indexStore/model' -const logger = loggerService.withContext('KnowledgeService') -const SEARCH_TOKEN_PATTERN = /[\p{L}\p{N}_]+/u /** - * Fetch this many × the requested result count as index candidates. The index - * store only filters by material state; the item-visibility filter (missing / - * other-base / not-completed) runs afterwards in the caller and can drop matches, - * so over-fetching keeps the final set from shrinking below topK. + * Facade of the knowledge feature: registers the job handlers, runs boot-time + * recovery, and delegates every public operation to the module that owns it — + * base lifecycle (base/), write-side orchestration (ingestion/), and the read + * side (query/). Holds no domain logic of its own. */ -const KNOWLEDGE_SEARCH_OVERFETCH_FACTOR = 5 -/** Hard ceiling on fetched candidates, bounding the brute-force vector scan and rerank cost regardless of topK. */ -const KNOWLEDGE_SEARCH_CANDIDATE_CAP = 200 - -/** Max characters {@link KnowledgeService.readConcept} returns in one slice, so a large document can't flood the agent's context. */ -const CONCEPT_READ_MAX_CHARS = 20_000 -/** Default returned-match cap for {@link KnowledgeService.grepConcept} (the agent may raise it up to {@link CONCEPT_GREP_MAX_MATCHES}). */ -const CONCEPT_GREP_DEFAULT_MAX_MATCHES = 50 -/** Hard ceiling on returned grep matches, bounding the response size. */ -const CONCEPT_GREP_MAX_MATCHES = 200 -/** Characters of context kept on each side of a grep match in its snippet. */ -const CONCEPT_GREP_SNIPPET_PAD = 60 -/** - * Max characters of any single line {@link KnowledgeService.grepConcept} runs the pattern over. Matching one bounded - * line at a time keeps a catastrophic-backtracking pattern (e.g. `(a+)+$`) from freezing the main-process event loop - * by spanning the whole document; matches past this point on an over-long line are not scanned. This only removes the - * whole-document blow-up — a pathological pattern can still backtrack exponentially within one 2000-char line, so it - * is not RE2/ripgrep-grade linear-time matching. Reuses the same 2000-char value as the filesystem grep tool (there - * it truncates displayed lines; here it bounds the text the pattern actually runs over). - */ -const CONCEPT_GREP_MAX_LINE_CHARS = 2000 -/** Hard ceiling on nodes {@link KnowledgeService.getOrganizationTree} returns, bounding the response for a huge base. */ -export const KNOWLEDGE_TREE_MAX_NODES = 1000 - -/** Verbatim slice of a knowledge concept's indexed text, addressed by Concept ID (the material's relative path, OKF §2). */ -export interface KnowledgeConceptContent { - conceptId: string - title: string - itemType: KnowledgeItemType - /** Length of the full document text, so the caller can tell when a slice is partial and page through it. */ - totalChars: number - charStart: number - charEnd: number - content: string - /** True when the returned slice was capped at {@link CONCEPT_READ_MAX_CHARS} and the document continues past `charEnd`. */ - truncated: boolean -} - -/** One exact-pattern match within a concept's indexed text. */ -export interface KnowledgeConceptGrepMatch { - /** 1-based line number of the match start within the document text. */ - line: number - charStart: number - charEnd: number - /** The matched text with a little surrounding context (see {@link CONCEPT_GREP_SNIPPET_PAD}). */ - snippet: string -} - -/** Result of grepping a knowledge concept's indexed text for a regular expression. */ -export interface KnowledgeConceptGrep { - conceptId: string - title: string - itemType: KnowledgeItemType - /** Total matches found in the document (may exceed `matches.length` when the cap was hit). */ - totalMatches: number - matches: KnowledgeConceptGrepMatch[] -} - -/** - * One node of a knowledge base's organization tree, emitted in pre-order DFS so - * the flat list reads top-down like an outline. `depth` (0 at the base root) - * carries the hierarchy without a recursive shape. `conceptId` is set only for a - * readable leaf (a completed file/url/note) so it can be passed to kb_read - * (read or grep mode); directories and not-yet-indexed leaves have none. - */ -export interface KnowledgeTreeNode { - depth: number - title: string - itemType: KnowledgeItemType - status: KnowledgeItemStatus - conceptId?: string -} - -/** - * A knowledge base's organization tree — the logical groupId hierarchy of - * knowledge_item (directories as folders, file/url/note as leaves), NOT the flat - * physical `raw/` layout. The synthesized view OKF surfaces as an `index.md`. - */ -export interface KnowledgeOrganizationTree { - baseId: string - /** - * Count of non-deleting items in the base. May exceed `nodes.length` for two reasons: the node - * list hit the {@link KNOWLEDGE_TREE_MAX_NODES} cap, or a `maxDepth` filter dropped deeper items - * (which are still counted here but not emitted as nodes). - */ - totalItems: number - /** - * True only when the emitted node list was capped at {@link KNOWLEDGE_TREE_MAX_NODES} — it does - * NOT flag `maxDepth` filtering. A reliable "whole tree returned" check is therefore - * `truncated === false` AND no `maxDepth` was passed. - */ - truncated: boolean - nodes: KnowledgeTreeNode[] -} - -/** - * Result of a Concept ID-addressed mutation ({@link KnowledgeService.deleteConcepts} / - * {@link KnowledgeService.refreshConcepts}). `applied` are the Concept IDs that resolved - * to a visible document and were acted on; `notFound` are those that did not resolve in - * this base (a no-op, reported back so the agent can re-check the id rather than assume success). - */ -export interface KnowledgeConceptMutationResult { - applied: string[] - notFound: string[] -} - -/** - * Concept ID (relative path, OKF §2) of a search hit's source document, or - * undefined when it has none: a directory (no material), or a url/note whose - * snapshot relativePath was not captured yet. A completed leaf normally has one, - * so this only swallows the rare unindexed-snapshot case rather than throwing - * out of the whole search. - */ -function deriveConceptId(item: KnowledgeItem): string | undefined { - if (item.type === 'directory') { - return undefined - } - try { - return toMaterialRelativePath(item) - } catch (error) { - // A completed url/note with no snapshot relativePath is an invariant violation. Swallow it (one - // unfollowable hit must not sink the whole search) but leave a diagnostic trail rather than a silent drop. - logger.warn('deriveConceptId: completed item has no material relativePath', { - itemId: item.id, - type: item.type, - error: error instanceof Error ? error.message : String(error) - }) - return undefined - } -} - -/** Compile a kb_read grep-mode pattern (always global; case-insensitive unless told otherwise), turning a bad pattern into a validation error. */ -function compileGrepRegex(pattern: string, ignoreCase: boolean): RegExp { - try { - return new RegExp(pattern, ignoreCase ? 'gi' : 'g') - } catch (error) { - const message = error instanceof Error ? error.message : String(error) - throw DataApiErrorFactory.validation({ pattern: [message] }, `Invalid kb_read regular expression: ${message}`) - } -} - -/** - * Scan `text` for every match of the global `regex`, one line at a time, returning the - * total count and the first `limit` matches with 1-based line numbers, document-absolute - * offsets, and padded snippets. Each line is matched independently and truncated at - * {@link CONCEPT_GREP_MAX_LINE_CHARS}, so a single regex evaluation never runs over more - * than one bounded line — a catastrophic-backtracking pattern therefore cannot freeze the - * main process by spanning the whole document (this mirrors the line-oriented fallback in - * the filesystem grep tool). Anchors (`^`/`$`) consequently bind to each line, and a match - * cannot span lines. `lastIndex` is advanced past zero-width matches so an empty-matching - * pattern cannot spin. `regex` MUST carry the global flag (the caller compiles it so); it - * is reset at the start of every line. - */ -function scanConceptMatches( - text: string, - regex: RegExp, - limit: number -): { totalMatches: number; matches: KnowledgeConceptGrepMatch[] } { - const matches: KnowledgeConceptGrepMatch[] = [] - let totalMatches = 0 - let lineNumber = 0 - // Absolute offset of the current line's first character within `text`. - let lineStart = 0 - - while (lineStart <= text.length) { - lineNumber++ - const newlineIndex = text.indexOf('\n', lineStart) - const lineEnd = newlineIndex === -1 ? text.length : newlineIndex - // Run the pattern over a single, truncated line so backtracking can't blow up across the - // whole document; a match past the per-line cap on an over-long line is dropped. - const line = text.slice(lineStart, Math.min(lineEnd, lineStart + CONCEPT_GREP_MAX_LINE_CHARS)) - - regex.lastIndex = 0 - for (let result = regex.exec(line); result !== null; result = regex.exec(line)) { - totalMatches++ - const matchLength = result[0].length - const start = lineStart + result.index - const end = start + matchLength - - if (matches.length < limit) { - matches.push({ - line: lineNumber, - charStart: start, - charEnd: end, - snippet: text.slice( - Math.max(0, start - CONCEPT_GREP_SNIPPET_PAD), - Math.min(text.length, end + CONCEPT_GREP_SNIPPET_PAD) - ) - }) - } - - // Zero-width match: bump lastIndex so exec() advances past the same position. - regex.lastIndex = result.index + (matchLength > 0 ? matchLength : 1) - } - - lineStart = lineEnd + 1 - } - - return { totalMatches, matches } -} - @Injectable('KnowledgeService') @ServicePhase(Phase.WhenReady) @DependsOn(['KnowledgeVectorStoreService', 'JobManager', 'FileProcessingService']) @@ -256,6 +43,8 @@ export class KnowledgeService extends BaseService { private readonly knowledgeLockManager = new KnowledgeLockManager() private readonly ingestionService = new KnowledgeIngestionService(this.knowledgeLockManager) private readonly baseAdmin = new KnowledgeBaseAdminService(this.knowledgeLockManager, this.ingestionService) + private readonly queryService = new KnowledgeQueryService() + private readonly conceptService = new KnowledgeConceptService(this.ingestionService) protected onInit(): void { const jobManager = application.get('JobManager') @@ -292,6 +81,15 @@ export class KnowledgeService extends BaseService { return await this.baseAdmin.restoreBase(dto) } + listBases(): KnowledgeBase[] { + return this.baseAdmin.listBases() + } + + /** Whether the user has any knowledge base at all — a cheap count (not a full list) for tool-availability gating. */ + hasAnyBase(): boolean { + return this.baseAdmin.hasAnyBase() + } + async addItems( baseId: string, items: KnowledgeAddItemInput[], @@ -308,497 +106,43 @@ export class KnowledgeService extends BaseService { await this.ingestionService.reindexItems(baseId, itemIds) } - listBases(): KnowledgeBase[] { - return this.baseAdmin.listBases() - } - - /** Whether the user has any knowledge base at all — a cheap count (not a full list) for tool-availability gating. */ - hasAnyBase(): boolean { - return this.baseAdmin.hasAnyBase() - } - listRootItems(baseId: string): KnowledgeItem[] { - return knowledgeItemService.getRootItemsByBaseId(baseId) + return this.queryService.listRootItems(baseId) } - @TraceMethod({ spanName: 'Knowledge.search', tag: 'Knowledge' }) async search(baseId: string, query: string): Promise { - assertBaseCanRunRuntimeOperation(baseId, 'search') - - if (!SEARCH_TOKEN_PATTERN.test(query)) { - throw DataApiErrorFactory.validation( - { query: ['Query has no searchable tokens'] }, - 'Query has no searchable tokens' - ) - } - - const base = knowledgeBaseService.getById(baseId) - // Stored search mode and the index store's mode are the same enum now, so no mapping. - // Vector/hybrid retrieval needs an embedding model; a base without one is BM25-only - // regardless of its stored preference. - const mode = isCompletedVectorKnowledgeBase(base) ? base.searchMode : 'bm25' - // BM25 is lexical only; skip the embedding round-trip when the query won't use it. - const queryEmbedding = mode === 'bm25' ? undefined : await embedKnowledgeQuery(base, query) - - const resolvedTopK = base.documentCount ?? 10 - const candidateLimit = Math.min(resolvedTopK * KNOWLEDGE_SEARCH_OVERFETCH_FACTOR, KNOWLEDGE_SEARCH_CANDIDATE_CAP) - - const vectorStoreService = application.get('KnowledgeVectorStoreService') - const store = await vectorStoreService.getIndexStore(base) - const matches = await this.runStoreOperation(store, baseId, 'search', () => - store.search({ - queryText: query, - queryEmbedding, - mode, - topK: candidateLimit, - alpha: base.hybridAlpha - }) - ) - - const scoreKind = getInitialSearchScoreKind(base) - const visibleSearchResults = this.toVisibleSearchResults(baseId, matches, scoreKind) - const topResults = this.trimToTopK(visibleSearchResults, resolvedTopK, baseId) - - if (base.rerankModelId) { - const rerankedResults = await rerankKnowledgeSearchResults(base, query, topResults) - return withSearchRanks(applyRelevanceThreshold(rerankedResults, base.threshold)) - } - - return withSearchRanks(applyRelevanceThreshold(topResults, base.threshold)) + return await this.queryService.search(baseId, query) } async listItemChunks(baseId: string, itemId: string): Promise { - const knowledgeBaseId = toKnowledgeBaseId(baseId) - const knowledgeItemId = toKnowledgeItemId(itemId) - assertBaseCanRunRuntimeOperation(knowledgeBaseId, 'listItemChunks') - const item = await this.assertItemCanRunChunkOperation(knowledgeBaseId, knowledgeItemId, 'list chunks') - this.assertCompletedContainerHasNoDeletingChildren(knowledgeBaseId, item) - - const base = knowledgeBaseService.getById(knowledgeBaseId) - const leafItems = knowledgeItemService.getSubtreeItems(knowledgeBaseId, [knowledgeItemId], { - includeRoots: true, - leafOnly: true - }) - if (leafItems.length === 0) { - return [] - } - - const vectorStoreService = application.get('KnowledgeVectorStoreService') - const store = await vectorStoreService.getIndexStore(base) - const chunkGroups = await this.runStoreOperation(store, knowledgeBaseId, 'listItemChunks', () => - Promise.all( - leafItems.map(async (leafItem) => { - const units = await store.listMaterialUnits(leafItem.id) - return units.map( - (unit): KnowledgeItemChunk => ({ - id: unit.unitId, - itemId: leafItem.id, - content: unit.text, - metadata: { - itemId: leafItem.id, - itemType: leafItem.type, - source: leafItem.data.source, - chunkIndex: unit.unitIndex, - tokenCount: estimateTokenCount(unit.text) - } - }) - ) - }) - ) - ) - - return chunkGroups.flat() + return await this.queryService.listItemChunks(baseId, itemId) } - /** - * Read a knowledge concept's indexed text by its Concept ID (the material's - * relative path, OKF §2), optionally a `[charStart, charEnd)` slice. The slice - * is capped at {@link CONCEPT_READ_MAX_CHARS}; `totalChars` + `truncated` let - * the caller page through a longer document. Throws NOT_FOUND when the Concept - * ID does not resolve to a readable, visible document in this base. - */ async readConcept( baseId: string, conceptId: string, range?: { charStart?: number; charEnd?: number } ): Promise { - const { item, text } = await this.resolveConcept(baseId, conceptId, 'readConcept') - - const totalChars = text.length - const start = Math.min(Math.max(range?.charStart ?? 0, 0), totalChars) - // Where the caller would have ended without the cap (an omitted charEnd reads to the document end). - const naturalEnd = Math.min(range?.charEnd ?? totalChars, totalChars) - const end = Math.max(start, Math.min(naturalEnd, start + CONCEPT_READ_MAX_CHARS)) - - return { - conceptId, - title: getKnowledgeItemDisplayTitle(item), - itemType: item.type, - totalChars, - charStart: start, - charEnd: end, - content: text.slice(start, end), - truncated: end < naturalEnd - } + return await this.conceptService.readConcept(baseId, conceptId, range) } - /** - * Search a knowledge concept's indexed text for a regular expression, returning - * the total match count and the first `maxMatches` matches (line, offsets, a - * padded snippet). The pattern is compiled global, case-insensitive by default; - * an invalid pattern throws a validation error. Throws NOT_FOUND when the - * Concept ID does not resolve to a readable, visible document in this base. - */ async grepConcept( baseId: string, conceptId: string, options: { pattern: string; ignoreCase?: boolean; maxMatches?: number } ): Promise { - const { item, text } = await this.resolveConcept(baseId, conceptId, 'grepConcept') - - const limit = Math.min( - Math.max(options.maxMatches ?? CONCEPT_GREP_DEFAULT_MAX_MATCHES, 1), - CONCEPT_GREP_MAX_MATCHES - ) - const regex = compileGrepRegex(options.pattern, options.ignoreCase ?? true) - const { totalMatches, matches } = scanConceptMatches(text, regex, limit) - - return { - conceptId, - title: getKnowledgeItemDisplayTitle(item), - itemType: item.type, - totalMatches, - matches - } + return await this.conceptService.grepConcept(baseId, conceptId, options) } - /** - * Delete knowledge documents addressed by Concept ID (the deep-read/kb_manage - * write counterpart to {@link readConcept}). Each Concept ID is resolved to its - * knowledge item and deleted via {@link deleteItems} (which expands to the - * outermost selected subtree); an id that does not resolve to a visible document - * in this base is reported in `notFound` rather than failing the whole batch. - */ async deleteConcepts(baseId: string, conceptIds: string[]): Promise { - const { found, notFound } = await this.resolveConceptItemIds(baseId, conceptIds, 'deleteConcepts') - if (found.length > 0) { - await this.deleteItems( - baseId, - found.map((entry) => entry.itemId) - ) - } - return { applied: found.map((entry) => entry.conceptId), notFound } + return await this.conceptService.deleteConcepts(baseId, conceptIds) } - /** - * Re-index knowledge documents addressed by Concept ID. Each Concept ID is - * resolved to its knowledge item and re-indexed via {@link reindexItems}; an id - * that does not resolve to a visible document in this base is reported in - * `notFound` rather than failing the whole batch. - */ async refreshConcepts(baseId: string, conceptIds: string[]): Promise { - const { found, notFound } = await this.resolveConceptItemIds(baseId, conceptIds, 'refreshConcepts') - if (found.length > 0) { - await this.reindexItems( - baseId, - found.map((entry) => entry.itemId) - ) - } - return { applied: found.map((entry) => entry.conceptId), notFound } + return await this.conceptService.refreshConcepts(baseId, conceptIds) } - /** - * Resolve a batch of Concept IDs to their knowledge item ids for a Concept - * ID-addressed mutation. Mirrors {@link resolveConcept}'s identity boundary — - * each relative path is looked up in this base's index store, then re-validated - * against the visible knowledge_item (same base, completed) — but resolves many - * ids and partitions them into resolved (`found`) vs unresolved (`notFound`) - * instead of throwing, so one bad id does not sink the batch. Duplicate ids in - * the input are collapsed to a single resolution. - */ - private async resolveConceptItemIds( - baseId: string, - conceptIds: string[], - operation: string - ): Promise<{ found: Array<{ conceptId: string; itemId: string }>; notFound: string[] }> { - const base = knowledgeBaseService.getById(baseId) - const vectorStoreService = application.get('KnowledgeVectorStoreService') - const store = await vectorStoreService.getIndexStore(base) - - const found: Array<{ conceptId: string; itemId: string }> = [] - const notFound: string[] = [] - const seen = new Set() - - for (const conceptId of conceptIds) { - if (seen.has(conceptId)) { - continue - } - seen.add(conceptId) - - const ref = await this.runStoreOperation(store, baseId, operation, () => - store.getMaterialByRelativePath(conceptId) - ) - // Identity re-check: the resolved material id must still be a visible item in this base. - const item = ref ? this.loadVisibleItems(baseId, [ref.materialId]).get(ref.materialId) : undefined - if (ref && item) { - found.push({ conceptId, itemId: ref.materialId }) - } else { - notFound.push(conceptId) - } - } - - return { found, notFound } - } - - /** - * Resolve a Concept ID to its content text for the deep-read tools. The tools - * address by relative path (a transparent DB key, never an fs path), so this - * re-validates the resolved material against the visible knowledge_item (same - * base, completed) before returning any text — the identity boundary that keeps - * a relative path from reaching another base's or a deleted item's content. - * Throws NOT_FOUND when the concept does not resolve to a readable, visible document. - */ - private async resolveConcept( - baseId: string, - conceptId: string, - operation: string - ): Promise<{ item: KnowledgeItem; text: string }> { - assertBaseCanRunRuntimeOperation(baseId, operation) - - const base = knowledgeBaseService.getById(baseId) - const vectorStoreService = application.get('KnowledgeVectorStoreService') - const store = await vectorStoreService.getIndexStore(base) - - const ref = await this.runStoreOperation(store, baseId, operation, () => store.getMaterialByRelativePath(conceptId)) - if (!ref) { - throw DataApiErrorFactory.notFound('Knowledge concept', conceptId) - } - - const item = this.loadVisibleItems(baseId, [ref.materialId]).get(ref.materialId) - if (!item) { - throw DataApiErrorFactory.notFound('Knowledge concept', conceptId) - } - - const text = await this.runStoreOperation(store, baseId, operation, () => store.readMaterialContent(ref.materialId)) - if (text == null) { - // The material resolved and the item is visible + completed, yet it has no content row — an - // invariant violation or a reindex TOCTOU race, NOT a bad conceptId. Use a distinct resource so the - // tool layer (conceptLookupError) can steer "retry shortly" instead of "verify the conceptId", which - // cannot fix a missing content row. The resource string mirrors KNOWLEDGE_CONCEPT_CONTENT_NOT_FOUND_RESOURCE - // in knowledgeLookup.ts. - logger.warn('resolveConcept: visible completed item has no content row', { baseId, conceptId, operation }) - throw DataApiErrorFactory.notFound('Knowledge concept content', conceptId) - } - - return { item, text } - } - - /** - * Build a knowledge base's organization tree from the `knowledge_item.groupId` - * hierarchy — directories are folders, file/url/note are leaves carrying a - * readable `conceptId` (when completed) for kb_read. This is the - * logical org layer the OKF `index.md` surfaces, deliberately decoupled from - * the flat physical `raw/` layout: no material scan, no path joins. Emitted as - * a pre-order DFS node list (each node's `depth` carries the hierarchy), - * capped at {@link KNOWLEDGE_TREE_MAX_NODES}. Throws NOT_FOUND if the base does - * not exist; available regardless of the base's runtime (search) state. - */ getOrganizationTree(baseId: string, options: { maxDepth?: number } = {}): KnowledgeOrganizationTree { - knowledgeBaseService.getById(baseId) - - const items = knowledgeItemService.getItemsByBaseId(baseId) - - // Index children by their parent groupId (null = base root); each bucket keeps - // getItemsByBaseId's createdAt/id order, so siblings emit in a stable order. - const childrenByGroupId = new Map() - for (const item of items) { - const key = item.groupId ?? null - const bucket = childrenByGroupId.get(key) - if (bucket) { - bucket.push(item) - } else { - childrenByGroupId.set(key, [item]) - } - } - - const maxDepth = options.maxDepth - const nodes: KnowledgeTreeNode[] = [] - let truncated = false - - const walk = (groupId: string | null, depth: number): void => { - if (truncated || (maxDepth !== undefined && depth > maxDepth)) { - return - } - for (const item of childrenByGroupId.get(groupId) ?? []) { - if (nodes.length >= KNOWLEDGE_TREE_MAX_NODES) { - truncated = true - return - } - nodes.push({ - depth, - title: getKnowledgeItemDisplayTitle(item), - itemType: item.type, - status: item.status, - // Only a completed leaf is readable; directories and pending leaves carry no Concept ID. - conceptId: item.type !== 'directory' && item.status === 'completed' ? deriveConceptId(item) : undefined - }) - if (item.type === 'directory') { - walk(item.id, depth + 1) - } - } - } - - walk(null, 0) - - return { baseId, totalItems: items.length, truncated, nodes } - } - - /** - * Turn raw index matches into visible search results: fetch each match's - * knowledge item once, drop any that is missing, in another base, or not - * completed, and reconstruct the chunk metadata (item type / source from the - * item; chunk index from the unit; token count recomputed from the body). - */ - private toVisibleSearchResults( - baseId: string, - matches: KnowledgeIndexSearchMatch[], - scoreKind: KnowledgeSearchResult['scoreKind'] - ): KnowledgeSearchResult[] { - const itemsById = this.loadVisibleItems( - baseId, - matches.map((match) => match.materialId) - ) - - const results: KnowledgeSearchResult[] = [] - for (const match of matches) { - const item = itemsById.get(match.materialId) - if (!item) { - continue - } - results.push({ - pageContent: match.text, - score: match.score, - scoreKind, - rank: results.length + 1, - metadata: { - itemId: match.materialId, - itemType: item.type, - source: item.data.source, - chunkIndex: match.unitIndex, - tokenCount: estimateTokenCount(match.text) - }, - itemId: match.materialId, - chunkId: match.unitId, - // Concept ID + title so a hit can be followed up with kb_read. - conceptId: deriveConceptId(item), - title: getKnowledgeItemDisplayTitle(item) - }) - } - return results - } - - /** Keep the highest-scored `topK` visible results, discarding the over-fetched tail. */ - private trimToTopK(results: KnowledgeSearchResult[], topK: number, baseId: string): KnowledgeSearchResult[] { - if (results.length <= topK) { - return results - } - logger.debug('Trimmed over-fetched knowledge search results to topK', { - baseId, - visibleCandidates: results.length, - topK - }) - return results.slice(0, topK) - } - - /** Fetch the distinct items behind the matches, keeping only those visible in this base (same base, completed). */ - private loadVisibleItems(baseId: string, materialIds: string[]): Map { - const uniqueIds = [...new Set(materialIds)] - const visibleItems = new Map() - - for (const materialId of uniqueIds) { - try { - const item = knowledgeItemService.getById(materialId) - if (item.baseId === baseId && item.status === 'completed') { - visibleItems.set(materialId, item) - } - } catch (error) { - if (isDataApiError(error) && error.code === ErrorCode.NOT_FOUND) { - continue - } - throw error - } - } - - return visibleItems - } - - /** - * Run a per-base index-store interaction, translating the error raised when the - * store is closed mid-flight — a concurrent {@link deleteBase} or app shutdown - * closed the driver — into a defined, retryable DataApiError instead of leaking - * the opaque driver-level error to the renderer. Genuine query errors rethrow - * unchanged. - */ - private async runStoreOperation( - store: KnowledgeIndexStore, - baseId: string, - operation: string, - run: () => Promise - ): Promise { - try { - return await run() - } catch (error) { - if (store.isClosed()) { - logger.warn('Knowledge index store was closed during operation', { baseId, operation }) - throw DataApiErrorFactory.invalidOperation( - operation, - `Knowledge base '${baseId}' index store was closed during ${operation}; retry the operation` - ) - } - throw error - } - } - - private async getRootItemsInBase(baseId: string, itemIds: string[]): Promise { - const rootIds = [...new Set(itemIds)] - const items = await Promise.all(rootIds.map((itemId) => knowledgeItemService.getById(itemId))) - const invalidItem = items.find((item) => item.baseId !== baseId) - - if (invalidItem) { - throw new Error(`Knowledge item '${invalidItem.id}' does not belong to base '${baseId}'`) - } - - return items - } - - private async assertItemCanRunChunkOperation( - baseId: string, - itemId: string, - operation: 'list chunks' | 'delete chunk' - ): Promise { - const [item] = await this.getRootItemsInBase(baseId, [itemId]) - - if (item.status !== 'completed') { - throw DataApiErrorFactory.validation( - { item: [`Knowledge item '${itemId}' must be completed before ${operation}`] }, - `Cannot ${operation} for a non-completed knowledge item` - ) - } - - return item - } - - private assertCompletedContainerHasNoDeletingChildren(baseId: string, item: KnowledgeItem): void { - if (item.type !== 'directory') { - return - } - - const subtreeItems = knowledgeItemService.getSubtreeItems(baseId, [item.id]) - if (subtreeItems.some((item) => item.status === 'deleting')) { - throw DataApiErrorFactory.validation( - { item: [`Knowledge item subtree '${item.id}' is being deleted`] }, - 'Cannot list chunks for a deleting knowledge item' - ) - } + return this.conceptService.getOrganizationTree(baseId, options) } } diff --git a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts index eddc9ff23eb..4e398ebe485 100644 --- a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts +++ b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts @@ -185,7 +185,8 @@ vi.mock('../storage/pathStorage', async () => { } }) -const { KnowledgeService, KNOWLEDGE_TREE_MAX_NODES } = await import('../KnowledgeService') +const { KnowledgeService } = await import('../KnowledgeService') +const { KNOWLEDGE_TREE_MAX_NODES } = await import('../query/KnowledgeConceptService') const NOTE_ITEM_ID = '0198f3f2-7d1a-7abc-8def-123456789abc' const DELETING_NOTE_ITEM_ID = '0198f3f2-7d1b-7abc-8def-123456789abc' diff --git a/src/main/features/knowledge/query/KnowledgeConceptService.ts b/src/main/features/knowledge/query/KnowledgeConceptService.ts new file mode 100644 index 00000000000..12c24a0d670 --- /dev/null +++ b/src/main/features/knowledge/query/KnowledgeConceptService.ts @@ -0,0 +1,433 @@ +import { application } from '@application' +import { knowledgeBaseService } from '@data/services/KnowledgeBaseService' +import { knowledgeItemService } from '@data/services/KnowledgeItemService' +import { loggerService } from '@logger' +import { DataApiErrorFactory } from '@shared/data/api' +import type { KnowledgeItem, KnowledgeItemStatus, KnowledgeItemType } from '@shared/data/types/knowledge' +import { getKnowledgeItemDisplayTitle } from '@shared/data/types/knowledge' + +import { assertBaseCanRunRuntimeOperation } from '../base/guards' +import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' +import { runStoreOperation } from './storeOperation' +import { deriveConceptId, loadVisibleItems } from './visibility' + +const logger = loggerService.withContext('Knowledge:ConceptService') + +/** Max characters {@link KnowledgeConceptService.readConcept} returns in one slice, so a large document can't flood the agent's context. */ +const CONCEPT_READ_MAX_CHARS = 20_000 +/** Default returned-match cap for {@link KnowledgeConceptService.grepConcept} (the agent may raise it up to {@link CONCEPT_GREP_MAX_MATCHES}). */ +const CONCEPT_GREP_DEFAULT_MAX_MATCHES = 50 +/** Hard ceiling on returned grep matches, bounding the response size. */ +const CONCEPT_GREP_MAX_MATCHES = 200 +/** Characters of context kept on each side of a grep match in its snippet. */ +const CONCEPT_GREP_SNIPPET_PAD = 60 +/** + * Max characters of any single line {@link KnowledgeConceptService.grepConcept} runs the pattern over. Matching one + * bounded line at a time keeps a catastrophic-backtracking pattern (e.g. `(a+)+$`) from freezing the main-process + * event loop by spanning the whole document; matches past this point on an over-long line are not scanned. This only + * removes the whole-document blow-up — a pathological pattern can still backtrack exponentially within one 2000-char + * line, so it is not RE2/ripgrep-grade linear-time matching. Reuses the same 2000-char value as the filesystem grep + * tool (there it truncates displayed lines; here it bounds the text the pattern actually runs over). + */ +const CONCEPT_GREP_MAX_LINE_CHARS = 2000 +/** Hard ceiling on nodes {@link KnowledgeConceptService.getOrganizationTree} returns, bounding the response for a huge base. */ +export const KNOWLEDGE_TREE_MAX_NODES = 1000 + +/** Verbatim slice of a knowledge concept's indexed text, addressed by Concept ID (the material's relative path, OKF §2). */ +export interface KnowledgeConceptContent { + conceptId: string + title: string + itemType: KnowledgeItemType + /** Length of the full document text, so the caller can tell when a slice is partial and page through it. */ + totalChars: number + charStart: number + charEnd: number + content: string + /** True when the returned slice was capped at {@link CONCEPT_READ_MAX_CHARS} and the document continues past `charEnd`. */ + truncated: boolean +} + +/** One exact-pattern match within a concept's indexed text. */ +export interface KnowledgeConceptGrepMatch { + /** 1-based line number of the match start within the document text. */ + line: number + charStart: number + charEnd: number + /** The matched text with a little surrounding context (see {@link CONCEPT_GREP_SNIPPET_PAD}). */ + snippet: string +} + +/** Result of grepping a knowledge concept's indexed text for a regular expression. */ +export interface KnowledgeConceptGrep { + conceptId: string + title: string + itemType: KnowledgeItemType + /** Total matches found in the document (may exceed `matches.length` when the cap was hit). */ + totalMatches: number + matches: KnowledgeConceptGrepMatch[] +} + +/** + * One node of a knowledge base's organization tree, emitted in pre-order DFS so + * the flat list reads top-down like an outline. `depth` (0 at the base root) + * carries the hierarchy without a recursive shape. `conceptId` is set only for a + * readable leaf (a completed file/url/note) so it can be passed to kb_read + * (read or grep mode); directories and not-yet-indexed leaves have none. + */ +export interface KnowledgeTreeNode { + depth: number + title: string + itemType: KnowledgeItemType + status: KnowledgeItemStatus + conceptId?: string +} + +/** + * A knowledge base's organization tree — the logical groupId hierarchy of + * knowledge_item (directories as folders, file/url/note as leaves), NOT the flat + * physical `raw/` layout. The synthesized view OKF surfaces as an `index.md`. + */ +export interface KnowledgeOrganizationTree { + baseId: string + /** + * Count of non-deleting items in the base. May exceed `nodes.length` for two reasons: the node + * list hit the {@link KNOWLEDGE_TREE_MAX_NODES} cap, or a `maxDepth` filter dropped deeper items + * (which are still counted here but not emitted as nodes). + */ + totalItems: number + /** + * True only when the emitted node list was capped at {@link KNOWLEDGE_TREE_MAX_NODES} — it does + * NOT flag `maxDepth` filtering. A reliable "whole tree returned" check is therefore + * `truncated === false` AND no `maxDepth` was passed. + */ + truncated: boolean + nodes: KnowledgeTreeNode[] +} + +/** + * Result of a Concept ID-addressed mutation ({@link KnowledgeConceptService.deleteConcepts} / + * {@link KnowledgeConceptService.refreshConcepts}). `applied` are the Concept IDs that resolved + * to a visible document and were acted on; `notFound` are those that did not resolve in + * this base (a no-op, reported back so the agent can re-check the id rather than assume success). + */ +export interface KnowledgeConceptMutationResult { + applied: string[] + notFound: string[] +} + +/** Compile a kb_read grep-mode pattern (always global; case-insensitive unless told otherwise), turning a bad pattern into a validation error. */ +function compileGrepRegex(pattern: string, ignoreCase: boolean): RegExp { + try { + return new RegExp(pattern, ignoreCase ? 'gi' : 'g') + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + throw DataApiErrorFactory.validation({ pattern: [message] }, `Invalid kb_read regular expression: ${message}`) + } +} + +/** + * Scan `text` for every match of the global `regex`, one line at a time, returning the + * total count and the first `limit` matches with 1-based line numbers, document-absolute + * offsets, and padded snippets. Each line is matched independently and truncated at + * {@link CONCEPT_GREP_MAX_LINE_CHARS}, so a single regex evaluation never runs over more + * than one bounded line — a catastrophic-backtracking pattern therefore cannot freeze the + * main process by spanning the whole document (this mirrors the line-oriented fallback in + * the filesystem grep tool). Anchors (`^`/`$`) consequently bind to each line, and a match + * cannot span lines. `lastIndex` is advanced past zero-width matches so an empty-matching + * pattern cannot spin. `regex` MUST carry the global flag (the caller compiles it so); it + * is reset at the start of every line. + */ +function scanConceptMatches( + text: string, + regex: RegExp, + limit: number +): { totalMatches: number; matches: KnowledgeConceptGrepMatch[] } { + const matches: KnowledgeConceptGrepMatch[] = [] + let totalMatches = 0 + let lineNumber = 0 + // Absolute offset of the current line's first character within `text`. + let lineStart = 0 + + while (lineStart <= text.length) { + lineNumber++ + const newlineIndex = text.indexOf('\n', lineStart) + const lineEnd = newlineIndex === -1 ? text.length : newlineIndex + // Run the pattern over a single, truncated line so backtracking can't blow up across the + // whole document; a match past the per-line cap on an over-long line is dropped. + const line = text.slice(lineStart, Math.min(lineEnd, lineStart + CONCEPT_GREP_MAX_LINE_CHARS)) + + regex.lastIndex = 0 + for (let result = regex.exec(line); result !== null; result = regex.exec(line)) { + totalMatches++ + const matchLength = result[0].length + const start = lineStart + result.index + const end = start + matchLength + + if (matches.length < limit) { + matches.push({ + line: lineNumber, + charStart: start, + charEnd: end, + snippet: text.slice( + Math.max(0, start - CONCEPT_GREP_SNIPPET_PAD), + Math.min(text.length, end + CONCEPT_GREP_SNIPPET_PAD) + ) + }) + } + + // Zero-width match: bump lastIndex so exec() advances past the same position. + regex.lastIndex = result.index + (matchLength > 0 ? matchLength : 1) + } + + lineStart = lineEnd + 1 + } + + return { totalMatches, matches } +} + +/** Concept ID-addressed agent tool surface: deep read (read/grep), organization tree, and concept-level delete/refresh. */ +export class KnowledgeConceptService { + constructor(private readonly ingestionService: KnowledgeIngestionService) {} + + /** + * Read a knowledge concept's indexed text by its Concept ID (the material's + * relative path, OKF §2), optionally a `[charStart, charEnd)` slice. The slice + * is capped at {@link CONCEPT_READ_MAX_CHARS}; `totalChars` + `truncated` let + * the caller page through a longer document. Throws NOT_FOUND when the Concept + * ID does not resolve to a readable, visible document in this base. + */ + async readConcept( + baseId: string, + conceptId: string, + range?: { charStart?: number; charEnd?: number } + ): Promise { + const { item, text } = await this.resolveConcept(baseId, conceptId, 'readConcept') + + const totalChars = text.length + const start = Math.min(Math.max(range?.charStart ?? 0, 0), totalChars) + // Where the caller would have ended without the cap (an omitted charEnd reads to the document end). + const naturalEnd = Math.min(range?.charEnd ?? totalChars, totalChars) + const end = Math.max(start, Math.min(naturalEnd, start + CONCEPT_READ_MAX_CHARS)) + + return { + conceptId, + title: getKnowledgeItemDisplayTitle(item), + itemType: item.type, + totalChars, + charStart: start, + charEnd: end, + content: text.slice(start, end), + truncated: end < naturalEnd + } + } + + /** + * Search a knowledge concept's indexed text for a regular expression, returning + * the total match count and the first `maxMatches` matches (line, offsets, a + * padded snippet). The pattern is compiled global, case-insensitive by default; + * an invalid pattern throws a validation error. Throws NOT_FOUND when the + * Concept ID does not resolve to a readable, visible document in this base. + */ + async grepConcept( + baseId: string, + conceptId: string, + options: { pattern: string; ignoreCase?: boolean; maxMatches?: number } + ): Promise { + const { item, text } = await this.resolveConcept(baseId, conceptId, 'grepConcept') + + const limit = Math.min( + Math.max(options.maxMatches ?? CONCEPT_GREP_DEFAULT_MAX_MATCHES, 1), + CONCEPT_GREP_MAX_MATCHES + ) + const regex = compileGrepRegex(options.pattern, options.ignoreCase ?? true) + const { totalMatches, matches } = scanConceptMatches(text, regex, limit) + + return { + conceptId, + title: getKnowledgeItemDisplayTitle(item), + itemType: item.type, + totalMatches, + matches + } + } + + /** + * Delete knowledge documents addressed by Concept ID (the deep-read/kb_manage + * write counterpart to {@link readConcept}). Each Concept ID is resolved to its + * knowledge item and deleted via the ingestion service's `deleteItems` (which + * expands to the outermost selected subtree); an id that does not resolve to a + * visible document in this base is reported in `notFound` rather than failing + * the whole batch. + */ + async deleteConcepts(baseId: string, conceptIds: string[]): Promise { + const { found, notFound } = await this.resolveConceptItemIds(baseId, conceptIds, 'deleteConcepts') + if (found.length > 0) { + await this.ingestionService.deleteItems( + baseId, + found.map((entry) => entry.itemId) + ) + } + return { applied: found.map((entry) => entry.conceptId), notFound } + } + + /** + * Re-index knowledge documents addressed by Concept ID. Each Concept ID is + * resolved to its knowledge item and re-indexed via the ingestion service's + * `reindexItems`; an id that does not resolve to a visible document in this + * base is reported in `notFound` rather than failing the whole batch. + */ + async refreshConcepts(baseId: string, conceptIds: string[]): Promise { + const { found, notFound } = await this.resolveConceptItemIds(baseId, conceptIds, 'refreshConcepts') + if (found.length > 0) { + await this.ingestionService.reindexItems( + baseId, + found.map((entry) => entry.itemId) + ) + } + return { applied: found.map((entry) => entry.conceptId), notFound } + } + + /** + * Build a knowledge base's organization tree from the `knowledge_item.groupId` + * hierarchy — directories are folders, file/url/note are leaves carrying a + * readable `conceptId` (when completed) for kb_read. This is the + * logical org layer the OKF `index.md` surfaces, deliberately decoupled from + * the flat physical `raw/` layout: no material scan, no path joins. Emitted as + * a pre-order DFS node list (each node's `depth` carries the hierarchy), + * capped at {@link KNOWLEDGE_TREE_MAX_NODES}. Throws NOT_FOUND if the base does + * not exist; available regardless of the base's runtime (search) state. + */ + getOrganizationTree(baseId: string, options: { maxDepth?: number } = {}): KnowledgeOrganizationTree { + knowledgeBaseService.getById(baseId) + + const items = knowledgeItemService.getItemsByBaseId(baseId) + + // Index children by their parent groupId (null = base root); each bucket keeps + // getItemsByBaseId's createdAt/id order, so siblings emit in a stable order. + const childrenByGroupId = new Map() + for (const item of items) { + const key = item.groupId ?? null + const bucket = childrenByGroupId.get(key) + if (bucket) { + bucket.push(item) + } else { + childrenByGroupId.set(key, [item]) + } + } + + const maxDepth = options.maxDepth + const nodes: KnowledgeTreeNode[] = [] + let truncated = false + + const walk = (groupId: string | null, depth: number): void => { + if (truncated || (maxDepth !== undefined && depth > maxDepth)) { + return + } + for (const item of childrenByGroupId.get(groupId) ?? []) { + if (nodes.length >= KNOWLEDGE_TREE_MAX_NODES) { + truncated = true + return + } + nodes.push({ + depth, + title: getKnowledgeItemDisplayTitle(item), + itemType: item.type, + status: item.status, + // Only a completed leaf is readable; directories and pending leaves carry no Concept ID. + conceptId: item.type !== 'directory' && item.status === 'completed' ? deriveConceptId(item) : undefined + }) + if (item.type === 'directory') { + walk(item.id, depth + 1) + } + } + } + + walk(null, 0) + + return { baseId, totalItems: items.length, truncated, nodes } + } + + /** + * Resolve a batch of Concept IDs to their knowledge item ids for a Concept + * ID-addressed mutation. Mirrors {@link resolveConcept}'s identity boundary — + * each relative path is looked up in this base's index store, then re-validated + * against the visible knowledge_item (same base, completed) — but resolves many + * ids and partitions them into resolved (`found`) vs unresolved (`notFound`) + * instead of throwing, so one bad id does not sink the batch. Duplicate ids in + * the input are collapsed to a single resolution. + */ + private async resolveConceptItemIds( + baseId: string, + conceptIds: string[], + operation: string + ): Promise<{ found: Array<{ conceptId: string; itemId: string }>; notFound: string[] }> { + const base = knowledgeBaseService.getById(baseId) + const vectorStoreService = application.get('KnowledgeVectorStoreService') + const store = await vectorStoreService.getIndexStore(base) + + const found: Array<{ conceptId: string; itemId: string }> = [] + const notFound: string[] = [] + const seen = new Set() + + for (const conceptId of conceptIds) { + if (seen.has(conceptId)) { + continue + } + seen.add(conceptId) + + const ref = await runStoreOperation(store, baseId, operation, () => store.getMaterialByRelativePath(conceptId)) + // Identity re-check: the resolved material id must still be a visible item in this base. + const item = ref ? loadVisibleItems(baseId, [ref.materialId]).get(ref.materialId) : undefined + if (ref && item) { + found.push({ conceptId, itemId: ref.materialId }) + } else { + notFound.push(conceptId) + } + } + + return { found, notFound } + } + + /** + * Resolve a Concept ID to its content text for the deep-read tools. The tools + * address by relative path (a transparent DB key, never an fs path), so this + * re-validates the resolved material against the visible knowledge_item (same + * base, completed) before returning any text — the identity boundary that keeps + * a relative path from reaching another base's or a deleted item's content. + * Throws NOT_FOUND when the concept does not resolve to a readable, visible document. + */ + private async resolveConcept( + baseId: string, + conceptId: string, + operation: string + ): Promise<{ item: KnowledgeItem; text: string }> { + assertBaseCanRunRuntimeOperation(baseId, operation) + + const base = knowledgeBaseService.getById(baseId) + const vectorStoreService = application.get('KnowledgeVectorStoreService') + const store = await vectorStoreService.getIndexStore(base) + + const ref = await runStoreOperation(store, baseId, operation, () => store.getMaterialByRelativePath(conceptId)) + if (!ref) { + throw DataApiErrorFactory.notFound('Knowledge concept', conceptId) + } + + const item = loadVisibleItems(baseId, [ref.materialId]).get(ref.materialId) + if (!item) { + throw DataApiErrorFactory.notFound('Knowledge concept', conceptId) + } + + const text = await runStoreOperation(store, baseId, operation, () => store.readMaterialContent(ref.materialId)) + if (text == null) { + // The material resolved and the item is visible + completed, yet it has no content row — an + // invariant violation or a reindex TOCTOU race, NOT a bad conceptId. Use a distinct resource so the + // tool layer (conceptLookupError) can steer "retry shortly" instead of "verify the conceptId", which + // cannot fix a missing content row. The resource string mirrors KNOWLEDGE_CONCEPT_CONTENT_NOT_FOUND_RESOURCE + // in knowledgeLookup.ts. + logger.warn('resolveConcept: visible completed item has no content row', { baseId, conceptId, operation }) + throw DataApiErrorFactory.notFound('Knowledge concept content', conceptId) + } + + return { item, text } + } +} diff --git a/src/main/features/knowledge/query/KnowledgeQueryService.ts b/src/main/features/knowledge/query/KnowledgeQueryService.ts new file mode 100644 index 00000000000..da327bdf989 --- /dev/null +++ b/src/main/features/knowledge/query/KnowledgeQueryService.ts @@ -0,0 +1,226 @@ +import { application } from '@application' +import { knowledgeBaseService } from '@data/services/KnowledgeBaseService' +import { knowledgeItemService } from '@data/services/KnowledgeItemService' +import { loggerService } from '@logger' +import { TraceMethod } from '@mcp-trace/trace-core' +import { DataApiErrorFactory } from '@shared/data/api' +import type { KnowledgeItem, KnowledgeItemChunk, KnowledgeSearchResult } from '@shared/data/types/knowledge' +import { getKnowledgeItemDisplayTitle, isCompletedVectorKnowledgeBase } from '@shared/data/types/knowledge' +import { estimateTokenCount } from 'tokenx' + +import { assertBaseCanRunRuntimeOperation } from '../base/guards' +import { embedKnowledgeQuery } from '../indexing/embed' +import { rerankKnowledgeSearchResults } from '../indexing/rerank' +import { toKnowledgeBaseId, toKnowledgeItemId } from '../types' +import type { KnowledgeIndexSearchMatch } from '../vectorstore/indexStore/model' +import { applyRelevanceThreshold, getInitialSearchScoreKind, withSearchRanks } from './search' +import { runStoreOperation } from './storeOperation' +import { deriveConceptId, loadVisibleItems } from './visibility' + +const logger = loggerService.withContext('Knowledge:QueryService') +const SEARCH_TOKEN_PATTERN = /[\p{L}\p{N}_]+/u +/** + * Fetch this many × the requested result count as index candidates. The index + * store only filters by material state; the item-visibility filter (missing / + * other-base / not-completed) runs afterwards in the caller and can drop matches, + * so over-fetching keeps the final set from shrinking below topK. + */ +const KNOWLEDGE_SEARCH_OVERFETCH_FACTOR = 5 +/** Hard ceiling on fetched candidates, bounding the brute-force vector scan and rerank cost regardless of topK. */ +const KNOWLEDGE_SEARCH_CANDIDATE_CAP = 200 + +/** Read side of the knowledge feature: index search (with visibility filtering + rerank) and chunk/item listing. */ +export class KnowledgeQueryService { + @TraceMethod({ spanName: 'Knowledge.search', tag: 'Knowledge' }) + async search(baseId: string, query: string): Promise { + assertBaseCanRunRuntimeOperation(baseId, 'search') + + if (!SEARCH_TOKEN_PATTERN.test(query)) { + throw DataApiErrorFactory.validation( + { query: ['Query has no searchable tokens'] }, + 'Query has no searchable tokens' + ) + } + + const base = knowledgeBaseService.getById(baseId) + // Stored search mode and the index store's mode are the same enum now, so no mapping. + // Vector/hybrid retrieval needs an embedding model; a base without one is BM25-only + // regardless of its stored preference. + const mode = isCompletedVectorKnowledgeBase(base) ? base.searchMode : 'bm25' + // BM25 is lexical only; skip the embedding round-trip when the query won't use it. + const queryEmbedding = mode === 'bm25' ? undefined : await embedKnowledgeQuery(base, query) + + const resolvedTopK = base.documentCount ?? 10 + const candidateLimit = Math.min(resolvedTopK * KNOWLEDGE_SEARCH_OVERFETCH_FACTOR, KNOWLEDGE_SEARCH_CANDIDATE_CAP) + + const vectorStoreService = application.get('KnowledgeVectorStoreService') + const store = await vectorStoreService.getIndexStore(base) + const matches = await runStoreOperation(store, baseId, 'search', () => + store.search({ + queryText: query, + queryEmbedding, + mode, + topK: candidateLimit, + alpha: base.hybridAlpha + }) + ) + + const scoreKind = getInitialSearchScoreKind(base) + const visibleSearchResults = this.toVisibleSearchResults(baseId, matches, scoreKind) + const topResults = this.trimToTopK(visibleSearchResults, resolvedTopK, baseId) + + if (base.rerankModelId) { + const rerankedResults = await rerankKnowledgeSearchResults(base, query, topResults) + return withSearchRanks(applyRelevanceThreshold(rerankedResults, base.threshold)) + } + + return withSearchRanks(applyRelevanceThreshold(topResults, base.threshold)) + } + + async listItemChunks(baseId: string, itemId: string): Promise { + const knowledgeBaseId = toKnowledgeBaseId(baseId) + const knowledgeItemId = toKnowledgeItemId(itemId) + assertBaseCanRunRuntimeOperation(knowledgeBaseId, 'listItemChunks') + const item = await this.assertItemCanRunChunkOperation(knowledgeBaseId, knowledgeItemId, 'list chunks') + this.assertCompletedContainerHasNoDeletingChildren(knowledgeBaseId, item) + + const base = knowledgeBaseService.getById(knowledgeBaseId) + const leafItems = knowledgeItemService.getSubtreeItems(knowledgeBaseId, [knowledgeItemId], { + includeRoots: true, + leafOnly: true + }) + if (leafItems.length === 0) { + return [] + } + + const vectorStoreService = application.get('KnowledgeVectorStoreService') + const store = await vectorStoreService.getIndexStore(base) + const chunkGroups = await runStoreOperation(store, knowledgeBaseId, 'listItemChunks', () => + Promise.all( + leafItems.map(async (leafItem) => { + const units = await store.listMaterialUnits(leafItem.id) + return units.map( + (unit): KnowledgeItemChunk => ({ + id: unit.unitId, + itemId: leafItem.id, + content: unit.text, + metadata: { + itemId: leafItem.id, + itemType: leafItem.type, + source: leafItem.data.source, + chunkIndex: unit.unitIndex, + tokenCount: estimateTokenCount(unit.text) + } + }) + ) + }) + ) + ) + + return chunkGroups.flat() + } + + listRootItems(baseId: string): KnowledgeItem[] { + return knowledgeItemService.getRootItemsByBaseId(baseId) + } + + /** + * Turn raw index matches into visible search results: fetch each match's + * knowledge item once, drop any that is missing, in another base, or not + * completed, and reconstruct the chunk metadata (item type / source from the + * item; chunk index from the unit; token count recomputed from the body). + */ + private toVisibleSearchResults( + baseId: string, + matches: KnowledgeIndexSearchMatch[], + scoreKind: KnowledgeSearchResult['scoreKind'] + ): KnowledgeSearchResult[] { + const itemsById = loadVisibleItems( + baseId, + matches.map((match) => match.materialId) + ) + + const results: KnowledgeSearchResult[] = [] + for (const match of matches) { + const item = itemsById.get(match.materialId) + if (!item) { + continue + } + results.push({ + pageContent: match.text, + score: match.score, + scoreKind, + rank: results.length + 1, + metadata: { + itemId: match.materialId, + itemType: item.type, + source: item.data.source, + chunkIndex: match.unitIndex, + tokenCount: estimateTokenCount(match.text) + }, + itemId: match.materialId, + chunkId: match.unitId, + // Concept ID + title so a hit can be followed up with kb_read. + conceptId: deriveConceptId(item), + title: getKnowledgeItemDisplayTitle(item) + }) + } + return results + } + + /** Keep the highest-scored `topK` visible results, discarding the over-fetched tail. */ + private trimToTopK(results: KnowledgeSearchResult[], topK: number, baseId: string): KnowledgeSearchResult[] { + if (results.length <= topK) { + return results + } + logger.debug('Trimmed over-fetched knowledge search results to topK', { + baseId, + visibleCandidates: results.length, + topK + }) + return results.slice(0, topK) + } + + private async getRootItemsInBase(baseId: string, itemIds: string[]): Promise { + const rootIds = [...new Set(itemIds)] + const items = await Promise.all(rootIds.map((itemId) => knowledgeItemService.getById(itemId))) + const invalidItem = items.find((item) => item.baseId !== baseId) + + if (invalidItem) { + throw new Error(`Knowledge item '${invalidItem.id}' does not belong to base '${baseId}'`) + } + + return items + } + + private async assertItemCanRunChunkOperation( + baseId: string, + itemId: string, + operation: 'list chunks' | 'delete chunk' + ): Promise { + const [item] = await this.getRootItemsInBase(baseId, [itemId]) + + if (item.status !== 'completed') { + throw DataApiErrorFactory.validation( + { item: [`Knowledge item '${itemId}' must be completed before ${operation}`] }, + `Cannot ${operation} for a non-completed knowledge item` + ) + } + + return item + } + + private assertCompletedContainerHasNoDeletingChildren(baseId: string, item: KnowledgeItem): void { + if (item.type !== 'directory') { + return + } + + const subtreeItems = knowledgeItemService.getSubtreeItems(baseId, [item.id]) + if (subtreeItems.some((item) => item.status === 'deleting')) { + throw DataApiErrorFactory.validation( + { item: [`Knowledge item subtree '${item.id}' is being deleted`] }, + 'Cannot list chunks for a deleting knowledge item' + ) + } + } +} diff --git a/src/main/features/knowledge/query/storeOperation.ts b/src/main/features/knowledge/query/storeOperation.ts new file mode 100644 index 00000000000..53184d7cbc3 --- /dev/null +++ b/src/main/features/knowledge/query/storeOperation.ts @@ -0,0 +1,33 @@ +import { loggerService } from '@logger' +import { DataApiErrorFactory } from '@shared/data/api' + +import type { KnowledgeIndexStore } from '../vectorstore/indexStore/KnowledgeIndexStore' + +const logger = loggerService.withContext('Knowledge:Query') + +/** + * Run a per-base index-store interaction, translating the error raised when the + * store is closed mid-flight — a concurrent `deleteBase` or app shutdown + * closed the driver — into a defined, retryable DataApiError instead of leaking + * the opaque driver-level error to the renderer. Genuine query errors rethrow + * unchanged. + */ +export async function runStoreOperation( + store: KnowledgeIndexStore, + baseId: string, + operation: string, + run: () => Promise +): Promise { + try { + return await run() + } catch (error) { + if (store.isClosed()) { + logger.warn('Knowledge index store was closed during operation', { baseId, operation }) + throw DataApiErrorFactory.invalidOperation( + operation, + `Knowledge base '${baseId}' index store was closed during ${operation}; retry the operation` + ) + } + throw error + } +} diff --git a/src/main/features/knowledge/query/visibility.ts b/src/main/features/knowledge/query/visibility.ts new file mode 100644 index 00000000000..11b8518b280 --- /dev/null +++ b/src/main/features/knowledge/query/visibility.ts @@ -0,0 +1,55 @@ +import { knowledgeItemService } from '@data/services/KnowledgeItemService' +import { loggerService } from '@logger' +import { ErrorCode, isDataApiError } from '@shared/data/api' +import type { KnowledgeItem } from '@shared/data/types/knowledge' + +import { toMaterialRelativePath } from '../indexing/materialFields' + +const logger = loggerService.withContext('Knowledge:Query') + +/** + * Concept ID (relative path, OKF §2) of a search hit's source document, or + * undefined when it has none: a directory (no material), or a url/note whose + * snapshot relativePath was not captured yet. A completed leaf normally has one, + * so this only swallows the rare unindexed-snapshot case rather than throwing + * out of the whole search. + */ +export function deriveConceptId(item: KnowledgeItem): string | undefined { + if (item.type === 'directory') { + return undefined + } + try { + return toMaterialRelativePath(item) + } catch (error) { + // A completed url/note with no snapshot relativePath is an invariant violation. Swallow it (one + // unfollowable hit must not sink the whole search) but leave a diagnostic trail rather than a silent drop. + logger.warn('deriveConceptId: completed item has no material relativePath', { + itemId: item.id, + type: item.type, + error: error instanceof Error ? error.message : String(error) + }) + return undefined + } +} + +/** Fetch the distinct items behind the matches, keeping only those visible in this base (same base, completed). */ +export function loadVisibleItems(baseId: string, materialIds: string[]): Map { + const uniqueIds = [...new Set(materialIds)] + const visibleItems = new Map() + + for (const materialId of uniqueIds) { + try { + const item = knowledgeItemService.getById(materialId) + if (item.baseId === baseId && item.status === 'completed') { + visibleItems.set(materialId, item) + } + } catch (error) { + if (isDataApiError(error) && error.code === ErrorCode.NOT_FOUND) { + continue + } + throw error + } + } + + return visibleItems +} From 26c1c09bf7ef99bb00e3ef7979e77be4ee4dc41f Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Thu, 2 Jul 2026 18:48:34 +0800 Subject: [PATCH 05/37] docs(knowledge): add feature README Document the pipeline main axis (sources -> readers -> indexing -> vectorstore, driven by tasks/), the directory map, the five jobs and their chaining/recovery semantics, the item status flow, and what the per-base lock does (and does not) protect. Co-Authored-By: Claude Fable 5 Signed-off-by: eeee0717 --- src/main/features/knowledge/README.md | 76 +++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 src/main/features/knowledge/README.md diff --git a/src/main/features/knowledge/README.md b/src/main/features/knowledge/README.md new file mode 100644 index 00000000000..7a615f311bc --- /dev/null +++ b/src/main/features/knowledge/README.md @@ -0,0 +1,76 @@ +# Knowledge Feature + +Per-base knowledge library: ingest sources (files, directories, urls, notes), convert them to +markdown, chunk + embed the text, and persist everything into a per-base `index.sqlite` +(better-sqlite3 + sqlite-vec) that serves hybrid vector/BM25 search and the Concept ID-addressed +agent tools (`kb_search` / `kb_read` / `kb_tree` / `kb_manage`). + +## Pipeline + +The top-level directories spell out the ingestion pipeline in stage order: + +``` + input preprocess index persist + ┌──────────────┐ ┌────────────────┐ ┌───────────────┐ ┌───────────────┐ + │ sources/ │ ───> │ readers/ │ ──> │ indexing/ │ ──> │ vectorstore/ │ + │ expand dirs, │ │ file → md text │ │ chunk, embed, │ │ index.sqlite │ + │ url/note │ │ (pdf, docx, …) │ │ rerank │ │ (per base) │ + │ snapshots │ └────────────────┘ └───────────────┘ └───────────────┘ + └──────────────┘ heavy conversions (MinerU/PaddleOCR/…) run out-of-process + via FileProcessingService, polled by a knowledge job +``` + +Jobs in `tasks/` drive the stages; `ingestion/` decides which jobs to enqueue; `query/` reads the +result back out. Nothing in a stage directory enqueues jobs or mutates item status — that is +orchestration, and it lives in `ingestion/` and `tasks/`. + +## Directory map + +| Directory | Role | +| --- | --- | +| `KnowledgeService.ts` | Lifecycle facade: registers job handlers, runs boot recovery, delegates every public method. No domain logic. | +| `base/` | Base lifecycle: create (with rollback), delete, restore, list + the shared failed-base guard (`guards.ts`). | +| `ingestion/` | Write-side orchestration: admission checks, item creation, add-conflict resolution, job enqueueing, boot recovery. | +| `sources/` | Input stage: directory expansion, url fetch (Jina reader), url/note snapshot capture, OKF frontmatter. | +| `readers/` | Preprocess stage: file → markdown/text `Document[]` readers (pdf/docx/epub/…). | +| `indexing/` | Index stage: offset-preserving splitter + chunker, `AiService` embedding/rerank wrappers, material field derivation. | +| `vectorstore/` | Persist stage: per-base `index.sqlite` lifecycle (`KnowledgeVectorStoreService`) and the store itself (`indexStore/`, synchronous better-sqlite3 driver). | +| `query/` | Read side: hybrid search with visibility filtering (`KnowledgeQueryService`), Concept ID tool surface (`KnowledgeConceptService`). | +| `tasks/` | Job handlers — the pipeline executors (see below). | +| `cleanup/` | Subtree purge (vectors + files + rows) and index space reclamation. | +| `storage/` | `raw/` path allocation: collision-free names, reservation, base file paths. | +| `items.ts` / `types.ts` / `types/` | Shared item predicates + source probing; branded ids, queue names, idempotency keys. | + +## Jobs + +All jobs run on the per-base queue `base.{baseId}`; idempotency keys prevent double-enqueues. + +| Job | Does | Enqueued by | +| --- | --- | --- | +| `knowledge.prepare-root` | Expand a directory root into child items, then enqueue leaf indexing. | `ingestion` (add), reindex handler | +| `knowledge.index-documents` | Read → chunk → embed → `rebuildMaterial` in one store transaction. | `ingestion`, prepare-root, fp-check | +| `knowledge.check-file-processing-result` | Poll a FileProcessingService job (5s delay per round); on success enqueue indexing. | `ingestion` (files needing conversion) | +| `knowledge.delete-subtree` | Cancel active jobs → delete vectors → delete files → delete rows. | `ingestion` (delete), boot recovery | +| `knowledge.reindex-subtree` | Verify source → delete vectors → reset statuses → re-enqueue indexing. | `ingestion` (reindex) | + +Indexing jobs declare `recovery: 'abandon'` — an app restart never silently resumes them (that +would auto-spend the paid embedding API); boot recovery parks interrupted items at `failed` +instead. Delete/reindex jobs use `recovery: 'retry'`. + +Item status flow: `preparing` (directory) / `processing` → `completed` | `failed`; any status → +`deleting` → row removed. `reading`/`embedding` are transient sub-phases surfaced while the index +job runs. + +## Concurrency + +`KnowledgeLockManager` is a per-base **application-level** mutex serializing multi-step business +invariants that span the main DB, the index store, and the filesystem (e.g. add's +read-conflicts-then-create-rows sequence). It is not about protecting SQLite itself — the per-base +driver is synchronous, and single statements are atomic. Handlers acquire the lock only around the +mutation section, never across slow I/O (fetch, read, embed). + +## Related docs + +- Data-layer selection and patterns: [docs/references/data/README.md](../../../../docs/references/data/README.md) +- Concept ID = material relative path (OKF §2): the addressing primitive for `kb_read`/`kb_manage`; + resolved against the index store and re-validated against the visible `knowledge_item`. From 89229bfa95d3dfa2da387a6d1b1ea5495bdc064f Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Thu, 2 Jul 2026 19:04:21 +0800 Subject: [PATCH 06/37] refactor(knowledge): flatten single/thin-file directories to feature root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit storage/, base/, and cleanup/ each wrapped only 1-2 files around a directory that added no grouping value — naming-conventions.md §4.4 promotes to a subdirectory only when a topic actually owns multiple files, and storage/ (a lone pathStorage.ts) had no precedent elsewhere in the codebase for a non-bucket single-file topic directory. - storage/pathStorage.ts -> pathStorage.ts - base/KnowledgeBaseAdminService.ts -> KnowledgeBaseAdminService.ts - base/guards.ts -> baseGuards.ts (renamed: 'guards.ts' alone at the feature root lost the disambiguating parent directory name) - cleanup/{subtreePurge,vectorCleanup}.ts -> root All four are cross-cutting (consumed by ingestion/ and 3+ task handlers, not owned by a single pipeline stage), matching the existing items.ts/types.ts root-file precedent. Brings knowledge/ from 11 top-level subdirectories down to 8, closer to the sibling fileProcessing/ (5) and apiGateway/ (4) feature modules. Pure move + import rewrite + two internal import-depth fixes in the moved files themselves; no behavior change. Co-Authored-By: Claude Fable 5 Signed-off-by: eeee0717 --- .../data/migration/v2/migrators/KnowledgeMigrator.ts | 2 +- .../v2/migrators/KnowledgeVectorMigrator.ts | 4 ++-- .../__tests__/KnowledgeVectorMigrator.test.ts | 2 +- .../{base => }/KnowledgeBaseAdminService.ts | 12 ++++++------ src/main/features/knowledge/KnowledgeService.ts | 4 ++-- src/main/features/knowledge/README.md | 7 ++++--- .../knowledge/__tests__/KnowledgeService.test.ts | 6 +++--- src/main/features/knowledge/__tests__/items.test.ts | 6 +++--- .../{storage => }/__tests__/pathStorage.test.ts | 0 .../{cleanup => }/__tests__/vectorCleanup.test.ts | 0 .../knowledge/{base/guards.ts => baseGuards.ts} | 0 .../knowledge/ingestion/KnowledgeIngestionService.ts | 6 +++--- src/main/features/knowledge/items.ts | 2 +- .../features/knowledge/{storage => }/pathStorage.ts | 0 .../knowledge/query/KnowledgeConceptService.ts | 2 +- .../knowledge/query/KnowledgeQueryService.ts | 2 +- .../knowledge/readers/KnowledgeFileReader.ts | 2 +- .../knowledge/readers/KnowledgeSnapshotReader.ts | 2 +- .../knowledge/sources/__tests__/directory.test.ts | 6 +++--- .../knowledge/sources/__tests__/noteSnapshot.test.ts | 6 +++--- .../knowledge/sources/__tests__/urlSnapshot.test.ts | 6 +++--- src/main/features/knowledge/sources/directory.ts | 2 +- src/main/features/knowledge/sources/noteSnapshot.ts | 2 +- src/main/features/knowledge/sources/prepare.ts | 2 +- src/main/features/knowledge/sources/urlSnapshot.ts | 2 +- .../features/knowledge/{cleanup => }/subtreePurge.ts | 10 +++++----- .../knowledge/tasks/__tests__/jobHandlerTestUtils.ts | 6 +++--- .../tasks/checkFileProcessingResultJobHandler.ts | 2 +- .../knowledge/tasks/deleteSubtreeJobHandler.ts | 4 ++-- .../knowledge/tasks/indexDocumentsJobHandler.ts | 2 +- .../knowledge/tasks/prepareRootJobHandler.ts | 4 ++-- .../knowledge/tasks/reindexSubtreeJobHandler.ts | 4 ++-- .../knowledge/{cleanup => }/vectorCleanup.ts | 0 .../vectorstore/KnowledgeVectorStoreService.ts | 2 +- .../__tests__/KnowledgeVectorStoreService.test.ts | 2 +- 35 files changed, 61 insertions(+), 60 deletions(-) rename src/main/features/knowledge/{base => }/KnowledgeBaseAdminService.ts (96%) rename src/main/features/knowledge/{storage => }/__tests__/pathStorage.test.ts (100%) rename src/main/features/knowledge/{cleanup => }/__tests__/vectorCleanup.test.ts (100%) rename src/main/features/knowledge/{base/guards.ts => baseGuards.ts} (100%) rename src/main/features/knowledge/{storage => }/pathStorage.ts (100%) rename src/main/features/knowledge/{cleanup => }/subtreePurge.ts (93%) rename src/main/features/knowledge/{cleanup => }/vectorCleanup.ts (100%) diff --git a/src/main/data/migration/v2/migrators/KnowledgeMigrator.ts b/src/main/data/migration/v2/migrators/KnowledgeMigrator.ts index b6366529512..5d8051e1989 100644 --- a/src/main/data/migration/v2/migrators/KnowledgeMigrator.ts +++ b/src/main/data/migration/v2/migrators/KnowledgeMigrator.ts @@ -12,7 +12,7 @@ import { loggerService } from '@logger' import { needsProcessedArtifactReservation, reserveImportedFileRelativePath -} from '@main/features/knowledge/storage/pathStorage' +} from '@main/features/knowledge/pathStorage' import { sanitizeFilename } from '@main/utils/file' import { copy, ensureDir } from '@main/utils/file/fs' import type { ExecuteResult, PrepareResult, ValidateResult, ValidationError } from '@shared/data/migration/v2/types' diff --git a/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts b/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts index 5688dea9098..46845134a1b 100644 --- a/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts +++ b/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts @@ -15,7 +15,7 @@ import { assertSafeKnowledgeRelativePath, collectKnowledgeReservedRelativePaths, reserveImportedFileRelativePath -} from '@main/features/knowledge/storage/pathStorage' +} from '@main/features/knowledge/pathStorage' import { type BetterSqlite3Driver, openBetterSqlite3IndexDriver @@ -48,7 +48,7 @@ import { const logger = loggerService.withContext('KnowledgeVectorMigrator') // Runtime vector store + material layout — source of truth: -// src/main/features/knowledge/storage/pathStorage.ts +// src/main/features/knowledge/pathStorage.ts // (CHERRY_META_DIR / VECTOR_STORE_FILE / MATERIAL_ROOT_DIR). Runtime opens // {knowledgeBaseDir}/{baseId}/.cherry/index.sqlite by the migrated (new) base id, and resolves // every material's bytes at {knowledgeBaseDir}/{baseId}/raw/{relativePath}, so the migrator must diff --git a/src/main/data/migration/v2/migrators/__tests__/KnowledgeVectorMigrator.test.ts b/src/main/data/migration/v2/migrators/__tests__/KnowledgeVectorMigrator.test.ts index 90772577201..e06d5a56918 100644 --- a/src/main/data/migration/v2/migrators/__tests__/KnowledgeVectorMigrator.test.ts +++ b/src/main/data/migration/v2/migrators/__tests__/KnowledgeVectorMigrator.test.ts @@ -73,7 +73,7 @@ function createTempRoot() { } // Mirrors the runtime vector store layout in -// src/main/features/knowledge/storage/pathStorage.ts: {root}/{baseId}/.cherry/index.sqlite. +// src/main/features/knowledge/pathStorage.ts: {root}/{baseId}/.cherry/index.sqlite. // Read-back assertions use this so they fail if the migrator ever writes to a path the runtime // would not open — the exact bug this regression guards against. function runtimeVectorStorePath(baseId: string): string { diff --git a/src/main/features/knowledge/base/KnowledgeBaseAdminService.ts b/src/main/features/knowledge/KnowledgeBaseAdminService.ts similarity index 96% rename from src/main/features/knowledge/base/KnowledgeBaseAdminService.ts rename to src/main/features/knowledge/KnowledgeBaseAdminService.ts index 0e30fd50c62..069e8d62cf2 100644 --- a/src/main/features/knowledge/base/KnowledgeBaseAdminService.ts +++ b/src/main/features/knowledge/KnowledgeBaseAdminService.ts @@ -14,18 +14,18 @@ import { type RestoreKnowledgeBaseResult } from '@shared/data/types/knowledge' -import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' -import { classifyKnowledgeItemSource } from '../items' -import type { KnowledgeLockManager } from '../KnowledgeLockManager' -import { getKnowledgeBaseFilePath } from '../storage/pathStorage' -import { narrowKnowledgeJobInput } from '../tasks/utils/jobInput' +import type { KnowledgeIngestionService } from './ingestion/KnowledgeIngestionService' +import { classifyKnowledgeItemSource } from './items' +import type { KnowledgeLockManager } from './KnowledgeLockManager' +import { getKnowledgeBaseFilePath } from './pathStorage' +import { narrowKnowledgeJobInput } from './tasks/utils/jobInput' import { KNOWLEDGE_ACTIVE_JOB_LIMIT, KNOWLEDGE_ACTIVE_JOB_STATUSES, KNOWLEDGE_JOB_TYPES, knowledgeQueueName, toKnowledgeBaseId -} from '../types' +} from './types' const logger = loggerService.withContext('Knowledge:BaseAdmin') const KNOWLEDGE_JOB_TYPE_SET = new Set(KNOWLEDGE_JOB_TYPES) diff --git a/src/main/features/knowledge/KnowledgeService.ts b/src/main/features/knowledge/KnowledgeService.ts index 97e9b738a44..43f14162efc 100644 --- a/src/main/features/knowledge/KnowledgeService.ts +++ b/src/main/features/knowledge/KnowledgeService.ts @@ -13,7 +13,7 @@ import type { RestoreKnowledgeBaseResult } from '@shared/data/types/knowledge' -import { KnowledgeBaseAdminService } from './base/KnowledgeBaseAdminService' +import { KnowledgeBaseAdminService } from './KnowledgeBaseAdminService' import { KnowledgeIngestionService } from './ingestion/KnowledgeIngestionService' import { KnowledgeLockManager } from './KnowledgeLockManager' import type { @@ -33,7 +33,7 @@ import { createReindexSubtreeJobHandler } from './tasks/reindexSubtreeJobHandler /** * Facade of the knowledge feature: registers the job handlers, runs boot-time * recovery, and delegates every public operation to the module that owns it — - * base lifecycle (base/), write-side orchestration (ingestion/), and the read + * base lifecycle (KnowledgeBaseAdminService), write-side orchestration (ingestion/), and the read * side (query/). Holds no domain logic of its own. */ @Injectable('KnowledgeService') diff --git a/src/main/features/knowledge/README.md b/src/main/features/knowledge/README.md index 7a615f311bc..26f888c24c5 100644 --- a/src/main/features/knowledge/README.md +++ b/src/main/features/knowledge/README.md @@ -29,7 +29,8 @@ orchestration, and it lives in `ingestion/` and `tasks/`. | Directory | Role | | --- | --- | | `KnowledgeService.ts` | Lifecycle facade: registers job handlers, runs boot recovery, delegates every public method. No domain logic. | -| `base/` | Base lifecycle: create (with rollback), delete, restore, list + the shared failed-base guard (`guards.ts`). | +| `KnowledgeBaseAdminService.ts` | Base lifecycle: create (with rollback), delete, restore, list. | +| `baseGuards.ts` | Shared failed-base guard, used by both the write and read side. | | `ingestion/` | Write-side orchestration: admission checks, item creation, add-conflict resolution, job enqueueing, boot recovery. | | `sources/` | Input stage: directory expansion, url fetch (Jina reader), url/note snapshot capture, OKF frontmatter. | | `readers/` | Preprocess stage: file → markdown/text `Document[]` readers (pdf/docx/epub/…). | @@ -37,8 +38,8 @@ orchestration, and it lives in `ingestion/` and `tasks/`. | `vectorstore/` | Persist stage: per-base `index.sqlite` lifecycle (`KnowledgeVectorStoreService`) and the store itself (`indexStore/`, synchronous better-sqlite3 driver). | | `query/` | Read side: hybrid search with visibility filtering (`KnowledgeQueryService`), Concept ID tool surface (`KnowledgeConceptService`). | | `tasks/` | Job handlers — the pipeline executors (see below). | -| `cleanup/` | Subtree purge (vectors + files + rows) and index space reclamation. | -| `storage/` | `raw/` path allocation: collision-free names, reservation, base file paths. | +| `subtreePurge.ts` / `vectorCleanup.ts` | Subtree purge (vectors + files + rows) and index space reclamation — shared by ingestion and the delete/reindex/prepare-root handlers. | +| `pathStorage.ts` | `raw/` path allocation: collision-free names, reservation, base file paths. | | `items.ts` / `types.ts` / `types/` | Shared item predicates + source probing; branded ids, queue names, idempotency keys. | ## Jobs diff --git a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts index 4e398ebe485..19764814c6f 100644 --- a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts +++ b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts @@ -10,7 +10,7 @@ import { } from '@shared/data/types/knowledge' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import type * as PathStorage from '../storage/pathStorage' +import type * as PathStorage from '../pathStorage' const { cancelManyMock, @@ -174,8 +174,8 @@ vi.mock('../indexing/rerank', () => ({ rerankKnowledgeSearchResults: rerankKnowledgeSearchResultsMock })) -vi.mock('../storage/pathStorage', async () => { - const actual = await vi.importActual('../storage/pathStorage') +vi.mock('../pathStorage', async () => { + const actual = await vi.importActual('../pathStorage') return { ...actual, copyFileIntoKnowledgeBaseAt: copyFileIntoKnowledgeBaseAtMock, diff --git a/src/main/features/knowledge/__tests__/items.test.ts b/src/main/features/knowledge/__tests__/items.test.ts index bd6b1b26872..93563ca992f 100644 --- a/src/main/features/knowledge/__tests__/items.test.ts +++ b/src/main/features/knowledge/__tests__/items.test.ts @@ -1,15 +1,15 @@ import type { KnowledgeItem, KnowledgeItemOf } from '@shared/data/types/knowledge' import { beforeEach, describe, expect, it, vi } from 'vitest' -import type * as PathStorage from '../storage/pathStorage' +import type * as PathStorage from '../pathStorage' const { probeKnowledgeFileMock, probeKnowledgeSourcePathMock } = vi.hoisted(() => ({ probeKnowledgeFileMock: vi.fn(), probeKnowledgeSourcePathMock: vi.fn() })) -vi.mock('../storage/pathStorage', async () => { - const actual = await vi.importActual('../storage/pathStorage') +vi.mock('../pathStorage', async () => { + const actual = await vi.importActual('../pathStorage') return { ...actual, probeKnowledgeFile: probeKnowledgeFileMock, diff --git a/src/main/features/knowledge/storage/__tests__/pathStorage.test.ts b/src/main/features/knowledge/__tests__/pathStorage.test.ts similarity index 100% rename from src/main/features/knowledge/storage/__tests__/pathStorage.test.ts rename to src/main/features/knowledge/__tests__/pathStorage.test.ts diff --git a/src/main/features/knowledge/cleanup/__tests__/vectorCleanup.test.ts b/src/main/features/knowledge/__tests__/vectorCleanup.test.ts similarity index 100% rename from src/main/features/knowledge/cleanup/__tests__/vectorCleanup.test.ts rename to src/main/features/knowledge/__tests__/vectorCleanup.test.ts diff --git a/src/main/features/knowledge/base/guards.ts b/src/main/features/knowledge/baseGuards.ts similarity index 100% rename from src/main/features/knowledge/base/guards.ts rename to src/main/features/knowledge/baseGuards.ts diff --git a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts index 64453bea7da..8db521f365e 100644 --- a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts +++ b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts @@ -20,8 +20,8 @@ import { } from '@shared/data/types/knowledge' import { knowledgeSupportedFileExts } from '@shared/utils/file' -import { assertBaseCanRunRuntimeOperation } from '../base/guards' -import { cancelActiveKnowledgeSubtreeJobs, purgeKnowledgeSubtreeWithinLock } from '../cleanup/subtreePurge' +import { assertBaseCanRunRuntimeOperation } from '../baseGuards' +import { cancelActiveKnowledgeSubtreeJobs, purgeKnowledgeSubtreeWithinLock } from '../subtreePurge' import { classifyKnowledgeItemSource, isContainerKnowledgeItem } from '../items' import type { KnowledgeLockManager } from '../KnowledgeLockManager' import { planKnowledgeItemSource } from '../sources/sourcePlanning' @@ -35,7 +35,7 @@ import { getProcessedMarkdownRelativePath, needsProcessedArtifactReservation, reserveImportedFileRelativePath -} from '../storage/pathStorage' +} from '../pathStorage' import { cancelJobOrThrow } from '../tasks/utils/cancel' import { type KnowledgeBaseId, diff --git a/src/main/features/knowledge/items.ts b/src/main/features/knowledge/items.ts index 4fdbcc8aeec..8fce570571b 100644 --- a/src/main/features/knowledge/items.ts +++ b/src/main/features/knowledge/items.ts @@ -2,7 +2,7 @@ import type { PathReadability } from '@main/utils/file/fs' import type { KnowledgeItem } from '@shared/data/types/knowledge' import type { ContainerKnowledgeItem, IndexableKnowledgeItem } from './types/items' -import { probeKnowledgeFile, probeKnowledgeSourcePath } from './storage/pathStorage' +import { probeKnowledgeFile, probeKnowledgeSourcePath } from './pathStorage' export function isIndexableKnowledgeItem(item: KnowledgeItem): item is IndexableKnowledgeItem { return item.type === 'file' || item.type === 'url' || item.type === 'note' diff --git a/src/main/features/knowledge/storage/pathStorage.ts b/src/main/features/knowledge/pathStorage.ts similarity index 100% rename from src/main/features/knowledge/storage/pathStorage.ts rename to src/main/features/knowledge/pathStorage.ts diff --git a/src/main/features/knowledge/query/KnowledgeConceptService.ts b/src/main/features/knowledge/query/KnowledgeConceptService.ts index 12c24a0d670..38a5fdb1de3 100644 --- a/src/main/features/knowledge/query/KnowledgeConceptService.ts +++ b/src/main/features/knowledge/query/KnowledgeConceptService.ts @@ -6,7 +6,7 @@ import { DataApiErrorFactory } from '@shared/data/api' import type { KnowledgeItem, KnowledgeItemStatus, KnowledgeItemType } from '@shared/data/types/knowledge' import { getKnowledgeItemDisplayTitle } from '@shared/data/types/knowledge' -import { assertBaseCanRunRuntimeOperation } from '../base/guards' +import { assertBaseCanRunRuntimeOperation } from '../baseGuards' import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' import { runStoreOperation } from './storeOperation' import { deriveConceptId, loadVisibleItems } from './visibility' diff --git a/src/main/features/knowledge/query/KnowledgeQueryService.ts b/src/main/features/knowledge/query/KnowledgeQueryService.ts index da327bdf989..ff1fb78fdb9 100644 --- a/src/main/features/knowledge/query/KnowledgeQueryService.ts +++ b/src/main/features/knowledge/query/KnowledgeQueryService.ts @@ -8,7 +8,7 @@ import type { KnowledgeItem, KnowledgeItemChunk, KnowledgeSearchResult } from '@ import { getKnowledgeItemDisplayTitle, isCompletedVectorKnowledgeBase } from '@shared/data/types/knowledge' import { estimateTokenCount } from 'tokenx' -import { assertBaseCanRunRuntimeOperation } from '../base/guards' +import { assertBaseCanRunRuntimeOperation } from '../baseGuards' import { embedKnowledgeQuery } from '../indexing/embed' import { rerankKnowledgeSearchResults } from '../indexing/rerank' import { toKnowledgeBaseId, toKnowledgeItemId } from '../types' diff --git a/src/main/features/knowledge/readers/KnowledgeFileReader.ts b/src/main/features/knowledge/readers/KnowledgeFileReader.ts index bb431faaa4a..7dd43d98715 100644 --- a/src/main/features/knowledge/readers/KnowledgeFileReader.ts +++ b/src/main/features/knowledge/readers/KnowledgeFileReader.ts @@ -10,7 +10,7 @@ import { MarkdownReader } from '@vectorstores/readers/markdown' import { PDFReader } from '@vectorstores/readers/pdf' import { TextFileReader } from '@vectorstores/readers/text' -import { getKnowledgeBaseFilePath } from '../storage/pathStorage' +import { getKnowledgeBaseFilePath } from '../pathStorage' import { DraftsExportReader } from './files/DraftsExportReader' import { EpubReader } from './files/EpubReader' diff --git a/src/main/features/knowledge/readers/KnowledgeSnapshotReader.ts b/src/main/features/knowledge/readers/KnowledgeSnapshotReader.ts index 8472c1acafa..7b16c9e3a0f 100644 --- a/src/main/features/knowledge/readers/KnowledgeSnapshotReader.ts +++ b/src/main/features/knowledge/readers/KnowledgeSnapshotReader.ts @@ -3,7 +3,7 @@ import type { KnowledgeItemOf, KnowledgeSourceMetadata } from '@shared/data/type import { Document } from '@vectorstores/core' import { stripOkfFrontmatter } from '../sources/okfFrontmatter' -import { getKnowledgeBaseFilePath } from '../storage/pathStorage' +import { getKnowledgeBaseFilePath } from '../pathStorage' /** * Read a url or note item from its captured on-disk snapshot — never the network diff --git a/src/main/features/knowledge/sources/__tests__/directory.test.ts b/src/main/features/knowledge/sources/__tests__/directory.test.ts index 0d6e48a21ff..575dfb6c645 100644 --- a/src/main/features/knowledge/sources/__tests__/directory.test.ts +++ b/src/main/features/knowledge/sources/__tests__/directory.test.ts @@ -4,7 +4,7 @@ import path from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' -import type * as PathStorage from '../../storage/pathStorage' +import type * as PathStorage from '../../pathStorage' const copyFileIntoKnowledgeBaseAtMock = vi.hoisted(() => vi.fn(async (_baseId: string, _externalPath: string, relativePath: string) => relativePath) @@ -15,8 +15,8 @@ vi.mock('@application', async () => { return mockApplicationFactory() }) -vi.mock('../../storage/pathStorage', async () => { - const actual = await vi.importActual('../../storage/pathStorage') +vi.mock('../../pathStorage', async () => { + const actual = await vi.importActual('../../pathStorage') return { ...actual, copyFileIntoKnowledgeBaseAt: copyFileIntoKnowledgeBaseAtMock diff --git a/src/main/features/knowledge/sources/__tests__/noteSnapshot.test.ts b/src/main/features/knowledge/sources/__tests__/noteSnapshot.test.ts index 5c2c145904e..fbd90da35fd 100644 --- a/src/main/features/knowledge/sources/__tests__/noteSnapshot.test.ts +++ b/src/main/features/knowledge/sources/__tests__/noteSnapshot.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' -import type * as PathStorage from '../../storage/pathStorage' +import type * as PathStorage from '../../pathStorage' const { writeFileIntoKnowledgeBaseAtMock } = vi.hoisted(() => ({ writeFileIntoKnowledgeBaseAtMock: vi.fn() @@ -19,8 +19,8 @@ vi.mock('@logger', () => ({ // Keep reserveImportedFileRelativePath real (it is pure); only stub the disk write so // the test exercises name derivation + dedupe without touching the filesystem. -vi.mock('../../storage/pathStorage', async () => { - const actual = await vi.importActual('../../storage/pathStorage') +vi.mock('../../pathStorage', async () => { + const actual = await vi.importActual('../../pathStorage') return { ...actual, writeFileIntoKnowledgeBaseAt: writeFileIntoKnowledgeBaseAtMock } }) diff --git a/src/main/features/knowledge/sources/__tests__/urlSnapshot.test.ts b/src/main/features/knowledge/sources/__tests__/urlSnapshot.test.ts index 33e19616877..9a1747d9ae9 100644 --- a/src/main/features/knowledge/sources/__tests__/urlSnapshot.test.ts +++ b/src/main/features/knowledge/sources/__tests__/urlSnapshot.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' -import type * as PathStorage from '../../storage/pathStorage' +import type * as PathStorage from '../../pathStorage' const { writeFileIntoKnowledgeBaseAtMock } = vi.hoisted(() => ({ writeFileIntoKnowledgeBaseAtMock: vi.fn() @@ -19,8 +19,8 @@ vi.mock('@logger', () => ({ // Keep reserveImportedFileRelativePath real (it is pure); only stub the disk write so // the test exercises name derivation + dedupe without touching the filesystem. -vi.mock('../../storage/pathStorage', async () => { - const actual = await vi.importActual('../../storage/pathStorage') +vi.mock('../../pathStorage', async () => { + const actual = await vi.importActual('../../pathStorage') return { ...actual, writeFileIntoKnowledgeBaseAt: writeFileIntoKnowledgeBaseAtMock } }) diff --git a/src/main/features/knowledge/sources/directory.ts b/src/main/features/knowledge/sources/directory.ts index 41cbd739421..f67f21da089 100644 --- a/src/main/features/knowledge/sources/directory.ts +++ b/src/main/features/knowledge/sources/directory.ts @@ -5,7 +5,7 @@ import { nextFreeKnowledgeRelativePath } from '@main/utils/knowledge' import type { DirectoryItemData, FileItemData, KnowledgeItem } from '@shared/data/types/knowledge' import { knowledgeSupportedFileExts } from '@shared/utils/file' -import { copyFileIntoKnowledgeBaseAt } from '../storage/pathStorage' +import { copyFileIntoKnowledgeBaseAt } from '../pathStorage' const KNOWLEDGE_SUPPORTED_FILE_EXT_SET = new Set(knowledgeSupportedFileExts) diff --git a/src/main/features/knowledge/sources/noteSnapshot.ts b/src/main/features/knowledge/sources/noteSnapshot.ts index 26f225180f8..7dbc034cd10 100644 --- a/src/main/features/knowledge/sources/noteSnapshot.ts +++ b/src/main/features/knowledge/sources/noteSnapshot.ts @@ -1,6 +1,6 @@ import { sanitizeFilename } from '@shared/utils/file/filename' -import { reserveImportedFileRelativePath, writeFileIntoKnowledgeBaseAt } from '../storage/pathStorage' +import { reserveImportedFileRelativePath, writeFileIntoKnowledgeBaseAt } from '../pathStorage' import { serializeOkfFrontmatter } from './okfFrontmatter' const SNAPSHOT_TITLE_MAX = 80 diff --git a/src/main/features/knowledge/sources/prepare.ts b/src/main/features/knowledge/sources/prepare.ts index 2adf04ca1fb..ea58ba5ae07 100644 --- a/src/main/features/knowledge/sources/prepare.ts +++ b/src/main/features/knowledge/sources/prepare.ts @@ -9,7 +9,7 @@ import { import type { IndexableKnowledgeItem } from '../types/items' import { isContainerKnowledgeItem, isIndexableKnowledgeItem } from '../items' -import { collectKnowledgeReservedRelativePaths } from '../storage/pathStorage' +import { collectKnowledgeReservedRelativePaths } from '../pathStorage' import { expandDirectoryOwnerToTree, type ExpandedDirectoryNode } from './directory' const logger = loggerService.withContext('KnowledgePrepare') diff --git a/src/main/features/knowledge/sources/urlSnapshot.ts b/src/main/features/knowledge/sources/urlSnapshot.ts index 114817eb67d..535db11ab9c 100644 --- a/src/main/features/knowledge/sources/urlSnapshot.ts +++ b/src/main/features/knowledge/sources/urlSnapshot.ts @@ -1,6 +1,6 @@ import { sanitizeFilename } from '@shared/utils/file/filename' -import { reserveImportedFileRelativePath, writeFileIntoKnowledgeBaseAt } from '../storage/pathStorage' +import { reserveImportedFileRelativePath, writeFileIntoKnowledgeBaseAt } from '../pathStorage' import { serializeOkfFrontmatter } from './okfFrontmatter' const SNAPSHOT_TITLE_MAX = 80 diff --git a/src/main/features/knowledge/cleanup/subtreePurge.ts b/src/main/features/knowledge/subtreePurge.ts similarity index 93% rename from src/main/features/knowledge/cleanup/subtreePurge.ts rename to src/main/features/knowledge/subtreePurge.ts index c69aa1e0150..27b5034153c 100644 --- a/src/main/features/knowledge/cleanup/subtreePurge.ts +++ b/src/main/features/knowledge/subtreePurge.ts @@ -2,16 +2,16 @@ import { application } from '@application' import { knowledgeItemService } from '@data/services/KnowledgeItemService' import type { KnowledgeBase, KnowledgeItem } from '@shared/data/types/knowledge' -import { cancelJobOrThrow } from '../tasks/utils/cancel' -import { narrowKnowledgeJobInput } from '../tasks/utils/jobInput' +import { cancelJobOrThrow } from './tasks/utils/cancel' +import { narrowKnowledgeJobInput } from './tasks/utils/jobInput' import { KNOWLEDGE_ACTIVE_JOB_LIMIT, KNOWLEDGE_ACTIVE_JOB_STATUSES, knowledgeQueueName, toKnowledgeBaseId -} from '../types' -import { isIndexableKnowledgeItem } from '../items' -import { deleteKnowledgeItemFilesBestEffort } from '../storage/pathStorage' +} from './types' +import { isIndexableKnowledgeItem } from './items' +import { deleteKnowledgeItemFilesBestEffort } from './pathStorage' import { deleteKnowledgeItemVectors } from './vectorCleanup' /** diff --git a/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts b/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts index 8a878e84f68..05debfbc1cf 100644 --- a/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts +++ b/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts @@ -4,7 +4,7 @@ import type { KnowledgeBase, KnowledgeItemOf } from '@shared/data/types/knowledg import { MockMainCacheServiceUtils } from '@test-mocks/main/CacheService' import { beforeEach, vi } from 'vitest' -import type * as PathStorage from '../../storage/pathStorage' +import type * as PathStorage from '../../pathStorage' const mocks = vi.hoisted(() => ({ cancelMock: vi.fn(), @@ -151,8 +151,8 @@ vi.mock('../../sources/noteSnapshot', () => ({ captureNoteSnapshotFile: captureNoteSnapshotFileMock })) -vi.mock('../../storage/pathStorage', async () => { - const actual = await vi.importActual('../../storage/pathStorage') +vi.mock('../../pathStorage', async () => { + const actual = await vi.importActual('../../pathStorage') return { ...actual, // Stub the best-effort cleanup the handlers call. Its swallow-on-failure diff --git a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts index 75bc22df1a9..5bbf4a42268 100644 --- a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts +++ b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts @@ -15,7 +15,7 @@ import type { FileProcessingJobPayload } from '../../fileProcessing/tasks/shared import type { KnowledgeLockManager } from '../KnowledgeLockManager' import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' import { knowledgeQueueName, toKnowledgeBaseId, toKnowledgeItemId } from '../types' -import { toKnowledgeRelativePath } from '../storage/pathStorage' +import { toKnowledgeRelativePath } from '../pathStorage' import type { KnowledgeCheckFileProcessingResultPayload } from './jobTypes' import { cancelJobOrThrow } from './utils/cancel' import { isDataApiNotFoundError, markKnowledgeItemFailedOnSettled } from './utils/settled' diff --git a/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts b/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts index d48964da3bf..bbe90646868 100644 --- a/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts +++ b/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts @@ -7,8 +7,8 @@ import type { JobHandler } from '@main/core/job/types' import type { KnowledgeLockManager } from '../KnowledgeLockManager' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId } from '../types' -import { cancelActiveKnowledgeSubtreeJobs, purgeKnowledgeSubtreeWithinLock } from '../cleanup/subtreePurge' -import { reclaimKnowledgeIndexSpace } from '../cleanup/vectorCleanup' +import { cancelActiveKnowledgeSubtreeJobs, purgeKnowledgeSubtreeWithinLock } from '../subtreePurge' +import { reclaimKnowledgeIndexSpace } from '../vectorCleanup' import type { KnowledgeDeleteSubtreePayload } from './jobTypes' const logger = loggerService.withContext('Knowledge:DeleteSubtreeJobHandler') diff --git a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts index b66a328bce6..7ef813546da 100644 --- a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts +++ b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts @@ -19,7 +19,7 @@ import { isIndexableKnowledgeItem } from '../items' import { captureNoteSnapshotFile } from '../sources/noteSnapshot' import { fetchKnowledgeWebPage } from '../sources/url' import { captureUrlSnapshotFile } from '../sources/urlSnapshot' -import { collectKnowledgeReservedRelativePaths } from '../storage/pathStorage' +import { collectKnowledgeReservedRelativePaths } from '../pathStorage' import { hashEmbeddingText } from '../vectorstore/indexStore/hashing' import type { RebuildMaterialEmbeddingInput, RebuildMaterialInput } from '../vectorstore/indexStore/model' import type { KnowledgeIndexDocumentsPayload } from './jobTypes' diff --git a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts index 0456263bc82..ce88271556f 100644 --- a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts +++ b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts @@ -10,10 +10,10 @@ import type { KnowledgeLockManager } from '../KnowledgeLockManager' import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId, toKnowledgeItemId } from '../types' import { markUnscheduledKnowledgeItemsFailed } from '../ingestion/statusCleanup' -import { deleteKnowledgeItemVectors } from '../cleanup/vectorCleanup' +import { deleteKnowledgeItemVectors } from '../vectorCleanup' import { isIndexableKnowledgeItem } from '../items' import { prepareKnowledgeItem } from '../sources/prepare' -import { deleteKnowledgeItemFilesBestEffort } from '../storage/pathStorage' +import { deleteKnowledgeItemFilesBestEffort } from '../pathStorage' import type { KnowledgePrepareRootPayload } from './jobTypes' import { isDataApiNotFoundError, markKnowledgeItemFailedOnSettled } from './utils/settled' diff --git a/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts b/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts index 7d34cd5f0cb..6b5d3f4aa4d 100644 --- a/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts +++ b/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts @@ -18,9 +18,9 @@ import { toKnowledgeBaseId, toKnowledgeItemId } from '../types' -import { deleteKnowledgeItemVectors } from '../cleanup/vectorCleanup' +import { deleteKnowledgeItemVectors } from '../vectorCleanup' import { canKnowledgeItemRebuildSource, isContainerKnowledgeItem } from '../items' -import { deleteKnowledgeItemFilesBestEffort } from '../storage/pathStorage' +import { deleteKnowledgeItemFilesBestEffort } from '../pathStorage' import type { KnowledgeReindexSubtreePayload } from './jobTypes' import { narrowKnowledgeJobInput } from './utils/jobInput' diff --git a/src/main/features/knowledge/cleanup/vectorCleanup.ts b/src/main/features/knowledge/vectorCleanup.ts similarity index 100% rename from src/main/features/knowledge/cleanup/vectorCleanup.ts rename to src/main/features/knowledge/vectorCleanup.ts diff --git a/src/main/features/knowledge/vectorstore/KnowledgeVectorStoreService.ts b/src/main/features/knowledge/vectorstore/KnowledgeVectorStoreService.ts index df4f78ce959..d2168d1443c 100644 --- a/src/main/features/knowledge/vectorstore/KnowledgeVectorStoreService.ts +++ b/src/main/features/knowledge/vectorstore/KnowledgeVectorStoreService.ts @@ -8,7 +8,7 @@ import type { CompletedKnowledgeBase, KnowledgeBase } from '@shared/data/types/k import { isCompletedKnowledgeBase } from '@shared/data/types/knowledge' import { isIndexableKnowledgeItem } from '../items' -import { deleteKnowledgeBaseDir, getKnowledgeVectorStoreFilePathSync } from '../storage/pathStorage' +import { deleteKnowledgeBaseDir, getKnowledgeVectorStoreFilePathSync } from '../pathStorage' import { openBetterSqlite3IndexDriver } from './indexStore/BetterSqlite3Driver' import { betterSqlite3VectorIndex } from './indexStore/BetterSqlite3VectorIndex' import { ensureIndexMeta, hasAnyMaterial, readIndexSchemaVersion } from './indexStore/indexMeta' diff --git a/src/main/features/knowledge/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts b/src/main/features/knowledge/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts index 15c7c28b149..39b2640db95 100644 --- a/src/main/features/knowledge/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts +++ b/src/main/features/knowledge/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts @@ -98,7 +98,7 @@ vi.mock('@data/services/KnowledgeItemService', () => ({ knowledgeItemService: { getItemsByBaseId: getItemsByBaseIdMock } })) -vi.mock('../../storage/pathStorage', () => ({ +vi.mock('../../pathStorage', () => ({ getKnowledgeVectorStoreFilePathSync: getPathSyncMock, deleteKnowledgeBaseDir: deleteDirMock })) From 0bd58aba6c68b65a9cad8e058a6af63dce6aa9a5 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Thu, 2 Jul 2026 19:20:14 +0800 Subject: [PATCH 07/37] style(knowledge): fix import ordering after structure refactor Auto-fixed by eslint/biome across the flattened knowledge/ layout; no logic changes. Signed-off-by: eeee0717 --- .../migration/v2/migrators/KnowledgeVectorMigrator.ts | 11 ++++------- src/main/features/knowledge/KnowledgeService.ts | 2 +- .../knowledge/ingestion/KnowledgeIngestionService.ts | 4 ++-- src/main/features/knowledge/items.ts | 2 +- .../knowledge/readers/KnowledgeSnapshotReader.ts | 2 +- src/main/features/knowledge/sources/prepare.ts | 2 +- src/main/features/knowledge/subtreePurge.ts | 4 ++-- .../checkFileProcessingResultJobHandler.test.ts | 4 ++-- .../tasks/__tests__/prepareRootJobHandler.test.ts | 4 ++-- .../tasks/__tests__/reindexSubtreeJobHandler.test.ts | 4 ++-- .../tasks/checkFileProcessingResultJobHandler.ts | 4 ++-- .../knowledge/tasks/deleteSubtreeJobHandler.ts | 2 +- .../knowledge/tasks/indexDocumentsJobHandler.ts | 10 +++++----- .../features/knowledge/tasks/prepareRootJobHandler.ts | 8 ++++---- .../knowledge/tasks/reindexSubtreeJobHandler.ts | 6 +++--- 15 files changed, 33 insertions(+), 36 deletions(-) diff --git a/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts b/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts index 46845134a1b..2b7ec2f8f33 100644 --- a/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts +++ b/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts @@ -4,18 +4,15 @@ import path from 'node:path' import { knowledgeBaseTable, knowledgeItemTable } from '@data/db/schemas/knowledge' import { loggerService } from '@logger' import { DOCUMENT_SEPARATOR } from '@main/features/knowledge/indexing/chunk' -import { - type MaterialFieldSource, - toMaterialRelativePath -} from '@main/features/knowledge/indexing/materialFields' -import { deriveNoteSnapshotSlug } from '@main/features/knowledge/sources/noteSnapshot' -import { serializeOkfFrontmatter } from '@main/features/knowledge/sources/okfFrontmatter' -import { deriveUrlSnapshotSlug, deriveUrlSnapshotTitle } from '@main/features/knowledge/sources/urlSnapshot' +import { type MaterialFieldSource, toMaterialRelativePath } from '@main/features/knowledge/indexing/materialFields' import { assertSafeKnowledgeRelativePath, collectKnowledgeReservedRelativePaths, reserveImportedFileRelativePath } from '@main/features/knowledge/pathStorage' +import { deriveNoteSnapshotSlug } from '@main/features/knowledge/sources/noteSnapshot' +import { serializeOkfFrontmatter } from '@main/features/knowledge/sources/okfFrontmatter' +import { deriveUrlSnapshotSlug, deriveUrlSnapshotTitle } from '@main/features/knowledge/sources/urlSnapshot' import { type BetterSqlite3Driver, openBetterSqlite3IndexDriver diff --git a/src/main/features/knowledge/KnowledgeService.ts b/src/main/features/knowledge/KnowledgeService.ts index 43f14162efc..2f9e9fca191 100644 --- a/src/main/features/knowledge/KnowledgeService.ts +++ b/src/main/features/knowledge/KnowledgeService.ts @@ -13,8 +13,8 @@ import type { RestoreKnowledgeBaseResult } from '@shared/data/types/knowledge' -import { KnowledgeBaseAdminService } from './KnowledgeBaseAdminService' import { KnowledgeIngestionService } from './ingestion/KnowledgeIngestionService' +import { KnowledgeBaseAdminService } from './KnowledgeBaseAdminService' import { KnowledgeLockManager } from './KnowledgeLockManager' import type { KnowledgeConceptContent, diff --git a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts index 8db521f365e..b1461366f98 100644 --- a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts +++ b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts @@ -21,10 +21,8 @@ import { import { knowledgeSupportedFileExts } from '@shared/utils/file' import { assertBaseCanRunRuntimeOperation } from '../baseGuards' -import { cancelActiveKnowledgeSubtreeJobs, purgeKnowledgeSubtreeWithinLock } from '../subtreePurge' import { classifyKnowledgeItemSource, isContainerKnowledgeItem } from '../items' import type { KnowledgeLockManager } from '../KnowledgeLockManager' -import { planKnowledgeItemSource } from '../sources/sourcePlanning' import { assertKnowledgeFileTargetAvailable, collectKnowledgeReservedRelativePaths, @@ -36,6 +34,8 @@ import { needsProcessedArtifactReservation, reserveImportedFileRelativePath } from '../pathStorage' +import { planKnowledgeItemSource } from '../sources/sourcePlanning' +import { cancelActiveKnowledgeSubtreeJobs, purgeKnowledgeSubtreeWithinLock } from '../subtreePurge' import { cancelJobOrThrow } from '../tasks/utils/cancel' import { type KnowledgeBaseId, diff --git a/src/main/features/knowledge/items.ts b/src/main/features/knowledge/items.ts index 8fce570571b..1d3e7531d19 100644 --- a/src/main/features/knowledge/items.ts +++ b/src/main/features/knowledge/items.ts @@ -1,8 +1,8 @@ import type { PathReadability } from '@main/utils/file/fs' import type { KnowledgeItem } from '@shared/data/types/knowledge' -import type { ContainerKnowledgeItem, IndexableKnowledgeItem } from './types/items' import { probeKnowledgeFile, probeKnowledgeSourcePath } from './pathStorage' +import type { ContainerKnowledgeItem, IndexableKnowledgeItem } from './types/items' export function isIndexableKnowledgeItem(item: KnowledgeItem): item is IndexableKnowledgeItem { return item.type === 'file' || item.type === 'url' || item.type === 'note' diff --git a/src/main/features/knowledge/readers/KnowledgeSnapshotReader.ts b/src/main/features/knowledge/readers/KnowledgeSnapshotReader.ts index 7b16c9e3a0f..0db8830c542 100644 --- a/src/main/features/knowledge/readers/KnowledgeSnapshotReader.ts +++ b/src/main/features/knowledge/readers/KnowledgeSnapshotReader.ts @@ -2,8 +2,8 @@ import { read } from '@main/utils/file/fs' import type { KnowledgeItemOf, KnowledgeSourceMetadata } from '@shared/data/types/knowledge' import { Document } from '@vectorstores/core' -import { stripOkfFrontmatter } from '../sources/okfFrontmatter' import { getKnowledgeBaseFilePath } from '../pathStorage' +import { stripOkfFrontmatter } from '../sources/okfFrontmatter' /** * Read a url or note item from its captured on-disk snapshot — never the network diff --git a/src/main/features/knowledge/sources/prepare.ts b/src/main/features/knowledge/sources/prepare.ts index ea58ba5ae07..269f1e9fe02 100644 --- a/src/main/features/knowledge/sources/prepare.ts +++ b/src/main/features/knowledge/sources/prepare.ts @@ -7,9 +7,9 @@ import { type KnowledgeItemType } from '@shared/data/types/knowledge' -import type { IndexableKnowledgeItem } from '../types/items' import { isContainerKnowledgeItem, isIndexableKnowledgeItem } from '../items' import { collectKnowledgeReservedRelativePaths } from '../pathStorage' +import type { IndexableKnowledgeItem } from '../types/items' import { expandDirectoryOwnerToTree, type ExpandedDirectoryNode } from './directory' const logger = loggerService.withContext('KnowledgePrepare') diff --git a/src/main/features/knowledge/subtreePurge.ts b/src/main/features/knowledge/subtreePurge.ts index 27b5034153c..35de995fd83 100644 --- a/src/main/features/knowledge/subtreePurge.ts +++ b/src/main/features/knowledge/subtreePurge.ts @@ -2,6 +2,8 @@ import { application } from '@application' import { knowledgeItemService } from '@data/services/KnowledgeItemService' import type { KnowledgeBase, KnowledgeItem } from '@shared/data/types/knowledge' +import { isIndexableKnowledgeItem } from './items' +import { deleteKnowledgeItemFilesBestEffort } from './pathStorage' import { cancelJobOrThrow } from './tasks/utils/cancel' import { narrowKnowledgeJobInput } from './tasks/utils/jobInput' import { @@ -10,8 +12,6 @@ import { knowledgeQueueName, toKnowledgeBaseId } from './types' -import { isIndexableKnowledgeItem } from './items' -import { deleteKnowledgeItemFilesBestEffort } from './pathStorage' import { deleteKnowledgeItemVectors } from './vectorCleanup' /** diff --git a/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts index 4c869e94b3d..4bc6caa454d 100644 --- a/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts @@ -13,12 +13,12 @@ import { createJobSnapshot, FILE_ITEM_ID, getJobMock, + ingestionService, knowledgeItemGetByIdMock, knowledgeItemUpdateIndexedRelativePathMock, knowledgeItemUpdateStatusMock, knowledgeLockManager, - PROCESSED_RELATIVE_PATH, - ingestionService + PROCESSED_RELATIVE_PATH } from './jobHandlerTestUtils' function createFileProcessingJobSnapshot(overrides: Partial> = {}) { diff --git a/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts index 7b422ed8c00..c6c44627483 100644 --- a/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts @@ -10,14 +10,14 @@ import { deleteKnowledgeItemFilesBestEffortMock, deleteMaterialsMock, getJobMock, + ingestionService, knowledgeItemGetByIdMock, knowledgeItemGetSubtreeItemsMock, knowledgeItemSetSubtreeStatusMock, knowledgeItemUpdateStatusMock, knowledgeLockManager, prepareKnowledgeItemMock, - scheduleItemMock, - ingestionService + scheduleItemMock } from './jobHandlerTestUtils' describe('prepare-root job handler', () => { diff --git a/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts index 4e783a53549..d7f0e52f173 100644 --- a/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts @@ -14,6 +14,7 @@ import { deleteMaterialsMock, FILE_ITEM_ID, getJobMock, + ingestionService, knowledgeItemGetSubtreeItemsMock, knowledgeItemSetSubtreeStatusMock, knowledgeItemUpdateStatusMock, @@ -21,8 +22,7 @@ import { listMock, loggerWarnMock, probeKnowledgeSourcePathMock, - scheduleItemMock, - ingestionService + scheduleItemMock } from './jobHandlerTestUtils' describe('reindex-subtree job handler', () => { diff --git a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts index 5bbf4a42268..0ead847b6ce 100644 --- a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts +++ b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts @@ -12,10 +12,10 @@ import { getFileProcessingMarkdownArtifactPath } from '../../fileProcessing/persistence/artifacts' import type { FileProcessingJobPayload } from '../../fileProcessing/tasks/shared' -import type { KnowledgeLockManager } from '../KnowledgeLockManager' import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' -import { knowledgeQueueName, toKnowledgeBaseId, toKnowledgeItemId } from '../types' +import type { KnowledgeLockManager } from '../KnowledgeLockManager' import { toKnowledgeRelativePath } from '../pathStorage' +import { knowledgeQueueName, toKnowledgeBaseId, toKnowledgeItemId } from '../types' import type { KnowledgeCheckFileProcessingResultPayload } from './jobTypes' import { cancelJobOrThrow } from './utils/cancel' import { isDataApiNotFoundError, markKnowledgeItemFailedOnSettled } from './utils/settled' diff --git a/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts b/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts index bbe90646868..3d10221ab3a 100644 --- a/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts +++ b/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts @@ -6,8 +6,8 @@ import { loggerService } from '@logger' import type { JobHandler } from '@main/core/job/types' import type { KnowledgeLockManager } from '../KnowledgeLockManager' -import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId } from '../types' import { cancelActiveKnowledgeSubtreeJobs, purgeKnowledgeSubtreeWithinLock } from '../subtreePurge' +import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId } from '../types' import { reclaimKnowledgeIndexSpace } from '../vectorCleanup' import type { KnowledgeDeleteSubtreePayload } from './jobTypes' diff --git a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts index 7ef813546da..e91efb2140c 100644 --- a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts +++ b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts @@ -8,18 +8,18 @@ import type { JobContext, JobHandler } from '@main/core/job/types' import type { KnowledgeBase } from '@shared/data/types/knowledge' import { isCompletedVectorKnowledgeBase } from '@shared/data/types/knowledge' -import type { KnowledgeLockManager } from '../KnowledgeLockManager' -import { loadKnowledgeItemDocuments } from '../readers/KnowledgeReader' -import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId } from '../types' -import type { IndexableKnowledgeItem } from '../types/items' import { type ChunkedKnowledgeContent, chunkKnowledgeDocuments } from '../indexing/chunk' import { embedKnowledgeTexts } from '../indexing/embed' import { toMaterialRelativePath } from '../indexing/materialFields' import { isIndexableKnowledgeItem } from '../items' +import type { KnowledgeLockManager } from '../KnowledgeLockManager' +import { collectKnowledgeReservedRelativePaths } from '../pathStorage' +import { loadKnowledgeItemDocuments } from '../readers/KnowledgeReader' import { captureNoteSnapshotFile } from '../sources/noteSnapshot' import { fetchKnowledgeWebPage } from '../sources/url' import { captureUrlSnapshotFile } from '../sources/urlSnapshot' -import { collectKnowledgeReservedRelativePaths } from '../pathStorage' +import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId } from '../types' +import type { IndexableKnowledgeItem } from '../types/items' import { hashEmbeddingText } from '../vectorstore/indexStore/hashing' import type { RebuildMaterialEmbeddingInput, RebuildMaterialInput } from '../vectorstore/indexStore/model' import type { KnowledgeIndexDocumentsPayload } from './jobTypes' diff --git a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts index ce88271556f..9c64c984347 100644 --- a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts +++ b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts @@ -6,14 +6,14 @@ import { loggerService } from '@logger' import type { JobContext, JobHandler } from '@main/core/job/types' import type { KnowledgeItem } from '@shared/data/types/knowledge' -import type { KnowledgeLockManager } from '../KnowledgeLockManager' import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' -import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId, toKnowledgeItemId } from '../types' import { markUnscheduledKnowledgeItemsFailed } from '../ingestion/statusCleanup' -import { deleteKnowledgeItemVectors } from '../vectorCleanup' import { isIndexableKnowledgeItem } from '../items' -import { prepareKnowledgeItem } from '../sources/prepare' +import type { KnowledgeLockManager } from '../KnowledgeLockManager' import { deleteKnowledgeItemFilesBestEffort } from '../pathStorage' +import { prepareKnowledgeItem } from '../sources/prepare' +import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId, toKnowledgeItemId } from '../types' +import { deleteKnowledgeItemVectors } from '../vectorCleanup' import type { KnowledgePrepareRootPayload } from './jobTypes' import { isDataApiNotFoundError, markKnowledgeItemFailedOnSettled } from './utils/settled' diff --git a/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts b/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts index 6b5d3f4aa4d..5dc29284016 100644 --- a/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts +++ b/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts @@ -8,8 +8,10 @@ import type { JobHandler, JobSettledEvent } from '@main/core/job/types' import type { JobSnapshot } from '@shared/data/api/schemas/jobs' import { KNOWLEDGE_ITEM_ERROR_INDEXING_INTERRUPTED, type KnowledgeItemStatus } from '@shared/data/types/knowledge' -import type { KnowledgeLockManager } from '../KnowledgeLockManager' import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' +import { canKnowledgeItemRebuildSource, isContainerKnowledgeItem } from '../items' +import type { KnowledgeLockManager } from '../KnowledgeLockManager' +import { deleteKnowledgeItemFilesBestEffort } from '../pathStorage' import { KNOWLEDGE_ACTIVE_JOB_LIMIT, KNOWLEDGE_ACTIVE_JOB_STATUSES, @@ -19,8 +21,6 @@ import { toKnowledgeItemId } from '../types' import { deleteKnowledgeItemVectors } from '../vectorCleanup' -import { canKnowledgeItemRebuildSource, isContainerKnowledgeItem } from '../items' -import { deleteKnowledgeItemFilesBestEffort } from '../pathStorage' import type { KnowledgeReindexSubtreePayload } from './jobTypes' import { narrowKnowledgeJobInput } from './utils/jobInput' From f0b1bec2f7e054246d72079cf2117251d7e9bdb5 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Thu, 2 Jul 2026 19:53:27 +0800 Subject: [PATCH 08/37] refactor(knowledge): group feature into domain directories Top level now reads as one facade plus five domains: - pipeline/ gathers the four stage dirs (sources/readers/indexing/ vectorstore) under one roof; nothing inside enqueues jobs or mutates item status - base/ owns the per-base domain: KnowledgeBaseAdminService, baseGuards, KnowledgeLockManager - subtreePurge moves into ingestion/ (write-side teardown), vectorCleanup into pipeline/vectorstore/ (pure store maintenance) - types/items.ts merges into items.ts, removing the single-file types/ dir - pathStorage/items/types stay at the root as shared cross-domain vocabulary Pure moves + import rewrites, zero behavior change. README directory map and pipeline diagram updated to match. Signed-off-by: eeee0717 --- docs/references/data/database-construction.md | 2 +- src/main/core/application/serviceRegistry.ts | 2 +- .../v2/migrators/KnowledgeVectorMigrator.ts | 27 +++++++------- .../__tests__/KnowledgeVectorMigrator.test.ts | 8 ++--- .../features/knowledge/KnowledgeService.ts | 4 +-- src/main/features/knowledge/README.md | 36 +++++++++---------- .../__tests__/KnowledgeService.test.ts | 2 +- .../{ => base}/KnowledgeBaseAdminService.ts | 12 +++---- .../{ => base}/KnowledgeLockManager.ts | 0 .../__tests__/KnowledgeLockManager.test.ts | 0 .../knowledge/{ => base}/baseGuards.ts | 0 .../ingestion/KnowledgeIngestionService.ts | 8 ++--- .../knowledge/{ => ingestion}/subtreePurge.ts | 12 +++---- src/main/features/knowledge/items.ts | 10 ++++-- .../indexing/__tests__/chunk.test.ts | 0 .../indexing/__tests__/embed.test.ts | 0 .../indexing/__tests__/materialFields.test.ts | 0 .../indexing/__tests__/rerank.test.ts | 0 .../indexing/__tests__/splitter.test.ts | 0 .../{ => pipeline}/indexing/chunk.ts | 0 .../{ => pipeline}/indexing/embed.ts | 0 .../{ => pipeline}/indexing/materialFields.ts | 0 .../{ => pipeline}/indexing/rerank.ts | 0 .../{ => pipeline}/indexing/splitter.ts | 0 .../readers/KnowledgeFileReader.ts | 2 +- .../{ => pipeline}/readers/KnowledgeReader.ts | 0 .../readers/KnowledgeSnapshotReader.ts | 2 +- .../__tests__/KnowledgeReaderMetadata.test.ts | 0 .../readers/__tests__/ReaderFactory.test.ts | 0 .../readers/files/DraftsExportReader.ts | 0 .../readers/files/EpubReader.ts | 0 .../__tests__/DraftsExportReader.test.ts | 0 .../files/__tests__/EpubReader.test.ts | 0 .../sources/__tests__/directory.test.ts | 6 ++-- .../sources/__tests__/noteSnapshot.test.ts | 6 ++-- .../sources/__tests__/okfFrontmatter.test.ts | 0 .../sources/__tests__/prepare.test.ts | 0 .../sources/__tests__/sourcePlanning.test.ts | 0 .../sources/__tests__/url.test.ts | 0 .../sources/__tests__/urlSnapshot.test.ts | 6 ++-- .../{ => pipeline}/sources/directory.ts | 2 +- .../{ => pipeline}/sources/noteSnapshot.ts | 2 +- .../{ => pipeline}/sources/okfFrontmatter.ts | 0 .../{ => pipeline}/sources/prepare.ts | 5 ++- .../{ => pipeline}/sources/sourcePlanning.ts | 2 +- .../knowledge/{ => pipeline}/sources/url.ts | 0 .../{ => pipeline}/sources/urlSnapshot.ts | 2 +- .../KnowledgeVectorStoreService.ts | 4 +-- .../KnowledgeVectorStoreService.test.ts | 2 +- .../__tests__/vectorCleanup.test.ts | 0 .../indexStore/BetterSqlite3Driver.ts | 0 .../indexStore/BetterSqlite3VectorIndex.ts | 0 .../indexStore/KnowledgeIndexStore.ts | 0 .../__tests__/BetterSqlite3Driver.test.ts | 0 .../BetterSqlite3VectorIndex.test.ts | 0 .../KnowledgeIndexStore.integration.test.ts | 0 .../__tests__/KnowledgeIndexStore.rrf.test.ts | 0 .../KnowledgeIndexStore.search.test.ts | 0 .../__tests__/KnowledgeIndexStore.test.ts | 0 .../indexStore/__tests__/ftsQuery.test.ts | 0 .../indexStore/__tests__/hashing.test.ts | 0 .../indexStore/__tests__/indexMeta.test.ts | 0 .../indexStore/__tests__/schema.test.ts | 0 .../indexStore/__tests__/vectorBlob.test.ts | 0 .../vectorstore/indexStore/ftsQuery.ts | 0 .../vectorstore/indexStore/hashing.ts | 0 .../vectorstore/indexStore/indexMeta.ts | 0 .../vectorstore/indexStore/model.ts | 0 .../vectorstore/indexStore/schema.ts | 0 .../vectorstore/indexStore/types.ts | 0 .../vectorstore/indexStore/vectorBlob.ts | 0 .../vectorstore}/vectorCleanup.ts | 0 .../query/KnowledgeConceptService.ts | 2 +- .../knowledge/query/KnowledgeQueryService.ts | 8 ++--- .../knowledge/query/storeOperation.ts | 2 +- .../features/knowledge/query/visibility.ts | 2 +- .../indexDocumentsJobHandler.test.ts | 4 +-- .../tasks/__tests__/jobHandlerTestUtils.ts | 12 +++---- .../checkFileProcessingResultJobHandler.ts | 2 +- .../tasks/deleteSubtreeJobHandler.ts | 6 ++-- .../tasks/indexDocumentsJobHandler.ts | 22 ++++++------ .../knowledge/tasks/prepareRootJobHandler.ts | 6 ++-- .../tasks/reindexSubtreeJobHandler.ts | 4 +-- src/main/features/knowledge/types/items.ts | 6 ---- 84 files changed, 119 insertions(+), 121 deletions(-) rename src/main/features/knowledge/{ => base}/KnowledgeBaseAdminService.ts (97%) rename src/main/features/knowledge/{ => base}/KnowledgeLockManager.ts (100%) rename src/main/features/knowledge/{ => base}/__tests__/KnowledgeLockManager.test.ts (100%) rename src/main/features/knowledge/{ => base}/baseGuards.ts (100%) rename src/main/features/knowledge/{ => ingestion}/subtreePurge.ts (92%) rename src/main/features/knowledge/{ => pipeline}/indexing/__tests__/chunk.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/indexing/__tests__/embed.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/indexing/__tests__/materialFields.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/indexing/__tests__/rerank.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/indexing/__tests__/splitter.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/indexing/chunk.ts (100%) rename src/main/features/knowledge/{ => pipeline}/indexing/embed.ts (100%) rename src/main/features/knowledge/{ => pipeline}/indexing/materialFields.ts (100%) rename src/main/features/knowledge/{ => pipeline}/indexing/rerank.ts (100%) rename src/main/features/knowledge/{ => pipeline}/indexing/splitter.ts (100%) rename src/main/features/knowledge/{ => pipeline}/readers/KnowledgeFileReader.ts (97%) rename src/main/features/knowledge/{ => pipeline}/readers/KnowledgeReader.ts (100%) rename src/main/features/knowledge/{ => pipeline}/readers/KnowledgeSnapshotReader.ts (96%) rename src/main/features/knowledge/{ => pipeline}/readers/__tests__/KnowledgeReaderMetadata.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/readers/__tests__/ReaderFactory.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/readers/files/DraftsExportReader.ts (100%) rename src/main/features/knowledge/{ => pipeline}/readers/files/EpubReader.ts (100%) rename src/main/features/knowledge/{ => pipeline}/readers/files/__tests__/DraftsExportReader.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/readers/files/__tests__/EpubReader.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/sources/__tests__/directory.test.ts (98%) rename src/main/features/knowledge/{ => pipeline}/sources/__tests__/noteSnapshot.test.ts (95%) rename src/main/features/knowledge/{ => pipeline}/sources/__tests__/okfFrontmatter.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/sources/__tests__/prepare.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/sources/__tests__/sourcePlanning.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/sources/__tests__/url.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/sources/__tests__/urlSnapshot.test.ts (96%) rename src/main/features/knowledge/{ => pipeline}/sources/directory.ts (98%) rename src/main/features/knowledge/{ => pipeline}/sources/noteSnapshot.ts (98%) rename src/main/features/knowledge/{ => pipeline}/sources/okfFrontmatter.ts (100%) rename src/main/features/knowledge/{ => pipeline}/sources/prepare.ts (96%) rename src/main/features/knowledge/{ => pipeline}/sources/sourcePlanning.ts (98%) rename src/main/features/knowledge/{ => pipeline}/sources/url.ts (100%) rename src/main/features/knowledge/{ => pipeline}/sources/urlSnapshot.ts (98%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/KnowledgeVectorStoreService.ts (99%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts (99%) rename src/main/features/knowledge/{ => pipeline/vectorstore}/__tests__/vectorCleanup.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/BetterSqlite3Driver.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/BetterSqlite3VectorIndex.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/KnowledgeIndexStore.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/__tests__/BetterSqlite3Driver.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/__tests__/BetterSqlite3VectorIndex.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/__tests__/KnowledgeIndexStore.integration.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/__tests__/KnowledgeIndexStore.rrf.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/__tests__/KnowledgeIndexStore.search.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/__tests__/KnowledgeIndexStore.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/__tests__/ftsQuery.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/__tests__/hashing.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/__tests__/indexMeta.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/__tests__/schema.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/__tests__/vectorBlob.test.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/ftsQuery.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/hashing.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/indexMeta.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/model.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/schema.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/types.ts (100%) rename src/main/features/knowledge/{ => pipeline}/vectorstore/indexStore/vectorBlob.ts (100%) rename src/main/features/knowledge/{ => pipeline/vectorstore}/vectorCleanup.ts (100%) delete mode 100644 src/main/features/knowledge/types/items.ts diff --git a/docs/references/data/database-construction.md b/docs/references/data/database-construction.md index e6d85615140..3b1114f5f57 100644 --- a/docs/references/data/database-construction.md +++ b/docs/references/data/database-construction.md @@ -100,7 +100,7 @@ A table rebuild (drizzle's `INSERT…SELECT` drops the implicit rowid) **and `VA ### Knowledge `search_text_fts` follows the same rule -`src/main/features/knowledge/vectorstore/indexStore/schema.ts` keys `search_text_fts` on a stable `fts_rowid` column too (assigned by the `search_text_ai` trigger; `content_rowid='fts_rowid'`). It is a **separate per-base `index.sqlite`** (not the main DB, not drizzle-managed, not in `CUSTOM_SQL_STATEMENTS`), but the same hazard applies: its `reclaim()` path runs `VACUUM` to return freed pages to the OS after a large delete, which renumbers the implicit rowid — keying on `fts_rowid` keeps the external-content index aligned by construction. The regression guard is `KnowledgeIndexStore.test.ts` → "keeps search_text_fts aligned after a rowid-reshuffling rebuild". +`src/main/features/knowledge/pipeline/vectorstore/indexStore/schema.ts` keys `search_text_fts` on a stable `fts_rowid` column too (assigned by the `search_text_ai` trigger; `content_rowid='fts_rowid'`). It is a **separate per-base `index.sqlite`** (not the main DB, not drizzle-managed, not in `CUSTOM_SQL_STATEMENTS`), but the same hazard applies: its `reclaim()` path runs `VACUUM` to return freed pages to the OS after a large delete, which renumbers the implicit rowid — keying on `fts_rowid` keeps the external-content index aligned by construction. The regression guard is `KnowledgeIndexStore.test.ts` → "keeps search_text_fts aligned after a rowid-reshuffling rebuild". ## 5. Testing the build diff --git a/src/main/core/application/serviceRegistry.ts b/src/main/core/application/serviceRegistry.ts index 7647eb0661d..8ef611b906d 100644 --- a/src/main/core/application/serviceRegistry.ts +++ b/src/main/core/application/serviceRegistry.ts @@ -21,7 +21,7 @@ import { WindowManager } from '@main/core/window/WindowManager' import { ApiGatewayService } from '@main/features/apiGateway/ApiGatewayService' import { FileProcessingService, TesseractRuntimeService } from '@main/features/fileProcessing' import { KnowledgeService } from '@main/features/knowledge' -import { KnowledgeVectorStoreService } from '@main/features/knowledge/vectorstore/KnowledgeVectorStoreService' +import { KnowledgeVectorStoreService } from '@main/features/knowledge/pipeline/vectorstore/KnowledgeVectorStoreService' import { IpcApiService } from '@main/ipc/IpcApiService' import { AnalyticsService } from '@main/services/AnalyticsService' import { AppMenuService } from '@main/services/AppMenuService' diff --git a/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts b/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts index 2b7ec2f8f33..f4fd006b398 100644 --- a/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts +++ b/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts @@ -3,26 +3,29 @@ import path from 'node:path' import { knowledgeBaseTable, knowledgeItemTable } from '@data/db/schemas/knowledge' import { loggerService } from '@logger' -import { DOCUMENT_SEPARATOR } from '@main/features/knowledge/indexing/chunk' -import { type MaterialFieldSource, toMaterialRelativePath } from '@main/features/knowledge/indexing/materialFields' import { assertSafeKnowledgeRelativePath, collectKnowledgeReservedRelativePaths, reserveImportedFileRelativePath } from '@main/features/knowledge/pathStorage' -import { deriveNoteSnapshotSlug } from '@main/features/knowledge/sources/noteSnapshot' -import { serializeOkfFrontmatter } from '@main/features/knowledge/sources/okfFrontmatter' -import { deriveUrlSnapshotSlug, deriveUrlSnapshotTitle } from '@main/features/knowledge/sources/urlSnapshot' +import { DOCUMENT_SEPARATOR } from '@main/features/knowledge/pipeline/indexing/chunk' +import { + type MaterialFieldSource, + toMaterialRelativePath +} from '@main/features/knowledge/pipeline/indexing/materialFields' +import { deriveNoteSnapshotSlug } from '@main/features/knowledge/pipeline/sources/noteSnapshot' +import { serializeOkfFrontmatter } from '@main/features/knowledge/pipeline/sources/okfFrontmatter' +import { deriveUrlSnapshotSlug, deriveUrlSnapshotTitle } from '@main/features/knowledge/pipeline/sources/urlSnapshot' import { type BetterSqlite3Driver, openBetterSqlite3IndexDriver -} from '@main/features/knowledge/vectorstore/indexStore/BetterSqlite3Driver' -import { betterSqlite3VectorIndex } from '@main/features/knowledge/vectorstore/indexStore/BetterSqlite3VectorIndex' -import { hashEmbeddingText } from '@main/features/knowledge/vectorstore/indexStore/hashing' -import { ensureIndexMeta } from '@main/features/knowledge/vectorstore/indexStore/indexMeta' -import { KnowledgeIndexStore } from '@main/features/knowledge/vectorstore/indexStore/KnowledgeIndexStore' -import type { RebuildMaterialInput } from '@main/features/knowledge/vectorstore/indexStore/model' -import { createKnowledgeIndexSchema } from '@main/features/knowledge/vectorstore/indexStore/schema' +} from '@main/features/knowledge/pipeline/vectorstore/indexStore/BetterSqlite3Driver' +import { betterSqlite3VectorIndex } from '@main/features/knowledge/pipeline/vectorstore/indexStore/BetterSqlite3VectorIndex' +import { hashEmbeddingText } from '@main/features/knowledge/pipeline/vectorstore/indexStore/hashing' +import { ensureIndexMeta } from '@main/features/knowledge/pipeline/vectorstore/indexStore/indexMeta' +import { KnowledgeIndexStore } from '@main/features/knowledge/pipeline/vectorstore/indexStore/KnowledgeIndexStore' +import type { RebuildMaterialInput } from '@main/features/knowledge/pipeline/vectorstore/indexStore/model' +import { createKnowledgeIndexSchema } from '@main/features/knowledge/pipeline/vectorstore/indexStore/schema' import type { ExecuteResult, PrepareResult, ValidateResult, ValidationError } from '@shared/data/migration/v2/types' import { KNOWLEDGE_BASE_ERROR_MISSING_EMBEDDING_MODEL, diff --git a/src/main/data/migration/v2/migrators/__tests__/KnowledgeVectorMigrator.test.ts b/src/main/data/migration/v2/migrators/__tests__/KnowledgeVectorMigrator.test.ts index e06d5a56918..47d26473c9b 100644 --- a/src/main/data/migration/v2/migrators/__tests__/KnowledgeVectorMigrator.test.ts +++ b/src/main/data/migration/v2/migrators/__tests__/KnowledgeVectorMigrator.test.ts @@ -2,10 +2,10 @@ import * as fs from 'node:fs' import * as os from 'node:os' import path from 'node:path' -import { stripOkfFrontmatter } from '@main/features/knowledge/sources/okfFrontmatter' -import { hashEmbeddingText } from '@main/features/knowledge/vectorstore/indexStore/hashing' -import { KnowledgeIndexStore } from '@main/features/knowledge/vectorstore/indexStore/KnowledgeIndexStore' -import { encodeVectorBlob } from '@main/features/knowledge/vectorstore/indexStore/vectorBlob' +import { stripOkfFrontmatter } from '@main/features/knowledge/pipeline/sources/okfFrontmatter' +import { hashEmbeddingText } from '@main/features/knowledge/pipeline/vectorstore/indexStore/hashing' +import { KnowledgeIndexStore } from '@main/features/knowledge/pipeline/vectorstore/indexStore/KnowledgeIndexStore' +import { encodeVectorBlob } from '@main/features/knowledge/pipeline/vectorstore/indexStore/vectorBlob' import { KNOWLEDGE_BASE_ERROR_MISSING_EMBEDDING_MODEL, KNOWLEDGE_BASE_ERROR_MISSING_VECTOR_STORE, diff --git a/src/main/features/knowledge/KnowledgeService.ts b/src/main/features/knowledge/KnowledgeService.ts index 2f9e9fca191..c1eac65bf8c 100644 --- a/src/main/features/knowledge/KnowledgeService.ts +++ b/src/main/features/knowledge/KnowledgeService.ts @@ -13,9 +13,9 @@ import type { RestoreKnowledgeBaseResult } from '@shared/data/types/knowledge' +import { KnowledgeBaseAdminService } from './base/KnowledgeBaseAdminService' +import { KnowledgeLockManager } from './base/KnowledgeLockManager' import { KnowledgeIngestionService } from './ingestion/KnowledgeIngestionService' -import { KnowledgeBaseAdminService } from './KnowledgeBaseAdminService' -import { KnowledgeLockManager } from './KnowledgeLockManager' import type { KnowledgeConceptContent, KnowledgeConceptGrep, diff --git a/src/main/features/knowledge/README.md b/src/main/features/knowledge/README.md index 26f888c24c5..a3d3b45fc97 100644 --- a/src/main/features/knowledge/README.md +++ b/src/main/features/knowledge/README.md @@ -7,21 +7,21 @@ agent tools (`kb_search` / `kb_read` / `kb_tree` / `kb_manage`). ## Pipeline -The top-level directories spell out the ingestion pipeline in stage order: +`pipeline/` spells out the ingestion pipeline in stage order: ``` - input preprocess index persist - ┌──────────────┐ ┌────────────────┐ ┌───────────────┐ ┌───────────────┐ - │ sources/ │ ───> │ readers/ │ ──> │ indexing/ │ ──> │ vectorstore/ │ - │ expand dirs, │ │ file → md text │ │ chunk, embed, │ │ index.sqlite │ - │ url/note │ │ (pdf, docx, …) │ │ rerank │ │ (per base) │ - │ snapshots │ └────────────────┘ └───────────────┘ └───────────────┘ - └──────────────┘ heavy conversions (MinerU/PaddleOCR/…) run out-of-process - via FileProcessingService, polled by a knowledge job + input preprocess index persist + ┌──────────────┐ ┌────────────────┐ ┌───────────────┐ ┌───────────────┐ +pipeline/ │ sources/ │ ───> │ readers/ │ ──> │ indexing/ │ ──> │ vectorstore/ │ + │ expand dirs, │ │ file → md text │ │ chunk, embed, │ │ index.sqlite │ + │ url/note │ │ (pdf, docx, …) │ │ rerank │ │ (per base) │ + │ snapshots │ └────────────────┘ └───────────────┘ └───────────────┘ + └──────────────┘ heavy conversions (MinerU/PaddleOCR/…) run out-of-process + via FileProcessingService, polled by a knowledge job ``` Jobs in `tasks/` drive the stages; `ingestion/` decides which jobs to enqueue; `query/` reads the -result back out. Nothing in a stage directory enqueues jobs or mutates item status — that is +result back out. Nothing under `pipeline/` enqueues jobs or mutates item status — that is orchestration, and it lives in `ingestion/` and `tasks/`. ## Directory map @@ -29,18 +29,16 @@ orchestration, and it lives in `ingestion/` and `tasks/`. | Directory | Role | | --- | --- | | `KnowledgeService.ts` | Lifecycle facade: registers job handlers, runs boot recovery, delegates every public method. No domain logic. | -| `KnowledgeBaseAdminService.ts` | Base lifecycle: create (with rollback), delete, restore, list. | -| `baseGuards.ts` | Shared failed-base guard, used by both the write and read side. | -| `ingestion/` | Write-side orchestration: admission checks, item creation, add-conflict resolution, job enqueueing, boot recovery. | -| `sources/` | Input stage: directory expansion, url fetch (Jina reader), url/note snapshot capture, OKF frontmatter. | -| `readers/` | Preprocess stage: file → markdown/text `Document[]` readers (pdf/docx/epub/…). | -| `indexing/` | Index stage: offset-preserving splitter + chunker, `AiService` embedding/rerank wrappers, material field derivation. | -| `vectorstore/` | Persist stage: per-base `index.sqlite` lifecycle (`KnowledgeVectorStoreService`) and the store itself (`indexStore/`, synchronous better-sqlite3 driver). | +| `base/` | Per-base domain: lifecycle admin (`KnowledgeBaseAdminService` — create with rollback, delete, restore, list), failed-base guard (`baseGuards.ts`), per-base mutation lock (`KnowledgeLockManager`). | +| `ingestion/` | Write-side orchestration: admission checks, item creation, add-conflict resolution, job enqueueing, subtree purge (`subtreePurge.ts`), boot recovery. | +| `pipeline/sources/` | Input stage: directory expansion, url fetch (Jina reader), url/note snapshot capture, OKF frontmatter. | +| `pipeline/readers/` | Preprocess stage: file → markdown/text `Document[]` readers (pdf/docx/epub/…). | +| `pipeline/indexing/` | Index stage: offset-preserving splitter + chunker, `AiService` embedding/rerank wrappers, material field derivation. | +| `pipeline/vectorstore/` | Persist stage: per-base `index.sqlite` lifecycle (`KnowledgeVectorStoreService`), the store itself (`indexStore/`, synchronous better-sqlite3 driver), vector deletion + index space reclamation (`vectorCleanup.ts`). | | `query/` | Read side: hybrid search with visibility filtering (`KnowledgeQueryService`), Concept ID tool surface (`KnowledgeConceptService`). | | `tasks/` | Job handlers — the pipeline executors (see below). | -| `subtreePurge.ts` / `vectorCleanup.ts` | Subtree purge (vectors + files + rows) and index space reclamation — shared by ingestion and the delete/reindex/prepare-root handlers. | | `pathStorage.ts` | `raw/` path allocation: collision-free names, reservation, base file paths. | -| `items.ts` / `types.ts` / `types/` | Shared item predicates + source probing; branded ids, queue names, idempotency keys. | +| `items.ts` / `types.ts` | Shared item vocabulary (type aliases, predicates, source probing); branded ids, queue names, idempotency keys. | ## Jobs diff --git a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts index 19764814c6f..b2fcb0b724b 100644 --- a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts +++ b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts @@ -170,7 +170,7 @@ vi.mock('@data/services/KnowledgeItemService', () => ({ } })) -vi.mock('../indexing/rerank', () => ({ +vi.mock('../pipeline/indexing/rerank', () => ({ rerankKnowledgeSearchResults: rerankKnowledgeSearchResultsMock })) diff --git a/src/main/features/knowledge/KnowledgeBaseAdminService.ts b/src/main/features/knowledge/base/KnowledgeBaseAdminService.ts similarity index 97% rename from src/main/features/knowledge/KnowledgeBaseAdminService.ts rename to src/main/features/knowledge/base/KnowledgeBaseAdminService.ts index 069e8d62cf2..2ae6daaf435 100644 --- a/src/main/features/knowledge/KnowledgeBaseAdminService.ts +++ b/src/main/features/knowledge/base/KnowledgeBaseAdminService.ts @@ -14,18 +14,18 @@ import { type RestoreKnowledgeBaseResult } from '@shared/data/types/knowledge' -import type { KnowledgeIngestionService } from './ingestion/KnowledgeIngestionService' -import { classifyKnowledgeItemSource } from './items' -import type { KnowledgeLockManager } from './KnowledgeLockManager' -import { getKnowledgeBaseFilePath } from './pathStorage' -import { narrowKnowledgeJobInput } from './tasks/utils/jobInput' +import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' +import { classifyKnowledgeItemSource } from '../items' +import { getKnowledgeBaseFilePath } from '../pathStorage' +import { narrowKnowledgeJobInput } from '../tasks/utils/jobInput' import { KNOWLEDGE_ACTIVE_JOB_LIMIT, KNOWLEDGE_ACTIVE_JOB_STATUSES, KNOWLEDGE_JOB_TYPES, knowledgeQueueName, toKnowledgeBaseId -} from './types' +} from '../types' +import type { KnowledgeLockManager } from './KnowledgeLockManager' const logger = loggerService.withContext('Knowledge:BaseAdmin') const KNOWLEDGE_JOB_TYPE_SET = new Set(KNOWLEDGE_JOB_TYPES) diff --git a/src/main/features/knowledge/KnowledgeLockManager.ts b/src/main/features/knowledge/base/KnowledgeLockManager.ts similarity index 100% rename from src/main/features/knowledge/KnowledgeLockManager.ts rename to src/main/features/knowledge/base/KnowledgeLockManager.ts diff --git a/src/main/features/knowledge/__tests__/KnowledgeLockManager.test.ts b/src/main/features/knowledge/base/__tests__/KnowledgeLockManager.test.ts similarity index 100% rename from src/main/features/knowledge/__tests__/KnowledgeLockManager.test.ts rename to src/main/features/knowledge/base/__tests__/KnowledgeLockManager.test.ts diff --git a/src/main/features/knowledge/baseGuards.ts b/src/main/features/knowledge/base/baseGuards.ts similarity index 100% rename from src/main/features/knowledge/baseGuards.ts rename to src/main/features/knowledge/base/baseGuards.ts diff --git a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts index b1461366f98..ca44ad3d579 100644 --- a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts +++ b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts @@ -20,9 +20,9 @@ import { } from '@shared/data/types/knowledge' import { knowledgeSupportedFileExts } from '@shared/utils/file' -import { assertBaseCanRunRuntimeOperation } from '../baseGuards' +import { assertBaseCanRunRuntimeOperation } from '../base/baseGuards' +import type { KnowledgeLockManager } from '../base/KnowledgeLockManager' import { classifyKnowledgeItemSource, isContainerKnowledgeItem } from '../items' -import type { KnowledgeLockManager } from '../KnowledgeLockManager' import { assertKnowledgeFileTargetAvailable, collectKnowledgeReservedRelativePaths, @@ -34,8 +34,7 @@ import { needsProcessedArtifactReservation, reserveImportedFileRelativePath } from '../pathStorage' -import { planKnowledgeItemSource } from '../sources/sourcePlanning' -import { cancelActiveKnowledgeSubtreeJobs, purgeKnowledgeSubtreeWithinLock } from '../subtreePurge' +import { planKnowledgeItemSource } from '../pipeline/sources/sourcePlanning' import { cancelJobOrThrow } from '../tasks/utils/cancel' import { type KnowledgeBaseId, @@ -52,6 +51,7 @@ import { } from '../types' import { resolveKnowledgeAddConflicts } from './addConflicts' import { markUnscheduledKnowledgeItemsFailed } from './statusCleanup' +import { cancelActiveKnowledgeSubtreeJobs, purgeKnowledgeSubtreeWithinLock } from './subtreePurge' const logger = loggerService.withContext('Knowledge:IngestionService') // Keep poll jobs delayed enough to avoid hot-looping while remote processors are still working. diff --git a/src/main/features/knowledge/subtreePurge.ts b/src/main/features/knowledge/ingestion/subtreePurge.ts similarity index 92% rename from src/main/features/knowledge/subtreePurge.ts rename to src/main/features/knowledge/ingestion/subtreePurge.ts index 35de995fd83..97f005f1295 100644 --- a/src/main/features/knowledge/subtreePurge.ts +++ b/src/main/features/knowledge/ingestion/subtreePurge.ts @@ -2,17 +2,17 @@ import { application } from '@application' import { knowledgeItemService } from '@data/services/KnowledgeItemService' import type { KnowledgeBase, KnowledgeItem } from '@shared/data/types/knowledge' -import { isIndexableKnowledgeItem } from './items' -import { deleteKnowledgeItemFilesBestEffort } from './pathStorage' -import { cancelJobOrThrow } from './tasks/utils/cancel' -import { narrowKnowledgeJobInput } from './tasks/utils/jobInput' +import { isIndexableKnowledgeItem } from '../items' +import { deleteKnowledgeItemFilesBestEffort } from '../pathStorage' +import { deleteKnowledgeItemVectors } from '../pipeline/vectorstore/vectorCleanup' +import { cancelJobOrThrow } from '../tasks/utils/cancel' +import { narrowKnowledgeJobInput } from '../tasks/utils/jobInput' import { KNOWLEDGE_ACTIVE_JOB_LIMIT, KNOWLEDGE_ACTIVE_JOB_STATUSES, knowledgeQueueName, toKnowledgeBaseId -} from './types' -import { deleteKnowledgeItemVectors } from './vectorCleanup' +} from '../types' /** * Cancel any in-flight job touching the given subtree (the roots plus their diff --git a/src/main/features/knowledge/items.ts b/src/main/features/knowledge/items.ts index 1d3e7531d19..c2d7b2c8cdd 100644 --- a/src/main/features/knowledge/items.ts +++ b/src/main/features/knowledge/items.ts @@ -1,8 +1,12 @@ import type { PathReadability } from '@main/utils/file/fs' -import type { KnowledgeItem } from '@shared/data/types/knowledge' +import type { KnowledgeItem, KnowledgeItemOf } from '@shared/data/types/knowledge' import { probeKnowledgeFile, probeKnowledgeSourcePath } from './pathStorage' -import type { ContainerKnowledgeItem, IndexableKnowledgeItem } from './types/items' + +export type IndexableKnowledgeItem = KnowledgeItemOf<'file' | 'url' | 'note'> + +export type ContainerKnowledgeItem = KnowledgeItemOf<'directory'> +export type ContainerKnowledgeItemType = ContainerKnowledgeItem['type'] export function isIndexableKnowledgeItem(item: KnowledgeItem): item is IndexableKnowledgeItem { return item.type === 'file' || item.type === 'url' || item.type === 'note' @@ -41,7 +45,7 @@ export async function classifyKnowledgeItemSource( /** * Whether a knowledge item can rebuild from a still-readable source. Gates reindex both at admission - * (`KnowledgeService.assertSubtreesCanReindex`) and inside the reindex job's mutation lock right + * (`KnowledgeIngestionService.assertSubtreesCanReindex`) and inside the reindex job's mutation lock right * before the delete — a vanished or unverifiable source must never wipe vectors with nothing to * rebuild from. Admission additionally distinguishes the two via {@link classifyKnowledgeItemSource}. */ diff --git a/src/main/features/knowledge/indexing/__tests__/chunk.test.ts b/src/main/features/knowledge/pipeline/indexing/__tests__/chunk.test.ts similarity index 100% rename from src/main/features/knowledge/indexing/__tests__/chunk.test.ts rename to src/main/features/knowledge/pipeline/indexing/__tests__/chunk.test.ts diff --git a/src/main/features/knowledge/indexing/__tests__/embed.test.ts b/src/main/features/knowledge/pipeline/indexing/__tests__/embed.test.ts similarity index 100% rename from src/main/features/knowledge/indexing/__tests__/embed.test.ts rename to src/main/features/knowledge/pipeline/indexing/__tests__/embed.test.ts diff --git a/src/main/features/knowledge/indexing/__tests__/materialFields.test.ts b/src/main/features/knowledge/pipeline/indexing/__tests__/materialFields.test.ts similarity index 100% rename from src/main/features/knowledge/indexing/__tests__/materialFields.test.ts rename to src/main/features/knowledge/pipeline/indexing/__tests__/materialFields.test.ts diff --git a/src/main/features/knowledge/indexing/__tests__/rerank.test.ts b/src/main/features/knowledge/pipeline/indexing/__tests__/rerank.test.ts similarity index 100% rename from src/main/features/knowledge/indexing/__tests__/rerank.test.ts rename to src/main/features/knowledge/pipeline/indexing/__tests__/rerank.test.ts diff --git a/src/main/features/knowledge/indexing/__tests__/splitter.test.ts b/src/main/features/knowledge/pipeline/indexing/__tests__/splitter.test.ts similarity index 100% rename from src/main/features/knowledge/indexing/__tests__/splitter.test.ts rename to src/main/features/knowledge/pipeline/indexing/__tests__/splitter.test.ts diff --git a/src/main/features/knowledge/indexing/chunk.ts b/src/main/features/knowledge/pipeline/indexing/chunk.ts similarity index 100% rename from src/main/features/knowledge/indexing/chunk.ts rename to src/main/features/knowledge/pipeline/indexing/chunk.ts diff --git a/src/main/features/knowledge/indexing/embed.ts b/src/main/features/knowledge/pipeline/indexing/embed.ts similarity index 100% rename from src/main/features/knowledge/indexing/embed.ts rename to src/main/features/knowledge/pipeline/indexing/embed.ts diff --git a/src/main/features/knowledge/indexing/materialFields.ts b/src/main/features/knowledge/pipeline/indexing/materialFields.ts similarity index 100% rename from src/main/features/knowledge/indexing/materialFields.ts rename to src/main/features/knowledge/pipeline/indexing/materialFields.ts diff --git a/src/main/features/knowledge/indexing/rerank.ts b/src/main/features/knowledge/pipeline/indexing/rerank.ts similarity index 100% rename from src/main/features/knowledge/indexing/rerank.ts rename to src/main/features/knowledge/pipeline/indexing/rerank.ts diff --git a/src/main/features/knowledge/indexing/splitter.ts b/src/main/features/knowledge/pipeline/indexing/splitter.ts similarity index 100% rename from src/main/features/knowledge/indexing/splitter.ts rename to src/main/features/knowledge/pipeline/indexing/splitter.ts diff --git a/src/main/features/knowledge/readers/KnowledgeFileReader.ts b/src/main/features/knowledge/pipeline/readers/KnowledgeFileReader.ts similarity index 97% rename from src/main/features/knowledge/readers/KnowledgeFileReader.ts rename to src/main/features/knowledge/pipeline/readers/KnowledgeFileReader.ts index 7dd43d98715..8d718c5a09b 100644 --- a/src/main/features/knowledge/readers/KnowledgeFileReader.ts +++ b/src/main/features/knowledge/pipeline/readers/KnowledgeFileReader.ts @@ -10,7 +10,7 @@ import { MarkdownReader } from '@vectorstores/readers/markdown' import { PDFReader } from '@vectorstores/readers/pdf' import { TextFileReader } from '@vectorstores/readers/text' -import { getKnowledgeBaseFilePath } from '../pathStorage' +import { getKnowledgeBaseFilePath } from '../../pathStorage' import { DraftsExportReader } from './files/DraftsExportReader' import { EpubReader } from './files/EpubReader' diff --git a/src/main/features/knowledge/readers/KnowledgeReader.ts b/src/main/features/knowledge/pipeline/readers/KnowledgeReader.ts similarity index 100% rename from src/main/features/knowledge/readers/KnowledgeReader.ts rename to src/main/features/knowledge/pipeline/readers/KnowledgeReader.ts diff --git a/src/main/features/knowledge/readers/KnowledgeSnapshotReader.ts b/src/main/features/knowledge/pipeline/readers/KnowledgeSnapshotReader.ts similarity index 96% rename from src/main/features/knowledge/readers/KnowledgeSnapshotReader.ts rename to src/main/features/knowledge/pipeline/readers/KnowledgeSnapshotReader.ts index 0db8830c542..d45d9dbcf75 100644 --- a/src/main/features/knowledge/readers/KnowledgeSnapshotReader.ts +++ b/src/main/features/knowledge/pipeline/readers/KnowledgeSnapshotReader.ts @@ -2,7 +2,7 @@ import { read } from '@main/utils/file/fs' import type { KnowledgeItemOf, KnowledgeSourceMetadata } from '@shared/data/types/knowledge' import { Document } from '@vectorstores/core' -import { getKnowledgeBaseFilePath } from '../pathStorage' +import { getKnowledgeBaseFilePath } from '../../pathStorage' import { stripOkfFrontmatter } from '../sources/okfFrontmatter' /** diff --git a/src/main/features/knowledge/readers/__tests__/KnowledgeReaderMetadata.test.ts b/src/main/features/knowledge/pipeline/readers/__tests__/KnowledgeReaderMetadata.test.ts similarity index 100% rename from src/main/features/knowledge/readers/__tests__/KnowledgeReaderMetadata.test.ts rename to src/main/features/knowledge/pipeline/readers/__tests__/KnowledgeReaderMetadata.test.ts diff --git a/src/main/features/knowledge/readers/__tests__/ReaderFactory.test.ts b/src/main/features/knowledge/pipeline/readers/__tests__/ReaderFactory.test.ts similarity index 100% rename from src/main/features/knowledge/readers/__tests__/ReaderFactory.test.ts rename to src/main/features/knowledge/pipeline/readers/__tests__/ReaderFactory.test.ts diff --git a/src/main/features/knowledge/readers/files/DraftsExportReader.ts b/src/main/features/knowledge/pipeline/readers/files/DraftsExportReader.ts similarity index 100% rename from src/main/features/knowledge/readers/files/DraftsExportReader.ts rename to src/main/features/knowledge/pipeline/readers/files/DraftsExportReader.ts diff --git a/src/main/features/knowledge/readers/files/EpubReader.ts b/src/main/features/knowledge/pipeline/readers/files/EpubReader.ts similarity index 100% rename from src/main/features/knowledge/readers/files/EpubReader.ts rename to src/main/features/knowledge/pipeline/readers/files/EpubReader.ts diff --git a/src/main/features/knowledge/readers/files/__tests__/DraftsExportReader.test.ts b/src/main/features/knowledge/pipeline/readers/files/__tests__/DraftsExportReader.test.ts similarity index 100% rename from src/main/features/knowledge/readers/files/__tests__/DraftsExportReader.test.ts rename to src/main/features/knowledge/pipeline/readers/files/__tests__/DraftsExportReader.test.ts diff --git a/src/main/features/knowledge/readers/files/__tests__/EpubReader.test.ts b/src/main/features/knowledge/pipeline/readers/files/__tests__/EpubReader.test.ts similarity index 100% rename from src/main/features/knowledge/readers/files/__tests__/EpubReader.test.ts rename to src/main/features/knowledge/pipeline/readers/files/__tests__/EpubReader.test.ts diff --git a/src/main/features/knowledge/sources/__tests__/directory.test.ts b/src/main/features/knowledge/pipeline/sources/__tests__/directory.test.ts similarity index 98% rename from src/main/features/knowledge/sources/__tests__/directory.test.ts rename to src/main/features/knowledge/pipeline/sources/__tests__/directory.test.ts index 575dfb6c645..fb0000b9a62 100644 --- a/src/main/features/knowledge/sources/__tests__/directory.test.ts +++ b/src/main/features/knowledge/pipeline/sources/__tests__/directory.test.ts @@ -4,7 +4,7 @@ import path from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' -import type * as PathStorage from '../../pathStorage' +import type * as PathStorage from '../../../pathStorage' const copyFileIntoKnowledgeBaseAtMock = vi.hoisted(() => vi.fn(async (_baseId: string, _externalPath: string, relativePath: string) => relativePath) @@ -15,8 +15,8 @@ vi.mock('@application', async () => { return mockApplicationFactory() }) -vi.mock('../../pathStorage', async () => { - const actual = await vi.importActual('../../pathStorage') +vi.mock('../../../pathStorage', async () => { + const actual = await vi.importActual('../../../pathStorage') return { ...actual, copyFileIntoKnowledgeBaseAt: copyFileIntoKnowledgeBaseAtMock diff --git a/src/main/features/knowledge/sources/__tests__/noteSnapshot.test.ts b/src/main/features/knowledge/pipeline/sources/__tests__/noteSnapshot.test.ts similarity index 95% rename from src/main/features/knowledge/sources/__tests__/noteSnapshot.test.ts rename to src/main/features/knowledge/pipeline/sources/__tests__/noteSnapshot.test.ts index fbd90da35fd..c76cf0629b9 100644 --- a/src/main/features/knowledge/sources/__tests__/noteSnapshot.test.ts +++ b/src/main/features/knowledge/pipeline/sources/__tests__/noteSnapshot.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' -import type * as PathStorage from '../../pathStorage' +import type * as PathStorage from '../../../pathStorage' const { writeFileIntoKnowledgeBaseAtMock } = vi.hoisted(() => ({ writeFileIntoKnowledgeBaseAtMock: vi.fn() @@ -19,8 +19,8 @@ vi.mock('@logger', () => ({ // Keep reserveImportedFileRelativePath real (it is pure); only stub the disk write so // the test exercises name derivation + dedupe without touching the filesystem. -vi.mock('../../pathStorage', async () => { - const actual = await vi.importActual('../../pathStorage') +vi.mock('../../../pathStorage', async () => { + const actual = await vi.importActual('../../../pathStorage') return { ...actual, writeFileIntoKnowledgeBaseAt: writeFileIntoKnowledgeBaseAtMock } }) diff --git a/src/main/features/knowledge/sources/__tests__/okfFrontmatter.test.ts b/src/main/features/knowledge/pipeline/sources/__tests__/okfFrontmatter.test.ts similarity index 100% rename from src/main/features/knowledge/sources/__tests__/okfFrontmatter.test.ts rename to src/main/features/knowledge/pipeline/sources/__tests__/okfFrontmatter.test.ts diff --git a/src/main/features/knowledge/sources/__tests__/prepare.test.ts b/src/main/features/knowledge/pipeline/sources/__tests__/prepare.test.ts similarity index 100% rename from src/main/features/knowledge/sources/__tests__/prepare.test.ts rename to src/main/features/knowledge/pipeline/sources/__tests__/prepare.test.ts diff --git a/src/main/features/knowledge/sources/__tests__/sourcePlanning.test.ts b/src/main/features/knowledge/pipeline/sources/__tests__/sourcePlanning.test.ts similarity index 100% rename from src/main/features/knowledge/sources/__tests__/sourcePlanning.test.ts rename to src/main/features/knowledge/pipeline/sources/__tests__/sourcePlanning.test.ts diff --git a/src/main/features/knowledge/sources/__tests__/url.test.ts b/src/main/features/knowledge/pipeline/sources/__tests__/url.test.ts similarity index 100% rename from src/main/features/knowledge/sources/__tests__/url.test.ts rename to src/main/features/knowledge/pipeline/sources/__tests__/url.test.ts diff --git a/src/main/features/knowledge/sources/__tests__/urlSnapshot.test.ts b/src/main/features/knowledge/pipeline/sources/__tests__/urlSnapshot.test.ts similarity index 96% rename from src/main/features/knowledge/sources/__tests__/urlSnapshot.test.ts rename to src/main/features/knowledge/pipeline/sources/__tests__/urlSnapshot.test.ts index 9a1747d9ae9..79e40f54365 100644 --- a/src/main/features/knowledge/sources/__tests__/urlSnapshot.test.ts +++ b/src/main/features/knowledge/pipeline/sources/__tests__/urlSnapshot.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' -import type * as PathStorage from '../../pathStorage' +import type * as PathStorage from '../../../pathStorage' const { writeFileIntoKnowledgeBaseAtMock } = vi.hoisted(() => ({ writeFileIntoKnowledgeBaseAtMock: vi.fn() @@ -19,8 +19,8 @@ vi.mock('@logger', () => ({ // Keep reserveImportedFileRelativePath real (it is pure); only stub the disk write so // the test exercises name derivation + dedupe without touching the filesystem. -vi.mock('../../pathStorage', async () => { - const actual = await vi.importActual('../../pathStorage') +vi.mock('../../../pathStorage', async () => { + const actual = await vi.importActual('../../../pathStorage') return { ...actual, writeFileIntoKnowledgeBaseAt: writeFileIntoKnowledgeBaseAtMock } }) diff --git a/src/main/features/knowledge/sources/directory.ts b/src/main/features/knowledge/pipeline/sources/directory.ts similarity index 98% rename from src/main/features/knowledge/sources/directory.ts rename to src/main/features/knowledge/pipeline/sources/directory.ts index f67f21da089..fff213d39dc 100644 --- a/src/main/features/knowledge/sources/directory.ts +++ b/src/main/features/knowledge/pipeline/sources/directory.ts @@ -5,7 +5,7 @@ import { nextFreeKnowledgeRelativePath } from '@main/utils/knowledge' import type { DirectoryItemData, FileItemData, KnowledgeItem } from '@shared/data/types/knowledge' import { knowledgeSupportedFileExts } from '@shared/utils/file' -import { copyFileIntoKnowledgeBaseAt } from '../pathStorage' +import { copyFileIntoKnowledgeBaseAt } from '../../pathStorage' const KNOWLEDGE_SUPPORTED_FILE_EXT_SET = new Set(knowledgeSupportedFileExts) diff --git a/src/main/features/knowledge/sources/noteSnapshot.ts b/src/main/features/knowledge/pipeline/sources/noteSnapshot.ts similarity index 98% rename from src/main/features/knowledge/sources/noteSnapshot.ts rename to src/main/features/knowledge/pipeline/sources/noteSnapshot.ts index 7dbc034cd10..b794d28fcb3 100644 --- a/src/main/features/knowledge/sources/noteSnapshot.ts +++ b/src/main/features/knowledge/pipeline/sources/noteSnapshot.ts @@ -1,6 +1,6 @@ import { sanitizeFilename } from '@shared/utils/file/filename' -import { reserveImportedFileRelativePath, writeFileIntoKnowledgeBaseAt } from '../pathStorage' +import { reserveImportedFileRelativePath, writeFileIntoKnowledgeBaseAt } from '../../pathStorage' import { serializeOkfFrontmatter } from './okfFrontmatter' const SNAPSHOT_TITLE_MAX = 80 diff --git a/src/main/features/knowledge/sources/okfFrontmatter.ts b/src/main/features/knowledge/pipeline/sources/okfFrontmatter.ts similarity index 100% rename from src/main/features/knowledge/sources/okfFrontmatter.ts rename to src/main/features/knowledge/pipeline/sources/okfFrontmatter.ts diff --git a/src/main/features/knowledge/sources/prepare.ts b/src/main/features/knowledge/pipeline/sources/prepare.ts similarity index 96% rename from src/main/features/knowledge/sources/prepare.ts rename to src/main/features/knowledge/pipeline/sources/prepare.ts index 269f1e9fe02..0f14f136564 100644 --- a/src/main/features/knowledge/sources/prepare.ts +++ b/src/main/features/knowledge/pipeline/sources/prepare.ts @@ -7,9 +7,8 @@ import { type KnowledgeItemType } from '@shared/data/types/knowledge' -import { isContainerKnowledgeItem, isIndexableKnowledgeItem } from '../items' -import { collectKnowledgeReservedRelativePaths } from '../pathStorage' -import type { IndexableKnowledgeItem } from '../types/items' +import { type IndexableKnowledgeItem, isContainerKnowledgeItem, isIndexableKnowledgeItem } from '../../items' +import { collectKnowledgeReservedRelativePaths } from '../../pathStorage' import { expandDirectoryOwnerToTree, type ExpandedDirectoryNode } from './directory' const logger = loggerService.withContext('KnowledgePrepare') diff --git a/src/main/features/knowledge/sources/sourcePlanning.ts b/src/main/features/knowledge/pipeline/sources/sourcePlanning.ts similarity index 98% rename from src/main/features/knowledge/sources/sourcePlanning.ts rename to src/main/features/knowledge/pipeline/sources/sourcePlanning.ts index bdf2f64111c..ef5dc4e6a74 100644 --- a/src/main/features/knowledge/sources/sourcePlanning.ts +++ b/src/main/features/knowledge/pipeline/sources/sourcePlanning.ts @@ -3,7 +3,7 @@ import type { KnowledgeBase } from '@shared/data/types/knowledge' import type { KnowledgeItem } from '@shared/data/types/knowledge' import { knowledgeFileProcessingExts } from '@shared/utils/file' -import { isContainerKnowledgeItem, isIndexableKnowledgeItem } from '../items' +import { isContainerKnowledgeItem, isIndexableKnowledgeItem } from '../../items' const KNOWLEDGE_FILE_PROCESSING_EXT_SET = new Set(knowledgeFileProcessingExts) diff --git a/src/main/features/knowledge/sources/url.ts b/src/main/features/knowledge/pipeline/sources/url.ts similarity index 100% rename from src/main/features/knowledge/sources/url.ts rename to src/main/features/knowledge/pipeline/sources/url.ts diff --git a/src/main/features/knowledge/sources/urlSnapshot.ts b/src/main/features/knowledge/pipeline/sources/urlSnapshot.ts similarity index 98% rename from src/main/features/knowledge/sources/urlSnapshot.ts rename to src/main/features/knowledge/pipeline/sources/urlSnapshot.ts index 535db11ab9c..651c45e3f0f 100644 --- a/src/main/features/knowledge/sources/urlSnapshot.ts +++ b/src/main/features/knowledge/pipeline/sources/urlSnapshot.ts @@ -1,6 +1,6 @@ import { sanitizeFilename } from '@shared/utils/file/filename' -import { reserveImportedFileRelativePath, writeFileIntoKnowledgeBaseAt } from '../pathStorage' +import { reserveImportedFileRelativePath, writeFileIntoKnowledgeBaseAt } from '../../pathStorage' import { serializeOkfFrontmatter } from './okfFrontmatter' const SNAPSHOT_TITLE_MAX = 80 diff --git a/src/main/features/knowledge/vectorstore/KnowledgeVectorStoreService.ts b/src/main/features/knowledge/pipeline/vectorstore/KnowledgeVectorStoreService.ts similarity index 99% rename from src/main/features/knowledge/vectorstore/KnowledgeVectorStoreService.ts rename to src/main/features/knowledge/pipeline/vectorstore/KnowledgeVectorStoreService.ts index d2168d1443c..bad056f63dc 100644 --- a/src/main/features/knowledge/vectorstore/KnowledgeVectorStoreService.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/KnowledgeVectorStoreService.ts @@ -7,8 +7,8 @@ import { DataApiErrorFactory } from '@shared/data/api' import type { CompletedKnowledgeBase, KnowledgeBase } from '@shared/data/types/knowledge' import { isCompletedKnowledgeBase } from '@shared/data/types/knowledge' -import { isIndexableKnowledgeItem } from '../items' -import { deleteKnowledgeBaseDir, getKnowledgeVectorStoreFilePathSync } from '../pathStorage' +import { isIndexableKnowledgeItem } from '../../items' +import { deleteKnowledgeBaseDir, getKnowledgeVectorStoreFilePathSync } from '../../pathStorage' import { openBetterSqlite3IndexDriver } from './indexStore/BetterSqlite3Driver' import { betterSqlite3VectorIndex } from './indexStore/BetterSqlite3VectorIndex' import { ensureIndexMeta, hasAnyMaterial, readIndexSchemaVersion } from './indexStore/indexMeta' diff --git a/src/main/features/knowledge/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts b/src/main/features/knowledge/pipeline/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts similarity index 99% rename from src/main/features/knowledge/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts rename to src/main/features/knowledge/pipeline/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts index 39b2640db95..09b036d519a 100644 --- a/src/main/features/knowledge/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts @@ -98,7 +98,7 @@ vi.mock('@data/services/KnowledgeItemService', () => ({ knowledgeItemService: { getItemsByBaseId: getItemsByBaseIdMock } })) -vi.mock('../../pathStorage', () => ({ +vi.mock('../../../pathStorage', () => ({ getKnowledgeVectorStoreFilePathSync: getPathSyncMock, deleteKnowledgeBaseDir: deleteDirMock })) diff --git a/src/main/features/knowledge/__tests__/vectorCleanup.test.ts b/src/main/features/knowledge/pipeline/vectorstore/__tests__/vectorCleanup.test.ts similarity index 100% rename from src/main/features/knowledge/__tests__/vectorCleanup.test.ts rename to src/main/features/knowledge/pipeline/vectorstore/__tests__/vectorCleanup.test.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/BetterSqlite3Driver.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/BetterSqlite3Driver.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/BetterSqlite3Driver.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/BetterSqlite3Driver.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/BetterSqlite3VectorIndex.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/BetterSqlite3VectorIndex.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/BetterSqlite3VectorIndex.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/BetterSqlite3VectorIndex.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/KnowledgeIndexStore.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/KnowledgeIndexStore.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/KnowledgeIndexStore.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/KnowledgeIndexStore.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/__tests__/BetterSqlite3Driver.test.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/BetterSqlite3Driver.test.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/__tests__/BetterSqlite3Driver.test.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/BetterSqlite3Driver.test.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/__tests__/BetterSqlite3VectorIndex.test.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/BetterSqlite3VectorIndex.test.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/__tests__/BetterSqlite3VectorIndex.test.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/BetterSqlite3VectorIndex.test.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/__tests__/KnowledgeIndexStore.integration.test.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.integration.test.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/__tests__/KnowledgeIndexStore.integration.test.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.integration.test.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/__tests__/KnowledgeIndexStore.rrf.test.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.rrf.test.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/__tests__/KnowledgeIndexStore.rrf.test.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.rrf.test.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/__tests__/KnowledgeIndexStore.search.test.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.search.test.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/__tests__/KnowledgeIndexStore.search.test.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.search.test.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/__tests__/KnowledgeIndexStore.test.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.test.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/__tests__/KnowledgeIndexStore.test.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.test.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/__tests__/ftsQuery.test.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/ftsQuery.test.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/__tests__/ftsQuery.test.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/ftsQuery.test.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/__tests__/hashing.test.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/hashing.test.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/__tests__/hashing.test.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/hashing.test.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/__tests__/indexMeta.test.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/indexMeta.test.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/__tests__/indexMeta.test.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/indexMeta.test.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/__tests__/schema.test.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/schema.test.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/__tests__/schema.test.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/schema.test.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/__tests__/vectorBlob.test.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/vectorBlob.test.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/__tests__/vectorBlob.test.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/vectorBlob.test.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/ftsQuery.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/ftsQuery.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/ftsQuery.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/ftsQuery.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/hashing.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/hashing.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/hashing.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/hashing.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/indexMeta.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/indexMeta.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/indexMeta.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/indexMeta.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/model.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/model.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/model.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/model.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/schema.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/schema.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/schema.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/schema.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/types.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/types.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/types.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/types.ts diff --git a/src/main/features/knowledge/vectorstore/indexStore/vectorBlob.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/vectorBlob.ts similarity index 100% rename from src/main/features/knowledge/vectorstore/indexStore/vectorBlob.ts rename to src/main/features/knowledge/pipeline/vectorstore/indexStore/vectorBlob.ts diff --git a/src/main/features/knowledge/vectorCleanup.ts b/src/main/features/knowledge/pipeline/vectorstore/vectorCleanup.ts similarity index 100% rename from src/main/features/knowledge/vectorCleanup.ts rename to src/main/features/knowledge/pipeline/vectorstore/vectorCleanup.ts diff --git a/src/main/features/knowledge/query/KnowledgeConceptService.ts b/src/main/features/knowledge/query/KnowledgeConceptService.ts index 38a5fdb1de3..477d8c0ad5b 100644 --- a/src/main/features/knowledge/query/KnowledgeConceptService.ts +++ b/src/main/features/knowledge/query/KnowledgeConceptService.ts @@ -6,7 +6,7 @@ import { DataApiErrorFactory } from '@shared/data/api' import type { KnowledgeItem, KnowledgeItemStatus, KnowledgeItemType } from '@shared/data/types/knowledge' import { getKnowledgeItemDisplayTitle } from '@shared/data/types/knowledge' -import { assertBaseCanRunRuntimeOperation } from '../baseGuards' +import { assertBaseCanRunRuntimeOperation } from '../base/baseGuards' import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' import { runStoreOperation } from './storeOperation' import { deriveConceptId, loadVisibleItems } from './visibility' diff --git a/src/main/features/knowledge/query/KnowledgeQueryService.ts b/src/main/features/knowledge/query/KnowledgeQueryService.ts index ff1fb78fdb9..f157b4d422b 100644 --- a/src/main/features/knowledge/query/KnowledgeQueryService.ts +++ b/src/main/features/knowledge/query/KnowledgeQueryService.ts @@ -8,11 +8,11 @@ import type { KnowledgeItem, KnowledgeItemChunk, KnowledgeSearchResult } from '@ import { getKnowledgeItemDisplayTitle, isCompletedVectorKnowledgeBase } from '@shared/data/types/knowledge' import { estimateTokenCount } from 'tokenx' -import { assertBaseCanRunRuntimeOperation } from '../baseGuards' -import { embedKnowledgeQuery } from '../indexing/embed' -import { rerankKnowledgeSearchResults } from '../indexing/rerank' +import { assertBaseCanRunRuntimeOperation } from '../base/baseGuards' +import { embedKnowledgeQuery } from '../pipeline/indexing/embed' +import { rerankKnowledgeSearchResults } from '../pipeline/indexing/rerank' +import type { KnowledgeIndexSearchMatch } from '../pipeline/vectorstore/indexStore/model' import { toKnowledgeBaseId, toKnowledgeItemId } from '../types' -import type { KnowledgeIndexSearchMatch } from '../vectorstore/indexStore/model' import { applyRelevanceThreshold, getInitialSearchScoreKind, withSearchRanks } from './search' import { runStoreOperation } from './storeOperation' import { deriveConceptId, loadVisibleItems } from './visibility' diff --git a/src/main/features/knowledge/query/storeOperation.ts b/src/main/features/knowledge/query/storeOperation.ts index 53184d7cbc3..90796413cc7 100644 --- a/src/main/features/knowledge/query/storeOperation.ts +++ b/src/main/features/knowledge/query/storeOperation.ts @@ -1,7 +1,7 @@ import { loggerService } from '@logger' import { DataApiErrorFactory } from '@shared/data/api' -import type { KnowledgeIndexStore } from '../vectorstore/indexStore/KnowledgeIndexStore' +import type { KnowledgeIndexStore } from '../pipeline/vectorstore/indexStore/KnowledgeIndexStore' const logger = loggerService.withContext('Knowledge:Query') diff --git a/src/main/features/knowledge/query/visibility.ts b/src/main/features/knowledge/query/visibility.ts index 11b8518b280..184bff737b8 100644 --- a/src/main/features/knowledge/query/visibility.ts +++ b/src/main/features/knowledge/query/visibility.ts @@ -3,7 +3,7 @@ import { loggerService } from '@logger' import { ErrorCode, isDataApiError } from '@shared/data/api' import type { KnowledgeItem } from '@shared/data/types/knowledge' -import { toMaterialRelativePath } from '../indexing/materialFields' +import { toMaterialRelativePath } from '../pipeline/indexing/materialFields' const logger = loggerService.withContext('Knowledge:Query') diff --git a/src/main/features/knowledge/tasks/__tests__/indexDocumentsJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/indexDocumentsJobHandler.test.ts index df0a0a471b1..704ee95ebd8 100644 --- a/src/main/features/knowledge/tasks/__tests__/indexDocumentsJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/indexDocumentsJobHandler.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest' -import { hashEmbeddingText } from '../../vectorstore/indexStore/hashing' -import type { RebuildMaterialInput } from '../../vectorstore/indexStore/model' +import { hashEmbeddingText } from '../../pipeline/vectorstore/indexStore/hashing' +import type { RebuildMaterialInput } from '../../pipeline/vectorstore/indexStore/model' import { captureNoteSnapshotFileMock, captureUrlSnapshotFileMock, diff --git a/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts b/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts index 05debfbc1cf..cbcd8af3a47 100644 --- a/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts +++ b/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts @@ -131,23 +131,23 @@ vi.mock('@data/services/KnowledgeItemService', () => ({ } })) -vi.mock('../../readers/KnowledgeReader', () => ({ +vi.mock('../../pipeline/readers/KnowledgeReader', () => ({ loadKnowledgeItemDocuments: loadKnowledgeItemDocumentsMock })) -vi.mock('../../sources/prepare', () => ({ +vi.mock('../../pipeline/sources/prepare', () => ({ prepareKnowledgeItem: prepareKnowledgeItemMock })) -vi.mock('../../sources/url', () => ({ +vi.mock('../../pipeline/sources/url', () => ({ fetchKnowledgeWebPage: fetchKnowledgeWebPageMock })) -vi.mock('../../sources/urlSnapshot', () => ({ +vi.mock('../../pipeline/sources/urlSnapshot', () => ({ captureUrlSnapshotFile: captureUrlSnapshotFileMock })) -vi.mock('../../sources/noteSnapshot', () => ({ +vi.mock('../../pipeline/sources/noteSnapshot', () => ({ captureNoteSnapshotFile: captureNoteSnapshotFileMock })) @@ -167,7 +167,7 @@ vi.mock('../../pathStorage', async () => { } }) -vi.mock('../../indexing/embed', () => ({ +vi.mock('../../pipeline/indexing/embed', () => ({ embedKnowledgeTexts: embedKnowledgeTextsMock })) diff --git a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts index 0ead847b6ce..d46cedde168 100644 --- a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts +++ b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts @@ -12,8 +12,8 @@ import { getFileProcessingMarkdownArtifactPath } from '../../fileProcessing/persistence/artifacts' import type { FileProcessingJobPayload } from '../../fileProcessing/tasks/shared' +import type { KnowledgeLockManager } from '../base/KnowledgeLockManager' import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' -import type { KnowledgeLockManager } from '../KnowledgeLockManager' import { toKnowledgeRelativePath } from '../pathStorage' import { knowledgeQueueName, toKnowledgeBaseId, toKnowledgeItemId } from '../types' import type { KnowledgeCheckFileProcessingResultPayload } from './jobTypes' diff --git a/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts b/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts index 3d10221ab3a..675dacf3f70 100644 --- a/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts +++ b/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts @@ -5,10 +5,10 @@ import { knowledgeItemService } from '@data/services/KnowledgeItemService' import { loggerService } from '@logger' import type { JobHandler } from '@main/core/job/types' -import type { KnowledgeLockManager } from '../KnowledgeLockManager' -import { cancelActiveKnowledgeSubtreeJobs, purgeKnowledgeSubtreeWithinLock } from '../subtreePurge' +import type { KnowledgeLockManager } from '../base/KnowledgeLockManager' +import { cancelActiveKnowledgeSubtreeJobs, purgeKnowledgeSubtreeWithinLock } from '../ingestion/subtreePurge' +import { reclaimKnowledgeIndexSpace } from '../pipeline/vectorstore/vectorCleanup' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId } from '../types' -import { reclaimKnowledgeIndexSpace } from '../vectorCleanup' import type { KnowledgeDeleteSubtreePayload } from './jobTypes' const logger = loggerService.withContext('Knowledge:DeleteSubtreeJobHandler') diff --git a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts index e91efb2140c..af2a4971f05 100644 --- a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts +++ b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts @@ -8,20 +8,20 @@ import type { JobContext, JobHandler } from '@main/core/job/types' import type { KnowledgeBase } from '@shared/data/types/knowledge' import { isCompletedVectorKnowledgeBase } from '@shared/data/types/knowledge' -import { type ChunkedKnowledgeContent, chunkKnowledgeDocuments } from '../indexing/chunk' -import { embedKnowledgeTexts } from '../indexing/embed' -import { toMaterialRelativePath } from '../indexing/materialFields' +import type { KnowledgeLockManager } from '../base/KnowledgeLockManager' +import type { IndexableKnowledgeItem } from '../items' import { isIndexableKnowledgeItem } from '../items' -import type { KnowledgeLockManager } from '../KnowledgeLockManager' import { collectKnowledgeReservedRelativePaths } from '../pathStorage' -import { loadKnowledgeItemDocuments } from '../readers/KnowledgeReader' -import { captureNoteSnapshotFile } from '../sources/noteSnapshot' -import { fetchKnowledgeWebPage } from '../sources/url' -import { captureUrlSnapshotFile } from '../sources/urlSnapshot' +import { type ChunkedKnowledgeContent, chunkKnowledgeDocuments } from '../pipeline/indexing/chunk' +import { embedKnowledgeTexts } from '../pipeline/indexing/embed' +import { toMaterialRelativePath } from '../pipeline/indexing/materialFields' +import { loadKnowledgeItemDocuments } from '../pipeline/readers/KnowledgeReader' +import { captureNoteSnapshotFile } from '../pipeline/sources/noteSnapshot' +import { fetchKnowledgeWebPage } from '../pipeline/sources/url' +import { captureUrlSnapshotFile } from '../pipeline/sources/urlSnapshot' +import { hashEmbeddingText } from '../pipeline/vectorstore/indexStore/hashing' +import type { RebuildMaterialEmbeddingInput, RebuildMaterialInput } from '../pipeline/vectorstore/indexStore/model' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId } from '../types' -import type { IndexableKnowledgeItem } from '../types/items' -import { hashEmbeddingText } from '../vectorstore/indexStore/hashing' -import type { RebuildMaterialEmbeddingInput, RebuildMaterialInput } from '../vectorstore/indexStore/model' import type { KnowledgeIndexDocumentsPayload } from './jobTypes' import { isDataApiNotFoundError, markKnowledgeItemFailedOnSettled } from './utils/settled' diff --git a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts index 9c64c984347..f3ed9fc4ae9 100644 --- a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts +++ b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts @@ -6,14 +6,14 @@ import { loggerService } from '@logger' import type { JobContext, JobHandler } from '@main/core/job/types' import type { KnowledgeItem } from '@shared/data/types/knowledge' +import type { KnowledgeLockManager } from '../base/KnowledgeLockManager' import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' import { markUnscheduledKnowledgeItemsFailed } from '../ingestion/statusCleanup' import { isIndexableKnowledgeItem } from '../items' -import type { KnowledgeLockManager } from '../KnowledgeLockManager' import { deleteKnowledgeItemFilesBestEffort } from '../pathStorage' -import { prepareKnowledgeItem } from '../sources/prepare' +import { prepareKnowledgeItem } from '../pipeline/sources/prepare' +import { deleteKnowledgeItemVectors } from '../pipeline/vectorstore/vectorCleanup' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId, toKnowledgeItemId } from '../types' -import { deleteKnowledgeItemVectors } from '../vectorCleanup' import type { KnowledgePrepareRootPayload } from './jobTypes' import { isDataApiNotFoundError, markKnowledgeItemFailedOnSettled } from './utils/settled' diff --git a/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts b/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts index 5dc29284016..425f35c247c 100644 --- a/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts +++ b/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts @@ -8,10 +8,11 @@ import type { JobHandler, JobSettledEvent } from '@main/core/job/types' import type { JobSnapshot } from '@shared/data/api/schemas/jobs' import { KNOWLEDGE_ITEM_ERROR_INDEXING_INTERRUPTED, type KnowledgeItemStatus } from '@shared/data/types/knowledge' +import type { KnowledgeLockManager } from '../base/KnowledgeLockManager' import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' import { canKnowledgeItemRebuildSource, isContainerKnowledgeItem } from '../items' -import type { KnowledgeLockManager } from '../KnowledgeLockManager' import { deleteKnowledgeItemFilesBestEffort } from '../pathStorage' +import { deleteKnowledgeItemVectors } from '../pipeline/vectorstore/vectorCleanup' import { KNOWLEDGE_ACTIVE_JOB_LIMIT, KNOWLEDGE_ACTIVE_JOB_STATUSES, @@ -20,7 +21,6 @@ import { toKnowledgeBaseId, toKnowledgeItemId } from '../types' -import { deleteKnowledgeItemVectors } from '../vectorCleanup' import type { KnowledgeReindexSubtreePayload } from './jobTypes' import { narrowKnowledgeJobInput } from './utils/jobInput' diff --git a/src/main/features/knowledge/types/items.ts b/src/main/features/knowledge/types/items.ts deleted file mode 100644 index e6c57392b11..00000000000 --- a/src/main/features/knowledge/types/items.ts +++ /dev/null @@ -1,6 +0,0 @@ -import type { KnowledgeItemOf } from '@shared/data/types/knowledge' - -export type IndexableKnowledgeItem = KnowledgeItemOf<'file' | 'url' | 'note'> - -export type ContainerKnowledgeItem = KnowledgeItemOf<'directory'> -export type ContainerKnowledgeItemType = ContainerKnowledgeItem['type'] From 5a81409308910cc70362c282cc3c06df58a01147 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 11:15:40 +0800 Subject: [PATCH 09/37] refactor(knowledge-query): reuse admission-guard result and sync store calls assertBaseCanRunRuntimeOperation now returns the fetched KnowledgeBase so search()/listItemChunks() no longer re-fetch it, and search()'s token check and chunk-access guard are folded into the FTS layer's own tokenizer / knowledgeItemService.getById instead of duplicating them. listItemChunks() also drops its now-pointless Promise.all/await wrapper now that KnowledgeIndexStore's per-material reads are synchronous (better-sqlite3). Signed-off-by: eeee0717 Signed-off-by: eeee0717 --- .../features/knowledge/base/baseGuards.ts | 10 ++- .../knowledge/query/KnowledgeQueryService.ts | 77 +++++++------------ .../knowledge/query/storeOperation.ts | 2 +- .../features/knowledge/query/visibility.ts | 4 +- 4 files changed, 39 insertions(+), 54 deletions(-) diff --git a/src/main/features/knowledge/base/baseGuards.ts b/src/main/features/knowledge/base/baseGuards.ts index feeeb75c321..deed694056b 100644 --- a/src/main/features/knowledge/base/baseGuards.ts +++ b/src/main/features/knowledge/base/baseGuards.ts @@ -1,12 +1,16 @@ import { knowledgeBaseService } from '@data/services/KnowledgeBaseService' import { DataApiErrorFactory } from '@shared/data/api' +import type { KnowledgeBase } from '@shared/data/types/knowledge' -/** A base in `failed` state must be restored before any runtime operation (search, add, reindex, deep-read). */ -export function assertBaseCanRunRuntimeOperation(baseId: string, operation: string): void { +/** + * A base in `failed` state must be restored before any runtime operation (search, add, reindex, + * deep-read). Returns the fetched base so synchronous callers need not re-fetch it. + */ +export function assertBaseCanRunRuntimeOperation(baseId: string, operation: string): KnowledgeBase { const base = knowledgeBaseService.getById(baseId) if (base.status !== 'failed') { - return + return base } throw DataApiErrorFactory.validation( diff --git a/src/main/features/knowledge/query/KnowledgeQueryService.ts b/src/main/features/knowledge/query/KnowledgeQueryService.ts index f157b4d422b..af47218a349 100644 --- a/src/main/features/knowledge/query/KnowledgeQueryService.ts +++ b/src/main/features/knowledge/query/KnowledgeQueryService.ts @@ -11,6 +11,7 @@ import { estimateTokenCount } from 'tokenx' import { assertBaseCanRunRuntimeOperation } from '../base/baseGuards' import { embedKnowledgeQuery } from '../pipeline/indexing/embed' import { rerankKnowledgeSearchResults } from '../pipeline/indexing/rerank' +import { extractFtsTokens } from '../pipeline/vectorstore/indexStore/ftsQuery' import type { KnowledgeIndexSearchMatch } from '../pipeline/vectorstore/indexStore/model' import { toKnowledgeBaseId, toKnowledgeItemId } from '../types' import { applyRelevanceThreshold, getInitialSearchScoreKind, withSearchRanks } from './search' @@ -18,7 +19,6 @@ import { runStoreOperation } from './storeOperation' import { deriveConceptId, loadVisibleItems } from './visibility' const logger = loggerService.withContext('Knowledge:QueryService') -const SEARCH_TOKEN_PATTERN = /[\p{L}\p{N}_]+/u /** * Fetch this many × the requested result count as index candidates. The index * store only filters by material state; the item-visibility filter (missing / @@ -33,16 +33,16 @@ const KNOWLEDGE_SEARCH_CANDIDATE_CAP = 200 export class KnowledgeQueryService { @TraceMethod({ spanName: 'Knowledge.search', tag: 'Knowledge' }) async search(baseId: string, query: string): Promise { - assertBaseCanRunRuntimeOperation(baseId, 'search') + const base = assertBaseCanRunRuntimeOperation(baseId, 'search') - if (!SEARCH_TOKEN_PATTERN.test(query)) { + // Same tokenization the FTS layer uses: no token means no BM25 hit is even possible. + if (extractFtsTokens(query).length === 0) { throw DataApiErrorFactory.validation( { query: ['Query has no searchable tokens'] }, 'Query has no searchable tokens' ) } - const base = knowledgeBaseService.getById(baseId) // Stored search mode and the index store's mode are the same enum now, so no mapping. // Vector/hybrid retrieval needs an embedding model; a base without one is BM25-only // regardless of its stored preference. @@ -69,19 +69,15 @@ export class KnowledgeQueryService { const visibleSearchResults = this.toVisibleSearchResults(baseId, matches, scoreKind) const topResults = this.trimToTopK(visibleSearchResults, resolvedTopK, baseId) - if (base.rerankModelId) { - const rerankedResults = await rerankKnowledgeSearchResults(base, query, topResults) - return withSearchRanks(applyRelevanceThreshold(rerankedResults, base.threshold)) - } - - return withSearchRanks(applyRelevanceThreshold(topResults, base.threshold)) + const rerankedResults = await rerankKnowledgeSearchResults(base, query, topResults) + return withSearchRanks(applyRelevanceThreshold(rerankedResults, base.threshold)) } async listItemChunks(baseId: string, itemId: string): Promise { const knowledgeBaseId = toKnowledgeBaseId(baseId) const knowledgeItemId = toKnowledgeItemId(itemId) assertBaseCanRunRuntimeOperation(knowledgeBaseId, 'listItemChunks') - const item = await this.assertItemCanRunChunkOperation(knowledgeBaseId, knowledgeItemId, 'list chunks') + const item = this.assertItemCanRunChunkOperation(knowledgeBaseId, knowledgeItemId) this.assertCompletedContainerHasNoDeletingChildren(knowledgeBaseId, item) const base = knowledgeBaseService.getById(knowledgeBaseId) @@ -96,25 +92,23 @@ export class KnowledgeQueryService { const vectorStoreService = application.get('KnowledgeVectorStoreService') const store = await vectorStoreService.getIndexStore(base) const chunkGroups = await runStoreOperation(store, knowledgeBaseId, 'listItemChunks', () => - Promise.all( - leafItems.map(async (leafItem) => { - const units = await store.listMaterialUnits(leafItem.id) - return units.map( - (unit): KnowledgeItemChunk => ({ - id: unit.unitId, + leafItems.map((leafItem) => { + const units = store.listMaterialUnits(leafItem.id) + return units.map( + (unit): KnowledgeItemChunk => ({ + id: unit.unitId, + itemId: leafItem.id, + content: unit.text, + metadata: { itemId: leafItem.id, - content: unit.text, - metadata: { - itemId: leafItem.id, - itemType: leafItem.type, - source: leafItem.data.source, - chunkIndex: unit.unitIndex, - tokenCount: estimateTokenCount(unit.text) - } - }) - ) - }) - ) + itemType: leafItem.type, + source: leafItem.data.source, + chunkIndex: unit.unitIndex, + tokenCount: estimateTokenCount(unit.text) + } + }) + ) + }) ) return chunkGroups.flat() @@ -181,29 +175,16 @@ export class KnowledgeQueryService { return results.slice(0, topK) } - private async getRootItemsInBase(baseId: string, itemIds: string[]): Promise { - const rootIds = [...new Set(itemIds)] - const items = await Promise.all(rootIds.map((itemId) => knowledgeItemService.getById(itemId))) - const invalidItem = items.find((item) => item.baseId !== baseId) - - if (invalidItem) { - throw new Error(`Knowledge item '${invalidItem.id}' does not belong to base '${baseId}'`) + private assertItemCanRunChunkOperation(baseId: string, itemId: string): KnowledgeItem { + const item = knowledgeItemService.getById(itemId) + if (item.baseId !== baseId) { + throw new Error(`Knowledge item '${itemId}' does not belong to base '${baseId}'`) } - return items - } - - private async assertItemCanRunChunkOperation( - baseId: string, - itemId: string, - operation: 'list chunks' | 'delete chunk' - ): Promise { - const [item] = await this.getRootItemsInBase(baseId, [itemId]) - if (item.status !== 'completed') { throw DataApiErrorFactory.validation( - { item: [`Knowledge item '${itemId}' must be completed before ${operation}`] }, - `Cannot ${operation} for a non-completed knowledge item` + { item: [`Knowledge item '${itemId}' must be completed before list chunks`] }, + 'Cannot list chunks for a non-completed knowledge item' ) } diff --git a/src/main/features/knowledge/query/storeOperation.ts b/src/main/features/knowledge/query/storeOperation.ts index 90796413cc7..bbc26a4bdb9 100644 --- a/src/main/features/knowledge/query/storeOperation.ts +++ b/src/main/features/knowledge/query/storeOperation.ts @@ -16,7 +16,7 @@ export async function runStoreOperation( store: KnowledgeIndexStore, baseId: string, operation: string, - run: () => Promise + run: () => T | Promise ): Promise { try { return await run() diff --git a/src/main/features/knowledge/query/visibility.ts b/src/main/features/knowledge/query/visibility.ts index 184bff737b8..19aa98fc342 100644 --- a/src/main/features/knowledge/query/visibility.ts +++ b/src/main/features/knowledge/query/visibility.ts @@ -1,9 +1,9 @@ import { knowledgeItemService } from '@data/services/KnowledgeItemService' import { loggerService } from '@logger' -import { ErrorCode, isDataApiError } from '@shared/data/api' import type { KnowledgeItem } from '@shared/data/types/knowledge' import { toMaterialRelativePath } from '../pipeline/indexing/materialFields' +import { isDataApiNotFoundError } from '../tasks/utils/settled' const logger = loggerService.withContext('Knowledge:Query') @@ -44,7 +44,7 @@ export function loadVisibleItems(baseId: string, materialIds: string[]): Map Date: Fri, 3 Jul 2026 11:16:05 +0800 Subject: [PATCH 10/37] refactor(knowledge): reuse admission-guard return value in addItems/resolveConcept assertBaseCanRunRuntimeOperation already fetches the KnowledgeBase; addItems() and resolveConcept() immediately re-fetched it by id right after with nothing but a synchronous check in between, so use the guard's return value instead. Signed-off-by: eeee0717 Signed-off-by: eeee0717 --- .../features/knowledge/ingestion/KnowledgeIngestionService.ts | 4 +--- src/main/features/knowledge/query/KnowledgeConceptService.ts | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts index ca44ad3d579..259cde0e733 100644 --- a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts +++ b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts @@ -69,14 +69,12 @@ export class KnowledgeIngestionService { inputs: KnowledgeAddItemInput[], conflictStrategy: KnowledgeAddConflictStrategy = DEFAULT_KNOWLEDGE_ADD_CONFLICT_STRATEGY ): Promise { - assertBaseCanRunRuntimeOperation(baseId, 'addItems') + const base = assertBaseCanRunRuntimeOperation(baseId, 'addItems') if (inputs.length === 0) { return { status: 'added' } } - const base = knowledgeBaseService.getById(baseId) - // rename (the default, and every internal caller — restore/migrator): keep all, // auto-rename on collision. detect/replace first resolve same-name conflicts // against the existing root items and earlier items in the same batch. diff --git a/src/main/features/knowledge/query/KnowledgeConceptService.ts b/src/main/features/knowledge/query/KnowledgeConceptService.ts index 477d8c0ad5b..c8716ac9d92 100644 --- a/src/main/features/knowledge/query/KnowledgeConceptService.ts +++ b/src/main/features/knowledge/query/KnowledgeConceptService.ts @@ -401,9 +401,8 @@ export class KnowledgeConceptService { conceptId: string, operation: string ): Promise<{ item: KnowledgeItem; text: string }> { - assertBaseCanRunRuntimeOperation(baseId, operation) + const base = assertBaseCanRunRuntimeOperation(baseId, operation) - const base = knowledgeBaseService.getById(baseId) const vectorStoreService = application.get('KnowledgeVectorStoreService') const store = await vectorStoreService.getIndexStore(base) From 311a734b86cefa86e88020962b9ae273547c892e Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 11:16:29 +0800 Subject: [PATCH 11/37] refactor(knowledge-ingestion): dedupe status-cleanup log key markUnscheduledKnowledgeItemsFailed's MarkFailedInput.logContextKey was always passed as the literal 'scheduleError' by both call sites, so drop the field and hardcode the key in the two log payloads it built. Signed-off-by: eeee0717 Signed-off-by: eeee0717 --- .../knowledge/ingestion/KnowledgeIngestionService.ts | 3 +-- .../ingestion/__tests__/statusCleanup.integration.test.ts | 3 +-- src/main/features/knowledge/ingestion/statusCleanup.ts | 5 ++--- src/main/features/knowledge/tasks/prepareRootJobHandler.ts | 3 +-- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts index 259cde0e733..5c06ccc44c3 100644 --- a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts +++ b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts @@ -612,8 +612,7 @@ export class KnowledgeIngestionService { errorMessage: message, failedStatusError: `Failed to schedule knowledge item job: ${message}`, logger, - logMessage: 'Failed to mark unscheduled knowledge item after addItems scheduling failure', - logContextKey: 'scheduleError' + logMessage: 'Failed to mark unscheduled knowledge item after addItems scheduling failure' }) } } diff --git a/src/main/features/knowledge/ingestion/__tests__/statusCleanup.integration.test.ts b/src/main/features/knowledge/ingestion/__tests__/statusCleanup.integration.test.ts index 50dc19a9b15..5ed491bae9b 100644 --- a/src/main/features/knowledge/ingestion/__tests__/statusCleanup.integration.test.ts +++ b/src/main/features/knowledge/ingestion/__tests__/statusCleanup.integration.test.ts @@ -118,8 +118,7 @@ describe('markUnscheduledKnowledgeItemsFailed integration', () => { info: vi.fn(), warn: vi.fn() } as unknown as LoggerService, - logMessage: 'Failed to mark unscheduled item', - logContextKey: 'scheduleError' + logMessage: 'Failed to mark unscheduled item' }) } finally { updateStatusSpy.mockRestore() diff --git a/src/main/features/knowledge/ingestion/statusCleanup.ts b/src/main/features/knowledge/ingestion/statusCleanup.ts index 48b5e72759f..ea75eeec875 100644 --- a/src/main/features/knowledge/ingestion/statusCleanup.ts +++ b/src/main/features/knowledge/ingestion/statusCleanup.ts @@ -10,7 +10,6 @@ type MarkFailedInput = { failedStatusError: string logger: LoggerService logMessage: string - logContextKey: string } export function markUnscheduledKnowledgeItemsFailed(input: MarkFailedInput): void { @@ -33,7 +32,7 @@ export function markUnscheduledKnowledgeItemsFailed(input: MarkFailedInput): voi { baseId: input.baseId, itemId: item.id, - [input.logContextKey]: input.errorMessage + scheduleError: input.errorMessage } ) } @@ -50,7 +49,7 @@ export function markUnscheduledKnowledgeItemsFailed(input: MarkFailedInput): voi { baseId: input.baseId, itemId: item.id, - [input.logContextKey]: input.errorMessage + scheduleError: input.errorMessage } ) } diff --git a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts index f3ed9fc4ae9..5c47c445012 100644 --- a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts +++ b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts @@ -190,7 +190,6 @@ function markUnscheduledLeafItemsFailed( errorMessage: message, failedStatusError: `Failed to schedule knowledge child item job: ${message}`, logger, - logMessage: 'Failed to mark unscheduled knowledge child item after prepare-root scheduling failure', - logContextKey: 'scheduleError' + logMessage: 'Failed to mark unscheduled knowledge child item after prepare-root scheduling failure' }) } From c3dcae6506e50d42b1a2b06af348d1c220d01d63 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 11:16:40 +0800 Subject: [PATCH 12/37] refactor(knowledge-ingestion): batch the reindex admission check across roots assertSubtreesCanReindex queried each root's subtree and classified its source one at a time in a sequential loop. rootItemIds always come from getOutermostSelectedItemIds, which guarantees the roots are mutually non-descendant, so a single batched getSubtreeItems(..., rootItemIds) call covers every root's subtree in one query, and the per-root source classifications can run concurrently via Promise.all instead of sequential awaits. Signed-off-by: eeee0717 Signed-off-by: eeee0717 --- .../ingestion/KnowledgeIngestionService.ts | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts index 5c06ccc44c3..72c4c132580 100644 --- a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts +++ b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts @@ -435,38 +435,38 @@ export class KnowledgeIngestionService { } private async assertSubtreesCanReindex(baseId: string, rootItemIds: string[]): Promise { - const blockingStatusCounts = new Map() + // rootItemIds comes from getOutermostSelectedItemIds, which guarantees the roots are mutually + // non-descendant (disjoint subtrees), so one batched query's union equals the per-root sum. + const subtreeItems = knowledgeItemService.getSubtreeItems(baseId, rootItemIds, { includeRoots: true }) + const rootIdSet = new Set(rootItemIds) + const roots = subtreeItems.filter((item) => rootIdSet.has(item.id)) + + // Reindex deletes the subtree's vectors before re-reading the source (reindexSubtreeJobHandler), + // so a root whose source is gone would lose its vectors with nothing to rebuild from — reject up + // front. Only the root's own source matters: a directory is rescanned from data.source and its + // children recreated (never read from their raw/ files), a file leaf reads its own raw/ file, and + // note/url always rebuild from the DB / network. A v1-migrated folder child reindexed on its own + // is a file root whose raw/ file never existed, so this rejects it too. Distinguish a genuinely + // missing source (delete-and-re-add) from one we could not verify (transient/permission error, + // which should retry rather than be destroyed). + const sourceStates = await Promise.all(roots.map((root) => classifyKnowledgeItemSource(baseId, root))) + const missingSourceItemIds: string[] = [] const unverifiableSourceItemIds: string[] = [] - - for (const rootItemId of rootItemIds) { - const subtreeItems = knowledgeItemService.getSubtreeItems(baseId, [rootItemId], { includeRoots: true }) - - // Reindex deletes the subtree's vectors before re-reading the source (reindexSubtreeJobHandler), - // so a root whose source is gone would lose its vectors with nothing to rebuild from — reject up - // front. Only the root's own source matters: a directory is rescanned from data.source and its - // children recreated (never read from their raw/ files), a file leaf reads its own raw/ file, and - // note/url always rebuild from the DB / network. A v1-migrated folder child reindexed on its own - // is a file root whose raw/ file never existed, so this rejects it too. Distinguish a genuinely - // missing source (delete-and-re-add) from one we could not verify (transient/permission error, - // which should retry rather than be destroyed). - const root = subtreeItems.find((item) => item.id === rootItemId) - if (root) { - const sourceState = await classifyKnowledgeItemSource(baseId, root) - if (sourceState === 'missing') { - missingSourceItemIds.push(rootItemId) - } else if (sourceState === 'unverifiable') { - unverifiableSourceItemIds.push(rootItemId) - } + roots.forEach((root, index) => { + if (sourceStates[index] === 'missing') { + missingSourceItemIds.push(root.id) + } else if (sourceStates[index] === 'unverifiable') { + unverifiableSourceItemIds.push(root.id) } + }) - for (const item of subtreeItems) { - if (REINDEX_ALLOWED_STATUSES.has(item.status)) { - continue - } - - blockingStatusCounts.set(item.status, (blockingStatusCounts.get(item.status) ?? 0) + 1) + const blockingStatusCounts = new Map() + for (const item of subtreeItems) { + if (REINDEX_ALLOWED_STATUSES.has(item.status)) { + continue } + blockingStatusCounts.set(item.status, (blockingStatusCounts.get(item.status) ?? 0) + 1) } if (missingSourceItemIds.length > 0) { From 6420ed45a040678c4493f798b2c0ff83dbe9e608 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 11:16:59 +0800 Subject: [PATCH 13/37] refactor(knowledge-tasks): route progress reports through the typed reporter checkFileProcessingResultJobHandler called ctx.reportProgress(...) directly with raw payload literals in 8 places, instead of the typed reportKnowledgeProgress(ctx, ...) helper every other handler already uses. All payloads already matched KnowledgeProgressDetail, so this is a straight call-site swap. Signed-off-by: eeee0717 Signed-off-by: eeee0717 --- .../checkFileProcessingResultJobHandler.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts index d46cedde168..c42e302f26b 100644 --- a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts +++ b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts @@ -15,7 +15,7 @@ import type { FileProcessingJobPayload } from '../../fileProcessing/tasks/shared import type { KnowledgeLockManager } from '../base/KnowledgeLockManager' import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' import { toKnowledgeRelativePath } from '../pathStorage' -import { knowledgeQueueName, toKnowledgeBaseId, toKnowledgeItemId } from '../types' +import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId, toKnowledgeItemId } from '../types' import type { KnowledgeCheckFileProcessingResultPayload } from './jobTypes' import { cancelJobOrThrow } from './utils/cancel' import { isDataApiNotFoundError, markKnowledgeItemFailedOnSettled } from './utils/settled' @@ -59,13 +59,13 @@ export function createCheckFileProcessingResultJobHandler( if (!snapshot) { markItemFailed(itemId, `File processing job not found: ${fileProcessingJobId}`) - ctx.reportProgress(100, { stage: 'failed' }) + reportKnowledgeProgress(ctx, 100, { stage: 'failed' }) return } if (!isExpectedFileProcessingJob(snapshot, itemId)) { markItemFailed(itemId, `Invalid file processing job for knowledge item: ${fileProcessingJobId}`) - ctx.reportProgress(100, { stage: 'failed' }) + reportKnowledgeProgress(ctx, 100, { stage: 'failed' }) return } @@ -73,7 +73,7 @@ export function createCheckFileProcessingResultJobHandler( if (Date.now() - firstScheduledAt >= FILE_PROCESSING_MAX_WAIT_MS) { await cancelJobOrThrow(fileProcessingJobId, 'knowledge-file-processing-timeout') markItemFailed(itemId, `File processing job ${fileProcessingJobId} did not finish within 30 minutes`) - ctx.reportProgress(100, { stage: 'failed' }) + reportKnowledgeProgress(ctx, 100, { stage: 'failed' }) return } @@ -97,14 +97,14 @@ export function createCheckFileProcessingResultJobHandler( itemId, `File processing job ${fileProcessingJobId} ${snapshot.status}: ${getFileProcessingFailureMessage(snapshot)}` ) - ctx.reportProgress(100, { stage: 'failed' }) + reportKnowledgeProgress(ctx, 100, { stage: 'failed' }) return } const indexedRelativePath = parseMarkdownArtifactRelativePathOrNull(baseId, snapshot) if (!indexedRelativePath) { markItemFailed(itemId, `Invalid file processing result for job ${fileProcessingJobId}`) - ctx.reportProgress(100, { stage: 'failed' }) + reportKnowledgeProgress(ctx, 100, { stage: 'failed' }) return } @@ -124,7 +124,7 @@ export function createCheckFileProcessingResultJobHandler( if (!canContinue) { return } - ctx.reportProgress(100, { stage: 'done' }) + reportKnowledgeProgress(ctx, 100, { stage: 'done' }) }, async onSettled(event) { @@ -144,11 +144,11 @@ function reportWaitingProgress( ): void { const childProgress = application.get('CacheService').getShared(`${JOB_PROGRESS_KEY_PREFIX}${fileProcessingJobId}`) if (!childProgress) { - ctx.reportProgress(0, { stage: 'waiting', pollRound }) + reportKnowledgeProgress(ctx, 0, { stage: 'waiting', pollRound }) return } - ctx.reportProgress(childProgress.progress, { + reportKnowledgeProgress(ctx, childProgress.progress, { stage: 'waiting', pollRound, fileProcessingJobId, From 5130c82924403ac57019d1241199a5bbec2b994e Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 11:17:15 +0800 Subject: [PATCH 14/37] refactor(knowledge-index): drop libsql-era async from KnowledgeIndexStore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit better-sqlite3 is a synchronous driver; rebuildMaterial, listMaterialUnits, getMaterialByRelativePath, readMaterialContent, search, reclaimSpace, close, and listExistingEmbeddingHashes were still declared async/Promise from the old libsql-backed store, with zero real awaits in their bodies (only deleteMaterials keeps async — it genuinely yields via setImmediate for large batch deletes). Converting them to plain sync calls is behavior-preserving: await on a non-Promise value is a no-op. Also collapse listExistingEmbeddingHashes and assertEmbeddingCoverage's identical batched SELECT ... IN (...) scans into one shared selectExistingEmbeddingHashes helper parameterized over the SqliteExecutor (driver vs. transaction) each call site already had. Test suites updated in lockstep: Vitest's .rejects/.resolves matchers require an actual Promise, so assertions on the now-synchronous methods switch to expect(() => call()).toThrow(...) / plain expect(call()).toEqual(...), and the now-pointless await at every call site is dropped. Signed-off-by: eeee0717 Signed-off-by: eeee0717 --- .../indexStore/KnowledgeIndexStore.ts | 57 ++--- .../KnowledgeIndexStore.integration.test.ts | 12 +- .../__tests__/KnowledgeIndexStore.rrf.test.ts | 8 +- .../KnowledgeIndexStore.search.test.ts | 74 +++---- .../__tests__/KnowledgeIndexStore.test.ts | 198 ++++++++---------- 5 files changed, 169 insertions(+), 180 deletions(-) diff --git a/src/main/features/knowledge/pipeline/vectorstore/indexStore/KnowledgeIndexStore.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/KnowledgeIndexStore.ts index 9f664ba1adb..00621ea5ead 100644 --- a/src/main/features/knowledge/pipeline/vectorstore/indexStore/KnowledgeIndexStore.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/indexStore/KnowledgeIndexStore.ts @@ -7,7 +7,14 @@ import type { KnowledgeSearchUnit, RebuildMaterialInput } from './model' -import type { SqliteDriver, SqliteReclaimOutcome, SqliteTransaction, SqlValue, VectorIndex } from './types' +import type { + SqliteDriver, + SqliteExecutor, + SqliteReclaimOutcome, + SqliteTransaction, + SqlValue, + VectorIndex +} from './types' import { encodeVectorBlob } from './vectorBlob' /** RRF constant (1-indexed rank), matching the legacy hybrid fusion. */ @@ -45,7 +52,7 @@ export class KnowledgeIndexStore { * transaction so a crash or error can never leave old and new units mixed, and * an insert failure rolls back without destroying the prior index (§5.2). */ - async rebuildMaterial(materialId: string, input: RebuildMaterialInput): Promise { + rebuildMaterial(materialId: string, input: RebuildMaterialInput): void { // usesEmbeddings: false means a BM25-only rebuild — step 6 below writes whatever // `embeddings` holds unconditionally (only the step 6b coverage check is gated on // the flag), so a caller bug that sets both would silently write orphan vectors @@ -291,25 +298,12 @@ export class KnowledgeIndexStore { * re-reads (the hash is now absent) and re-embeds it. A stale "present" therefore * self-corrects rather than leaving a unit silently absent from vector search. */ - async listExistingEmbeddingHashes(hashes: string[]): Promise> { - const existing = new Set() - // Chunk to stay well under SQLite's bound-parameter limit for large materials. - for (let i = 0; i < hashes.length; i += EMBEDDING_HASH_QUERY_BATCH) { - const batch = hashes.slice(i, i + EMBEDDING_HASH_QUERY_BATCH) - const placeholders = batch.map(() => '?').join(', ') - const result = this.driver.execute( - `SELECT embedding_text_hash FROM embedding WHERE embedding_text_hash IN (${placeholders})`, - batch - ) - for (const row of result.rows) { - existing.add(row.embedding_text_hash as string) - } - } - return existing + listExistingEmbeddingHashes(hashes: string[]): Set { + return this.selectExistingEmbeddingHashes(this.driver, hashes) } /** Read back a material's units (with body text), ordered by unit index. */ - async listMaterialUnits(materialId: string): Promise { + listMaterialUnits(materialId: string): KnowledgeSearchUnit[] { const result = this.driver.execute( `SELECT su.unit_id, su.material_id, su.unit_type, su.unit_index, su.title, su.char_start, su.char_end, st.text AS body FROM search_unit su @@ -347,7 +341,7 @@ export class KnowledgeIndexStore { * index, so this is the lookup behind the deep-read tools — they re-validate * the resolved material against the visible knowledge_item before reading. */ - async getMaterialByRelativePath(relativePath: string): Promise { + getMaterialByRelativePath(relativePath: string): KnowledgeMaterialRef | null { const result = this.driver.execute(`SELECT material_id, relative_path FROM material WHERE relative_path = ?`, [ relativePath ]) @@ -368,7 +362,7 @@ export class KnowledgeIndexStore { * were sliced from, so `text.slice(unit.charStart, unit.charEnd) === unit.text` * holds (the same invariant rebuildMaterial enforces at write time). */ - async readMaterialContent(materialId: string): Promise { + readMaterialContent(materialId: string): string | null { const result = this.driver.execute( `SELECT c.text AS text FROM material m @@ -390,7 +384,7 @@ export class KnowledgeIndexStore { * text of a unit is the search source for both lanes * (knowledge-technical-design.md §6). */ - async search(input: KnowledgeIndexSearchInput): Promise { + search(input: KnowledgeIndexSearchInput): KnowledgeIndexSearchMatch[] { if (input.mode === 'bm25') { return this.bm25Search(input.queryText, input.topK) } @@ -420,11 +414,11 @@ export class KnowledgeIndexStore { * whole-index segment merge — and the FTS table name lives here, with the schema, not * in the engine-neutral driver. */ - async reclaimSpace(): Promise { + reclaimSpace(): SqliteReclaimOutcome { return this.driver.reclaim([`INSERT INTO search_text_fts(search_text_fts) VALUES('optimize')`]) } - async close(): Promise { + close(): void { this.driver.close() } @@ -514,20 +508,27 @@ export class KnowledgeIndexStore { return result.rows.map((row) => toMatch(row, -Number(row.len))) } - /** Throw (rolling back the surrounding rebuild) if any unit hash has no embedding row. */ - private assertEmbeddingCoverage(tx: SqliteTransaction, materialId: string, hashes: string[]): void { - const missing = new Set(hashes) + /** Of the given embedding-text hashes, return those already stored — chunked to stay well under SQLite's bound-parameter limit. */ + private selectExistingEmbeddingHashes(executor: SqliteExecutor, hashes: string[]): Set { + const existing = new Set() for (let i = 0; i < hashes.length; i += EMBEDDING_HASH_QUERY_BATCH) { const batch = hashes.slice(i, i + EMBEDDING_HASH_QUERY_BATCH) const placeholders = batch.map(() => '?').join(', ') - const result = tx.execute( + const result = executor.execute( `SELECT embedding_text_hash FROM embedding WHERE embedding_text_hash IN (${placeholders})`, batch ) for (const row of result.rows) { - missing.delete(row.embedding_text_hash as string) + existing.add(row.embedding_text_hash as string) } } + return existing + } + + /** Throw (rolling back the surrounding rebuild) if any unit hash has no embedding row. */ + private assertEmbeddingCoverage(tx: SqliteTransaction, materialId: string, hashes: string[]): void { + const present = this.selectExistingEmbeddingHashes(tx, hashes) + const missing = new Set(hashes.filter((hash) => !present.has(hash))) if (missing.size > 0) { throw new Error( `Knowledge index rebuild for material ${materialId} left ${missing.size} unit embedding hash(es) without a vector (first: ${[...missing][0]})` diff --git a/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.integration.test.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.integration.test.ts index 25a14ea760d..4c57e7fd51e 100644 --- a/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.integration.test.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.integration.test.ts @@ -32,7 +32,7 @@ describe('KnowledgeIndexStore integration (real better-sqlite3)', () => { }) afterEach(async () => { - await store.close() + store.close() rmSync(tempDir, { recursive: true, force: true }) }) @@ -63,9 +63,9 @@ describe('KnowledgeIndexStore integration (real better-sqlite3)', () => { ) } - await store.rebuildMaterial('m1', input) + store.rebuildMaterial('m1', input) - const units = await store.listMaterialUnits('m1') + const units = store.listMaterialUnits('m1') expect(units).toHaveLength(chunks.length) units.forEach((unit, index) => { // §5.3: a unit's stored body is the exact content slice, never a rewritten copy. @@ -74,15 +74,15 @@ describe('KnowledgeIndexStore integration (real better-sqlite3)', () => { expect(unit.unitIndex).toBe(index) }) - const bm25Hits = await store.search({ queryText: 'embedding', mode: 'bm25', topK: 5 }) + const bm25Hits = store.search({ queryText: 'embedding', mode: 'bm25', topK: 5 }) expect(bm25Hits.length).toBeGreaterThan(0) expect(bm25Hits.every((hit) => hit.materialId === 'm1')).toBe(true) - const vectorHits = await store.search({ queryText: '', queryEmbedding: [1, 1, 0], mode: 'vector', topK: 5 }) + const vectorHits = store.search({ queryText: '', queryEmbedding: [1, 1, 0], mode: 'vector', topK: 5 }) expect(vectorHits.length).toBeGreaterThan(0) expect(vectorHits.every((hit) => hit.materialId === 'm1')).toBe(true) - const hybridHits = await store.search({ + const hybridHits = store.search({ queryText: 'embedding', queryEmbedding: [1, 1, 0], mode: 'hybrid', diff --git a/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.rrf.test.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.rrf.test.ts index f097484fd0f..d4a2308668a 100644 --- a/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.rrf.test.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.rrf.test.ts @@ -59,7 +59,7 @@ describe('KnowledgeIndexStore hybrid RRF fusion', () => { const { driver, limits } = createFakeDriver([], []) const store = new KnowledgeIndexStore(driver, vectorIndex) - await store.search({ queryText: 'query', queryEmbedding: [1, 0], mode: 'hybrid', topK: 4 }) + store.search({ queryText: 'query', queryEmbedding: [1, 0], mode: 'hybrid', topK: 4 }) expect(limits.vector).toBe(20) expect(limits.bm25).toBe(20) @@ -74,7 +74,7 @@ describe('KnowledgeIndexStore hybrid RRF fusion', () => { const store = new KnowledgeIndexStore(driver, vectorIndex) const alpha = 0.75 - const results = await store.search({ queryText: 'query', queryEmbedding: [1, 0], mode: 'hybrid', topK: 10, alpha }) + const results = store.search({ queryText: 'query', queryEmbedding: [1, 0], mode: 'hybrid', topK: 10, alpha }) // score = alpha/(RRF_K + vRank + 1) + (1 - alpha)/(RRF_K + bRank + 1), ranks 0-indexed. const scoreA = alpha / (RRF_K + 1) + (1 - alpha) / (RRF_K + 2) @@ -94,7 +94,7 @@ describe('KnowledgeIndexStore hybrid RRF fusion', () => { const { driver } = createFakeDriver([{ unit_id: 'A', material_id: 'mA', unit_index: 0, body: null, dist: 0 }], []) const store = new KnowledgeIndexStore(driver, vectorIndex) - await expect(store.search({ queryText: '', queryEmbedding: [1, 0], mode: 'vector', topK: 5 })).rejects.toThrow( + expect(() => store.search({ queryText: '', queryEmbedding: [1, 0], mode: 'vector', topK: 5 })).toThrow( 'missing the body text for unit A' ) }) @@ -107,7 +107,7 @@ describe('KnowledgeIndexStore hybrid RRF fusion', () => { ) const store = new KnowledgeIndexStore(driver, vectorIndex) - const results = await store.search({ queryText: 'query', queryEmbedding: [1, 0], mode: 'hybrid', topK: 10 }) + const results = store.search({ queryText: 'query', queryEmbedding: [1, 0], mode: 'hybrid', topK: 10 }) const scoreById = Object.fromEntries(results.map((match) => [match.unitId, match.score])) expect(scoreById.A).toBeCloseTo(scoreById.B, 12) diff --git a/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.search.test.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.search.test.ts index 9ee556f6953..8bee658d7a6 100644 --- a/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.search.test.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.search.test.ts @@ -24,7 +24,7 @@ describe('KnowledgeIndexStore.search', () => { }) afterEach(async () => { - await store.close() + store.close() rmSync(tempDir, { recursive: true, force: true }) }) @@ -39,33 +39,33 @@ describe('KnowledgeIndexStore.search', () => { }) it('vector mode ranks units by cosine similarity to the query embedding', async () => { - await indexMaterial('m1', 'a.md', 'apple pie', [1, 0, 0]) - await indexMaterial('m2', 'b.md', 'banana bread', [0, 1, 0]) + indexMaterial('m1', 'a.md', 'apple pie', [1, 0, 0]) + indexMaterial('m2', 'b.md', 'banana bread', [0, 1, 0]) - const matches = await store.search({ queryText: '', queryEmbedding: [0.95, 0.05, 0], mode: 'vector', topK: 10 }) + const matches = store.search({ queryText: '', queryEmbedding: [0.95, 0.05, 0], mode: 'vector', topK: 10 }) expect(matches.map((m) => m.materialId)).toEqual(['m1', 'm2']) expect(matches[0].score).toBeGreaterThan(matches[1].score) }) it('vector mode drops a degenerate zero-norm embedding instead of ranking it first', async () => { - await indexMaterial('m1', 'a.md', 'apple pie', [1, 0, 0]) + indexMaterial('m1', 'a.md', 'apple pie', [1, 0, 0]) // A zero vector has undefined cosine distance (sqlite-vec returns NaN, coerced to // NULL). Without the `dist IS NOT NULL` guard it sorts first under `ORDER BY dist` // and scores a perfect `1 - Number(null) = 1`, outranking the real hit — so it must // be excluded entirely. - await indexMaterial('m2', 'b.md', 'banana bread', [0, 0, 0]) + indexMaterial('m2', 'b.md', 'banana bread', [0, 0, 0]) - const matches = await store.search({ queryText: '', queryEmbedding: [1, 1, 0], mode: 'vector', topK: 10 }) + const matches = store.search({ queryText: '', queryEmbedding: [1, 1, 0], mode: 'vector', topK: 10 }) expect(matches.map((m) => m.materialId)).toEqual(['m1']) }) it('bm25 mode returns only units whose body matches the query tokens', async () => { - await indexMaterial('m1', 'a.md', 'apple pie', [1, 0, 0]) - await indexMaterial('m2', 'b.md', 'banana bread', [0, 1, 0]) + indexMaterial('m1', 'a.md', 'apple pie', [1, 0, 0]) + indexMaterial('m2', 'b.md', 'banana bread', [0, 1, 0]) - const matches = await store.search({ queryText: 'banana', mode: 'bm25', topK: 10 }) + const matches = store.search({ queryText: 'banana', mode: 'bm25', topK: 10 }) expect(matches.map((m) => m.materialId)).toEqual(['m2']) }) @@ -75,7 +75,7 @@ describe('KnowledgeIndexStore.search', () => { // been exercised against a real store — every other test in this file goes through // `indexMaterial`, which always supplies a vector. const text = 'lexical only content with no embedding model' - await store.rebuildMaterial('m1', { + store.rebuildMaterial('m1', { material: { relativePath: 'a.md' }, content: { text }, units: [{ unitType: 'chunk', unitIndex: 0, charStart: 0, charEnd: text.length }], @@ -83,33 +83,33 @@ describe('KnowledgeIndexStore.search', () => { embeddings: [] }) - const matches = await store.search({ queryText: 'lexical', mode: 'bm25', topK: 10 }) + const matches = store.search({ queryText: 'lexical', mode: 'bm25', topK: 10 }) expect(matches.map((m) => m.materialId)).toEqual(['m1']) }) it('bm25 mode returns nothing when the query has no usable token', async () => { - await indexMaterial('m1', 'a.md', 'apple pie', [1, 0, 0]) + indexMaterial('m1', 'a.md', 'apple pie', [1, 0, 0]) - expect(await store.search({ queryText: '!!!', mode: 'bm25', topK: 10 })).toEqual([]) + expect(store.search({ queryText: '!!!', mode: 'bm25', topK: 10 })).toEqual([]) }) it('bm25 mode falls back to a LIKE substring scan for short CJK queries the trigram FTS cannot index', async () => { - await indexMaterial('m1', 'a.md', '今天天气很好', [1, 0, 0]) - await indexMaterial('m2', 'b.md', '我喜欢编程', [0, 1, 0]) + indexMaterial('m1', 'a.md', '今天天气很好', [1, 0, 0]) + indexMaterial('m2', 'b.md', '我喜欢编程', [0, 1, 0]) // '天气' is 2 characters → produces no trigram → a bare MATCH returns nothing. - const matches = await store.search({ queryText: '天气', mode: 'bm25', topK: 10 }) + const matches = store.search({ queryText: '天气', mode: 'bm25', topK: 10 }) expect(matches.map((m) => m.materialId)).toEqual(['m1']) }) it('LIKE fallback ANDs every token, so a mixed short+long query still filters correctly', async () => { - await indexMaterial('m1', 'a.md', '系统 architecture overview', [1, 0, 0]) - await indexMaterial('m2', 'b.md', '系统 design notes', [0, 1, 0]) + indexMaterial('m1', 'a.md', '系统 architecture overview', [1, 0, 0]) + indexMaterial('m2', 'b.md', '系统 design notes', [0, 1, 0]) // The 2-char '系统' routes the whole query to LIKE; 'architecture' must still constrain it. - const matches = await store.search({ queryText: '系统 architecture', mode: 'bm25', topK: 10 }) + const matches = store.search({ queryText: '系统 architecture', mode: 'bm25', topK: 10 }) expect(matches.map((m) => m.materialId)).toEqual(['m1']) }) @@ -120,15 +120,15 @@ describe('KnowledgeIndexStore.search', () => { // the real-DB expectations below provably exercise the trigram index. expect(needsLikeFallback('天气预报')).toBe(false) - await indexMaterial('m1', 'a.md', '明天的天气预报说有雨', [1, 0, 0]) - await indexMaterial('m2', 'b.md', '我喜欢户外编程活动', [0, 1, 0]) + indexMaterial('m1', 'a.md', '明天的天气预报说有雨', [1, 0, 0]) + indexMaterial('m2', 'b.md', '我喜欢户外编程活动', [0, 1, 0]) - const matches = await store.search({ queryText: '天气预报', mode: 'bm25', topK: 10 }) + const matches = store.search({ queryText: '天气预报', mode: 'bm25', topK: 10 }) expect(matches.map((m) => m.materialId)).toEqual(['m1']) // A 3+ char CJK query whose trigrams appear nowhere must return empty via MATCH. expect(needsLikeFallback('量子计算')).toBe(false) - expect(await store.search({ queryText: '量子计算', mode: 'bm25', topK: 10 })).toEqual([]) + expect(store.search({ queryText: '量子计算', mode: 'bm25', topK: 10 })).toEqual([]) }) it('hybrid mode lifts a short-CJK LIKE-only hit above a closer vector-only competitor', async () => { @@ -136,10 +136,10 @@ describe('KnowledgeIndexStore.search', () => { // orthogonal in vector space but matches '天气' via the LIKE fallback. The BM25 // contribution must lift m1 above m2 — drop the LIKE fallback and the order // flips to ['m2', 'm1'], so this pins the fallback's effect on hybrid ranking. - await indexMaterial('m1', 'a.md', '今天天气', [0, 1, 0]) - await indexMaterial('m2', 'b.md', 'sunny day', [1, 0, 0]) + indexMaterial('m1', 'a.md', '今天天气', [0, 1, 0]) + indexMaterial('m2', 'b.md', 'sunny day', [1, 0, 0]) - const matches = await store.search({ + const matches = store.search({ queryText: '天气', queryEmbedding: [1, 0, 0], mode: 'hybrid', @@ -151,10 +151,10 @@ describe('KnowledgeIndexStore.search', () => { it('hybrid fusion ranks a unit hit by both lanes above one hit by a single lane', async () => { // Vector favors m1; BM25 favors m2. RRF should lift m2 because it appears in both lanes. - await indexMaterial('m1', 'a.md', 'apple pie', [1, 0, 0]) - await indexMaterial('m2', 'b.md', 'banana bread', [0, 1, 0]) + indexMaterial('m1', 'a.md', 'apple pie', [1, 0, 0]) + indexMaterial('m2', 'b.md', 'banana bread', [0, 1, 0]) - const matches = await store.search({ + const matches = store.search({ queryText: 'banana', queryEmbedding: [0.95, 0.05, 0], mode: 'hybrid', @@ -165,17 +165,17 @@ describe('KnowledgeIndexStore.search', () => { }) it('honors topK', async () => { - await indexMaterial('m1', 'a.md', 'alpha text', [1, 0, 0]) - await indexMaterial('m2', 'b.md', 'beta text', [0, 1, 0]) - await indexMaterial('m3', 'c.md', 'gamma text', [0, 0, 1]) + indexMaterial('m1', 'a.md', 'alpha text', [1, 0, 0]) + indexMaterial('m2', 'b.md', 'beta text', [0, 1, 0]) + indexMaterial('m3', 'c.md', 'gamma text', [0, 0, 1]) - expect(await store.search({ queryText: '', queryEmbedding: [1, 1, 1], mode: 'vector', topK: 2 })).toHaveLength(2) + expect(store.search({ queryText: '', queryEmbedding: [1, 1, 1], mode: 'vector', topK: 2 })).toHaveLength(2) }) it('rejects vector and hybrid search without a query embedding', async () => { - await indexMaterial('m1', 'a.md', 'apple pie', [1, 0, 0]) + indexMaterial('m1', 'a.md', 'apple pie', [1, 0, 0]) - await expect(store.search({ queryText: 'apple', mode: 'vector', topK: 5 })).rejects.toThrow(/query embedding/) - await expect(store.search({ queryText: 'apple', mode: 'hybrid', topK: 5 })).rejects.toThrow(/query embedding/) + expect(() => store.search({ queryText: 'apple', mode: 'vector', topK: 5 })).toThrow(/query embedding/) + expect(() => store.search({ queryText: 'apple', mode: 'hybrid', topK: 5 })).toThrow(/query embedding/) }) }) diff --git a/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.test.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.test.ts index 5d3ba7abbb1..4d1ce5d3345 100644 --- a/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.test.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/indexStore/__tests__/KnowledgeIndexStore.test.ts @@ -47,7 +47,7 @@ describe('KnowledgeIndexStore', () => { }) afterEach(async () => { - await store.close() + store.close() rmSync(tempDir, { recursive: true, force: true }) }) @@ -68,7 +68,7 @@ describe('KnowledgeIndexStore', () => { driver.execute(`INSERT INTO search_text_fts(search_text_fts, rank) VALUES('integrity-check', 1)`) it('persists material, content, units, search_text and embeddings, then lists units in order', async () => { - await store.rebuildMaterial( + store.rebuildMaterial( 'm1', buildInput('hello world wide', [ [0, 5], @@ -76,7 +76,7 @@ describe('KnowledgeIndexStore', () => { ]) ) - const units = await store.listMaterialUnits('m1') + const units = store.listMaterialUnits('m1') expect(units.map((u) => u.text)).toEqual(['hello', 'world wide']) expect(units.map((u) => u.unitIndex)).toEqual([0, 1]) @@ -91,7 +91,7 @@ describe('KnowledgeIndexStore', () => { it('keeps body text equal to the content slice (search §5.3 invariant)', async () => { const text = 'alpha beta gamma' - await store.rebuildMaterial( + store.rebuildMaterial( 'm1', buildInput(text, [ [0, 5], @@ -100,24 +100,24 @@ describe('KnowledgeIndexStore', () => { ]) ) - for (const unit of await store.listMaterialUnits('m1')) { + for (const unit of store.listMaterialUnits('m1')) { expect(unit.text).toBe(text.slice(unit.charStart, unit.charEnd)) } }) it('throws on a unit whose body row is missing instead of fabricating an empty chunk', async () => { - await store.rebuildMaterial('m1', buildInput('alpha beta', [[0, 5]])) + store.rebuildMaterial('m1', buildInput('alpha beta', [[0, 5]])) // Corrupt the store: drop the body row out from under the unit. The same // damage silently excludes the unit from search (INNER JOIN); the list lane // must fail loudly rather than add a third symptom (existing-but-empty chunk). driver.execute(`DELETE FROM search_text WHERE target_type = 'search_unit' AND kind = 'body'`) - await expect(store.listMaterialUnits('m1')).rejects.toThrow('missing the body text for unit') + expect(() => store.listMaterialUnits('m1')).toThrow('missing the body text for unit') }) it('atomically replaces all prior units on rebuild (no old/new mix)', async () => { - await store.rebuildMaterial( + store.rebuildMaterial( 'm1', buildInput('one two three', [ [0, 3], @@ -127,8 +127,8 @@ describe('KnowledgeIndexStore', () => { ) expect(await count('search_unit')).toBe(3) - await store.rebuildMaterial('m1', buildInput('solo', [[0, 4]])) - const units = await store.listMaterialUnits('m1') + store.rebuildMaterial('m1', buildInput('solo', [[0, 4]])) + const units = store.listMaterialUnits('m1') expect(units).toHaveLength(1) expect(units[0].text).toBe('solo') expect(await count('search_unit')).toBe(1) @@ -137,7 +137,7 @@ describe('KnowledgeIndexStore', () => { it('reuses one embedding row for units sharing identical body text', async () => { // Two units with the same slice → same embedding_text_hash → one embedding row. - await store.rebuildMaterial( + store.rebuildMaterial( 'm1', buildInput('dup', [ [0, 3], @@ -150,16 +150,16 @@ describe('KnowledgeIndexStore', () => { }) it('keeps the FTS index in sync across rebuilds', async () => { - await store.rebuildMaterial('m1', buildInput('the knowledge base', [[0, 18]])) + store.rebuildMaterial('m1', buildInput('the knowledge base', [[0, 18]])) expect(await ftsMatchCount('knowledge')).toBe(1) - await store.rebuildMaterial('m1', buildInput('a different subject', [[0, 19]])) + store.rebuildMaterial('m1', buildInput('a different subject', [[0, 19]])) expect(await ftsMatchCount('knowledge')).toBe(0) expect(await ftsMatchCount('different')).toBe(1) }) it('rolls back the whole rebuild on a mid-transaction failure', async () => { - await store.rebuildMaterial( + store.rebuildMaterial( 'm1', buildInput('keep this safe', [ [0, 4], @@ -179,9 +179,9 @@ describe('KnowledgeIndexStore', () => { usesEmbeddings: true, embeddings: [{ embeddingTextHash: hashEmbeddingText('broken'), vector: [0.1, 0.2, 0.3] }] } - await expect(store.rebuildMaterial('m1', broken)).rejects.toThrow() + expect(() => store.rebuildMaterial('m1', broken)).toThrow() - const units = await store.listMaterialUnits('m1') + const units = store.listMaterialUnits('m1') expect(units.map((u) => u.text)).toEqual(['keep', 'this']) expect(await count('search_unit')).toBe(2) }) @@ -196,10 +196,10 @@ describe('KnowledgeIndexStore', () => { [0, 5], [6, 16] ]) - await store.rebuildMaterial('m1', input) - await expect(store.rebuildMaterial('m1', input)).resolves.toBeUndefined() + store.rebuildMaterial('m1', input) + expect(store.rebuildMaterial('m1', input)).toBeUndefined() - const units = await store.listMaterialUnits('m1') + const units = store.listMaterialUnits('m1') expect(units.map((u) => u.text)).toEqual(['hello', 'world wide']) expect(await count('search_unit')).toBe(2) expect(await count('search_text')).toBe(2) @@ -211,9 +211,7 @@ describe('KnowledgeIndexStore', () => { it('rejects a unit whose charEnd lies beyond the content text', async () => { // slice() would clamp silently and persist the lying offset — the store must // fail loud at write time instead of corrupting offset-based readers later. - await expect(store.rebuildMaterial('m1', buildInput('short', [[0, 99]]))).rejects.toThrow( - 'beyond the content length' - ) + expect(() => store.rebuildMaterial('m1', buildInput('short', [[0, 99]]))).toThrow('beyond the content length') expect(await count('material')).toBe(0) expect(await count('search_unit')).toBe(0) @@ -230,14 +228,14 @@ describe('KnowledgeIndexStore', () => { usesEmbeddings: false, embeddings: [{ embeddingTextHash: hashEmbeddingText('lexical only'), vector: [0.1, 0.2, 0.3] }] } - await expect(store.rebuildMaterial('m1', contradiction)).rejects.toThrow('usesEmbeddings: false') + expect(() => store.rebuildMaterial('m1', contradiction)).toThrow('usesEmbeddings: false') expect(await count('material')).toBe(0) expect(await count('search_unit')).toBe(0) }) it('rolls back a rebuild that leaves a unit embedding hash without a vector', async () => { - await store.rebuildMaterial('m1', buildInput('keep this safe', [[0, 4]])) + store.rebuildMaterial('m1', buildInput('keep this safe', [[0, 4]])) // A caller that hashes different text than the store re-slices (offset/hash // drift) supplies no vector for the unit's body — the coverage check must @@ -249,10 +247,10 @@ describe('KnowledgeIndexStore', () => { usesEmbeddings: true, embeddings: [{ embeddingTextHash: hashEmbeddingText('not the sliced body'), vector: [0.1, 0.2, 0.3] }] } - await expect(store.rebuildMaterial('m1', drifted)).rejects.toThrow('without a vector') + expect(() => store.rebuildMaterial('m1', drifted)).toThrow('without a vector') // Prior index intact (transaction rolled back). - const units = await store.listMaterialUnits('m1') + const units = store.listMaterialUnits('m1') expect(units.map((u) => u.text)).toEqual(['keep']) }) @@ -275,21 +273,21 @@ describe('KnowledgeIndexStore', () => { missingOne.embeddings = missingOne.embeddings.filter( (embedding) => embedding.embeddingTextHash !== hashEmbeddingText(words[500]) ) - await expect(store.rebuildMaterial('m1', missingOne)).rejects.toThrow('without a vector') + expect(() => store.rebuildMaterial('m1', missingOne)).toThrow('without a vector') expect(await count('material')).toBe(0) - await expect(store.rebuildMaterial('m1', buildInput(text, ranges))).resolves.toBeUndefined() + expect(store.rebuildMaterial('m1', buildInput(text, ranges))).toBeUndefined() expect(await count('search_unit')).toBe(501) expect(await count('embedding')).toBe(501) }) it('deletes a material and its derived rows, sweeping the now-orphaned embedding and content', async () => { - await store.rebuildMaterial('m1', buildInput('the knowledge base', [[0, 18]])) + store.rebuildMaterial('m1', buildInput('the knowledge base', [[0, 18]])) expect(await ftsMatchCount('knowledge')).toBe(1) await store.deleteMaterials(['m1']) - expect(await store.listMaterialUnits('m1')).toEqual([]) + expect(store.listMaterialUnits('m1')).toEqual([]) expect(await count('material')).toBe(0) expect(await count('search_unit')).toBe(0) expect(await count('search_text')).toBe(0) @@ -300,8 +298,8 @@ describe('KnowledgeIndexStore', () => { }) it('shares one content row across materials with identical content', async () => { - await store.rebuildMaterial('m1', buildInput('shared content', [[0, 14]], 'a.md')) - await store.rebuildMaterial('m2', buildInput('shared content', [[0, 14]], 'b.md')) + store.rebuildMaterial('m1', buildInput('shared content', [[0, 14]], 'a.md')) + store.rebuildMaterial('m2', buildInput('shared content', [[0, 14]], 'b.md')) expect(await count('material')).toBe(2) expect(await count('content')).toBe(1) @@ -310,8 +308,8 @@ describe('KnowledgeIndexStore', () => { it('keeps a shared embedding reachable for the remaining material after deleting one sharer', async () => { // m1 and m2 index the identical body → one embedding row, referenced by both. - await store.rebuildMaterial('m1', buildInput('shared body text', [[0, 16]], 'a.md')) - await store.rebuildMaterial('m2', buildInput('shared body text', [[0, 16]], 'b.md')) + store.rebuildMaterial('m1', buildInput('shared body text', [[0, 16]], 'a.md')) + store.rebuildMaterial('m2', buildInput('shared body text', [[0, 16]], 'b.md')) expect(await count('embedding')).toBe(1) await store.deleteMaterials(['m1']) @@ -321,16 +319,16 @@ describe('KnowledgeIndexStore', () => { // embedding when m1 was deleted would fail this behavioral assertion — the bare row // count in the delete test above cannot catch that. expect(await count('embedding')).toBe(1) - const matches = await store.search({ queryText: '', queryEmbedding: [0.1, 0.2, 0.3], mode: 'vector', topK: 10 }) + const matches = store.search({ queryText: '', queryEmbedding: [0.1, 0.2, 0.3], mode: 'vector', topK: 10 }) expect(matches.map((m) => m.materialId)).toEqual(['m2']) }) it('deleteMaterials removes the whole batch in one pass, sweeps only true orphans, and keeps the survivor searchable', async () => { // m1, m2 will be deleted; m3 survives. m1 and m3 index the IDENTICAL body, so they // share one content row and one embedding row; m2 has its own unique body. - await store.rebuildMaterial('m1', buildInput('shared knowledge body', [[0, 21]], 'a.md', [0.1, 0.2, 0.3])) - await store.rebuildMaterial('m2', buildInput('unique orphan body', [[0, 18]], 'b.md', [0.4, 0.5, 0.6])) - await store.rebuildMaterial('m3', buildInput('shared knowledge body', [[0, 21]], 'c.md', [0.1, 0.2, 0.3])) + store.rebuildMaterial('m1', buildInput('shared knowledge body', [[0, 21]], 'a.md', [0.1, 0.2, 0.3])) + store.rebuildMaterial('m2', buildInput('unique orphan body', [[0, 18]], 'b.md', [0.4, 0.5, 0.6])) + store.rebuildMaterial('m3', buildInput('shared knowledge body', [[0, 21]], 'c.md', [0.1, 0.2, 0.3])) expect(await count('material')).toBe(3) expect(await count('content')).toBe(2) // shared (m1+m3) + unique (m2) expect(await count('embedding')).toBe(2) @@ -340,8 +338,8 @@ describe('KnowledgeIndexStore', () => { await store.deleteMaterials(['m1', 'm2', 'm1']) expect(driver.execute(`SELECT material_id FROM material`).rows.map((r) => r.material_id)).toEqual(['m3']) - expect(await store.listMaterialUnits('m1')).toEqual([]) - expect(await store.listMaterialUnits('m2')).toEqual([]) + expect(store.listMaterialUnits('m1')).toEqual([]) + expect(store.listMaterialUnits('m2')).toEqual([]) // The single end-of-batch GC must sweep m2's now-orphaned body/embedding/content while // keeping the body m3 still references — i.e. the same end state N per-material GCs gave. expect(await count('content')).toBe(1) @@ -352,18 +350,16 @@ describe('KnowledgeIndexStore', () => { // the survivor m3 — whose search_text row is untouched — would drop out of keyword // search even though it is still present. It must stay both bm25- and vector-reachable. expect(await ftsMatchCount('knowledge')).toBe(1) - expect((await store.search({ queryText: 'knowledge', mode: 'bm25', topK: 10 })).map((h) => h.materialId)).toEqual([ - 'm3' - ]) + expect(store.search({ queryText: 'knowledge', mode: 'bm25', topK: 10 }).map((h) => h.materialId)).toEqual(['m3']) expect( - (await store.search({ queryText: '', queryEmbedding: [0.1, 0.2, 0.3], mode: 'vector', topK: 10 })).map( - (h) => h.materialId - ) + store + .search({ queryText: '', queryEmbedding: [0.1, 0.2, 0.3], mode: 'vector', topK: 10 }) + .map((h) => h.materialId) ).toEqual(['m3']) }) it('deleteMaterials is a no-op for an empty batch', async () => { - await store.rebuildMaterial('m1', buildInput('keep me', [[0, 7]])) + store.rebuildMaterial('m1', buildInput('keep me', [[0, 7]])) await expect(store.deleteMaterials([])).resolves.toBeUndefined() expect(await count('material')).toBe(1) }) @@ -374,9 +370,9 @@ describe('KnowledgeIndexStore', () => { // periodic yields. Prove the loop hands control back: drive the MONOTONIC performance.now() // past the time budget on every read (deterministic — no real-clock dependency) and confirm // the loop schedules a macrotask (setImmediate) per material while still deleting correctly. - await store.rebuildMaterial('m1', buildInput('alpha body one', [[0, 14]], 'a.md')) - await store.rebuildMaterial('m2', buildInput('bravo body two', [[0, 14]], 'b.md')) - await store.rebuildMaterial('m3', buildInput('gamma body six', [[0, 14]], 'c.md')) + store.rebuildMaterial('m1', buildInput('alpha body one', [[0, 14]], 'a.md')) + store.rebuildMaterial('m2', buildInput('bravo body two', [[0, 14]], 'b.md')) + store.rebuildMaterial('m3', buildInput('gamma body six', [[0, 14]], 'c.md')) // setImmediate is spied (call-through), not stubbed, so the yields still resolve. let clock = 0 @@ -402,9 +398,9 @@ describe('KnowledgeIndexStore', () => { // DELETE_YIELD_BUDGET_MS must schedule zero macrotasks. Freezing the monotonic clock keeps // every delta at 0 — so this fails if the gate is ever replaced by an unconditional // yield-per-row (the exact regression the test above cannot, on its own, rule out). - await store.rebuildMaterial('m1', buildInput('alpha body one', [[0, 14]], 'a.md')) - await store.rebuildMaterial('m2', buildInput('bravo body two', [[0, 14]], 'b.md')) - await store.rebuildMaterial('m3', buildInput('gamma body six', [[0, 14]], 'c.md')) + store.rebuildMaterial('m1', buildInput('alpha body one', [[0, 14]], 'a.md')) + store.rebuildMaterial('m2', buildInput('bravo body two', [[0, 14]], 'b.md')) + store.rebuildMaterial('m3', buildInput('gamma body six', [[0, 14]], 'c.md')) const perfNow = vi.spyOn(performance, 'now').mockReturnValue(1000) const immediate = vi.spyOn(global, 'setImmediate') @@ -427,34 +423,32 @@ describe('KnowledgeIndexStore', () => { // because search_text_fts keys on the stable fts_rowid (not the implicit rowid), m2 — whose // search_text is untouched — stays aligned. It must remain reachable, and the index intact. const hugeText = 'knowledge body filler '.repeat(600_000) - await store.rebuildMaterial('m1', buildInput(hugeText, [[0, 20]], 'big.md')) - await store.rebuildMaterial('m2', buildInput('shared knowledge body', [[0, 21]], 'keep.md')) + store.rebuildMaterial('m1', buildInput(hugeText, [[0, 20]], 'big.md')) + store.rebuildMaterial('m2', buildInput('shared knowledge body', [[0, 21]], 'keep.md')) await store.deleteMaterials(['m1']) - const outcome = await store.reclaimSpace() + const outcome = store.reclaimSpace() expect(outcome.vacuumed).toBe(true) expect(outcome.reclaimedBytes).toBeGreaterThan(0) expect(ftsIntegrityCheck()).toBeDefined() // The survivor stays both keyword- and vector-reachable after the rewrite. expect(await ftsMatchCount('knowledge')).toBe(1) - expect((await store.search({ queryText: 'knowledge', mode: 'bm25', topK: 10 })).map((h) => h.materialId)).toEqual([ - 'm2' - ]) + expect(store.search({ queryText: 'knowledge', mode: 'bm25', topK: 10 }).map((h) => h.materialId)).toEqual(['m2']) expect( - (await store.search({ queryText: '', queryEmbedding: [0.1, 0.2, 0.3], mode: 'vector', topK: 10 })).map( - (h) => h.materialId - ) + store + .search({ queryText: '', queryEmbedding: [0.1, 0.2, 0.3], mode: 'vector', topK: 10 }) + .map((h) => h.materialId) ).toEqual(['m2']) }) it('reclaimSpace skips the VACUUM (truncates the WAL only) when the freelist is below the threshold', async () => { // A tiny delete frees far less than the absolute floor, so the whole-file-rewrite block // is not worth it — reclaim just checkpoints and reports nothing reclaimed. - await store.rebuildMaterial('m1', buildInput('small knowledge body', [[0, 20]], 'a.md')) + store.rebuildMaterial('m1', buildInput('small knowledge body', [[0, 20]], 'a.md')) await store.deleteMaterials(['m1']) - const outcome = await store.reclaimSpace() + const outcome = store.reclaimSpace() expect(outcome).toEqual({ vacuumed: false, reclaimedBytes: 0 }) }) @@ -467,20 +461,18 @@ describe('KnowledgeIndexStore', () => { // before. content stays 13 MB so the freelist clears the VACUUM threshold; the 2 MB // body makes the shadow segments measurable. const hugeText = 'knowledge body filler '.repeat(600_000) - await store.rebuildMaterial('m1', buildInput(hugeText, [[0, 2_000_000]], 'big.md')) - await store.rebuildMaterial('m2', buildInput('shared knowledge body', [[0, 21]], 'keep.md')) + store.rebuildMaterial('m1', buildInput(hugeText, [[0, 2_000_000]], 'big.md')) + store.rebuildMaterial('m2', buildInput('shared knowledge body', [[0, 21]], 'keep.md')) const ftsSegmentsBefore = await count('search_text_fts_data') await store.deleteMaterials(['m1']) - const outcome = await store.reclaimSpace() + const outcome = store.reclaimSpace() expect(outcome.vacuumed).toBe(true) // optimize merged and dropped m1's dead segments instead of leaving them behind. expect(await count('search_text_fts_data')).toBeLessThan(ftsSegmentsBefore) // The survivor is still keyword-searchable after the optimize + VACUUM. - expect((await store.search({ queryText: 'shared', mode: 'bm25', topK: 10 })).map((h) => h.materialId)).toEqual([ - 'm2' - ]) + expect(store.search({ queryText: 'shared', mode: 'bm25', topK: 10 }).map((h) => h.materialId)).toEqual(['m2']) // VACUUM renumbers implicit rowids; assert the external-content FTS did NOT desync. Keyed on the // stable fts_rowid it stays aligned by construction — verified with the reliable integrity check // (the default integrity-check would pass even on a desync). @@ -492,10 +484,10 @@ describe('KnowledgeIndexStore', () => { // search_text's implicit rowid, and an external-content FTS keyed on that rowid would then // silently point at the wrong rows. search_text_fts keys on the stable fts_rowid column instead. // The index MUST be populated before the reshuffle, or it cannot expose the bug. - await store.rebuildMaterial('m1', buildInput('alpha apple body', [[0, 16]], 'a.md')) - await store.rebuildMaterial('m2', buildInput('bravo banana body', [[0, 17]], 'b.md')) - await store.rebuildMaterial('m3', buildInput('charlie cherry body', [[0, 19]], 'c.md')) - await store.rebuildMaterial('m4', buildInput('delta date body', [[0, 15]], 'd.md')) + store.rebuildMaterial('m1', buildInput('alpha apple body', [[0, 16]], 'a.md')) + store.rebuildMaterial('m2', buildInput('bravo banana body', [[0, 17]], 'b.md')) + store.rebuildMaterial('m3', buildInput('charlie cherry body', [[0, 19]], 'c.md')) + store.rebuildMaterial('m4', buildInput('delta date body', [[0, 15]], 'd.md')) // Delete a middle material to leave a rowid hole — the precondition the reshuffle needs. await store.deleteMaterials(['m2']) expect(await ftsMatchCount('cherry')).toBe(1) @@ -513,17 +505,15 @@ describe('KnowledgeIndexStore', () => { // fts_rowid was copied through the rebuild, so the index stays aligned: the reliable check passes // and the survivors resolve to the right materials (a rowid-keyed index would return wrong/empty). expect(ftsIntegrityCheck()).toBeDefined() - expect((await store.search({ queryText: 'cherry', mode: 'bm25', topK: 10 })).map((h) => h.materialId)).toEqual([ - 'm3' - ]) - expect((await store.search({ queryText: 'date', mode: 'bm25', topK: 10 })).map((h) => h.materialId)).toEqual(['m4']) + expect(store.search({ queryText: 'cherry', mode: 'bm25', topK: 10 }).map((h) => h.materialId)).toEqual(['m3']) + expect(store.search({ queryText: 'date', mode: 'bm25', topK: 10 }).map((h) => h.materialId)).toEqual(['m4']) // The reshuffle reassigned implicit rowids while leaving fts_rowid untouched and unique. expect(await count('search_text')).toBe(3) expect(Number(driver.execute(`SELECT COUNT(DISTINCT fts_rowid) AS n FROM search_text`).rows[0].n)).toBe(3) }) it('integrity-check,1 catches a NULL fts_rowid desync (and proves the detector is live)', async () => { - await store.rebuildMaterial('m1', buildInput('orphan searchable body', [[0, 22]], 'a.md')) + store.rebuildMaterial('m1', buildInput('orphan searchable body', [[0, 22]], 'a.md')) expect(await ftsMatchCount('searchable')).toBe(1) // Simulate a future bulk-insert / restore path that bypassed the AFTER INSERT trigger and left @@ -540,9 +530,9 @@ describe('KnowledgeIndexStore', () => { // current MAX and then inserting REUSES that just-freed rowid — the highest-risk path for a // stale external-content entry: the new row lands on the exact key the deleted row had, so a // delete trigger that failed to tombstone it would surface the old content under the new row. - await store.rebuildMaterial('m1', buildInput('alpha apple body', [[0, 16]], 'a.md')) - await store.rebuildMaterial('m2', buildInput('bravo banana body', [[0, 17]], 'b.md')) - await store.rebuildMaterial('m3', buildInput('charlie cherry body', [[0, 19]], 'c.md')) + store.rebuildMaterial('m1', buildInput('alpha apple body', [[0, 16]], 'a.md')) + store.rebuildMaterial('m2', buildInput('bravo banana body', [[0, 17]], 'b.md')) + store.rebuildMaterial('m3', buildInput('charlie cherry body', [[0, 19]], 'c.md')) // m3 holds the current MAX fts_rowid. Delete it to free that rowid. const maxBefore = Number(driver.execute(`SELECT MAX(fts_rowid) AS m FROM search_text`).rows[0].m) @@ -550,7 +540,7 @@ describe('KnowledgeIndexStore', () => { expect(await ftsMatchCount('cherry')).toBe(0) // The next insert's MAX(fts_rowid)+1 reuses the value m3 just vacated. - await store.rebuildMaterial('m4', buildInput('delta dragon body', [[0, 17]], 'd.md')) + store.rebuildMaterial('m4', buildInput('delta dragon body', [[0, 17]], 'd.md')) const reused = Number( driver.execute(`SELECT fts_rowid FROM search_text WHERE text LIKE 'delta%'`).rows[0].fts_rowid ) @@ -558,49 +548,47 @@ describe('KnowledgeIndexStore', () => { // The reused rowid resolves to m4 (not stale m3 content), the deleted term stays gone, and the // external-content index is consistent under the reliable detector. - expect((await store.search({ queryText: 'dragon', mode: 'bm25', topK: 10 })).map((h) => h.materialId)).toEqual([ - 'm4' - ]) + expect(store.search({ queryText: 'dragon', mode: 'bm25', topK: 10 }).map((h) => h.materialId)).toEqual(['m4']) expect(await ftsMatchCount('cherry')).toBe(0) expect(ftsIntegrityCheck()).toBeDefined() }) it('keeps a shared embedding reachable for the other material after rebuilding the one that introduced it', async () => { // m1 and m2 index the identical body → one shared embedding row, referenced by both. - await store.rebuildMaterial('m1', buildInput('shared body text', [[0, 16]], 'a.md')) - await store.rebuildMaterial('m2', buildInput('shared body text', [[0, 16]], 'b.md')) + store.rebuildMaterial('m1', buildInput('shared body text', [[0, 16]], 'a.md')) + store.rebuildMaterial('m2', buildInput('shared body text', [[0, 16]], 'b.md')) expect(await count('embedding')).toBe(1) // Rebuild m1 with unrelated content carrying a distinct vector, so m1 no longer // references the shared embedding — only m2 does now. - await store.rebuildMaterial('m1', buildInput('rebuilt unrelated body', [[0, 22]], 'a.md', [0.9, 0.8, 0.7])) + store.rebuildMaterial('m1', buildInput('rebuilt unrelated body', [[0, 22]], 'a.md', [0.9, 0.8, 0.7])) // The shared embedding must survive (m2 still references it), alongside m1's new // one → 2 rows. GC dropping the now-singly-referenced shared embedding on rebuild // would unjoin m2 from vector search below — the bare row count cannot catch that, // but searching the shared vector can. expect(await count('embedding')).toBe(2) - const matches = await store.search({ queryText: '', queryEmbedding: [0.1, 0.2, 0.3], mode: 'vector', topK: 10 }) + const matches = store.search({ queryText: '', queryEmbedding: [0.1, 0.2, 0.3], mode: 'vector', topK: 10 }) expect(matches.map((m) => m.materialId)).toContain('m2') }) it('sweeps the orphaned embedding and content when a material is rebuilt with new content', async () => { - await store.rebuildMaterial('m1', buildInput('original body', [[0, 13]], 'a.md', [0.1, 0.2, 0.3])) + store.rebuildMaterial('m1', buildInput('original body', [[0, 13]], 'a.md', [0.1, 0.2, 0.3])) expect(await count('embedding')).toBe(1) expect(await count('content')).toBe(1) // Rebuild with unrelated content + a distinct vector; nothing else references the // old embedding or old content, so GC must remove both, leaving only the new ones. - await store.rebuildMaterial('m1', buildInput('replacement body', [[0, 16]], 'a.md', [0.9, 0.8, 0.7])) + store.rebuildMaterial('m1', buildInput('replacement body', [[0, 16]], 'a.md', [0.9, 0.8, 0.7])) expect(await count('embedding')).toBe(1) expect(await count('content')).toBe(1) - const units = await store.listMaterialUnits('m1') + const units = store.listMaterialUnits('m1') expect(units.map((u) => u.text)).toEqual(['replacement body']) }) it('listExistingEmbeddingHashes reports only the hashes already stored', async () => { - await store.rebuildMaterial( + store.rebuildMaterial( 'm1', buildInput('alpha bravo', [ [0, 5], @@ -610,7 +598,7 @@ describe('KnowledgeIndexStore', () => { const stored = hashEmbeddingText('alpha') const absent = hashEmbeddingText('charlie') - const existing = await store.listExistingEmbeddingHashes([stored, absent]) + const existing = store.listExistingEmbeddingHashes([stored, absent]) expect(existing.has(stored)).toBe(true) expect(existing.has(absent)).toBe(false) @@ -618,24 +606,24 @@ describe('KnowledgeIndexStore', () => { }) it('listExistingEmbeddingHashes returns an empty set for empty input', async () => { - expect((await store.listExistingEmbeddingHashes([])).size).toBe(0) + expect(store.listExistingEmbeddingHashes([]).size).toBe(0) }) describe('deep-read lookups', () => { it('resolves a material by its relative path (the Concept ID) and returns null for an unknown path', async () => { - await store.rebuildMaterial('m1', buildInput('hello world', [[0, 11]], 'docs/intro.md')) + store.rebuildMaterial('m1', buildInput('hello world', [[0, 11]], 'docs/intro.md')) - const ref = await store.getMaterialByRelativePath('docs/intro.md') + const ref = store.getMaterialByRelativePath('docs/intro.md') expect(ref).not.toBeNull() expect(ref?.materialId).toBe('m1') expect(ref?.relativePath).toBe('docs/intro.md') - expect(await store.getMaterialByRelativePath('docs/missing.md')).toBeNull() + expect(store.getMaterialByRelativePath('docs/missing.md')).toBeNull() }) it('reads back the full content text a material was indexed from', async () => { const text = 'the quick brown fox jumps over the lazy dog' - await store.rebuildMaterial( + store.rebuildMaterial( 'm1', buildInput(text, [ [0, 19], @@ -643,12 +631,12 @@ describe('KnowledgeIndexStore', () => { ]) ) - expect(await store.readMaterialContent('m1')).toBe(text) + expect(store.readMaterialContent('m1')).toBe(text) }) it('keeps content readable as the exact source the units were sliced from', async () => { const text = 'alpha beta gamma' - await store.rebuildMaterial( + store.rebuildMaterial( 'm1', buildInput(text, [ [0, 5], @@ -656,14 +644,14 @@ describe('KnowledgeIndexStore', () => { ]) ) - const content = await store.readMaterialContent('m1') - for (const unit of await store.listMaterialUnits('m1')) { + const content = store.readMaterialContent('m1') + for (const unit of store.listMaterialUnits('m1')) { expect(content?.slice(unit.charStart, unit.charEnd)).toBe(unit.text) } }) it('returns null content for an unknown material', async () => { - expect(await store.readMaterialContent('nope')).toBeNull() + expect(store.readMaterialContent('nope')).toBeNull() }) }) }) From f8132f38d10427d8b434f16da78209cc15a572a7 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 11:17:31 +0800 Subject: [PATCH 15/37] refactor(knowledge): drop now-pointless await on synchronous index-store calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-on to the KnowledgeIndexStore async→sync conversion: every call site still awaited rebuildMaterial/listExistingEmbeddingHashes/reclaimSpace/close, which oxlint's await-thenable rule now flags since the return values are no longer Promises. Drop the await at each site (indexDocumentsJobHandler, KnowledgeVectorStoreService.closeStoreInstance — now itself sync, vectorCleanup.reclaimKnowledgeIndexSpace, KnowledgeVectorMigrator), converting the wrapping function to sync too where it had no other awaits left. An unawaited call also changes how a mocked rejection surfaces: a mockRejectedValue Promise that is no longer awaited becomes an unhandled rejection instead of a synchronous throw, so a surrounding try/catch written for the now-sync method never fires. Test mocks for these methods switch from mockResolvedValue/mockRejectedValue to mockReturnValue/mockImplementation(() => { throw ... }) to match. Signed-off-by: eeee0717 Signed-off-by: eeee0717 --- .../v2/migrators/KnowledgeVectorMigrator.ts | 2 +- .../__tests__/KnowledgeVectorMigrator.test.ts | 24 ++++++++++++++----- .../__tests__/KnowledgeService.test.ts | 10 ++++---- .../KnowledgeVectorStoreService.ts | 8 +++---- .../KnowledgeVectorStoreService.test.ts | 4 +++- .../__tests__/vectorCleanup.test.ts | 14 ++++++----- .../pipeline/vectorstore/vectorCleanup.ts | 2 +- .../indexDocumentsJobHandler.test.ts | 8 ++++--- .../tasks/__tests__/jobHandlerTestUtils.ts | 6 ++--- .../tasks/indexDocumentsJobHandler.ts | 4 ++-- 10 files changed, 51 insertions(+), 31 deletions(-) diff --git a/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts b/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts index f4fd006b398..8d0d7fe3fda 100644 --- a/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts +++ b/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts @@ -991,7 +991,7 @@ export class KnowledgeVectorMigrator extends BaseMigrator { const store = new KnowledgeIndexStore(driver, betterSqlite3VectorIndex) for (const material of plan.materials) { - await store.rebuildMaterial(material.itemId, material.input) + store.rebuildMaterial(material.itemId, material.input) processedWork += 1 this.reportRebuildProgress(processedWork, totalWork) await yieldToEventLoop() diff --git a/src/main/data/migration/v2/migrators/__tests__/KnowledgeVectorMigrator.test.ts b/src/main/data/migration/v2/migrators/__tests__/KnowledgeVectorMigrator.test.ts index 47d26473c9b..3d1df3fed24 100644 --- a/src/main/data/migration/v2/migrators/__tests__/KnowledgeVectorMigrator.test.ts +++ b/src/main/data/migration/v2/migrators/__tests__/KnowledgeVectorMigrator.test.ts @@ -1513,7 +1513,9 @@ describe('KnowledgeVectorMigrator', () => { expect([...migrator.directoryItemsToDegrade]).toEqual([]) expect(materialItemIds(migrator)).toEqual([CHILD_A]) - vi.spyOn(KnowledgeIndexStore.prototype, 'rebuildMaterial').mockRejectedValueOnce(new Error('rebuild failed')) + vi.spyOn(KnowledgeIndexStore.prototype, 'rebuildMaterial').mockImplementationOnce(() => { + throw new Error('rebuild failed') + }) const executeResult = await migrator.execute(migrationCtx as any) expect(executeResult.success).toBe(true) @@ -1883,7 +1885,9 @@ describe('KnowledgeVectorMigrator', () => { const migrator = new KnowledgeVectorMigrator() as any expect((await migrator.prepare(migrationCtx as any)).success).toBe(true) - vi.spyOn(KnowledgeIndexStore.prototype, 'rebuildMaterial').mockRejectedValueOnce(new Error('rebuild failed')) + vi.spyOn(KnowledgeIndexStore.prototype, 'rebuildMaterial').mockImplementationOnce(() => { + throw new Error('rebuild failed') + }) // A per-base failure is non-fatal (P1-6): execute succeeds overall, the failed base is left // out of successfulBaseIds (so validate never checks it), and its error surfaces as a warning @@ -1944,7 +1948,9 @@ describe('KnowledgeVectorMigrator', () => { expect((await migrator.prepare(migrationCtx as any)).success).toBe(true) // The rebuild fails (the real error), sending execute into its catch block... - vi.spyOn(KnowledgeIndexStore.prototype, 'rebuildMaterial').mockRejectedValueOnce(new Error('rebuild failed')) + vi.spyOn(KnowledgeIndexStore.prototype, 'rebuildMaterial').mockImplementationOnce(() => { + throw new Error('rebuild failed') + }) // ...where the partial-store cleanup itself also throws (e.g. a Windows-locked index.sqlite). // The first call (pre-build target clear) must still succeed so the rebuild is reached. migrator.removeIndexStoreFiles = vi @@ -2046,7 +2052,9 @@ describe('KnowledgeVectorMigrator', () => { const migrator = new KnowledgeVectorMigrator() as any expect((await migrator.prepare(migrationCtx as any)).success).toBe(true) - vi.spyOn(KnowledgeIndexStore.prototype, 'rebuildMaterial').mockRejectedValueOnce(new Error('rebuild failed')) + vi.spyOn(KnowledgeIndexStore.prototype, 'rebuildMaterial').mockImplementationOnce(() => { + throw new Error('rebuild failed') + }) const executeResult = await migrator.execute(migrationCtx as any) expect(executeResult.success).toBe(true) @@ -2094,7 +2102,9 @@ describe('KnowledgeVectorMigrator', () => { // Fail the rebuild itself: the per-base catch marks the base failed when its store never // finished building (storePromoted stays false). - vi.spyOn(KnowledgeIndexStore.prototype, 'rebuildMaterial').mockRejectedValueOnce(new Error('rebuild failed')) + vi.spyOn(KnowledgeIndexStore.prototype, 'rebuildMaterial').mockImplementationOnce(() => { + throw new Error('rebuild failed') + }) const executeResult = await migrator.execute(migrationCtx as any) expect(executeResult.success).toBe(true) @@ -2182,7 +2192,9 @@ describe('KnowledgeVectorMigrator', () => { // real rebuild so its store is written for real. const realRebuild = KnowledgeIndexStore.prototype.rebuildMaterial vi.spyOn(KnowledgeIndexStore.prototype, 'rebuildMaterial') - .mockRejectedValueOnce(new Error('base a rebuild failed')) + .mockImplementationOnce(() => { + throw new Error('base a rebuild failed') + }) .mockImplementation(function (this: KnowledgeIndexStore, ...args: Parameters) { return realRebuild.apply(this, args) }) diff --git a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts index b2fcb0b724b..492d8ee3649 100644 --- a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts +++ b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts @@ -369,7 +369,7 @@ describe('KnowledgeService', () => { getMaterialByRelativePath: getMaterialByRelativePathMock, readMaterialContent: readMaterialContentMock }) - listMaterialUnitsMock.mockResolvedValue([]) + listMaterialUnitsMock.mockReturnValue([]) storeSearchMock.mockResolvedValue([]) getMaterialByRelativePathMock.mockResolvedValue(null) readMaterialContentMock.mockResolvedValue(null) @@ -2361,7 +2361,7 @@ describe('KnowledgeService', () => { it('lists chunks after checking item ownership', async () => { const service = new KnowledgeService() knowledgeItemGetByIdMock.mockReturnValue(createNoteItem('note-1', 'kb-1', null, 'completed')) - listMaterialUnitsMock.mockResolvedValueOnce([ + listMaterialUnitsMock.mockReturnValueOnce([ { unitId: 'chunk-1', materialId: 'note-1', @@ -2386,7 +2386,7 @@ describe('KnowledgeService', () => { knowledgeItemGetSubtreeItemsMock .mockReturnValueOnce([createNoteItem('note-1', 'kb-1', 'dir-1', 'completed')]) .mockReturnValueOnce([createNoteItem('note-1', 'kb-1', 'dir-1', 'completed')]) - listMaterialUnitsMock.mockResolvedValueOnce([ + listMaterialUnitsMock.mockReturnValueOnce([ { unitId: 'chunk-1', materialId: 'note-1', @@ -2438,7 +2438,9 @@ describe('KnowledgeService', () => { knowledgeItemGetByIdMock.mockReturnValue(createNoteItem('note-1', 'kb-1', null, 'completed')) getIndexStoreMock.mockResolvedValueOnce({ search: storeSearchMock, - listMaterialUnits: vi.fn().mockRejectedValue(new Error('Knowledge index store driver is closed')), + listMaterialUnits: vi.fn().mockImplementation(() => { + throw new Error('Knowledge index store driver is closed') + }), isClosed: () => true }) diff --git a/src/main/features/knowledge/pipeline/vectorstore/KnowledgeVectorStoreService.ts b/src/main/features/knowledge/pipeline/vectorstore/KnowledgeVectorStoreService.ts index bad056f63dc..8a021a55f12 100644 --- a/src/main/features/knowledge/pipeline/vectorstore/KnowledgeVectorStoreService.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/KnowledgeVectorStoreService.ts @@ -94,7 +94,7 @@ export class KnowledgeVectorStoreService extends BaseService { const store = this.instanceCache.get(baseId) try { - await this.closeStoreInstance(store) + this.closeStoreInstance(store) await deleteKnowledgeBaseDir(baseId) logger.info('Deleted knowledge index store', { baseId, hadCachedStore: Boolean(store) }) } finally { @@ -109,7 +109,7 @@ export class KnowledgeVectorStoreService extends BaseService { try { for (const [baseId, store] of this.instanceCache.entries()) { try { - await this.closeStoreInstance(store) + this.closeStoreInstance(store) } catch (error) { logger.error('Failed to close knowledge index store', error as Error, { baseId }) } @@ -198,10 +198,10 @@ export class KnowledgeVectorStoreService extends BaseService { } } - private async closeStoreInstance(store: KnowledgeIndexStore | undefined): Promise { + private closeStoreInstance(store: KnowledgeIndexStore | undefined): void { if (!store) { return } - await store.close() + store.close() } } diff --git a/src/main/features/knowledge/pipeline/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts b/src/main/features/knowledge/pipeline/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts index 09b036d519a..fb12ba1e868 100644 --- a/src/main/features/knowledge/pipeline/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts @@ -384,7 +384,9 @@ describe('KnowledgeVectorStoreService', () => { const first = await service.getIndexStore(createBase('kb-1')) const second = await service.getIndexStore(createBase('kb-2')) const closeError = new Error('close failed') - ;(first.close as unknown as ReturnType).mockRejectedValueOnce(closeError) + ;(first.close as unknown as ReturnType).mockImplementationOnce(() => { + throw closeError + }) await expect((service as any).onStop()).resolves.toBeUndefined() diff --git a/src/main/features/knowledge/pipeline/vectorstore/__tests__/vectorCleanup.test.ts b/src/main/features/knowledge/pipeline/vectorstore/__tests__/vectorCleanup.test.ts index 929bc1aec1d..1026bf6b3f6 100644 --- a/src/main/features/knowledge/pipeline/vectorstore/__tests__/vectorCleanup.test.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/__tests__/vectorCleanup.test.ts @@ -65,7 +65,7 @@ describe('deleteKnowledgeItemVectors', () => { reclaimSpace: reclaimSpaceMock }) deleteMaterialsMock.mockResolvedValue(undefined) - reclaimSpaceMock.mockResolvedValue({ vacuumed: false, reclaimedBytes: 0 }) + reclaimSpaceMock.mockReturnValue({ vacuumed: false, reclaimedBytes: 0 }) }) it('skips cleanup when no vector store exists', async () => { @@ -101,7 +101,7 @@ describe('reclaimKnowledgeIndexSpace', () => { deleteMaterials: deleteMaterialsMock, reclaimSpace: reclaimSpaceMock }) - reclaimSpaceMock.mockResolvedValue({ vacuumed: true, reclaimedBytes: 1024 }) + reclaimSpaceMock.mockReturnValue({ vacuumed: true, reclaimedBytes: 1024 }) }) it('skips reclaim when no vector store exists', async () => { @@ -121,7 +121,9 @@ describe('reclaimKnowledgeIndexSpace', () => { it('never throws when reclaim fails — the delete already succeeded', async () => { // Best-effort: a transient reclaim failure must not fail the delete job whose rows // and vectors are already gone; the freed pages just wait for a later index to reuse. - reclaimSpaceMock.mockRejectedValueOnce(new Error('database is locked')) + reclaimSpaceMock.mockImplementationOnce(() => { + throw new Error('database is locked') + }) await expect(reclaimKnowledgeIndexSpace(createBase())).resolves.toBeUndefined() // A transient failure stays a warn — not the corruption-class error path. @@ -133,9 +135,9 @@ describe('reclaimKnowledgeIndexSpace', () => { // reclaim's whole-file checkpoint/optimize/VACUUM is often the first op to touch the full file // post-delete, so SQLITE_CORRUPT/SQLITE_NOTADB surfaces here — it must be loud, not buried in the // benign "failed to reclaim" warn, yet still swallowed (the delete already succeeded). - reclaimSpaceMock.mockRejectedValueOnce( - Object.assign(new Error('database disk image is malformed'), { code: 'SQLITE_CORRUPT' }) - ) + reclaimSpaceMock.mockImplementationOnce(() => { + throw Object.assign(new Error('database disk image is malformed'), { code: 'SQLITE_CORRUPT' }) + }) await expect(reclaimKnowledgeIndexSpace(createBase())).resolves.toBeUndefined() expect(loggerErrorMock).toHaveBeenCalledWith( diff --git a/src/main/features/knowledge/pipeline/vectorstore/vectorCleanup.ts b/src/main/features/knowledge/pipeline/vectorstore/vectorCleanup.ts index 0d916567329..9396308f6db 100644 --- a/src/main/features/knowledge/pipeline/vectorstore/vectorCleanup.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/vectorCleanup.ts @@ -48,7 +48,7 @@ export async function reclaimKnowledgeIndexSpace(base: KnowledgeBase): Promise { loadKnowledgeItemDocumentsMock.mockResolvedValueOnce(distinctDocuments()) // 'bravo' is already in the index; reindexing must not re-embed it. const storedHash = hashEmbeddingText('bravo') - listExistingEmbeddingHashesMock.mockResolvedValueOnce(new Set([storedHash])) + listExistingEmbeddingHashesMock.mockReturnValueOnce(new Set([storedHash])) await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID, parentJobId: null })) @@ -116,7 +116,7 @@ describe('index-documents job handler', () => { knowledgeItemGetByIdMock.mockReturnValue(createNoteItem(NOTE_ITEM_ID)) knowledgeItemUpdateStatusMock.mockReturnValue(createNoteItem(NOTE_ITEM_ID)) loadKnowledgeItemDocumentsMock.mockResolvedValueOnce(distinctDocuments()) - listExistingEmbeddingHashesMock.mockResolvedValueOnce(new Set(DISTINCT_DOCS.map(hashEmbeddingText))) + listExistingEmbeddingHashesMock.mockReturnValueOnce(new Set(DISTINCT_DOCS.map(hashEmbeddingText))) await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID, parentJobId: null })) @@ -232,7 +232,9 @@ describe('index-documents job handler', () => { it('does not mark completed when vector replacement fails', async () => { const handler = createIndexDocumentsJobHandler(knowledgeLockManager as never) knowledgeItemGetByIdMock.mockReturnValue(createNoteItem(NOTE_ITEM_ID)) - rebuildMaterialMock.mockRejectedValueOnce(new Error('vector write failed')) + rebuildMaterialMock.mockImplementationOnce(() => { + throw new Error('vector write failed') + }) await expect( handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID, parentJobId: null })) diff --git a/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts b/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts index cbcd8af3a47..828d6a0ccdb 100644 --- a/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts +++ b/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts @@ -377,11 +377,11 @@ beforeEach(() => { } getIndexStoreMock.mockResolvedValue(indexStore) getIndexStoreIfExistsMock.mockResolvedValue(indexStore) - rebuildMaterialMock.mockResolvedValue(undefined) + rebuildMaterialMock.mockReturnValue(undefined) deleteMaterialsMock.mockResolvedValue(undefined) - reclaimSpaceMock.mockResolvedValue({ vacuumed: false, reclaimedBytes: 0 }) + reclaimSpaceMock.mockReturnValue({ vacuumed: false, reclaimedBytes: 0 }) // No vectors stored yet by default → every chunk is embedded (prior behavior). - listExistingEmbeddingHashesMock.mockResolvedValue(new Set()) + listExistingEmbeddingHashesMock.mockReturnValue(new Set()) embedKnowledgeTextsMock.mockImplementation(async (_base: KnowledgeBase, values: string[]) => values.map(fakeEmbedVector) ) diff --git a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts index af2a4971f05..8b9900f4f0e 100644 --- a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts +++ b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts @@ -270,7 +270,7 @@ async function buildRebuildMaterialInput( if (usesEmbeddings) { const vectorStoreService = application.get('KnowledgeVectorStoreService') const store = await vectorStoreService.getIndexStore(base) - const existingHashes = await store.listExistingEmbeddingHashes([...bodyByHash.keys()]) + const existingHashes = store.listExistingEmbeddingHashes([...bodyByHash.keys()]) const missing = [...bodyByHash.entries()].filter(([hash]) => !existingHashes.has(hash)) const vectors = await embedKnowledgeTexts( base, @@ -316,7 +316,7 @@ async function writeItemMaterial( const vectorStoreService = application.get('KnowledgeVectorStoreService') const store = await vectorStoreService.getIndexStore(base) - await store.rebuildMaterial(itemId, input) + store.rebuildMaterial(itemId, input) knowledgeItemService.updateStatus(itemId, 'completed') }) } From da6473656386886bb3ff5754b912ab0aca0f6642 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 11:17:43 +0800 Subject: [PATCH 16/37] perf(knowledge-index): cache prepared statements in BetterSqlite3Driver execute() called this.db.prepare(sql) on every call, recompiling the same SQL text repeatedly for hot paths like the batched embedding-hash scan. Cache compiled Statement objects by SQL text; better-sqlite3 transparently recompiles a cached statement if the underlying schema changes, and assertOpen() already guards against use after close. Signed-off-by: eeee0717 Signed-off-by: eeee0717 --- .../pipeline/vectorstore/indexStore/BetterSqlite3Driver.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/features/knowledge/pipeline/vectorstore/indexStore/BetterSqlite3Driver.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/BetterSqlite3Driver.ts index 840e9491073..783e90e4c78 100644 --- a/src/main/features/knowledge/pipeline/vectorstore/indexStore/BetterSqlite3Driver.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/indexStore/BetterSqlite3Driver.ts @@ -58,12 +58,17 @@ function toBindable(value: SqlValue): Bindable { */ export class BetterSqlite3Driver implements SqliteDriver { private closed = false + private readonly stmtCache = new Map() constructor(private readonly db: Database.Database) {} execute(sql: string, args: SqlValue[] = []): SqlQueryResult { this.assertOpen() - const stmt = this.db.prepare(sql) + let stmt = this.stmtCache.get(sql) + if (!stmt) { + stmt = this.db.prepare(sql) + this.stmtCache.set(sql, stmt) + } const bound = args.map(toBindable) // `reader` is true for statements that yield rows (SELECT, row-returning PRAGMA). // run() throws on those and all() throws on non-row statements, so split on it. From 77cf2a76169b451157f7b55758f15539d398dab8 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 11:17:54 +0800 Subject: [PATCH 17/37] refactor(knowledge): drop dead exports and duplicate types - items.ts: delete ContainerKnowledgeItemType (zero references) - pathStorage.ts: de-export getKnowledgeBaseMetaDir (only used in-file) - types.ts: de-export KnowledgeWorkflowJobType (only used in-file) - KnowledgeReader.ts: delete ReadableKnowledgeItem, a duplicate of items.ts's IndexableKnowledgeItem; loadKnowledgeItemDocuments now takes the latter - indexStore/types.ts: delete VectorMatch (zero consumers) Signed-off-by: eeee0717 Signed-off-by: eeee0717 --- src/main/features/knowledge/items.ts | 1 - src/main/features/knowledge/pathStorage.ts | 2 +- .../features/knowledge/pipeline/readers/KnowledgeReader.ts | 7 +++---- .../knowledge/pipeline/vectorstore/indexStore/types.ts | 6 ------ src/main/features/knowledge/types.ts | 2 +- 5 files changed, 5 insertions(+), 13 deletions(-) diff --git a/src/main/features/knowledge/items.ts b/src/main/features/knowledge/items.ts index c2d7b2c8cdd..494cc25897d 100644 --- a/src/main/features/knowledge/items.ts +++ b/src/main/features/knowledge/items.ts @@ -6,7 +6,6 @@ import { probeKnowledgeFile, probeKnowledgeSourcePath } from './pathStorage' export type IndexableKnowledgeItem = KnowledgeItemOf<'file' | 'url' | 'note'> export type ContainerKnowledgeItem = KnowledgeItemOf<'directory'> -export type ContainerKnowledgeItemType = ContainerKnowledgeItem['type'] export function isIndexableKnowledgeItem(item: KnowledgeItem): item is IndexableKnowledgeItem { return item.type === 'file' || item.type === 'url' || item.type === 'note' diff --git a/src/main/features/knowledge/pathStorage.ts b/src/main/features/knowledge/pathStorage.ts index 8319395e183..d447f2222ab 100644 --- a/src/main/features/knowledge/pathStorage.ts +++ b/src/main/features/knowledge/pathStorage.ts @@ -39,7 +39,7 @@ export function getKnowledgeMaterialDir(baseId: string): FilePath { return path.join(getKnowledgeBaseDir(baseId), MATERIAL_ROOT_DIR) as FilePath } -export function getKnowledgeBaseMetaDir(baseId: string): FilePath { +function getKnowledgeBaseMetaDir(baseId: string): FilePath { return path.join(getKnowledgeBaseDir(baseId), CHERRY_META_DIR) as FilePath } diff --git a/src/main/features/knowledge/pipeline/readers/KnowledgeReader.ts b/src/main/features/knowledge/pipeline/readers/KnowledgeReader.ts index b7866448309..5818bdeb02d 100644 --- a/src/main/features/knowledge/pipeline/readers/KnowledgeReader.ts +++ b/src/main/features/knowledge/pipeline/readers/KnowledgeReader.ts @@ -1,12 +1,11 @@ -import type { KnowledgeItem, KnowledgeItemOf } from '@shared/data/types/knowledge' +import type { KnowledgeItem } from '@shared/data/types/knowledge' import type { Document } from '@vectorstores/core' +import type { IndexableKnowledgeItem } from '../../items' import { loadFileDocuments } from './KnowledgeFileReader' import { loadSnapshotDocuments } from './KnowledgeSnapshotReader' -export type ReadableKnowledgeItem = KnowledgeItemOf<'file'> | KnowledgeItemOf<'url'> | KnowledgeItemOf<'note'> - -export async function loadKnowledgeItemDocuments(item: ReadableKnowledgeItem): Promise { +export async function loadKnowledgeItemDocuments(item: IndexableKnowledgeItem): Promise { switch (item.type) { case 'file': return await loadFileDocuments(item) diff --git a/src/main/features/knowledge/pipeline/vectorstore/indexStore/types.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/types.ts index 037d3eaab9a..5622290829d 100644 --- a/src/main/features/knowledge/pipeline/vectorstore/indexStore/types.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/indexStore/types.ts @@ -72,12 +72,6 @@ export interface SqliteDriver extends SqliteExecutor { close(): void } -/** One brute-force vector match: an embedding row and its distance to the query. */ -export interface VectorMatch { - embeddingTextHash: string - distance: number -} - /** * Engine-specific vector primitives. The store composes the brute-force scan * (`SELECT … ORDER BY dist LIMIT k`) over the plain-BLOB `embedding.vector_blob` diff --git a/src/main/features/knowledge/types.ts b/src/main/features/knowledge/types.ts index 7f431ab289a..9c7acdb01b3 100644 --- a/src/main/features/knowledge/types.ts +++ b/src/main/features/knowledge/types.ts @@ -1,6 +1,6 @@ import type { JobType } from '@main/core/job/jobRegistry' -export type KnowledgeWorkflowJobType = Extract +type KnowledgeWorkflowJobType = Extract export const KNOWLEDGE_JOB_TYPES = [ 'knowledge.prepare-root', 'knowledge.index-documents', From d4f93adecf0218861cf6916695466db08272d84b Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 11:18:04 +0800 Subject: [PATCH 18/37] refactor(knowledge-tasks): drop dead chunkItemDocuments wrapper chunkItemDocuments was a same-signature pass-through to chunkKnowledgeDocuments with no added behavior; call chunkKnowledgeDocuments directly. Signed-off-by: eeee0717 Signed-off-by: eeee0717 --- .../features/knowledge/tasks/indexDocumentsJobHandler.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts index 8b9900f4f0e..2d761438bc7 100644 --- a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts +++ b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts @@ -72,7 +72,7 @@ export function createIndexDocumentsJobHandler( // lock; these phases can be slow and do not mutate shared state. const readableItem = await ensureSnapshot(ctx, item, knowledgeLockManager) const documents = await readItemDocuments(ctx, readableItem) - const chunked = chunkItemDocuments(base, documents) + const chunked = chunkKnowledgeDocuments(base, documents) if (chunked.chunks.length === 0) { // Deliberate: the item still completes (an empty material is written) so the // UI doesn't show a stuck/failed item, but leave a trace — an image-only PDF @@ -238,10 +238,6 @@ async function ensureSnapshot( }) } -function chunkItemDocuments(base: KnowledgeBase, documents: LoadedDocuments): ChunkedKnowledgeContent { - return chunkKnowledgeDocuments(base, documents) -} - /** * Embed the distinct chunk bodies and assemble the atomic rebuild input. Bodies * are deduped by embedding-text hash so identical chunks are embedded once; the From 0b47187b9756f64815d1f00b0525147d541425fa Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 11:18:13 +0800 Subject: [PATCH 19/37] refactor(knowledge): drop unreachable branch in directory node expansion expandDirectoryNode's `if (node.type !== 'folder') return null` guard was unreachable: the preceding branch already returns for the 'file' case, so TS statically knows node.type is 'folder' at this point. Signed-off-by: eeee0717 Signed-off-by: eeee0717 --- src/main/features/knowledge/pipeline/sources/directory.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/features/knowledge/pipeline/sources/directory.ts b/src/main/features/knowledge/pipeline/sources/directory.ts index fff213d39dc..b7589873643 100644 --- a/src/main/features/knowledge/pipeline/sources/directory.ts +++ b/src/main/features/knowledge/pipeline/sources/directory.ts @@ -110,10 +110,6 @@ async function expandDirectoryNode( } } - if (node.type !== 'folder') { - return null - } - const children: ExpandedDirectoryNode[] = [] for (const child of node.children ?? []) { From 31c5e5ee12104755e461c36c216c6a3558afc295 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 11:18:37 +0800 Subject: [PATCH 20/37] refactor(knowledge-tasks): move prepare-root item expansion out of pipeline/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pipeline/sources/prepare.ts's prepareKnowledgeItem/prepareDirectoryForRuntime/ createDirectoryChildren/createRuntimeItem created knowledge_item rows and flipped item status directly — violating the feature README's invariant that pipeline/ code doesn't enqueue jobs or mutate item status. Its only consumer, prepareRootJobHandler's scanRootItem (already inside withBaseMutationLock), threaded that through two no-op pass-through params: onCreatedItem (always `() => {}`) and runMutation (always an identity `async (task) => await task()`), a leftover from an earlier abstraction layer. Move the file to tasks/prepareItem.ts (its only consumer already lives in tasks/) and drop the two dead params, calling knowledgeItemService directly (better-sqlite3 is synchronous, so no wrapper was ever needed). Also reuse purgeKnowledgeSubtreeWithinLock (already used by deleteSubtreeJobHandler) in deletePreviousLeafExpansion instead of a hand-rolled delete-vectors/best-effort-delete-files/delete-rows sequence — the two now do the exact same three steps in the same order, just no longer duplicated. purgeKnowledgeSubtreeWithinLock short-circuits for an empty subtree without calling deleteItemsByIds, unlike the old code which always called it (even with []); this is a no-op-equivalent behavior change (an empty-id-list delete does nothing either way), reflected in prepareRootJobHandler.test.ts's updated assertion. Signed-off-by: eeee0717 Signed-off-by: eeee0717 --- src/main/features/knowledge/README.md | 2 +- .../tasks/__tests__/jobHandlerTestUtils.ts | 2 +- .../__tests__/prepareItem.test.ts} | 41 ++------------- .../__tests__/prepareRootJobHandler.test.ts | 3 +- .../prepare.ts => tasks/prepareItem.ts} | 52 +++++-------------- .../knowledge/tasks/prepareRootJobHandler.ts | 16 ++---- 6 files changed, 25 insertions(+), 91 deletions(-) rename src/main/features/knowledge/{pipeline/sources/__tests__/prepare.test.ts => tasks/__tests__/prepareItem.test.ts} (86%) rename src/main/features/knowledge/{pipeline/sources/prepare.ts => tasks/prepareItem.ts} (69%) diff --git a/src/main/features/knowledge/README.md b/src/main/features/knowledge/README.md index a3d3b45fc97..bb1e2f18431 100644 --- a/src/main/features/knowledge/README.md +++ b/src/main/features/knowledge/README.md @@ -36,7 +36,7 @@ orchestration, and it lives in `ingestion/` and `tasks/`. | `pipeline/indexing/` | Index stage: offset-preserving splitter + chunker, `AiService` embedding/rerank wrappers, material field derivation. | | `pipeline/vectorstore/` | Persist stage: per-base `index.sqlite` lifecycle (`KnowledgeVectorStoreService`), the store itself (`indexStore/`, synchronous better-sqlite3 driver), vector deletion + index space reclamation (`vectorCleanup.ts`). | | `query/` | Read side: hybrid search with visibility filtering (`KnowledgeQueryService`), Concept ID tool surface (`KnowledgeConceptService`). | -| `tasks/` | Job handlers — the pipeline executors (see below). | +| `tasks/` | Job handlers — the pipeline executors (see below); `prepareItem.ts` is a prepare-root handler-private helper that expands a directory root into child items. | | `pathStorage.ts` | `raw/` path allocation: collision-free names, reservation, base file paths. | | `items.ts` / `types.ts` | Shared item vocabulary (type aliases, predicates, source probing); branded ids, queue names, idempotency keys. | diff --git a/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts b/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts index 828d6a0ccdb..bd655909982 100644 --- a/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts +++ b/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts @@ -135,7 +135,7 @@ vi.mock('../../pipeline/readers/KnowledgeReader', () => ({ loadKnowledgeItemDocuments: loadKnowledgeItemDocumentsMock })) -vi.mock('../../pipeline/sources/prepare', () => ({ +vi.mock('../prepareItem', () => ({ prepareKnowledgeItem: prepareKnowledgeItemMock })) diff --git a/src/main/features/knowledge/pipeline/sources/__tests__/prepare.test.ts b/src/main/features/knowledge/tasks/__tests__/prepareItem.test.ts similarity index 86% rename from src/main/features/knowledge/pipeline/sources/__tests__/prepare.test.ts rename to src/main/features/knowledge/tasks/__tests__/prepareItem.test.ts index 18c4e7765a7..40d88713c54 100644 --- a/src/main/features/knowledge/pipeline/sources/__tests__/prepare.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/prepareItem.test.ts @@ -34,23 +34,21 @@ vi.mock('@logger', () => ({ } })) -vi.mock('../directory', () => ({ +vi.mock('../../pipeline/sources/directory', () => ({ expandDirectoryOwnerToTree: expandDirectoryOwnerToTreeMock })) -import type { PrepareKnowledgeItemOptions } from '../prepare' +import type { PrepareKnowledgeItemOptions } from '../prepareItem' -const { prepareKnowledgeItem } = await import('../prepare') +const { prepareKnowledgeItem } = await import('../prepareItem') const baseId = 'kb-1' -function createPrepareOptions(item: KnowledgeItem, onCreatedItem = vi.fn()): PrepareKnowledgeItemOptions { +function createPrepareOptions(item: KnowledgeItem): PrepareKnowledgeItemOptions { const signal = new AbortController().signal return { baseId, item, - onCreatedItem, - runMutation: async (task) => await task(), signal } } @@ -221,37 +219,6 @@ describe('prepareKnowledgeItem', () => { expect(knowledgeItemUpdateDirectoryRelativePathMock).not.toHaveBeenCalled() }) - it('reports created children before marking them processing', async () => { - const root = createDirectoryItem('dir-root') - const fileChild: KnowledgeItem = { - id: 'file-child', - baseId, - groupId: root.id, - type: 'file', - data: { - source: '/docs/file-child.md', - relativePath: 'file-child.md' - }, - status: 'idle', - error: null, - createdAt: '2026-04-08T00:00:00.000Z', - updatedAt: '2026-04-08T00:00:00.000Z' - } - const onCreatedItem = vi.fn() - expandDirectoryOwnerToTreeMock.mockResolvedValueOnce({ - pathPrefix: 'dir-root', - children: [{ type: 'file', data: fileChild.data }] - }) - knowledgeItemCreateMock.mockReturnValueOnce(fileChild) - knowledgeItemUpdateStatusMock.mockImplementationOnce(() => { - throw new Error('status failed') - }) - - await expect(prepareKnowledgeItem(createPrepareOptions(root, onCreatedItem))).rejects.toThrow('status failed') - - expect(onCreatedItem).toHaveBeenCalledWith(fileChild) - }) - it('stops creating children when the runtime signal is aborted after expansion', async () => { const root = createDirectoryItem('dir-root') const controller = new AbortController() diff --git a/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts index c6c44627483..fbc73525347 100644 --- a/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts @@ -28,7 +28,8 @@ describe('prepare-root job handler', () => { await handler.execute(createCtx({ baseId: 'kb-1', itemId: 'dir-1' }, 'prepare-job')) expect(knowledgeItemGetSubtreeItemsMock).toHaveBeenCalledWith('kb-1', ['dir-1']) - expect(deleteItemsByIdsMock).toHaveBeenCalledWith('kb-1', []) + // Nothing to purge (no prior expansion) — purgeKnowledgeSubtreeWithinLock is a no-op for an empty subtree. + expect(deleteItemsByIdsMock).not.toHaveBeenCalled() expect(prepareKnowledgeItemMock).toHaveBeenCalledWith(expect.objectContaining({ baseId: 'kb-1' })) expect(scheduleItemMock).toHaveBeenCalledWith('kb-1', 'leaf-1', 'prepare-job') expect(handler.defaultQueue?.({ baseId: 'kb-1', itemId: 'dir-1' })).toBe('base.kb-1') diff --git a/src/main/features/knowledge/pipeline/sources/prepare.ts b/src/main/features/knowledge/tasks/prepareItem.ts similarity index 69% rename from src/main/features/knowledge/pipeline/sources/prepare.ts rename to src/main/features/knowledge/tasks/prepareItem.ts index 0f14f136564..54188c5f110 100644 --- a/src/main/features/knowledge/pipeline/sources/prepare.ts +++ b/src/main/features/knowledge/tasks/prepareItem.ts @@ -7,9 +7,9 @@ import { type KnowledgeItemType } from '@shared/data/types/knowledge' -import { type IndexableKnowledgeItem, isContainerKnowledgeItem, isIndexableKnowledgeItem } from '../../items' -import { collectKnowledgeReservedRelativePaths } from '../../pathStorage' -import { expandDirectoryOwnerToTree, type ExpandedDirectoryNode } from './directory' +import { type IndexableKnowledgeItem, isContainerKnowledgeItem, isIndexableKnowledgeItem } from '../items' +import { collectKnowledgeReservedRelativePaths } from '../pathStorage' +import { expandDirectoryOwnerToTree, type ExpandedDirectoryNode } from '../pipeline/sources/directory' const logger = loggerService.withContext('KnowledgePrepare') const EMPTY_DIRECTORY_ERROR = 'Directory contains no indexable files' @@ -17,16 +17,12 @@ const EMPTY_DIRECTORY_ERROR = 'Directory contains no indexable files' export interface PrepareKnowledgeItemOptions { baseId: string item: KnowledgeItem - onCreatedItem: (item: KnowledgeItem) => void - runMutation: (task: () => T) => Promise signal: AbortSignal } export async function prepareKnowledgeItem({ baseId, item, - onCreatedItem, - runMutation, signal }: PrepareKnowledgeItemOptions): Promise { signal.throwIfAborted() @@ -35,14 +31,12 @@ export async function prepareKnowledgeItem({ return [item] } - return await prepareDirectoryForRuntime(baseId, item, onCreatedItem, runMutation, signal) + return await prepareDirectoryForRuntime(baseId, item, signal) } async function prepareDirectoryForRuntime( baseId: string, item: KnowledgeItemOf<'directory'>, - onCreatedItem: (item: KnowledgeItem) => void, - runMutation: (task: () => T) => Promise, signal: AbortSignal ): Promise { // Exclude this container itself: on reindex it already owns its `relativePath` @@ -57,15 +51,15 @@ async function prepareDirectoryForRuntime( itemId: item.id, source: item.data.source }) - await runMutation(() => knowledgeItemService.updateStatus(item.id, 'failed', { error: EMPTY_DIRECTORY_ERROR })) + knowledgeItemService.updateStatus(item.id, 'failed', { error: EMPTY_DIRECTORY_ERROR }) return [] } // Pin the deduped `raw/` prefix the children were stored under onto the container, so the // UI shows the on-disk name (e.g. `docs_2`) and delete can remove the whole shell by it. - await runMutation(() => knowledgeItemService.updateDirectoryRelativePath(item.id, pathPrefix)) + knowledgeItemService.updateDirectoryRelativePath(item.id, pathPrefix) - return await createDirectoryChildren(baseId, item.id, children, onCreatedItem, runMutation, signal) + return await createDirectoryChildren(baseId, item.id, children, signal) } /** @@ -91,8 +85,6 @@ async function createDirectoryChildren( baseId: string, parentId: string, children: ExpandedDirectoryNode[], - onCreatedItem: (item: KnowledgeItem) => void, - runMutation: (task: () => T) => Promise, signal: AbortSignal ): Promise { const leafItems: IndexableKnowledgeItem[] = [] @@ -108,8 +100,6 @@ async function createDirectoryChildren( type: 'file', data: child.data }, - onCreatedItem, - runMutation, signal ) leafItems.push(createdFile) @@ -123,19 +113,10 @@ async function createDirectoryChildren( type: 'directory', data: child.data }, - onCreatedItem, - runMutation, signal ) - const childLeafItems = await createDirectoryChildren( - baseId, - createdDirectory.id, - child.children, - onCreatedItem, - runMutation, - signal - ) - await runMutation(() => knowledgeItemService.updateStatus(createdDirectory.id, 'processing')) + const childLeafItems = await createDirectoryChildren(baseId, createdDirectory.id, child.children, signal) + knowledgeItemService.updateStatus(createdDirectory.id, 'processing') leafItems.push(...childLeafItems) } @@ -145,19 +126,14 @@ async function createDirectoryChildren( async function createRuntimeItem( baseId: string, item: Extract, - onCreatedItem: (item: KnowledgeItem) => void, - runMutation: (task: () => TResult) => Promise, signal: AbortSignal ): Promise> { signal.throwIfAborted() - const createdItem = await runMutation(() => knowledgeItemService.create(baseId, item)) - onCreatedItem(createdItem) - - const processingItem = await runMutation(() => - isContainerKnowledgeItem(createdItem) - ? knowledgeItemService.updateStatus(createdItem.id, 'preparing') - : knowledgeItemService.updateStatus(createdItem.id, 'processing') - ) + const createdItem = knowledgeItemService.create(baseId, item) + + const processingItem = isContainerKnowledgeItem(createdItem) + ? knowledgeItemService.updateStatus(createdItem.id, 'preparing') + : knowledgeItemService.updateStatus(createdItem.id, 'processing') signal.throwIfAborted() return processingItem as KnowledgeItemOf diff --git a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts index 5c47c445012..6535c5e9caa 100644 --- a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts +++ b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts @@ -9,12 +9,10 @@ import type { KnowledgeItem } from '@shared/data/types/knowledge' import type { KnowledgeLockManager } from '../base/KnowledgeLockManager' import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' import { markUnscheduledKnowledgeItemsFailed } from '../ingestion/statusCleanup' -import { isIndexableKnowledgeItem } from '../items' -import { deleteKnowledgeItemFilesBestEffort } from '../pathStorage' -import { prepareKnowledgeItem } from '../pipeline/sources/prepare' -import { deleteKnowledgeItemVectors } from '../pipeline/vectorstore/vectorCleanup' +import { purgeKnowledgeSubtreeWithinLock } from '../ingestion/subtreePurge' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId, toKnowledgeItemId } from '../types' import type { KnowledgePrepareRootPayload } from './jobTypes' +import { prepareKnowledgeItem } from './prepareItem' import { isDataApiNotFoundError, markKnowledgeItemFailedOnSettled } from './utils/settled' const logger = loggerService.withContext('Knowledge:PrepareRootJobHandler') @@ -98,13 +96,7 @@ async function deletePreviousLeafExpansion( const base = knowledgeBaseService.getById(baseId) const descendants = knowledgeItemService.getSubtreeItems(baseId, [itemId]) const removableDescendants = descendants.filter((item) => item.status !== 'deleting') - const removableDescendantIds = removableDescendants.map((item) => item.id) - const removableLeafIds = removableDescendants.filter(isIndexableKnowledgeItem).map((item) => item.id) - - await deleteKnowledgeItemVectors(base, removableLeafIds) - // Best-effort: a file-removal failure must not abort the row deletion below. - await deleteKnowledgeItemFilesBestEffort(baseId, removableDescendants, { baseId, itemId }) - knowledgeItemService.deleteItemsByIds(baseId, removableDescendantIds) + await purgeKnowledgeSubtreeWithinLock(base, removableDescendants, { baseId, itemId }) }) } @@ -136,8 +128,6 @@ async function scanRootItem( const leaves = await prepareKnowledgeItem({ baseId, item: currentItem, - onCreatedItem: () => {}, - runMutation: async (task) => await task(), signal: ctx.signal }) if (leaves.length > 0) { From 9b69cc5f5dcf1000df658541df8f33590538a049 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 17:19:24 +0800 Subject: [PATCH 21/37] refactor(knowledge-tasks): narrow job handlers to a KnowledgeItemScheduler interface Job handler factories only need scheduleItem/scheduleIndexing/scheduleFileProcessingCheck, but took the full KnowledgeIngestionService type. Tests had to hard-cast their 3-method mocks with `as never` at every call site to paper over the mismatch. Extract the narrow interface and have KnowledgeIngestionService implement it so handlers depend only on what they use. Co-Authored-By: Claude Sonnet 5 Signed-off-by: eeee0717 --- .../ingestion/KnowledgeIngestionService.ts | 13 ++++++- ...heckFileProcessingResultJobHandler.test.ts | 36 +++++++++---------- .../tasks/__tests__/jobHandlerTestUtils.ts | 5 +-- .../__tests__/prepareRootJobHandler.test.ts | 20 +++++------ .../reindexSubtreeJobHandler.test.ts | 28 +++++++-------- .../checkFileProcessingResultJobHandler.ts | 4 +-- .../knowledge/tasks/prepareRootJobHandler.ts | 6 ++-- .../tasks/reindexSubtreeJobHandler.ts | 4 +-- 8 files changed, 64 insertions(+), 52 deletions(-) diff --git a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts index 72c4c132580..2e8e97d24f8 100644 --- a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts +++ b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts @@ -60,8 +60,19 @@ const KNOWLEDGE_SUPPORTED_FILE_EXT_SET = new Set(knowledgeSupportedFileE const REINDEX_ALLOWED_STATUSES = new Set(['completed', 'failed']) const DELETE_RECOVERY_ROOT_CHUNK_SIZE = 500 +export interface KnowledgeItemScheduler { + scheduleItem(baseId: KnowledgeBaseId, itemId: KnowledgeItemId, parentJobId?: string | null): Promise + scheduleIndexing(baseId: KnowledgeBaseId, itemId: KnowledgeItemId, parentJobId?: string | null): Promise + scheduleFileProcessingCheck( + baseId: KnowledgeBaseId, + itemId: KnowledgeItemId, + fileProcessingJobId: string, + options: { pollRound: number; firstScheduledAt: number; parentJobId: string | null } + ): Promise +} + /** Write-side orchestration: admission checks, item creation, conflict handling, and job enqueueing for the add/delete/reindex flows. */ -export class KnowledgeIngestionService { +export class KnowledgeIngestionService implements KnowledgeItemScheduler { constructor(private readonly knowledgeLockManager: KnowledgeLockManager) {} async addItems( diff --git a/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts index 4bc6caa454d..ba0f6ed5b23 100644 --- a/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts @@ -55,7 +55,7 @@ function createCheckPayload( describe('check-file-processing-result job handler', () => { it('declares the knowledge check job contract', () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) expect(handler.recovery).toBe('abandon') expect(handler.defaultQueue?.(createCheckPayload())).toBe('base.kb-1') @@ -69,7 +69,7 @@ describe('check-file-processing-result job handler', () => { }) it('reschedules delayed polling while file processing is active', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) getJobMock.mockResolvedValue(createFileProcessingJobSnapshot({ status: 'running' })) @@ -92,7 +92,7 @@ describe('check-file-processing-result job handler', () => { }) it('keeps polling follow-ups attached to the original workflow parent', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) getJobMock.mockResolvedValue(createFileProcessingJobSnapshot({ status: 'running' })) @@ -116,7 +116,7 @@ describe('check-file-processing-result job handler', () => { }) it('mirrors file-processing progress while polling', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) getJobMock.mockResolvedValue(createFileProcessingJobSnapshot({ status: 'running' })) MockMainCacheServiceUtils.setSharedCacheValue(`${JOB_PROGRESS_KEY_PREFIX}fp-job-1`, { @@ -145,7 +145,7 @@ describe('check-file-processing-result job handler', () => { }) it('marks the item failed when file processing exceeds the wait limit', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) getJobMock.mockResolvedValue(createFileProcessingJobSnapshot({ status: 'running' })) @@ -166,7 +166,7 @@ describe('check-file-processing-result job handler', () => { }) it('stores the processed artifact relative path and schedules indexing on completion', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) getJobMock.mockResolvedValue( createFileProcessingJobSnapshot({ @@ -187,7 +187,7 @@ describe('check-file-processing-result job handler', () => { }) it('schedules indexing under the original workflow parent after polling completion', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) getJobMock.mockResolvedValue( createFileProcessingJobSnapshot({ @@ -210,7 +210,7 @@ describe('check-file-processing-result job handler', () => { }) it('marks the item failed when the linked job is not the expected file-processing job', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) getJobMock.mockResolvedValue( createFileProcessingJobSnapshot({ @@ -235,7 +235,7 @@ describe('check-file-processing-result job handler', () => { }) it('skips attaching artifacts when the item becomes deleting before continuation side effects', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) knowledgeItemGetByIdMock .mockReturnValueOnce(createFileItem()) .mockReturnValueOnce(createFileItem(FILE_ITEM_ID, 'deleting')) @@ -257,7 +257,7 @@ describe('check-file-processing-result job handler', () => { }) it('marks the item failed when file processing fails', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) getJobMock.mockResolvedValue( createFileProcessingJobSnapshot({ @@ -277,7 +277,7 @@ describe('check-file-processing-result job handler', () => { }) it('marks the item failed when the completed output has no markdown artifact', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) getJobMock.mockResolvedValue( createFileProcessingJobSnapshot({ @@ -298,7 +298,7 @@ describe('check-file-processing-result job handler', () => { }) it('cancels linked file-processing work before skipping deleting items', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) knowledgeItemGetByIdMock.mockReturnValue(createFileItem(FILE_ITEM_ID, 'deleting')) await handler.execute(createCtx(createCheckPayload())) @@ -309,7 +309,7 @@ describe('check-file-processing-result job handler', () => { }) it('surfaces cancel failures before skipping deleting items', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) knowledgeItemGetByIdMock.mockReturnValue(createFileItem(FILE_ITEM_ID, 'deleting')) cancelMock.mockRejectedValueOnce(new Error('cancel failed')) @@ -320,7 +320,7 @@ describe('check-file-processing-result job handler', () => { }) it('surfaces cancel timeouts before skipping deleting items', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) knowledgeItemGetByIdMock.mockReturnValue(createFileItem(FILE_ITEM_ID, 'deleting')) cancelMock.mockResolvedValue({ outcome: 'timed-out' }) @@ -330,7 +330,7 @@ describe('check-file-processing-result job handler', () => { }) it('cancels linked file-processing work before skipping missing items', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) knowledgeItemGetByIdMock.mockImplementation(() => { throw DataApiErrorFactory.notFound('KnowledgeItem', FILE_ITEM_ID) }) @@ -343,7 +343,7 @@ describe('check-file-processing-result job handler', () => { }) it('onSettled marks active items failed when the check job fails', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) getJobMock.mockResolvedValue({ id: 'job-1', status: 'failed', @@ -385,7 +385,7 @@ describe('check-file-processing-result job handler', () => { }) it('onSettled falls back to the terminal status when a failed job has no error message', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) getJobMock.mockResolvedValue({ id: 'job-1', status: 'failed', @@ -427,7 +427,7 @@ describe('check-file-processing-result job handler', () => { }) it('onSettled marks a cancelled (app-quit) item failed with the reindex-to-finish message', async () => { - const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) getJobMock.mockResolvedValue({ id: 'job-1', status: 'cancelled', diff --git a/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts b/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts index bd655909982..2c6ea969a42 100644 --- a/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts +++ b/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts @@ -2,8 +2,9 @@ import type { JobContext } from '@main/core/job/types' import type { JobSnapshot } from '@shared/data/api/schemas/jobs' import type { KnowledgeBase, KnowledgeItemOf } from '@shared/data/types/knowledge' import { MockMainCacheServiceUtils } from '@test-mocks/main/CacheService' -import { beforeEach, vi } from 'vitest' +import { beforeEach, type Mocked, vi } from 'vitest' +import type { KnowledgeItemScheduler } from '../../ingestion/KnowledgeIngestionService' import type * as PathStorage from '../../pathStorage' const mocks = vi.hoisted(() => ({ @@ -337,7 +338,7 @@ export const knowledgeLockManager = { withBaseMutationLock: vi.fn(async (_baseId: string, task: () => Promise) => await task()) } -export const ingestionService = { +export const ingestionService: Mocked = { scheduleFileProcessingCheck: vi.fn(), scheduleIndexing: vi.fn(), scheduleItem: scheduleItemMock diff --git a/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts index fbc73525347..2a8483fc8a5 100644 --- a/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts @@ -22,7 +22,7 @@ import { describe('prepare-root job handler', () => { it('clears stale expansion and schedules recreated leaves', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService) knowledgeItemGetByIdMock.mockReturnValue(createDirectoryItem()) await handler.execute(createCtx({ baseId: 'kb-1', itemId: 'dir-1' }, 'prepare-job')) @@ -36,7 +36,7 @@ describe('prepare-root job handler', () => { }) it('clears stale expansion vectors before deleting rows', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService) const activeChild = createNoteItem('active-note', 'dir-1') knowledgeItemGetByIdMock.mockReturnValue(createDirectoryItem()) knowledgeItemGetSubtreeItemsMock.mockReturnValue([activeChild]) @@ -51,7 +51,7 @@ describe('prepare-root job handler', () => { }) it('routes stale-expansion cleanup through best-effort delete before deleting rows', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService) const activeChild = createNoteItem('active-note', 'dir-1') knowledgeItemGetByIdMock.mockReturnValue(createDirectoryItem()) knowledgeItemGetSubtreeItemsMock.mockReturnValue([activeChild]) @@ -70,7 +70,7 @@ describe('prepare-root job handler', () => { }) it('leaves deleting descendants for delete-subtree cleanup', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService) const activeChild = createNoteItem('active-note', 'dir-1') const deletingChild = createNoteItem('deleting-note', 'dir-1', 'deleting') knowledgeItemGetByIdMock.mockReturnValue(createDirectoryItem()) @@ -85,7 +85,7 @@ describe('prepare-root job handler', () => { }) it('skips expansion when the root becomes deleting inside the mutation lock', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService) knowledgeItemGetByIdMock .mockReturnValueOnce(createDirectoryItem()) .mockReturnValueOnce(createDirectoryItem('dir-1', 'deleting')) @@ -100,7 +100,7 @@ describe('prepare-root job handler', () => { }) it('keeps terminal failure from an empty expansion', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService) knowledgeItemGetByIdMock.mockReturnValue(createDirectoryItem()) prepareKnowledgeItemMock.mockResolvedValue([]) @@ -111,7 +111,7 @@ describe('prepare-root job handler', () => { }) it('marks unscheduled child leaves failed when enqueueing a child fails', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService) const leaves = [ createNoteItem('leaf-1', 'dir-1'), createNoteItem('leaf-2', 'dir-1'), @@ -138,7 +138,7 @@ describe('prepare-root job handler', () => { }) it('falls back to subtree failed status when marking an unscheduled leaf fails', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService) const leaves = [createNoteItem('leaf-1', 'dir-1'), createNoteItem('leaf-2', 'dir-1')] knowledgeItemGetByIdMock.mockReturnValue(createDirectoryItem()) prepareKnowledgeItemMock.mockResolvedValue(leaves) @@ -162,7 +162,7 @@ describe('prepare-root job handler', () => { }) it('reports unrecovered leaves when failed status cleanup also fails', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService) const leaves = [createNoteItem('leaf-1', 'dir-1')] knowledgeItemGetByIdMock.mockReturnValue(createDirectoryItem()) prepareKnowledgeItemMock.mockResolvedValue(leaves) @@ -182,7 +182,7 @@ describe('prepare-root job handler', () => { }) it('onSettled skips failed status when the item is deleting', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService) getJobMock.mockResolvedValue( createJobSnapshot({ id: 'prepare-job', diff --git a/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts index d7f0e52f173..fe73f586520 100644 --- a/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts @@ -27,7 +27,7 @@ import { describe('reindex-subtree job handler', () => { it('clears old artifacts, resets selected roots, and schedules selected roots', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService) const root = createDirectoryItem('dir-1') const child = createNoteItem('note-1', 'dir-1') knowledgeItemGetSubtreeItemsMock.mockImplementation( @@ -50,7 +50,7 @@ describe('reindex-subtree job handler', () => { }) it('routes container descendant cleanup through best-effort delete before deleting rows', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService) const root = createDirectoryItem('dir-1') const child = createNoteItem('note-1', 'dir-1') knowledgeItemGetSubtreeItemsMock.mockImplementation( @@ -75,7 +75,7 @@ describe('reindex-subtree job handler', () => { }) it('skips deleting subtrees before reset', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService) const root = createDirectoryItem('dir-1', 'deleting') const child = createNoteItem('note-1', 'dir-1', 'deleting') const ctx = createCtx({ baseId: 'kb-1', rootItemIds: ['dir-1'] }, 'reindex-job') @@ -93,7 +93,7 @@ describe('reindex-subtree job handler', () => { }) it('skips reset when the subtree becomes deleting inside the mutation lock', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService) const root = createDirectoryItem('dir-1') const child = createNoteItem('note-1', 'dir-1') const deletingChild = createNoteItem('note-1', 'dir-1', 'deleting') @@ -110,7 +110,7 @@ describe('reindex-subtree job handler', () => { }) it('leaves a root untouched when its source vanished before the mutation lock', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService) const root = createDirectoryItem('dir-1') const child = createNoteItem('note-1', 'dir-1') const ctx = createCtx({ baseId: 'kb-1', rootItemIds: ['dir-1'] }, 'reindex-job') @@ -140,7 +140,7 @@ describe('reindex-subtree job handler', () => { }) it('reindexes only the roots whose source still exists', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService) const presentRoot = createDirectoryItem('dir-1') const presentChild = createNoteItem('note-1', 'dir-1') const missingRoot = createDirectoryItem('dir-2') @@ -176,7 +176,7 @@ describe('reindex-subtree job handler', () => { }) it('clears old artifacts for selected leaf roots', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService) const root = createNoteItem('note-1') knowledgeItemGetSubtreeItemsMock.mockImplementation( (_baseId: string, _rootIds: string[], options: { includeRoots?: boolean; leafOnly?: boolean } = {}) => { @@ -194,7 +194,7 @@ describe('reindex-subtree job handler', () => { }) it('marks only unscheduled reset roots failed when rescheduling fails', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService) const firstRoot = createDirectoryItem('dir-1') const secondRoot = createDirectoryItem('dir-2') const firstChild = createNoteItem('note-1', 'dir-1') @@ -220,7 +220,7 @@ describe('reindex-subtree job handler', () => { }) it('never fails a left-untouched missing-source root when rescheduling a rebuildable root fails', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService) const presentRoot = createDirectoryItem('dir-1') const presentChild = createNoteItem('note-1', 'dir-1') const missingRoot = createDirectoryItem('dir-2') @@ -255,7 +255,7 @@ describe('reindex-subtree job handler', () => { }) it('stores the localized interruption code when a shutdown abort interrupts rescheduling', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService) const root = createDirectoryItem('dir-1') const child = createNoteItem('note-1', 'dir-1') knowledgeItemGetSubtreeItemsMock.mockImplementation( @@ -285,7 +285,7 @@ describe('reindex-subtree job handler', () => { }) it('onSettled marks active roots without follow-up jobs failed', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService) getJobMock.mockResolvedValue( createJobSnapshot({ id: 'reindex-job', @@ -317,7 +317,7 @@ describe('reindex-subtree job handler', () => { }) it('onSettled stores the localized interruption code when the reindex job was cancelled', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService) getJobMock.mockResolvedValue( createJobSnapshot({ id: 'reindex-job', @@ -351,7 +351,7 @@ describe('reindex-subtree job handler', () => { }) it('onSettled skips deleting roots and roots with follow-up jobs', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService) getJobMock.mockResolvedValue( createJobSnapshot({ id: 'reindex-job', @@ -388,7 +388,7 @@ describe('reindex-subtree job handler', () => { }) it('onSettled treats file-processing check jobs as follow-up jobs', async () => { - const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService as never) + const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService) getJobMock.mockResolvedValue( createJobSnapshot({ id: 'reindex-job', diff --git a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts index c42e302f26b..ca3112a3589 100644 --- a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts +++ b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts @@ -13,7 +13,7 @@ import { } from '../../fileProcessing/persistence/artifacts' import type { FileProcessingJobPayload } from '../../fileProcessing/tasks/shared' import type { KnowledgeLockManager } from '../base/KnowledgeLockManager' -import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' +import type { KnowledgeItemScheduler } from '../ingestion/KnowledgeIngestionService' import { toKnowledgeRelativePath } from '../pathStorage' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId, toKnowledgeItemId } from '../types' import type { KnowledgeCheckFileProcessingResultPayload } from './jobTypes' @@ -27,7 +27,7 @@ const FILE_PROCESSING_ITEM_UNAVAILABLE_CANCEL_REASON = 'knowledge-file-processin export function createCheckFileProcessingResultJobHandler( knowledgeLockManager: KnowledgeLockManager, - ingestionService: KnowledgeIngestionService + ingestionService: KnowledgeItemScheduler ): JobHandler { return { // Don't auto-resume on restart — a deliberate app quit must not re-spend the diff --git a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts index 6535c5e9caa..b48826f2193 100644 --- a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts +++ b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts @@ -7,7 +7,7 @@ import type { JobContext, JobHandler } from '@main/core/job/types' import type { KnowledgeItem } from '@shared/data/types/knowledge' import type { KnowledgeLockManager } from '../base/KnowledgeLockManager' -import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' +import type { KnowledgeItemScheduler } from '../ingestion/KnowledgeIngestionService' import { markUnscheduledKnowledgeItemsFailed } from '../ingestion/statusCleanup' import { purgeKnowledgeSubtreeWithinLock } from '../ingestion/subtreePurge' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId, toKnowledgeItemId } from '../types' @@ -19,7 +19,7 @@ const logger = loggerService.withContext('Knowledge:PrepareRootJobHandler') export function createPrepareRootJobHandler( knowledgeLockManager: KnowledgeLockManager, - ingestionService: KnowledgeIngestionService + ingestionService: KnowledgeItemScheduler ): JobHandler { return { // Don't auto-resume on restart — a deliberate app quit must not re-spend the @@ -140,7 +140,7 @@ async function scanRootItem( async function enqueueLeafItems( ctx: JobContext, leafItems: KnowledgeItem[], - ingestionService: KnowledgeIngestionService + ingestionService: KnowledgeItemScheduler ): Promise { const { baseId } = ctx.input diff --git a/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts b/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts index 425f35c247c..2ea1eb76f8c 100644 --- a/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts +++ b/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts @@ -9,7 +9,7 @@ import type { JobSnapshot } from '@shared/data/api/schemas/jobs' import { KNOWLEDGE_ITEM_ERROR_INDEXING_INTERRUPTED, type KnowledgeItemStatus } from '@shared/data/types/knowledge' import type { KnowledgeLockManager } from '../base/KnowledgeLockManager' -import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' +import type { KnowledgeItemScheduler } from '../ingestion/KnowledgeIngestionService' import { canKnowledgeItemRebuildSource, isContainerKnowledgeItem } from '../items' import { deleteKnowledgeItemFilesBestEffort } from '../pathStorage' import { deleteKnowledgeItemVectors } from '../pipeline/vectorstore/vectorCleanup' @@ -29,7 +29,7 @@ const REINDEX_RECOVERY_ACTIVE_STATUSES = new Set(['preparin export function createReindexSubtreeJobHandler( knowledgeLockManager: KnowledgeLockManager, - ingestionService: KnowledgeIngestionService + ingestionService: KnowledgeItemScheduler ): JobHandler { return { // Don't auto-resume on restart — a deliberate app quit must not re-spend the From 4926853c6a86d6a23cc78d7d5d9172a01bebdee6 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 17:19:39 +0800 Subject: [PATCH 22/37] fix(knowledge-directory): clean up orphaned raw bytes on directory expansion failure expandDirectoryOwnerToTree picks a dedup'd pathPrefix from the DB's existing relativePaths, then copies file bytes into raw//... before any knowledge_item row references them. A failure partway through (disk full, permission error, abort) left those bytes orphaned on disk with no DB row and no reserved name, so the next attempt would reselect the same pathPrefix and fail again with "Knowledge file already exists". Wrap the expansion in a try/catch that best-effort removes the whole pathPrefix directory before rethrowing, so a retry starts from a clean namespace. Co-Authored-By: Claude Sonnet 5 Signed-off-by: eeee0717 --- .../sources/__tests__/directory.test.ts | 50 +++++++++++++++++++ .../knowledge/pipeline/sources/directory.ts | 36 +++++++++---- 2 files changed, 76 insertions(+), 10 deletions(-) diff --git a/src/main/features/knowledge/pipeline/sources/__tests__/directory.test.ts b/src/main/features/knowledge/pipeline/sources/__tests__/directory.test.ts index fb0000b9a62..008cc48e2ed 100644 --- a/src/main/features/knowledge/pipeline/sources/__tests__/directory.test.ts +++ b/src/main/features/knowledge/pipeline/sources/__tests__/directory.test.ts @@ -2,6 +2,7 @@ import type * as NodeFs from 'node:fs' import type * as NodeOs from 'node:os' import path from 'node:path' +import type * as FsUtils from '@main/utils/file/fs' import { afterEach, describe, expect, it, vi } from 'vitest' import type * as PathStorage from '../../../pathStorage' @@ -9,12 +10,19 @@ import type * as PathStorage from '../../../pathStorage' const copyFileIntoKnowledgeBaseAtMock = vi.hoisted(() => vi.fn(async (_baseId: string, _externalPath: string, relativePath: string) => relativePath) ) +const removeDirMock = vi.hoisted(() => vi.fn()) vi.mock('@application', async () => { const { mockApplicationFactory } = await import('@test-mocks/main/application') return mockApplicationFactory() }) +vi.mock('@main/utils/file/fs', async () => { + const actual = await vi.importActual('@main/utils/file/fs') + removeDirMock.mockImplementation(actual.removeDir) + return { ...actual, removeDir: removeDirMock } +}) + vi.mock('../../../pathStorage', async () => { const actual = await vi.importActual('../../../pathStorage') return { @@ -24,6 +32,7 @@ vi.mock('../../../pathStorage', async () => { }) const { expandDirectoryOwnerToTree } = await import('../directory') +const { getKnowledgeBaseFilePath } = await import('../../../pathStorage') const realFs = await vi.importActual('node:fs') const realOs = await vi.importActual('node:os') @@ -323,6 +332,47 @@ describe('expandDirectoryOwnerToTree', () => { ]) }) + it('cleans up copied materials when a mid-expansion copy fails', async () => { + tempRoot = createTempRoot() + const rootDir = path.join(tempRoot, 'workspace') + realFs.mkdirSync(rootDir, { recursive: true }) + realFs.writeFileSync(path.join(rootDir, 'a.md'), '# a') + realFs.writeFileSync(path.join(rootDir, 'b.md'), '# b') + + copyFileIntoKnowledgeBaseAtMock.mockClear() + removeDirMock.mockClear() + const copyError = new Error('disk full') + copyFileIntoKnowledgeBaseAtMock + .mockImplementationOnce(async (_baseId, _externalPath, relativePath) => relativePath) + .mockImplementationOnce(async () => { + throw copyError + }) + + await expect( + expandDirectoryOwnerToTree( + { + id: 'dir-owner-1', + baseId: 'kb-1', + groupId: null, + type: 'directory', + data: { + source: rootDir + }, + status: 'idle', + error: null, + createdAt: '2026-04-08T00:00:00.000Z', + updatedAt: '2026-04-08T00:00:00.000Z' + }, + 'kb-1', + new Set(), + createSignal() + ) + ).rejects.toBe(copyError) + + expect(removeDirMock).toHaveBeenCalledTimes(1) + expect(removeDirMock).toHaveBeenCalledWith(getKnowledgeBaseFilePath('kb-1', 'workspace')) + }) + it('stops before reading when the runtime signal is already aborted', async () => { tempRoot = createTempRoot() const controller = new AbortController() diff --git a/src/main/features/knowledge/pipeline/sources/directory.ts b/src/main/features/knowledge/pipeline/sources/directory.ts index b7589873643..57179af673c 100644 --- a/src/main/features/knowledge/pipeline/sources/directory.ts +++ b/src/main/features/knowledge/pipeline/sources/directory.ts @@ -1,12 +1,15 @@ import fs from 'node:fs/promises' import path from 'node:path' +import { loggerService } from '@logger' +import { removeDir } from '@main/utils/file/fs' import { nextFreeKnowledgeRelativePath } from '@main/utils/knowledge' import type { DirectoryItemData, FileItemData, KnowledgeItem } from '@shared/data/types/knowledge' import { knowledgeSupportedFileExts } from '@shared/utils/file' -import { copyFileIntoKnowledgeBaseAt } from '../../pathStorage' +import { copyFileIntoKnowledgeBaseAt, getKnowledgeBaseFilePath } from '../../pathStorage' +const logger = loggerService.withContext('Knowledge:Directory') const KNOWLEDGE_SUPPORTED_FILE_EXT_SET = new Set(knowledgeSupportedFileExts) /** A scanned filesystem entry under a directory owner — only the fields this module reads. */ @@ -155,15 +158,28 @@ export async function expandDirectoryOwnerToTree( false // a directory basename is not a filename — keep any trailing ".ext" intact ) - const children = await readDirectoryTree(resolvedPath, signal) - const expandedChildren: ExpandedDirectoryNode[] = [] - - for (const child of children) { - const expandedChild = await expandDirectoryNode(baseId, pathPrefix, child, signal) - if (expandedChild) { - expandedChildren.push(expandedChild) + // Expansion durably copies bytes into `raw//...` before any `knowledge_item` + // row references them (child rows are only created once the whole tree succeeds). A + // mid-expansion failure (disk full, permission error, an aborted signal) must not leave + // orphaned bytes under a name the DB doesn't know is occupied — the next attempt would + // pick the same `pathPrefix` again and collide with its own partial copy. This function + // owns the namespace it just reserved, so it cleans it up on any failure. + try { + const children = await readDirectoryTree(resolvedPath, signal) + const expandedChildren: ExpandedDirectoryNode[] = [] + + for (const child of children) { + const expandedChild = await expandDirectoryNode(baseId, pathPrefix, child, signal) + if (expandedChild) { + expandedChildren.push(expandedChild) + } } - } - return { pathPrefix, children: expandedChildren } + return { pathPrefix, children: expandedChildren } + } catch (error) { + await removeDir(getKnowledgeBaseFilePath(baseId, pathPrefix)).catch((cleanupError) => { + logger.warn('Failed to clean up partially expanded directory materials', { baseId, pathPrefix, cleanupError }) + }) + throw error + } } From 9717d2e7305d10cfb9ad4af37286af41feaa5280 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 17:19:52 +0800 Subject: [PATCH 23/37] docs(knowledge): fix drifted references to legacy paths, class names, and job recovery semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several docs still pointed at the pre-v2-refactor layout: knowledge-service.md referenced a nonexistent src/main/services/knowledge path, a since-renamed KnowledgeWorkflowService class, and legacy colon-style IPC channel names instead of the current knowledge.* DataApi routes. workflow-architecture.md repeated the stale class name and an architecture diagram still labelled FileManager where knowledge-owned files now go through pathStorage.ts directly. operation-guards.md and README.md described indexing/reindex job handlers as recovery:'retry' when they're actually recovery:'abandon' (interrupted items are parked at failed by recoverInterruptedItems on boot, not silently resumed, to avoid re-spending embedding cost) — only delete-subtree is 'retry'. knowledge-technical-design.md's §4.7 still described joining search_text.rowid = search_text_fts.rowid, the implicit-rowid join #16447's A1 fix deliberately replaced with the stable fts_rowid proxy key. Co-Authored-By: Claude Sonnet 5 Signed-off-by: eeee0717 --- .../experiment/knowledge-technical-design.md | 2 +- .../references/knowledge/knowledge-service.md | 37 +++++++++---------- docs/references/knowledge/operation-guards.md | 2 +- .../knowledge/workflow-architecture.md | 8 ++-- src/main/features/knowledge/README.md | 6 +-- 5 files changed, 26 insertions(+), 29 deletions(-) diff --git a/docs/references/knowledge/experiment/knowledge-technical-design.md b/docs/references/knowledge/experiment/knowledge-technical-design.md index 46c505e6090..675f7a4a3a7 100644 --- a/docs/references/knowledge/experiment/knowledge-technical-design.md +++ b/docs/references/knowledge/experiment/knowledge-technical-design.md @@ -101,7 +101,7 @@ Unique on `(target_type, target_id, kind)`; both FTS and vectors enter through ` ### 4.7 search_text_fts -External-content FTS5 (trigram). **FTS hits must join back through `search_text.rowid = search_text_fts.rowid`** — `search_text_id` is a TEXT business key, not the FTS rowid. +External-content FTS5 (trigram). **FTS hits must join back through `search_text.fts_rowid = search_text_fts.rowid`** — `fts_rowid` is a stable integer surrogate column (not SQLite's implicit rowid), because VACUUM and any `INSERT...SELECT` rebuild renumber the implicit rowid and would silently desync an external-content index keyed on it. `search_text_id` is a separate TEXT business key, not the FTS rowid. ## 5. Index interface and implementation notes diff --git a/docs/references/knowledge/knowledge-service.md b/docs/references/knowledge/knowledge-service.md index eebb9e2c396..9a2301c512c 100644 --- a/docs/references/knowledge/knowledge-service.md +++ b/docs/references/knowledge/knowledge-service.md @@ -2,7 +2,7 @@ This document records the current v2 knowledge backend shape in the main process. -It covers the `src/main/services/knowledge` workflow path and the SQLite-backed data services. It does not describe the legacy `src/main/knowledge` service or the old `knowledge-base:*` IPC channels. +It covers the `src/main/features/knowledge` workflow path and the SQLite-backed data services. It does not describe the legacy `src/main/knowledge` service or the old `knowledge-base:*` IPC channels. For workflow guard details, see [Knowledge Operation Guards](./operation-guards.md). For the workflow architecture overview, see [Knowledge Workflow Architecture](./workflow-architecture.md). @@ -21,14 +21,13 @@ The current implementation is split into four responsibility areas: - Expose database-backed list/get operations and base metadata/config patch. - Do not perform vector-store mutations. 3. `KnowledgeService` - - Owns caller-facing runtime IPC workflow. - - Creates/deletes/restores bases through data services and vector store services. - - Registers Knowledge JobManager handlers. - - Holds the `KnowledgeWorkflowService` and `KnowledgeLockManager`. - - Collapses delete/reindex item inputs to top-level roots and enforces runtime guards. + - Thin lifecycle facade: registers Knowledge JobManager handlers, runs boot recovery, and delegates every public method to `base/`, `ingestion/`, and `query/`. + - `base/` (`KnowledgeBaseAdminService`) creates/deletes/restores bases through data services and vector store services, and owns the per-base `KnowledgeLockManager`. + - `ingestion/` (`KnowledgeIngestionService`) collapses delete/reindex item inputs to top-level roots, enforces runtime guards, and schedules the next workflow step. + - `query/` (`KnowledgeQueryService` / `KnowledgeConceptService`) serves search and the Concept ID tool surface. 4. Knowledge job handlers - Execute durable workflow stages through JobManager. - - Use `KnowledgeWorkflowService` for next-step scheduling. + - Use `KnowledgeIngestionService` for next-step scheduling. - Use `KnowledgeLockManager` for same-base mutations and vector cleanup. ```text @@ -39,13 +38,13 @@ caller caller -> preload knowledge IPC -> KnowledgeService - -> KnowledgeWorkflowService + -> KnowledgeIngestionService -> JobManager -> knowledge.prepare-root / knowledge.index-documents -> knowledge.delete-subtree / knowledge.reindex-subtree -> KnowledgeLockManager -> KnowledgeBaseService / KnowledgeItemService - -> KnowledgeVectorStoreService / FileManager + -> KnowledgeVectorStoreService ``` There is no current `KnowledgeRuntimeService` and no in-memory Knowledge queue. Durable work is owned by `JobManager`. @@ -104,21 +103,19 @@ reindex-items(baseId, itemIds) ## IPC Surface -`KnowledgeService` currently owns these public IPC entrypoints: +`KnowledgeService` currently owns these public IpcApi routes, defined in `src/shared/ipc/schemas/knowledge.ts` and handled in `src/main/ipc/handlers/knowledge.ts`: -- `knowledge:create-base` -- `knowledge:restore-base` -- `knowledge:delete-base` -- `knowledge:add-items` -- `knowledge:delete-items` -- `knowledge:reindex-items` -- `knowledge:search` -- `knowledge:list-item-chunks` +- `knowledge.create_base` +- `knowledge.restore_base` +- `knowledge.delete_base` +- `knowledge.add_items` +- `knowledge.delete_items` +- `knowledge.reindex_items` +- `knowledge.search` +- `knowledge.list_item_chunks` These IPC handlers are workflow-oriented. They validate payloads, call data services, and enqueue or execute runtime work internally. (The former `knowledge:delete-item-chunk` entrypoint was removed with the per-base index store cutover — chunks are derived index rows, replaced wholesale by reindexing.) -`KnowledgeService` also owns one orphaned v1 bridge entrypoint, `knowledge-base:delete`. Its only caller was the legacy Redux `store/knowledge` slice, which has now been removed, so this entrypoint is dead and pending cleanup. It routes to the same `delete-base` path. - The chunk IPC entrypoint is a runtime inspection helper: - `list-item-chunks` rejects failed bases. diff --git a/docs/references/knowledge/operation-guards.md b/docs/references/knowledge/operation-guards.md index c238ed26933..4ddd91bbfea 100644 --- a/docs/references/knowledge/operation-guards.md +++ b/docs/references/knowledge/operation-guards.md @@ -262,7 +262,7 @@ The child scheduling compensation mirrors `addItems`: once a child job was accep ## Shutdown -`KnowledgeService` does not cancel knowledge jobs during service shutdown. Knowledge job handlers use JobManager `recovery: 'retry'`, so unfinished pending, delayed, or running rows are left for JobManager startup recovery instead of being terminal-cancelled while their knowledge items still show active statuses. +`KnowledgeService` does not cancel knowledge jobs during service shutdown. The indexing handlers (`prepare-root`, `index-documents`, `check-file-processing-result`) and `reindex-subtree` use JobManager `recovery: 'abandon'` — an app restart never silently resumes them, which would otherwise re-spend the paid embedding API. `KnowledgeIngestionService.recoverInterruptedItems()` runs on startup and parks any item left in an active status by an interrupted job as `failed`. Only `delete-subtree` uses `recovery: 'retry'`, so unfinished pending, delayed, or running delete jobs are left for JobManager startup recovery instead of being terminal-cancelled. ## Review Checklist diff --git a/docs/references/knowledge/workflow-architecture.md b/docs/references/knowledge/workflow-architecture.md index d17893efc92..206e17bf152 100644 --- a/docs/references/knowledge/workflow-architecture.md +++ b/docs/references/knowledge/workflow-architecture.md @@ -8,20 +8,20 @@ Knowledge operations are modelled as a lightweight workflow rather than a single ```text API / user action - -> KnowledgeWorkflowService + -> KnowledgeIngestionService -> JobManager -> Knowledge job handlers -> KnowledgeLockManager - -> SQLite / vector store / FileManager + -> SQLite / index store / knowledge-owned files (raw/) ``` The design keeps three owners: -- `KnowledgeWorkflowService` decides the next workflow step. +- `KnowledgeIngestionService` decides the next workflow step. - `KnowledgeLockManager` serializes same-base mutations and cleanup. - Knowledge job handlers execute one durable stage and call the workflow service for the next step. -Helpers may own source planning, lifecycle writes, artifact refs, and FileProcessing adaptation. They should stay as modules until they need lifecycle-managed resources, IPC, timers, or long-lived state. +Helpers may own source planning, lifecycle writes, knowledge-owned raw files, and FileProcessing adaptation. They should stay as modules until they need lifecycle-managed resources, IPC, timers, or long-lived state. ## Workflow Entry Points diff --git a/src/main/features/knowledge/README.md b/src/main/features/knowledge/README.md index bb1e2f18431..627b323dc7b 100644 --- a/src/main/features/knowledge/README.md +++ b/src/main/features/knowledge/README.md @@ -52,9 +52,9 @@ All jobs run on the per-base queue `base.{baseId}`; idempotency keys prevent dou | `knowledge.delete-subtree` | Cancel active jobs → delete vectors → delete files → delete rows. | `ingestion` (delete), boot recovery | | `knowledge.reindex-subtree` | Verify source → delete vectors → reset statuses → re-enqueue indexing. | `ingestion` (reindex) | -Indexing jobs declare `recovery: 'abandon'` — an app restart never silently resumes them (that -would auto-spend the paid embedding API); boot recovery parks interrupted items at `failed` -instead. Delete/reindex jobs use `recovery: 'retry'`. +Indexing jobs and `knowledge.reindex-subtree` declare `recovery: 'abandon'` — an app restart never +silently resumes them (that would auto-spend the paid embedding API); boot recovery parks +interrupted items at `failed` instead. Only `knowledge.delete-subtree` uses `recovery: 'retry'`. Item status flow: `preparing` (directory) / `processing` → `completed` | `failed`; any status → `deleting` → row removed. `reading`/`embedding` are transient sub-phases surfaced while the index From 8fe92dc9afa0d5d9fb58561c86d0911d9201e358 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 17:21:47 +0800 Subject: [PATCH 24/37] refactor(knowledge-tasks): unify active-job cancellation into cancelActiveKnowledgeJobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent implementations of "cancel in-flight knowledge jobs (+ their linked file-processing job) in a base's queue" had drifted apart: KnowledgeBaseAdminService.cancelAllJobsForBase (base-wide, used by deleteBase) called jobManager.cancel() directly with no timeout handling and no truthy check on the linked fileProcessingJobId; subtreePurge.cancelActiveKnowledgeSubtreeJobs (subtree-scoped, used by delete-items/replace-on-add) used cancelJobOrThrow and did check fileProcessingJobId. Both run outside the base mutation lock (cancel awaits handler settlement, and the handlers themselves take that lock, so cancelling while holding it would deadlock). Introduce cancelActiveKnowledgeJobs(baseId, reason, options) in tasks/utils/cancel.ts covering both the base-wide and subtree-scoped shapes via an options object (rootItemIds + excludeJobId + onCancelTimeout: 'throw' | 'proceed'), and switch all three call sites (deleteBase, deleteSubtreeJobHandler, and KnowledgeIngestionService's replace-on-add path) onto it. Delete the now-dead cancelAllJobsForBase/cancelActiveKnowledgeSubtreeJobs plus their now-unused imports. Also documents KnowledgeItemScheduler's role, missed when that interface was extracted. ⚠️ Side-fix (not pure refactor): the base-wide path (deleteBase) now also skips cancelling an empty-string fileProcessingJobId, matching the subtree path's existing truthy check. Schema allows an empty string in that field in principle, though normal write paths never produce one, so this is a narrow-but-real behavior change, not just a rename. Co-Authored-By: Claude Sonnet 5 Signed-off-by: eeee0717 --- .../base/KnowledgeBaseAdminService.ts | 31 +-- .../ingestion/KnowledgeIngestionService.ts | 18 +- .../knowledge/ingestion/subtreePurge.ts | 73 ------- .../tasks/deleteSubtreeJobHandler.ts | 9 +- .../tasks/utils/__tests__/cancel.test.ts | 180 ++++++++++++++++++ .../features/knowledge/tasks/utils/cancel.ts | 100 ++++++++++ 6 files changed, 300 insertions(+), 111 deletions(-) create mode 100644 src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts diff --git a/src/main/features/knowledge/base/KnowledgeBaseAdminService.ts b/src/main/features/knowledge/base/KnowledgeBaseAdminService.ts index 2ae6daaf435..b5e99974879 100644 --- a/src/main/features/knowledge/base/KnowledgeBaseAdminService.ts +++ b/src/main/features/knowledge/base/KnowledgeBaseAdminService.ts @@ -17,18 +17,10 @@ import { import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionService' import { classifyKnowledgeItemSource } from '../items' import { getKnowledgeBaseFilePath } from '../pathStorage' -import { narrowKnowledgeJobInput } from '../tasks/utils/jobInput' -import { - KNOWLEDGE_ACTIVE_JOB_LIMIT, - KNOWLEDGE_ACTIVE_JOB_STATUSES, - KNOWLEDGE_JOB_TYPES, - knowledgeQueueName, - toKnowledgeBaseId -} from '../types' +import { cancelActiveKnowledgeJobs } from '../tasks/utils/cancel' import type { KnowledgeLockManager } from './KnowledgeLockManager' const logger = loggerService.withContext('Knowledge:BaseAdmin') -const KNOWLEDGE_JOB_TYPE_SET = new Set(KNOWLEDGE_JOB_TYPES) /** Knowledge base lifecycle: create (with rollback), delete, restore, and list — everything about the base row + its on-disk artifacts, not about items. */ export class KnowledgeBaseAdminService { @@ -72,7 +64,7 @@ export class KnowledgeBaseAdminService { } async deleteBase(baseId: string): Promise { - await this.cancelAllJobsForBase(baseId) + await cancelActiveKnowledgeJobs(baseId, 'delete-base', { onCancelTimeout: 'proceed' }) await this.knowledgeLockManager.withBaseMutationLock(baseId, async () => { try { @@ -198,25 +190,6 @@ export class KnowledgeBaseAdminService { return total > 0 } - private async cancelAllJobsForBase(baseId: string): Promise { - const jobManager = application.get('JobManager') - const activeJobs = await jobManager.list({ - queue: knowledgeQueueName(toKnowledgeBaseId(baseId)), - status: [...KNOWLEDGE_ACTIVE_JOB_STATUSES], - limit: KNOWLEDGE_ACTIVE_JOB_LIMIT - }) - const jobsToCancel = activeJobs.filter((job) => KNOWLEDGE_JOB_TYPE_SET.has(job.type)) - const linkedFileProcessingJobIds = activeJobs.flatMap((job) => { - const narrowed = narrowKnowledgeJobInput(job) - return narrowed?.type === 'knowledge.check-file-processing-result' ? [narrowed.input.fileProcessingJobId] : [] - }) - - await Promise.all([ - ...jobsToCancel.map((job) => jobManager.cancel(job.id, 'delete-base')), - ...linkedFileProcessingJobIds.map((jobId) => jobManager.cancel(jobId, 'delete-base')) - ]) - } - private toRestoreRuntimeInput(sourceBaseId: string, item: KnowledgeItem): KnowledgeAddItemInput { try { if (item.type === 'file') { diff --git a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts index 2e8e97d24f8..0d888e268a6 100644 --- a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts +++ b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts @@ -35,7 +35,7 @@ import { reserveImportedFileRelativePath } from '../pathStorage' import { planKnowledgeItemSource } from '../pipeline/sources/sourcePlanning' -import { cancelJobOrThrow } from '../tasks/utils/cancel' +import { cancelActiveKnowledgeJobs, cancelJobOrThrow } from '../tasks/utils/cancel' import { type KnowledgeBaseId, knowledgeDeleteSubtreeIdempotencyKey, @@ -51,7 +51,7 @@ import { } from '../types' import { resolveKnowledgeAddConflicts } from './addConflicts' import { markUnscheduledKnowledgeItemsFailed } from './statusCleanup' -import { cancelActiveKnowledgeSubtreeJobs, purgeKnowledgeSubtreeWithinLock } from './subtreePurge' +import { purgeKnowledgeSubtreeWithinLock } from './subtreePurge' const logger = loggerService.withContext('Knowledge:IngestionService') // Keep poll jobs delayed enough to avoid hot-looping while remote processors are still working. @@ -60,6 +60,11 @@ const KNOWLEDGE_SUPPORTED_FILE_EXT_SET = new Set(knowledgeSupportedFileE const REINDEX_ALLOWED_STATUSES = new Set(['completed', 'failed']) const DELETE_RECOVERY_ROOT_CHUNK_SIZE = 500 +/** + * The workflow re-entry seam job handlers call back into (workflow-architecture.md): expand a + * container, index a leaf, or poll a file-processing job. Handlers depend on exactly this surface, + * not the full `KnowledgeIngestionService`. + */ export interface KnowledgeItemScheduler { scheduleItem(baseId: KnowledgeBaseId, itemId: KnowledgeItemId, parentJobId?: string | null): Promise scheduleIndexing(baseId: KnowledgeBaseId, itemId: KnowledgeItemId, parentJobId?: string | null): Promise @@ -106,11 +111,10 @@ export class KnowledgeIngestionService implements KnowledgeItemScheduler { // holding it would deadlock. itemsToAdd = resolution.keptInputs if (resolution.conflictingExistingRootIds.length > 0) { - await cancelActiveKnowledgeSubtreeJobs( - base.id, - resolution.conflictingExistingRootIds, - 'knowledge-add-replace' - ) + await cancelActiveKnowledgeJobs(base.id, 'knowledge-add-replace', { + rootItemIds: resolution.conflictingExistingRootIds, + onCancelTimeout: 'throw' + }) } } } diff --git a/src/main/features/knowledge/ingestion/subtreePurge.ts b/src/main/features/knowledge/ingestion/subtreePurge.ts index 97f005f1295..316fe5efed3 100644 --- a/src/main/features/knowledge/ingestion/subtreePurge.ts +++ b/src/main/features/knowledge/ingestion/subtreePurge.ts @@ -1,56 +1,9 @@ -import { application } from '@application' import { knowledgeItemService } from '@data/services/KnowledgeItemService' import type { KnowledgeBase, KnowledgeItem } from '@shared/data/types/knowledge' import { isIndexableKnowledgeItem } from '../items' import { deleteKnowledgeItemFilesBestEffort } from '../pathStorage' import { deleteKnowledgeItemVectors } from '../pipeline/vectorstore/vectorCleanup' -import { cancelJobOrThrow } from '../tasks/utils/cancel' -import { narrowKnowledgeJobInput } from '../tasks/utils/jobInput' -import { - KNOWLEDGE_ACTIVE_JOB_LIMIT, - KNOWLEDGE_ACTIVE_JOB_STATUSES, - knowledgeQueueName, - toKnowledgeBaseId -} from '../types' - -/** - * Cancel any in-flight job touching the given subtree (the roots plus their - * descendants). MUST run OUTSIDE the base mutation lock: `cancel` awaits each - * running handler's settlement, and knowledge index/prepare/reindex handlers - * themselves acquire `withBaseMutationLock`, so cancelling while holding the lock - * would deadlock (the handler can never reach its abort check). This is why both - * the delete job and the replace-on-add path cancel first, then purge under the - * lock. - */ -export async function cancelActiveKnowledgeSubtreeJobs( - baseId: string, - rootItemIds: string[], - reason: string, - currentJobId?: string -): Promise { - const subtreeItems = knowledgeItemService.getSubtreeItems(baseId, rootItemIds, { includeRoots: true }) - const subtreeIds = new Set(subtreeItems.map((item) => item.id)) - if (subtreeIds.size === 0) { - return - } - - const jobManager = application.get('JobManager') - const activeJobs = await jobManager.list({ - queue: knowledgeQueueName(toKnowledgeBaseId(baseId)), - status: [...KNOWLEDGE_ACTIVE_JOB_STATUSES], - limit: KNOWLEDGE_ACTIVE_JOB_LIMIT - }) - const jobIds = activeJobs - .filter((job) => job.id !== currentJobId && jobTouchesSubtree(job, subtreeIds)) - .map((job) => job.id) - const fileProcessingJobIds = activeJobs.flatMap((job) => getFileProcessingJobIdsForTouchedSubtree(job, subtreeIds)) - - await Promise.all([ - ...jobIds.map((jobId) => cancelJobOrThrow(jobId, reason)), - ...fileProcessingJobIds.map((jobId) => cancelJobOrThrow(jobId, reason)) - ]) -} /** * Remove a resolved subtree's vectors, on-disk files, and DB rows, in that order. @@ -81,29 +34,3 @@ export async function purgeKnowledgeSubtreeWithinLock( knowledgeItemService.deleteItemsByIds(base.id, subtreeItemIds) } - -function jobTouchesSubtree(job: { type: string; input: unknown }, subtreeIds: Set): boolean { - const narrowed = narrowKnowledgeJobInput(job) - if (!narrowed) { - return false - } - if ('itemId' in narrowed.input && subtreeIds.has(narrowed.input.itemId)) { - return true - } - return 'rootItemIds' in narrowed.input && narrowed.input.rootItemIds.some((itemId) => subtreeIds.has(itemId)) -} - -function getFileProcessingJobIdsForTouchedSubtree( - job: { type: string; input: unknown }, - subtreeIds: Set -): string[] { - const narrowed = narrowKnowledgeJobInput(job) - if ( - narrowed?.type === 'knowledge.check-file-processing-result' && - subtreeIds.has(narrowed.input.itemId) && - narrowed.input.fileProcessingJobId - ) { - return [narrowed.input.fileProcessingJobId] - } - return [] -} diff --git a/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts b/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts index 675dacf3f70..726feda4682 100644 --- a/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts +++ b/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts @@ -6,10 +6,11 @@ import { loggerService } from '@logger' import type { JobHandler } from '@main/core/job/types' import type { KnowledgeLockManager } from '../base/KnowledgeLockManager' -import { cancelActiveKnowledgeSubtreeJobs, purgeKnowledgeSubtreeWithinLock } from '../ingestion/subtreePurge' +import { purgeKnowledgeSubtreeWithinLock } from '../ingestion/subtreePurge' import { reclaimKnowledgeIndexSpace } from '../pipeline/vectorstore/vectorCleanup' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId } from '../types' import type { KnowledgeDeleteSubtreePayload } from './jobTypes' +import { cancelActiveKnowledgeJobs } from './utils/cancel' const logger = loggerService.withContext('Knowledge:DeleteSubtreeJobHandler') @@ -43,7 +44,11 @@ export function createDeleteSubtreeJobHandler( } // Stop active work touching deleting rows before removing vectors and rows. - await cancelActiveKnowledgeSubtreeJobs(baseId, deletingSubtreeItemIds, 'knowledge-delete-subtree', ctx.jobId) + await cancelActiveKnowledgeJobs(baseId, 'knowledge-delete-subtree', { + rootItemIds: deletingSubtreeItemIds, + excludeJobId: ctx.jobId, + onCancelTimeout: 'throw' + }) // Cleanup is locked so no indexer can write vectors for rows being removed. await knowledgeLockManager.withBaseMutationLock(baseId, async () => { diff --git a/src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts b/src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts new file mode 100644 index 00000000000..cffa708083c --- /dev/null +++ b/src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts @@ -0,0 +1,180 @@ +import type { JobSnapshot } from '@shared/data/api/schemas/jobs' +import type { KnowledgeItem } from '@shared/data/types/knowledge' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { cancelMock, listMock, knowledgeItemGetSubtreeItemsMock } = vi.hoisted(() => ({ + cancelMock: vi.fn(), + listMock: vi.fn(), + knowledgeItemGetSubtreeItemsMock: vi.fn() +})) + +vi.mock('@application', async () => { + const { mockApplicationFactory } = await import('@test-mocks/main/application') + return mockApplicationFactory({ + JobManager: { + cancel: cancelMock, + list: listMock + } + } as Parameters[0]) +}) + +vi.mock('@data/services/KnowledgeItemService', () => ({ + knowledgeItemService: { + getSubtreeItems: knowledgeItemGetSubtreeItemsMock + } +})) + +const { cancelActiveKnowledgeJobs } = await import('../cancel') + +type JobSnapshotInput = Pick & Partial + +function createJobSnapshot(overrides: JobSnapshotInput): JobSnapshot { + return { + id: 'job-1', + status: 'running', + priority: 0, + queue: 'base.kb-1', + idempotencyKey: null, + scheduleId: null, + scheduledAt: '2026-04-08T00:00:00.000Z', + startedAt: '2026-04-08T00:00:00.000Z', + finishedAt: null, + attempt: 1, + maxAttempts: 3, + output: null, + error: null, + parentId: null, + cancelRequested: false, + metadata: {}, + timeoutMs: null, + createdAt: '2026-04-08T00:00:00.000Z', + updatedAt: '2026-04-08T00:00:00.000Z', + ...overrides + } +} + +function createItem(id: string): KnowledgeItem { + return { + id, + baseId: 'kb-1', + groupId: null, + type: 'note', + data: { source: 'note', content: 'text' }, + status: 'processing', + error: null, + createdAt: '2026-04-08T00:00:00.000Z', + updatedAt: '2026-04-08T00:00:00.000Z' + } +} + +describe('cancelActiveKnowledgeJobs', () => { + beforeEach(() => { + cancelMock.mockReset().mockResolvedValue({ outcome: 'cancelled' }) + listMock.mockReset().mockResolvedValue([]) + knowledgeItemGetSubtreeItemsMock.mockReset() + }) + + it('cancels every active knowledge job in the base queue when scoped base-wide', async () => { + listMock.mockResolvedValue([ + createJobSnapshot({ + id: 'prepare-job', + type: 'knowledge.prepare-root', + input: { baseId: 'kb-1', itemId: 'dir-1' } + }), + createJobSnapshot({ + id: 'check-job', + type: 'knowledge.check-file-processing-result', + input: { + baseId: 'kb-1', + itemId: 'file-1', + fileProcessingJobId: 'fp-job-1', + pollRound: 0, + firstScheduledAt: 1779811200000, + parentJobId: null + } + }) + ]) + + await cancelActiveKnowledgeJobs('kb-1', 'delete-base', { onCancelTimeout: 'proceed' }) + + expect(knowledgeItemGetSubtreeItemsMock).not.toHaveBeenCalled() + expect(cancelMock).toHaveBeenCalledWith('prepare-job', 'delete-base') + expect(cancelMock).toHaveBeenCalledWith('fp-job-1', 'delete-base') + }) + + it('returns without listing jobs when the requested subtree resolves empty', async () => { + knowledgeItemGetSubtreeItemsMock.mockReturnValue([]) + + await cancelActiveKnowledgeJobs('kb-1', 'knowledge-delete-subtree', { + rootItemIds: ['missing-root'], + onCancelTimeout: 'throw' + }) + + expect(listMock).not.toHaveBeenCalled() + expect(cancelMock).not.toHaveBeenCalled() + }) + + it('scopes cancellation to jobs touching the resolved subtree and excludes the given job id', async () => { + knowledgeItemGetSubtreeItemsMock.mockReturnValue([createItem('dir-1'), createItem('note-1')]) + listMock.mockResolvedValue([ + createJobSnapshot({ + id: 'current-job', + type: 'knowledge.delete-subtree', + input: { baseId: 'kb-1', rootItemIds: ['dir-1'] } + }), + createJobSnapshot({ + id: 'index-job', + type: 'knowledge.index-documents', + input: { baseId: 'kb-1', itemId: 'note-1', parentJobId: null } + }), + createJobSnapshot({ + id: 'unrelated-job', + type: 'knowledge.index-documents', + input: { baseId: 'kb-1', itemId: 'other', parentJobId: null } + }) + ]) + + await cancelActiveKnowledgeJobs('kb-1', 'knowledge-delete-subtree', { + rootItemIds: ['dir-1'], + excludeJobId: 'current-job', + onCancelTimeout: 'throw' + }) + + expect(cancelMock).toHaveBeenCalledTimes(1) + expect(cancelMock).toHaveBeenCalledWith('index-job', 'knowledge-delete-subtree') + }) + + it('throws when a scoped cancel times out', async () => { + knowledgeItemGetSubtreeItemsMock.mockReturnValue([createItem('note-1')]) + listMock.mockResolvedValue([ + createJobSnapshot({ + id: 'index-job', + type: 'knowledge.index-documents', + input: { baseId: 'kb-1', itemId: 'note-1', parentJobId: null } + }) + ]) + cancelMock.mockResolvedValue({ outcome: 'timed-out' }) + + await expect( + cancelActiveKnowledgeJobs('kb-1', 'knowledge-delete-subtree', { + rootItemIds: ['note-1'], + onCancelTimeout: 'throw' + }) + ).rejects.toThrow('Job cancel timed out: index-job') + }) + + it('does not throw on a cancel timeout when proceeding base-wide', async () => { + listMock.mockResolvedValue([ + createJobSnapshot({ + id: 'index-job', + type: 'knowledge.index-documents', + input: { baseId: 'kb-1', itemId: 'note-1', parentJobId: null } + }) + ]) + cancelMock.mockResolvedValue({ outcome: 'timed-out' }) + + await expect( + cancelActiveKnowledgeJobs('kb-1', 'delete-base', { onCancelTimeout: 'proceed' }) + ).resolves.toBeUndefined() + }) +}) diff --git a/src/main/features/knowledge/tasks/utils/cancel.ts b/src/main/features/knowledge/tasks/utils/cancel.ts index f1fc8ca9068..66117ac8ff3 100644 --- a/src/main/features/knowledge/tasks/utils/cancel.ts +++ b/src/main/features/knowledge/tasks/utils/cancel.ts @@ -1,4 +1,16 @@ import { application } from '@application' +import { knowledgeItemService } from '@data/services/KnowledgeItemService' + +import { + KNOWLEDGE_ACTIVE_JOB_LIMIT, + KNOWLEDGE_ACTIVE_JOB_STATUSES, + KNOWLEDGE_JOB_TYPES, + knowledgeQueueName, + toKnowledgeBaseId +} from '../../types' +import { narrowKnowledgeJobInput } from './jobInput' + +const KNOWLEDGE_JOB_TYPE_SET = new Set(KNOWLEDGE_JOB_TYPES) export async function cancelJobOrThrow(jobId: string, reason: string): Promise { const result = await application.get('JobManager').cancel(jobId, reason) @@ -6,3 +18,91 @@ export async function cancelJobOrThrow(jobId: string, reason: string): Promise { + const { rootItemIds, excludeJobId, onCancelTimeout } = options + + let subtreeItemIds: Set | undefined + if (rootItemIds) { + const subtreeItems = knowledgeItemService.getSubtreeItems(baseId, rootItemIds, { includeRoots: true }) + subtreeItemIds = new Set(subtreeItems.map((item) => item.id)) + if (subtreeItemIds.size === 0) { + return + } + } + + const jobManager = application.get('JobManager') + const activeJobs = await jobManager.list({ + queue: knowledgeQueueName(toKnowledgeBaseId(baseId)), + status: [...KNOWLEDGE_ACTIVE_JOB_STATUSES], + limit: KNOWLEDGE_ACTIVE_JOB_LIMIT + }) + + const jobIds = activeJobs + .filter((job) => job.id !== excludeJobId) + .filter((job) => (subtreeItemIds ? jobTouchesSubtree(job, subtreeItemIds) : KNOWLEDGE_JOB_TYPE_SET.has(job.type))) + .map((job) => job.id) + const fileProcessingJobIds = activeJobs.flatMap((job) => getLinkedFileProcessingJobIds(job, subtreeItemIds)) + + const cancelOne: (jobId: string) => Promise = + onCancelTimeout === 'throw' + ? (jobId) => cancelJobOrThrow(jobId, reason) + : (jobId) => jobManager.cancel(jobId, reason) + + await Promise.all([...jobIds, ...fileProcessingJobIds].map(cancelOne)) +} + +function jobTouchesSubtree(job: { type: string; input: unknown }, subtreeIds: Set): boolean { + const narrowed = narrowKnowledgeJobInput(job) + if (!narrowed) { + return false + } + if ('itemId' in narrowed.input && subtreeIds.has(narrowed.input.itemId)) { + return true + } + return 'rootItemIds' in narrowed.input && narrowed.input.rootItemIds.some((itemId) => subtreeIds.has(itemId)) +} + +/** Linked file-processing jobs for a knowledge job, optionally filtered to a subtree scope. */ +function getLinkedFileProcessingJobIds( + job: { type: string; input: unknown }, + subtreeIds: Set | undefined +): string[] { + const narrowed = narrowKnowledgeJobInput(job) + if ( + narrowed?.type === 'knowledge.check-file-processing-result' && + (!subtreeIds || subtreeIds.has(narrowed.input.itemId)) && + narrowed.input.fileProcessingJobId + ) { + return [narrowed.input.fileProcessingJobId] + } + return [] +} From c9960fc59e7df3f429ab15f1c9bf4d16cca4cbdd Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 17:24:50 +0800 Subject: [PATCH 25/37] refactor(knowledge-tasks): unify missing/deleting skip guards into resolveLiveKnowledgeItem/Subtree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Is this item/subtree gone (NOT_FOUND) or being deleted (status==='deleting'), and if so treat the operation as an idempotent no-op" was hand-rolled ~8 times across prepareRootJobHandler, indexDocumentsJobHandler, checkFileProcessingResultJobHandler, and reindexSubtreeJobHandler, with the classification logic (catch NOT_FOUND / check status==='deleting') repeated near-verbatim each time. Extract only that classification into tasks/utils/liveItem.ts (resolveLiveKnowledgeItem for a single item, resolveLiveKnowledgeSubtree for a resolved root+descendants set) and leave each call site's own logging text, progress stage, and control flow in place — operation-guards.md's convention of each operation keeping its own explicit flow is preserved; only the duplicated classification is deduped. checkFileProcessingResultJobHandler.shouldSkipMissingOrDeletingItem keeps its extra baseId ownership check (an explicit throw, not a skip) outside the shared helper, since that's specific to this one call site. Co-Authored-By: Claude Sonnet 5 Signed-off-by: eeee0717 --- .../checkFileProcessingResultJobHandler.ts | 41 ++++----- .../tasks/indexDocumentsJobHandler.ts | 50 ++++++----- .../knowledge/tasks/prepareRootJobHandler.ts | 44 +++++----- .../tasks/reindexSubtreeJobHandler.ts | 14 +-- .../tasks/utils/__tests__/liveItem.test.ts | 88 +++++++++++++++++++ .../knowledge/tasks/utils/liveItem.ts | 34 +++++++ 6 files changed, 198 insertions(+), 73 deletions(-) create mode 100644 src/main/features/knowledge/tasks/utils/__tests__/liveItem.test.ts create mode 100644 src/main/features/knowledge/tasks/utils/liveItem.ts diff --git a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts index ca3112a3589..743da2ee74a 100644 --- a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts +++ b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts @@ -18,7 +18,8 @@ import { toKnowledgeRelativePath } from '../pathStorage' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId, toKnowledgeItemId } from '../types' import type { KnowledgeCheckFileProcessingResultPayload } from './jobTypes' import { cancelJobOrThrow } from './utils/cancel' -import { isDataApiNotFoundError, markKnowledgeItemFailedOnSettled } from './utils/settled' +import { resolveLiveKnowledgeItem } from './utils/liveItem' +import { markKnowledgeItemFailedOnSettled } from './utils/settled' const logger = loggerService.withContext('Knowledge:CheckFileProcessingResultJobHandler') // Remote document processors can be slow, but a stale paid job should not poll forever. @@ -168,40 +169,32 @@ function isExpectedFileProcessingJob(snapshot: JobSnapshot, itemId: string): boo } function shouldSkipMissingOrDeletingItem(baseId: string, itemId: string, jobId: string): boolean { - try { - const item = knowledgeItemService.getById(itemId) - if (item.baseId !== baseId) { - throw new Error(`Knowledge item '${itemId}' does not belong to base '${baseId}'`) - } - if (item.status === 'deleting') { + const result = resolveLiveKnowledgeItem(itemId) + if ('skip' in result) { + if (result.skip === 'deleting') { logger.info('Skipping file-processing check for deleting item', { baseId, itemId, jobId }) - return true - } - return false - } catch (error) { - if (isDataApiNotFoundError(error)) { + } else { logger.info('Skipping file-processing check for missing item', { baseId, itemId, jobId }) - return true } - throw error + return true } + if (result.item.baseId !== baseId) { + throw new Error(`Knowledge item '${itemId}' does not belong to base '${baseId}'`) + } + return false } function markItemFailed(itemId: string, error: string): void { - try { - const item = knowledgeItemService.getById(itemId) - if (item.status === 'deleting') { + const result = resolveLiveKnowledgeItem(itemId) + if ('skip' in result) { + if (result.skip === 'deleting') { logger.info('Skipping mark failed for deleting item', { itemId, error }) - return - } - knowledgeItemService.updateStatus(itemId, 'failed', { error }) - } catch (updateError) { - if (isDataApiNotFoundError(updateError)) { + } else { logger.info('Skipping mark failed for missing item', { itemId, error }) - return } - throw updateError + return } + knowledgeItemService.updateStatus(itemId, 'failed', { error }) } function parseMarkdownArtifactRelativePathOrNull(baseId: string, snapshot: JobSnapshot): string | null { diff --git a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts index 2d761438bc7..ad97c32c399 100644 --- a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts +++ b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts @@ -23,6 +23,7 @@ import { hashEmbeddingText } from '../pipeline/vectorstore/indexStore/hashing' import type { RebuildMaterialEmbeddingInput, RebuildMaterialInput } from '../pipeline/vectorstore/indexStore/model' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId } from '../types' import type { KnowledgeIndexDocumentsPayload } from './jobTypes' +import { resolveLiveKnowledgeItem } from './utils/liveItem' import { isDataApiNotFoundError, markKnowledgeItemFailedOnSettled } from './utils/settled' const logger = loggerService.withContext('Knowledge:IndexDocumentsJobHandler') @@ -112,26 +113,9 @@ function loadIndexDocumentsInputOrSkip( ): LoadedIndexDocumentsInput | null { const { baseId, itemId } = ctx.input + let base: KnowledgeBase try { - const base = knowledgeBaseService.getById(baseId) - const item = knowledgeItemService.getById(itemId) - - if (item.status === 'deleting') { - logger.info('Skipping index-documents for deleting item', { baseId, itemId, jobId: ctx.jobId }) - reportKnowledgeProgress(ctx, 100, { stage: 'deleting', currentFile: 1, totalFiles: 1 }) - return null - } - - if (!isIndexableKnowledgeItem(item)) { - throw new Error(`indexDocumentsJobHandler received non-leaf knowledge item: id=${itemId} type=${item.type}`) - } - - if (item.status === 'completed') { - reportKnowledgeProgress(ctx, 100, { stage: 'already-completed', currentFile: 1, totalFiles: 1 }) - return null - } - - return { base, item } + base = knowledgeBaseService.getById(baseId) } catch (error) { if (isDataApiNotFoundError(error)) { logger.info('Skipping index-documents for missing base or item', { baseId, itemId, jobId: ctx.jobId }) @@ -140,6 +124,30 @@ function loadIndexDocumentsInputOrSkip( } throw error } + + const result = resolveLiveKnowledgeItem(itemId) + if ('skip' in result) { + if (result.skip === 'deleting') { + logger.info('Skipping index-documents for deleting item', { baseId, itemId, jobId: ctx.jobId }) + reportKnowledgeProgress(ctx, 100, { stage: 'deleting', currentFile: 1, totalFiles: 1 }) + } else { + logger.info('Skipping index-documents for missing base or item', { baseId, itemId, jobId: ctx.jobId }) + reportKnowledgeProgress(ctx, 100, { stage: 'item-gone', currentFile: 1, totalFiles: 1 }) + } + return null + } + const { item } = result + + if (!isIndexableKnowledgeItem(item)) { + throw new Error(`indexDocumentsJobHandler received non-leaf knowledge item: id=${itemId} type=${item.type}`) + } + + if (item.status === 'completed') { + reportKnowledgeProgress(ctx, 100, { stage: 'already-completed', currentFile: 1, totalFiles: 1 }) + return null + } + + return { base, item } } async function readItemDocuments( @@ -304,8 +312,8 @@ async function writeItemMaterial( await knowledgeLockManager.withBaseMutationLock(baseId, async () => { ctx.signal.throwIfAborted() - const latestItem = knowledgeItemService.getById(itemId) - if (latestItem.status === 'deleting') { + const result = resolveLiveKnowledgeItem(itemId) + if ('skip' in result) { logger.info('Skipping material rebuild for deleting item', { baseId, itemId, jobId: ctx.jobId }) return } diff --git a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts index b48826f2193..f183d6cd048 100644 --- a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts +++ b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts @@ -13,6 +13,7 @@ import { purgeKnowledgeSubtreeWithinLock } from '../ingestion/subtreePurge' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId, toKnowledgeItemId } from '../types' import type { KnowledgePrepareRootPayload } from './jobTypes' import { prepareKnowledgeItem } from './prepareItem' +import { resolveLiveKnowledgeItem } from './utils/liveItem' import { isDataApiNotFoundError, markKnowledgeItemFailedOnSettled } from './utils/settled' const logger = loggerService.withContext('Knowledge:PrepareRootJobHandler') @@ -68,15 +69,6 @@ function loadPrepareRootItemOrSkip(ctx: JobContext) try { knowledgeBaseService.getById(baseId) - const item = knowledgeItemService.getById(itemId) - - if (item.status === 'deleting') { - logger.info('Skipping prepare-root for deleting item', { baseId, itemId, jobId: ctx.jobId }) - reportKnowledgeProgress(ctx, 100, { stage: 'deleting' }) - return null - } - - return item } catch (error) { if (isDataApiNotFoundError(error)) { logger.info('Skipping prepare-root for missing base or item', { baseId, itemId, jobId: ctx.jobId }) @@ -85,6 +77,20 @@ function loadPrepareRootItemOrSkip(ctx: JobContext) } throw error } + + const result = resolveLiveKnowledgeItem(itemId) + if ('skip' in result) { + if (result.skip === 'deleting') { + logger.info('Skipping prepare-root for deleting item', { baseId, itemId, jobId: ctx.jobId }) + reportKnowledgeProgress(ctx, 100, { stage: 'deleting' }) + } else { + logger.info('Skipping prepare-root for missing base or item', { baseId, itemId, jobId: ctx.jobId }) + reportKnowledgeProgress(ctx, 100, { stage: 'item-gone' }) + } + return null + } + + return result.item } async function deletePreviousLeafExpansion( @@ -107,27 +113,21 @@ async function scanRootItem( const { baseId, itemId } = ctx.input return await knowledgeLockManager.withBaseMutationLock(baseId, async () => { - let currentItem: KnowledgeItem - try { - currentItem = knowledgeItemService.getById(itemId) - } catch (error) { - if (isDataApiNotFoundError(error)) { + const result = resolveLiveKnowledgeItem(itemId) + if ('skip' in result) { + if (result.skip === 'deleting') { + logger.info('Skipping prepare-root for deleting item before expansion', { baseId, itemId, jobId: ctx.jobId }) + reportKnowledgeProgress(ctx, 100, { stage: 'deleting' }) + } else { logger.info('Skipping prepare-root for missing item before expansion', { baseId, itemId, jobId: ctx.jobId }) reportKnowledgeProgress(ctx, 100, { stage: 'item-gone' }) - return [] } - throw error - } - - if (currentItem.status === 'deleting') { - logger.info('Skipping prepare-root for deleting item before expansion', { baseId, itemId, jobId: ctx.jobId }) - reportKnowledgeProgress(ctx, 100, { stage: 'deleting' }) return [] } const leaves = await prepareKnowledgeItem({ baseId, - item: currentItem, + item: result.item, signal: ctx.signal }) if (leaves.length > 0) { diff --git a/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts b/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts index 2ea1eb76f8c..215b0225a91 100644 --- a/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts +++ b/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts @@ -23,6 +23,7 @@ import { } from '../types' import type { KnowledgeReindexSubtreePayload } from './jobTypes' import { narrowKnowledgeJobInput } from './utils/jobInput' +import { resolveLiveKnowledgeSubtree } from './utils/liveItem' const logger = loggerService.withContext('Knowledge:ReindexSubtreeJobHandler') const REINDEX_RECOVERY_ACTIVE_STATUSES = new Set(['preparing', 'processing']) @@ -60,13 +61,14 @@ export function createReindexSubtreeJobHandler( // Reset vectors, expanded children, and root statuses as one base-level mutation. const resetResult = await knowledgeLockManager.withBaseMutationLock(baseId, async () => { const base = knowledgeBaseService.getById(baseId) - const rootItems = knowledgeItemService.getSubtreeItems(baseId, rootItemIds, { includeRoots: true }) // Re-check under the mutation lock so reindex cannot turn a just-deleted // subtree back into preparing/processing during cleanup/reset. - if (rootItems.some((item) => item.status === 'deleting')) { + const subtreeResult = resolveLiveKnowledgeSubtree(baseId, rootItemIds) + if ('skip' in subtreeResult) { logger.info('Skipping reindex-subtree reset for deleting subtree', { baseId, rootItemIds, jobId: ctx.jobId }) return { roots: [], skippedDeleting: true, skippedMissingSource: 0 } } + const rootItems = subtreeResult.items const selectedRoots = rootItems.filter((item) => rootItemIds.includes(item.id)) // Admission (assertSubtreesCanReindex) already rejected roots whose source is gone, but the @@ -174,12 +176,12 @@ export function createReindexSubtreeJobHandler( } function shouldSkipDeletingSubtreeReindex(baseId: string, rootItemIds: string[], jobId: string): boolean { - const subtreeItems = knowledgeItemService.getSubtreeItems(baseId, rootItemIds, { includeRoots: true }) - const hasDeletingItem = subtreeItems.some((item) => item.status === 'deleting') - if (hasDeletingItem) { + const result = resolveLiveKnowledgeSubtree(baseId, rootItemIds) + const isSkipped = 'skip' in result + if (isSkipped) { logger.info('Skipping reindex-subtree for deleting subtree', { baseId, rootItemIds, jobId }) } - return hasDeletingItem + return isSkipped } async function markReindexSubtreeFailedOnSettled(event: JobSettledEvent): Promise { diff --git a/src/main/features/knowledge/tasks/utils/__tests__/liveItem.test.ts b/src/main/features/knowledge/tasks/utils/__tests__/liveItem.test.ts new file mode 100644 index 00000000000..06b68e3cb8e --- /dev/null +++ b/src/main/features/knowledge/tasks/utils/__tests__/liveItem.test.ts @@ -0,0 +1,88 @@ +import { DataApiErrorFactory } from '@shared/data/api' +import type { KnowledgeItemOf } from '@shared/data/types/knowledge' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { knowledgeItemGetByIdMock, knowledgeItemGetSubtreeItemsMock } = vi.hoisted(() => ({ + knowledgeItemGetByIdMock: vi.fn(), + knowledgeItemGetSubtreeItemsMock: vi.fn() +})) + +vi.mock('@data/services/KnowledgeItemService', () => ({ + knowledgeItemService: { + getById: knowledgeItemGetByIdMock, + getSubtreeItems: knowledgeItemGetSubtreeItemsMock + } +})) + +const { resolveLiveKnowledgeItem, resolveLiveKnowledgeSubtree } = await import('../liveItem') + +function createItem( + id: string, + status: Exclude['status'], 'failed'> = 'processing' +): KnowledgeItemOf<'note'> { + return { + id, + baseId: 'kb-1', + groupId: null, + type: 'note', + data: { source: 'note', content: 'text' }, + status, + error: null, + createdAt: '2026-04-08T00:00:00.000Z', + updatedAt: '2026-04-08T00:00:00.000Z' + } +} + +describe('resolveLiveKnowledgeItem', () => { + beforeEach(() => { + knowledgeItemGetByIdMock.mockReset() + }) + + it('returns the item when it is live', () => { + const item = createItem('note-1') + knowledgeItemGetByIdMock.mockReturnValue(item) + + expect(resolveLiveKnowledgeItem('note-1')).toEqual({ item }) + }) + + it('classifies a deleting item as a deleting skip', () => { + knowledgeItemGetByIdMock.mockReturnValue(createItem('note-1', 'deleting')) + + expect(resolveLiveKnowledgeItem('note-1')).toEqual({ skip: 'deleting' }) + }) + + it('classifies a NOT_FOUND row as a missing skip', () => { + knowledgeItemGetByIdMock.mockImplementation(() => { + throw DataApiErrorFactory.notFound('KnowledgeItem', 'note-1') + }) + + expect(resolveLiveKnowledgeItem('note-1')).toEqual({ skip: 'missing' }) + }) + + it('rethrows a non-NOT_FOUND error', () => { + knowledgeItemGetByIdMock.mockImplementation(() => { + throw new Error('db busy') + }) + + expect(() => resolveLiveKnowledgeItem('note-1')).toThrow('db busy') + }) +}) + +describe('resolveLiveKnowledgeSubtree', () => { + beforeEach(() => { + knowledgeItemGetSubtreeItemsMock.mockReset() + }) + + it('returns the resolved items when none are deleting', () => { + const items = [createItem('dir-1'), createItem('note-1')] + knowledgeItemGetSubtreeItemsMock.mockReturnValue(items) + + expect(resolveLiveKnowledgeSubtree('kb-1', ['dir-1'])).toEqual({ items }) + }) + + it('skips when any resolved item is deleting', () => { + knowledgeItemGetSubtreeItemsMock.mockReturnValue([createItem('dir-1'), createItem('note-1', 'deleting')]) + + expect(resolveLiveKnowledgeSubtree('kb-1', ['dir-1'])).toEqual({ skip: 'deleting' }) + }) +}) diff --git a/src/main/features/knowledge/tasks/utils/liveItem.ts b/src/main/features/knowledge/tasks/utils/liveItem.ts new file mode 100644 index 00000000000..1e883cab8a4 --- /dev/null +++ b/src/main/features/knowledge/tasks/utils/liveItem.ts @@ -0,0 +1,34 @@ +import { knowledgeItemService } from '@data/services/KnowledgeItemService' +import type { KnowledgeItem } from '@shared/data/types/knowledge' + +import { isDataApiNotFoundError } from './settled' + +export type LiveKnowledgeItemSkipReason = 'missing' | 'deleting' +export type LiveKnowledgeItemResult = { item: KnowledgeItem } | { skip: LiveKnowledgeItemSkipReason } + +/** Classify a single item: found and live, or a skip (`item.status === 'deleting'`, or the row is NOT_FOUND). Does not check the base. */ +export function resolveLiveKnowledgeItem(itemId: string): LiveKnowledgeItemResult { + try { + const item = knowledgeItemService.getById(itemId) + if (item.status === 'deleting') { + return { skip: 'deleting' } + } + return { item } + } catch (error) { + if (isDataApiNotFoundError(error)) { + return { skip: 'missing' } + } + throw error + } +} + +export type LiveKnowledgeSubtreeResult = { items: KnowledgeItem[] } | { skip: 'deleting' } + +/** Classify a resolved subtree (roots + descendants): live, or a skip if any member is deleting. */ +export function resolveLiveKnowledgeSubtree(baseId: string, rootItemIds: string[]): LiveKnowledgeSubtreeResult { + const items = knowledgeItemService.getSubtreeItems(baseId, rootItemIds, { includeRoots: true }) + if (items.some((item) => item.status === 'deleting')) { + return { skip: 'deleting' } + } + return { items } +} From 4b10249ee1e271108c22a43ad0aec8d12a814e65 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 17:26:23 +0800 Subject: [PATCH 26/37] refactor(knowledge-data): compute active status directly in KnowledgeItemService.createActive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit create() hardcoded status: 'idle' on insert; both production call sites (prepareItem.ts, KnowledgeIngestionService.addItems) immediately followed it with updateStatus(id, isContainerKnowledgeItem(item) ? 'preparing' : 'processing') — the same "container -> preparing, leaf -> processing" rule duplicated twice, plus an extra write transaction in between that was never observably idle. Rename create() to createActive() and compute the active status inline from item.type (mirroring the existing item.type === 'directory' check in reindexSubtreeJobHandler, rather than importing the feature-layer isContainerKnowledgeItem into the data layer). Call reconcileContainers after insert to preserve the container-reactivation side effect that updateStatus's non-failed branch used to trigger — e.g. adding a live child under an already completed directory pulls the parent back to preparing/processing. Both call sites drop their now-redundant updateStatus follow-up call. Also fixes updateStatus's deleting-guard branch: it returned early inside the withWriteTx callback, but the method itself still fell through to the normal "Updated knowledge item status" info log — a blocked update looked identical to a successful one in the logs. Pull the blocked flag out of the transaction and log a distinct warn instead. ⚠️ Side-fix (not pure refactor): updateStatus's deleting-guard branch now logs a warn instead of a misleading "Updated" info line when a status update is blocked because the item is being deleted. Return value and DB state are unchanged — this only affects log output. Co-Authored-By: Claude Sonnet 5 Signed-off-by: eeee0717 --- .../data/services/KnowledgeItemService.ts | 25 ++++-- .../__tests__/KnowledgeItemService.test.ts | 70 +++++++++++++--- .../__tests__/KnowledgeService.test.ts | 81 ++++++++----------- .../ingestion/KnowledgeIngestionService.ts | 9 +-- .../tasks/__tests__/prepareItem.test.ts | 25 +++--- .../features/knowledge/tasks/prepareItem.ts | 10 +-- 6 files changed, 128 insertions(+), 92 deletions(-) diff --git a/src/main/data/services/KnowledgeItemService.ts b/src/main/data/services/KnowledgeItemService.ts index f25af098da7..e7349399c7d 100644 --- a/src/main/data/services/KnowledgeItemService.ts +++ b/src/main/data/services/KnowledgeItemService.ts @@ -245,7 +245,9 @@ export class KnowledgeItemService { return updatedRows.length } - create(baseId: string, item: CreateKnowledgeItemDto): KnowledgeItem { + /** Create an item already in its active status (`preparing` for a directory container, `processing` for a leaf) and reconcile its parent container so an add into an already-completed directory pulls it back into an active status. */ + createActive(baseId: string, item: CreateKnowledgeItemDto): KnowledgeItem { + const status: KnowledgeItemStatus = item.type === 'directory' ? 'preparing' : 'processing' const dbService = application.get('DbService') const row = dbService.withWriteTx((tx) => { this.validateGroupOwnerTx(tx, baseId, item.groupId) @@ -259,7 +261,7 @@ export class KnowledgeItemService { groupId: item.groupId ?? null, type: item.type, data: item.data, - status: 'idle', + status, error: null }) .returning() @@ -289,7 +291,8 @@ export class KnowledgeItemService { return insertedRow }) - logger.info('Created knowledge item', { baseId, id: row.id, type: row.type }) + this.reconcileContainers(baseId, [row.id, row.groupId]) + logger.info('Created active knowledge item', { baseId, id: row.id, type: row.type, status }) return rowToKnowledgeItem(row) } @@ -508,7 +511,7 @@ export class KnowledgeItemService { } const dbService = application.get('DbService') - const { item, startContainerIds } = dbService.withWriteTx((tx) => { + const { item, startContainerIds, blocked } = dbService.withWriteTx((tx) => { const [existingRow] = tx.select().from(knowledgeItemTable).where(eq(knowledgeItemTable.id, id)).limit(1).all() if (!existingRow) { @@ -518,7 +521,8 @@ export class KnowledgeItemService { if (existingRow.status === 'deleting' && status !== 'deleting') { return { item: rowToKnowledgeItem(existingRow), - startContainerIds: [] + startContainerIds: [], + blocked: true } } @@ -541,12 +545,17 @@ export class KnowledgeItemService { startContainerIds: status === 'failed' && updatedRow.type === 'directory' ? [existingRow.groupId] - : [updatedRow.id, existingRow.groupId] + : [updatedRow.id, existingRow.groupId], + blocked: false } }) - this.reconcileContainers(item.baseId, startContainerIds) - logger.info('Updated knowledge item status', { id, status }) + if (blocked) { + logger.warn('Skipped status update for deleting item', { id, attemptedStatus: status }) + } else { + this.reconcileContainers(item.baseId, startContainerIds) + logger.info('Updated knowledge item status', { id, status }) + } return item } diff --git a/src/main/data/services/__tests__/KnowledgeItemService.test.ts b/src/main/data/services/__tests__/KnowledgeItemService.test.ts index c7e2b8edeec..5f8503fe86d 100644 --- a/src/main/data/services/__tests__/KnowledgeItemService.test.ts +++ b/src/main/data/services/__tests__/KnowledgeItemService.test.ts @@ -7,6 +7,7 @@ import { ErrorCode } from '@shared/data/api' import type { CreateKnowledgeItemDto } from '@shared/data/types/knowledge' import { createUniqueModelId } from '@shared/data/types/model' import { setupTestDatabase } from '@test-helpers/db' +import { mockMainLoggerService } from '@test-mocks/MainLoggerService' import { eq } from 'drizzle-orm' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -437,19 +438,37 @@ describe('KnowledgeItemService', () => { }) describe('create', () => { - it('creates one knowledge item as idle', async () => { + it('creates a directory item as preparing', async () => { const item: CreateKnowledgeItemDto = { type: 'directory', data: { source: '/tmp/files' } } - const result = service.create(KNOWLEDGE_BASE_ID, item) + const result = service.createActive(KNOWLEDGE_BASE_ID, item) expect(result).toMatchObject({ baseId: KNOWLEDGE_BASE_ID, groupId: null, type: 'directory', - status: 'idle', + status: 'preparing', + error: null, + data: item.data + }) + }) + + it('creates a leaf item as processing', async () => { + const item: CreateKnowledgeItemDto = { + type: 'note', + data: { source: 'note', content: 'note' } + } + + const result = service.createActive(KNOWLEDGE_BASE_ID, item) + + expect(result).toMatchObject({ + baseId: KNOWLEDGE_BASE_ID, + groupId: null, + type: 'note', + status: 'processing', error: null, data: item.data }) @@ -458,7 +477,7 @@ describe('KnowledgeItemService', () => { it('accepts a group owner in the same base', async () => { await seedItem({ id: DIR_A_ID, type: 'directory', data: { source: '/a' } }) - const result = service.create(KNOWLEDGE_BASE_ID, { + const result = service.createActive(KNOWLEDGE_BASE_ID, { groupId: DIR_A_ID, type: 'note', data: { source: 'new grouped note', content: 'new grouped note' } @@ -481,7 +500,7 @@ describe('KnowledgeItemService', () => { let err: unknown try { - service.create(KNOWLEDGE_BASE_ID, { + service.createActive(KNOWLEDGE_BASE_ID, { groupId: DIR_A_ID, type: 'note', data: { source: 'child note', content: 'child note' } @@ -504,7 +523,7 @@ describe('KnowledgeItemService', () => { let err: unknown try { - service.create(KNOWLEDGE_BASE_ID, { + service.createActive(KNOWLEDGE_BASE_ID, { groupId: NOTE_OWNER_ID, type: 'note', data: { source: 'child note', content: 'child note' } @@ -525,7 +544,7 @@ describe('KnowledgeItemService', () => { it('rejects blank group owner ids before hitting foreign key constraints', async () => { let err: unknown try { - service.create(KNOWLEDGE_BASE_ID, { + service.createActive(KNOWLEDGE_BASE_ID, { groupId: ' ', type: 'note', data: { source: 'child note', content: 'child note' } @@ -546,7 +565,7 @@ describe('KnowledgeItemService', () => { it('translates missing base and missing group owner constraints', () => { let missingBaseErr: unknown try { - service.create('missing-base', { type: 'note', data: { source: 'note', content: 'note' } }) + service.createActive('missing-base', { type: 'note', data: { source: 'note', content: 'note' } }) } catch (e) { missingBaseErr = e } @@ -557,7 +576,7 @@ describe('KnowledgeItemService', () => { let missingOwnerErr: unknown try { - service.create(KNOWLEDGE_BASE_ID, { + service.createActive(KNOWLEDGE_BASE_ID, { groupId: 'missing-owner', type: 'note', data: { source: 'child note', content: 'child note' } @@ -624,7 +643,7 @@ describe('KnowledgeItemService', () => { }) it('creates a file knowledge item with a copied relative path', async () => { - const result = service.create(KNOWLEDGE_BASE_ID, { + const result = service.createActive(KNOWLEDGE_BASE_ID, { type: 'file', data: { source: '/docs/a.md', @@ -1016,6 +1035,8 @@ describe('KnowledgeItemService', () => { const seeded = await seedItem({ status: 'deleting' }) + mockMainLoggerService.warn.mockClear() + mockMainLoggerService.info.mockClear() const result = service.updateStatus(seeded.id, 'completed') @@ -1028,12 +1049,19 @@ describe('KnowledgeItemService', () => { status: 'deleting', error: null }) + expect(mockMainLoggerService.warn).toHaveBeenCalledWith( + 'Skipped status update for deleting item', + expect.objectContaining({ id: seeded.id, attemptedStatus: 'completed' }) + ) + expect(mockMainLoggerService.info).not.toHaveBeenCalledWith('Updated knowledge item status', expect.anything()) }) it('does not overwrite deleting items with failed status from settled jobs', async () => { const seeded = await seedItem({ status: 'deleting' }) + mockMainLoggerService.warn.mockClear() + mockMainLoggerService.info.mockClear() const result = service.updateStatus(seeded.id, 'failed', { error: 'cancelled' }) @@ -1046,6 +1074,11 @@ describe('KnowledgeItemService', () => { status: 'deleting', error: null }) + expect(mockMainLoggerService.warn).toHaveBeenCalledWith( + 'Skipped status update for deleting item', + expect.objectContaining({ id: seeded.id, attemptedStatus: 'failed' }) + ) + expect(mockMainLoggerService.info).not.toHaveBeenCalledWith('Updated knowledge item status', expect.anything()) }) }) @@ -1608,5 +1641,22 @@ describe('KnowledgeItemService', () => { await expect(getItemRow(DIR_A_ID)).resolves.toMatchObject({ status: 'completed', error: null }) }) + + it('pulls a completed container back to processing when createActive adds a new child', async () => { + await seedItem({ + id: DIR_ROOT_ID, + type: 'directory', + data: { source: '/docs' }, + status: 'completed' + }) + + service.createActive(KNOWLEDGE_BASE_ID, { + groupId: DIR_ROOT_ID, + type: 'note', + data: { source: 'note', content: 'note' } + }) + + await expect(getItemRow(DIR_ROOT_ID)).resolves.toMatchObject({ status: 'processing', error: null }) + }) }) }) diff --git a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts index 492d8ee3649..940f0479356 100644 --- a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts +++ b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts @@ -25,7 +25,7 @@ const { knowledgeBaseDeleteMock, knowledgeBaseGetByIdMock, knowledgeBaseListMock, - knowledgeItemCreateMock, + knowledgeItemCreateActiveMock, knowledgeItemDeleteMock, knowledgeItemGetDeletingRootGroupsMock, knowledgeItemFailInterruptedItemsMock, @@ -62,7 +62,7 @@ const { knowledgeBaseDeleteMock: vi.fn(), knowledgeBaseGetByIdMock: vi.fn(), knowledgeBaseListMock: vi.fn(), - knowledgeItemCreateMock: vi.fn(), + knowledgeItemCreateActiveMock: vi.fn(), knowledgeItemDeleteMock: vi.fn(), knowledgeItemGetDeletingRootGroupsMock: vi.fn(), knowledgeItemFailInterruptedItemsMock: vi.fn(), @@ -156,7 +156,7 @@ vi.mock('@data/services/KnowledgeBaseService', () => ({ vi.mock('@data/services/KnowledgeItemService', () => ({ knowledgeItemService: { - create: knowledgeItemCreateMock, + createActive: knowledgeItemCreateActiveMock, delete: knowledgeItemDeleteMock, getDeletingRootGroups: knowledgeItemGetDeletingRootGroupsMock, failInterruptedItems: knowledgeItemFailInterruptedItemsMock, @@ -328,13 +328,15 @@ describe('KnowledgeService', () => { copyFileIntoKnowledgeBaseAtMock.mockImplementation( async (_baseId: string, _sourcePath: string, relativePath: string) => relativePath ) - knowledgeItemCreateMock.mockImplementation((baseId: string, input: { type?: string; data: { source: string } }) => { - createdItemBaseIds.set(input.data.source, baseId) - if (input.type === 'file') { - return createFileItem(input.data.source, baseId, input.data.source) + knowledgeItemCreateActiveMock.mockImplementation( + (baseId: string, input: { type?: string; data: { source: string } }) => { + createdItemBaseIds.set(input.data.source, baseId) + if (input.type === 'file') { + return createFileItem(input.data.source, baseId, input.data.source, 'processing') + } + return createNoteItem(input.data.source, baseId, null, 'processing') } - return createNoteItem(input.data.source, baseId) - }) + ) knowledgeItemDeleteMock.mockReturnValue(undefined) deleteKnowledgeItemFilesBestEffortMock.mockResolvedValue(undefined) knowledgeItemGetDeletingRootGroupsMock.mockReturnValue([]) @@ -886,8 +888,7 @@ describe('KnowledgeService', () => { ...createFileItem('restored-file', 'restored-kb', '/docs/report.pdf', 'processing'), data: { source: '/docs/report.pdf', relativePath: 'report.pdf', indexedRelativePath: 'report.md' } } - knowledgeItemCreateMock.mockReturnValueOnce(restoredFile) - knowledgeItemUpdateStatusMock.mockReturnValueOnce(restoredFile) + knowledgeItemCreateActiveMock.mockReturnValueOnce(restoredFile) knowledgeItemGetByIdMock.mockReturnValue(restoredFile) await service.restoreBase({ @@ -903,7 +904,7 @@ describe('KnowledgeService', () => { ['restored-kb', '/mock/feature.knowledgebase.data/source-kb/raw/report.md', 'report.md'] ]) // The created item carries the artifact path. - expect(knowledgeItemCreateMock).toHaveBeenCalledWith( + expect(knowledgeItemCreateActiveMock).toHaveBeenCalledWith( 'restored-kb', expect.objectContaining({ type: 'file', @@ -947,7 +948,7 @@ describe('KnowledgeService', () => { 'example-page.md' ) // The created url item is pinned to the copied snapshot so first index reads it offline. - expect(knowledgeItemCreateMock).toHaveBeenCalledWith( + expect(knowledgeItemCreateActiveMock).toHaveBeenCalledWith( 'restored-kb', expect.objectContaining({ type: 'url', @@ -978,7 +979,7 @@ describe('KnowledgeService', () => { }) // No snapshot to carry: the restored url has no relativePath so first index re-captures it. - expect(knowledgeItemCreateMock).toHaveBeenCalledWith( + expect(knowledgeItemCreateActiveMock).toHaveBeenCalledWith( 'restored-kb', expect.objectContaining({ type: 'url', data: { source: 'https://example.com', url: 'https://example.com' } }) ) @@ -1005,11 +1006,9 @@ describe('KnowledgeService', () => { vi.useFakeTimers() vi.setSystemTime(new Date('2026-04-08T00:00:00.000Z')) const service = new KnowledgeService() - const createdFile = createFileItem('file-1', 'kb-1', '/docs/source.pdf') const processingFile = createFileItem('file-1', 'kb-1', '/docs/source.pdf', 'processing') knowledgeBaseGetByIdMock.mockReturnValue(createBase({ fileProcessorId: 'doc2x' })) - knowledgeItemCreateMock.mockReturnValueOnce(createdFile) - knowledgeItemUpdateStatusMock.mockReturnValueOnce(processingFile) + knowledgeItemCreateActiveMock.mockReturnValueOnce(processingFile) knowledgeItemGetByIdMock.mockReturnValueOnce(processingFile) await service.addItems('kb-1', [{ type: 'file', data: { source: '/docs/source.pdf', path: '/docs/source.pdf' } }]) @@ -1049,10 +1048,7 @@ describe('KnowledgeService', () => { it('auto-renames a duplicate uploaded file name instead of rejecting the import', async () => { const service = new KnowledgeService() knowledgeBaseGetByIdMock.mockReturnValue(createBase({ fileProcessorId: null })) - knowledgeItemCreateMock - .mockReturnValueOnce(createFileItem('file-1', 'kb-1', '/Users/me/a/notes.md')) - .mockReturnValueOnce(createFileItem('file-2', 'kb-1', '/Users/me/b/notes.md')) - knowledgeItemUpdateStatusMock + knowledgeItemCreateActiveMock .mockReturnValueOnce(createFileItem('file-1', 'kb-1', '/Users/me/a/notes.md', 'processing')) .mockReturnValueOnce(createFileItem('file-2', 'kb-1', '/Users/me/b/notes.md', 'processing')) knowledgeItemGetByIdMock @@ -1065,7 +1061,7 @@ describe('KnowledgeService', () => { ]) // Both imports land; the second's relativePath is deduped (`_N`) rather than refused. - expect(knowledgeItemCreateMock).toHaveBeenCalledTimes(2) + expect(knowledgeItemCreateActiveMock).toHaveBeenCalledTimes(2) expect(copyFileIntoKnowledgeBaseAtMock).toHaveBeenNthCalledWith(1, 'kb-1', '/Users/me/a/notes.md', 'notes.md') expect(copyFileIntoKnowledgeBaseAtMock).toHaveBeenNthCalledWith(2, 'kb-1', '/Users/me/b/notes.md', 'notes_1.md') }) @@ -1073,10 +1069,7 @@ describe('KnowledgeService', () => { it('auto-renames a file whose processed-markdown name would collide', async () => { const service = new KnowledgeService() knowledgeBaseGetByIdMock.mockReturnValue(createBase({ fileProcessorId: 'doc2x' })) - knowledgeItemCreateMock - .mockReturnValueOnce(createFileItem('file-1', 'kb-1', '/Users/me/a/brief.pdf')) - .mockReturnValueOnce(createFileItem('file-2', 'kb-1', '/Users/me/b/brief.docx')) - knowledgeItemUpdateStatusMock + knowledgeItemCreateActiveMock .mockReturnValueOnce(createFileItem('file-1', 'kb-1', '/Users/me/a/brief.pdf', 'processing')) .mockReturnValueOnce(createFileItem('file-2', 'kb-1', '/Users/me/b/brief.docx', 'processing')) knowledgeItemGetByIdMock @@ -1090,7 +1083,7 @@ describe('KnowledgeService', () => { // brief.pdf reserves brief.pdf + its brief.md output; brief.docx would also emit // brief.md, so it is bumped to brief_1.docx (whose brief_1.md sibling is free). - expect(knowledgeItemCreateMock).toHaveBeenCalledTimes(2) + expect(knowledgeItemCreateActiveMock).toHaveBeenCalledTimes(2) expect(copyFileIntoKnowledgeBaseAtMock).toHaveBeenNthCalledWith(1, 'kb-1', '/Users/me/a/brief.pdf', 'brief.pdf') expect(copyFileIntoKnowledgeBaseAtMock).toHaveBeenNthCalledWith(2, 'kb-1', '/Users/me/b/brief.docx', 'brief_1.docx') }) @@ -1126,7 +1119,7 @@ describe('KnowledgeService', () => { '/captured/example-page.md', 'example-page_1.md' ) - expect(knowledgeItemCreateMock).toHaveBeenCalledWith( + expect(knowledgeItemCreateActiveMock).toHaveBeenCalledWith( 'kb-1', expect.objectContaining({ type: 'url', @@ -1140,7 +1133,7 @@ describe('KnowledgeService', () => { knowledgeBaseGetByIdMock.mockReturnValue(createBase({ fileProcessorId: null })) knowledgeItemGetItemsByBaseIdMock.mockReturnValue([]) // The url restore copies its snapshot to raw/ before the row is created; that create fails. - knowledgeItemCreateMock.mockImplementationOnce(() => { + knowledgeItemCreateActiveMock.mockImplementationOnce(() => { throw new Error('db down') }) @@ -1178,8 +1171,7 @@ describe('KnowledgeService', () => { data: { source: 'Meeting notes', content: 'hello', relativePath: 'Meeting notes.md' } } ]) - knowledgeItemCreateMock.mockReturnValueOnce(createFileItem('file-1', 'kb-1', '/Users/me/Meeting notes.md')) - knowledgeItemUpdateStatusMock.mockReturnValueOnce( + knowledgeItemCreateActiveMock.mockReturnValueOnce( createFileItem('file-1', 'kb-1', '/Users/me/Meeting notes.md', 'processing') ) knowledgeItemGetByIdMock.mockReturnValueOnce( @@ -1198,7 +1190,7 @@ describe('KnowledgeService', () => { '/Users/me/Meeting notes.md', 'Meeting notes_1.md' ) - expect(knowledgeItemCreateMock).toHaveBeenCalledWith( + expect(knowledgeItemCreateActiveMock).toHaveBeenCalledWith( 'kb-1', expect.objectContaining({ type: 'file', @@ -1275,7 +1267,7 @@ describe('KnowledgeService', () => { service.addItems('kb-1', [{ type: 'file', data: { source: '/Users/me/app.exe', path: '/Users/me/app.exe' } }]) ).rejects.toThrow('Unsupported knowledge file type: /Users/me/app.exe') - expect(knowledgeItemCreateMock).not.toHaveBeenCalled() + expect(knowledgeItemCreateActiveMock).not.toHaveBeenCalled() expect(copyFileIntoKnowledgeBaseAtMock).not.toHaveBeenCalled() expect(fileProcessingStartJobMock).not.toHaveBeenCalled() }) @@ -1441,11 +1433,9 @@ describe('KnowledgeService', () => { it('schedules direct indexing for file items when the extension does not need file processing', async () => { const service = new KnowledgeService() - const createdFile = createFileItem('file-1', 'kb-1', '/docs/source.md') const processingFile = createFileItem('file-1', 'kb-1', '/docs/source.md', 'processing') knowledgeBaseGetByIdMock.mockReturnValue(createBase({ fileProcessorId: 'doc2x' })) - knowledgeItemCreateMock.mockReturnValueOnce(createdFile) - knowledgeItemUpdateStatusMock.mockReturnValueOnce(processingFile) + knowledgeItemCreateActiveMock.mockReturnValueOnce(processingFile) knowledgeItemGetByIdMock.mockReturnValueOnce(processingFile) await service.addItems('kb-1', [{ type: 'file', data: { source: '/docs/source.md', path: '/docs/source.md' } }]) @@ -1464,11 +1454,9 @@ describe('KnowledgeService', () => { it('schedules direct indexing for document files when the base has no file processor', async () => { const service = new KnowledgeService() - const createdFile = createFileItem('file-1', 'kb-1', '/docs/source.pdf') const processingFile = createFileItem('file-1', 'kb-1', '/docs/source.pdf', 'processing') knowledgeBaseGetByIdMock.mockReturnValue(createBase({ fileProcessorId: null })) - knowledgeItemCreateMock.mockReturnValueOnce(createdFile) - knowledgeItemUpdateStatusMock.mockReturnValueOnce(processingFile) + knowledgeItemCreateActiveMock.mockReturnValueOnce(processingFile) knowledgeItemGetByIdMock.mockReturnValueOnce(processingFile) await service.addItems('kb-1', [{ type: 'file', data: { source: '/docs/source.pdf', path: '/docs/source.pdf' } }]) @@ -1500,20 +1488,18 @@ describe('KnowledgeService', () => { ]) ).rejects.toThrow('enqueue failed') - expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith('note-1', 'processing') - expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith('note-2', 'processing') expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith('note-2', 'failed', { error: 'Failed to schedule knowledge item job: enqueue failed' }) expect(knowledgeItemUpdateStatusMock).not.toHaveBeenCalledWith('note-1', 'failed', expect.anything()) }) - it('rolls back every created addItems row when a status update fails', async () => { + it('rolls back already-created addItems rows when a later create fails mid-batch', async () => { const service = new KnowledgeService() - knowledgeItemUpdateStatusMock + knowledgeItemCreateActiveMock .mockReturnValueOnce(createNoteItem('note-1', 'kb-1', null, 'processing')) .mockImplementationOnce(() => { - throw new Error('status failed') + throw new Error('create failed') }) await expect( @@ -1521,16 +1507,17 @@ describe('KnowledgeService', () => { { type: 'note', data: { source: 'note-1', content: 'hello 1' } }, { type: 'note', data: { source: 'note-2', content: 'hello 2' } } ]) - ).rejects.toThrow('status failed') + ).rejects.toThrow('create failed') + // note-2's row was never created, so there is nothing to roll back for it. expect(knowledgeItemDeleteMock).toHaveBeenCalledWith('note-1') - expect(knowledgeItemDeleteMock).toHaveBeenCalledWith('note-2') + expect(knowledgeItemDeleteMock).not.toHaveBeenCalledWith('note-2') expect(enqueueMock).not.toHaveBeenCalled() }) it('runs best-effort copied-file cleanup and preserves the original addItems error', async () => { const service = new KnowledgeService() - knowledgeItemCreateMock.mockImplementationOnce(() => { + knowledgeItemCreateActiveMock.mockImplementationOnce(() => { throw new Error('create failed') }) diff --git a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts index 0d888e268a6..943a7fade7e 100644 --- a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts +++ b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts @@ -22,7 +22,7 @@ import { knowledgeSupportedFileExts } from '@shared/utils/file' import { assertBaseCanRunRuntimeOperation } from '../base/baseGuards' import type { KnowledgeLockManager } from '../base/KnowledgeLockManager' -import { classifyKnowledgeItemSource, isContainerKnowledgeItem } from '../items' +import { classifyKnowledgeItemSource } from '../items' import { assertKnowledgeFileTargetAvailable, collectKnowledgeReservedRelativePaths, @@ -145,13 +145,8 @@ export class KnowledgeIngestionService implements KnowledgeItemScheduler { if (createInput.type === 'file' || (createInput.type === 'url' && createInput.data.relativePath)) { copiedFileItems.push(createInput) } - const createdItem = knowledgeItemService.create(base.id, createInput) + const createdItem = knowledgeItemService.createActive(base.id, createInput) acceptedItems.push(createdItem) - const activeItem = knowledgeItemService.updateStatus( - createdItem.id, - isContainerKnowledgeItem(createdItem) ? 'preparing' : 'processing' - ) - acceptedItems[acceptedItems.length - 1] = activeItem } } catch (error) { this.rollbackAcceptedItems(base.id, acceptedItems, error) diff --git a/src/main/features/knowledge/tasks/__tests__/prepareItem.test.ts b/src/main/features/knowledge/tasks/__tests__/prepareItem.test.ts index 40d88713c54..db062ed5860 100644 --- a/src/main/features/knowledge/tasks/__tests__/prepareItem.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/prepareItem.test.ts @@ -3,14 +3,14 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const { expandDirectoryOwnerToTreeMock, - knowledgeItemCreateMock, + knowledgeItemCreateActiveMock, knowledgeItemUpdateStatusMock, knowledgeItemUpdateDirectoryRelativePathMock, knowledgeItemGetItemsByBaseIdMock, loggerWarnMock } = vi.hoisted(() => ({ expandDirectoryOwnerToTreeMock: vi.fn(), - knowledgeItemCreateMock: vi.fn(), + knowledgeItemCreateActiveMock: vi.fn(), knowledgeItemUpdateStatusMock: vi.fn(), knowledgeItemUpdateDirectoryRelativePathMock: vi.fn(), knowledgeItemGetItemsByBaseIdMock: vi.fn(), @@ -19,7 +19,7 @@ const { vi.mock('@data/services/KnowledgeItemService', () => ({ knowledgeItemService: { - create: knowledgeItemCreateMock, + createActive: knowledgeItemCreateActiveMock, updateStatus: knowledgeItemUpdateStatusMock, updateDirectoryRelativePath: knowledgeItemUpdateDirectoryRelativePathMock, getItemsByBaseId: knowledgeItemGetItemsByBaseIdMock @@ -104,13 +104,13 @@ describe('prepareKnowledgeItem', () => { expandDirectoryOwnerToTreeMock.mockResolvedValue({ pathPrefix: 'docs', children: [] }) knowledgeItemGetItemsByBaseIdMock.mockReturnValue([]) - knowledgeItemCreateMock.mockImplementation((_baseId: string, item: Partial) => ({ + knowledgeItemCreateActiveMock.mockImplementation((_baseId: string, item: Partial) => ({ id: `${item.type}-created`, baseId, groupId: item.groupId ?? null, type: item.type, data: item.data, - status: 'idle', + status: item.type === 'directory' ? 'preparing' : 'processing', error: null, createdAt: '2026-04-08T00:00:00.000Z', updatedAt: '2026-04-08T00:00:00.000Z' @@ -135,7 +135,7 @@ describe('prepareKnowledgeItem', () => { await expect(prepareKnowledgeItem(createPrepareOptions(note))).resolves.toEqual([note]) - expect(knowledgeItemCreateMock).not.toHaveBeenCalled() + expect(knowledgeItemCreateActiveMock).not.toHaveBeenCalled() expect(knowledgeItemUpdateStatusMock).not.toHaveBeenCalled() }) @@ -143,8 +143,7 @@ describe('prepareKnowledgeItem', () => { const root = createDirectoryItem('dir-root') const childDir = createDirectoryItem('dir-child', root.id) const childFile = createFileItem('file-child', childDir.id) - knowledgeItemCreateMock.mockReturnValueOnce(childDir).mockReturnValueOnce(childFile) - knowledgeItemUpdateStatusMock.mockReturnValueOnce(childDir).mockReturnValueOnce(childFile) + knowledgeItemCreateActiveMock.mockReturnValueOnce(childDir).mockReturnValueOnce(childFile) expandDirectoryOwnerToTreeMock.mockResolvedValueOnce({ pathPrefix: 'dir-root-prefix', children: [ @@ -166,19 +165,19 @@ describe('prepareKnowledgeItem', () => { expect(expandDirectoryOwnerToTreeMock).toHaveBeenCalledWith(root, baseId, expect.any(Set), options.signal) expect(knowledgeItemUpdateDirectoryRelativePathMock).toHaveBeenCalledWith(root.id, 'dir-root-prefix') - expect(knowledgeItemCreateMock).toHaveBeenNthCalledWith(1, baseId, { + expect(knowledgeItemCreateActiveMock).toHaveBeenNthCalledWith(1, baseId, { groupId: root.id, type: 'directory', data: childDir.data }) - expect(knowledgeItemCreateMock).toHaveBeenNthCalledWith(2, baseId, { + expect(knowledgeItemCreateActiveMock).toHaveBeenNthCalledWith(2, baseId, { groupId: childDir.id, type: 'file', data: childFile.data }) - expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith(childDir.id, 'preparing') + // `childDir` is a directory: it's created active as `preparing` by `createActive`, + // then flipped to `processing` once its own children finish being created. expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith(childDir.id, 'processing') - expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith(childFile.id, 'processing') }) it('excludes the container itself from the reserved names so a reindex keeps its own prefix', () => { @@ -246,7 +245,7 @@ describe('prepareKnowledgeItem', () => { }) ).rejects.toBe(abortError) - expect(knowledgeItemCreateMock).not.toHaveBeenCalled() + expect(knowledgeItemCreateActiveMock).not.toHaveBeenCalled() expect(knowledgeItemUpdateStatusMock).not.toHaveBeenCalled() }) diff --git a/src/main/features/knowledge/tasks/prepareItem.ts b/src/main/features/knowledge/tasks/prepareItem.ts index 54188c5f110..a562091b62f 100644 --- a/src/main/features/knowledge/tasks/prepareItem.ts +++ b/src/main/features/knowledge/tasks/prepareItem.ts @@ -7,7 +7,7 @@ import { type KnowledgeItemType } from '@shared/data/types/knowledge' -import { type IndexableKnowledgeItem, isContainerKnowledgeItem, isIndexableKnowledgeItem } from '../items' +import { type IndexableKnowledgeItem, isIndexableKnowledgeItem } from '../items' import { collectKnowledgeReservedRelativePaths } from '../pathStorage' import { expandDirectoryOwnerToTree, type ExpandedDirectoryNode } from '../pipeline/sources/directory' @@ -129,12 +129,8 @@ async function createRuntimeItem( signal: AbortSignal ): Promise> { signal.throwIfAborted() - const createdItem = knowledgeItemService.create(baseId, item) - - const processingItem = isContainerKnowledgeItem(createdItem) - ? knowledgeItemService.updateStatus(createdItem.id, 'preparing') - : knowledgeItemService.updateStatus(createdItem.id, 'processing') + const createdItem = knowledgeItemService.createActive(baseId, item) signal.throwIfAborted() - return processingItem as KnowledgeItemOf + return createdItem as KnowledgeItemOf } From 49b2cea09e606cd7dd14b538d9da5cd731ece3bd Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 17:26:40 +0800 Subject: [PATCH 27/37] test(knowledge-tasks): assert lock boundaries around fetch and embed in index-documents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit README.md documents that the base mutation lock must only wrap the mutation step of index-documents, never the network fetch or the embedding API call — but no test asserted the ordering, only that both eventually happened. withBaseMutationLock's test double is a plain pass-through (vi.fn(async (_baseId, task) => await task())), so mock.invocationCallOrder already lets us assert fetch/embed happen strictly before the lock is acquired; add those assertions to the two tests that already exercise this path. Co-Authored-By: Claude Sonnet 5 Signed-off-by: eeee0717 --- .../tasks/__tests__/indexDocumentsJobHandler.test.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/features/knowledge/tasks/__tests__/indexDocumentsJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/indexDocumentsJobHandler.test.ts index 81a050beea3..c7b2b449e54 100644 --- a/src/main/features/knowledge/tasks/__tests__/indexDocumentsJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/indexDocumentsJobHandler.test.ts @@ -61,6 +61,11 @@ describe('index-documents job handler', () => { ) expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith(NOTE_ITEM_ID, 'completed') expect(handler.defaultQueue?.({ baseId: 'kb-1', itemId: NOTE_ITEM_ID, parentJobId: null })).toBe('base.kb-1') + // README's lock-boundary contract: embedding (paid API call) must run before the + // mutation lock is taken, never inside it. + expect(embedKnowledgeTextsMock.mock.invocationCallOrder[0]).toBeLessThan( + knowledgeLockManager.withBaseMutationLock.mock.invocationCallOrder[0] + ) }) it('pairs every embedding vector with the hash of the body it was computed from', async () => { @@ -282,6 +287,11 @@ describe('index-documents job handler', () => { // item-id virtual placeholder — so it points at the bytes captureUrlSnapshotFile // wrote and agrees with what the v1→v2 migrator stamps for the same url. expect(lastRebuildInput().material.relativePath).toBe('example-page.md') + // README's lock-boundary contract: the network fetch must complete before the + // (first) mutation lock is taken, never inside it. + expect(fetchKnowledgeWebPageMock.mock.invocationCallOrder[0]).toBeLessThan( + knowledgeLockManager.withBaseMutationLock.mock.invocationCallOrder[0] + ) }) it('does not fetch a URL that already has a captured snapshot', async () => { From 16eabd559b08468ffa14ced3c5d4d2d4c36469e3 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 17:26:48 +0800 Subject: [PATCH 28/37] test(mocks): give the DbService withWriteTx mock real transaction semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mock's withWriteTx was a pure pass-through (fn(this.db)), even when setupTestDatabase() had attached a real better-sqlite3 connection via MockMainDbServiceUtils.setDb() — so tests never got the rollback-on-throw semantics that production's this.db.transaction(fn, { behavior: 'immediate' }) provides. README.md also documented the mock's signature as async ((fn: (tx) => Promise) => Promise), which doesn't match the synchronous production signature. Delegate to the real db's .transaction() when one is attached (detected via typeof db.transaction === 'function', which only real better-sqlite3 instances have — hand-written mock db literals don't), falling back to the plain pass-through otherwise. Update the README's documented signature to match the synchronous production one. Co-Authored-By: Claude Sonnet 5 Signed-off-by: eeee0717 --- tests/__mocks__/README.md | 4 ++-- tests/__mocks__/main/DbService.ts | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/__mocks__/README.md b/tests/__mocks__/README.md index 7a008f9f1fc..2077c6f1757 100644 --- a/tests/__mocks__/README.md +++ b/tests/__mocks__/README.md @@ -333,7 +333,7 @@ Database service providing access to the mock SQLite database. | Method | Signature | |--------|-----------| | `getDb` | `() => MockDb` | -| `withWriteTx` | `(fn: (tx) => Promise) => Promise` (passthrough — calls `fn(db)`) | +| `withWriteTx` | `(fn: (tx) => T) => T` (synchronous, mirrors production) | | `isReady` | `boolean` (getter) | ```typescript @@ -348,7 +348,7 @@ MockMainDbServiceUtils.getDefaultMockDb() MockMainDbServiceUtils.setDb(customMockDb) ``` -> **`withWriteTx`**: passthrough (`async (fn) => fn(this.db)`) — no mutex / BUSY retry. Use `vi.spyOn(dbServiceInstance, 'withWriteTx')` to inject custom behavior. Hand-rolled DbService mocks MUST include this method or production code throws `TypeError: dbService.withWriteTx is not a function`. +> **`withWriteTx`**: once a real DB is attached (`setupTestDatabase()` → `setDb()`), delegates to `db.transaction(fn, { behavior: 'immediate' })` for real transaction semantics (rollback on throw, etc.); in pure-mock mode (default db, no `setDb()`) falls through to `fn(this.db)` — no mutex / BUSY retry. Use `vi.spyOn(dbServiceInstance, 'withWriteTx')` to inject custom behavior. Hand-rolled DbService mocks MUST include this method or production code throws `TypeError: dbService.withWriteTx is not a function`. --- diff --git a/tests/__mocks__/main/DbService.ts b/tests/__mocks__/main/DbService.ts index f3a504c8350..944f82a583c 100644 --- a/tests/__mocks__/main/DbService.ts +++ b/tests/__mocks__/main/DbService.ts @@ -69,12 +69,19 @@ export class MockMainDbService { public getDb = vi.fn(() => this.db) /** - * Write transaction mock. Mirrors `DbService.withWriteTx`: synchronously passes - * the current db (or whatever was set via `setDb`) to the synchronous `fn` so - * tests exercising the write path do not need a real transaction. Tests can - * replace this mock with `vi.spyOn(...)` to assert call order, etc. + * Write transaction mock. Mirrors `DbService.withWriteTx`: when a real + * better-sqlite3 connection is attached (via `setDb`, e.g. `setupTestDatabase()`), + * delegates to its `.transaction()` for real transaction semantics (rollback on + * throw, etc.); otherwise falls through to the plain (non-transactional) db stub. + * Tests can replace this mock with `vi.spyOn(...)` to assert call order, etc. */ - public withWriteTx = vi.fn((fn: (tx: unknown) => T): T => fn(this.db)) + public withWriteTx = vi.fn((fn: (tx: unknown) => T): T => { + const db = this.db as { transaction?: (fn: (tx: unknown) => unknown, options?: unknown) => unknown } + if (typeof db?.transaction === 'function') { + return db.transaction(fn, { behavior: 'immediate' }) as T + } + return fn(this.db) + }) public get isReady() { return this._isReady From fca2350c2f7b1873624ccb44949f84c5020cce2b Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Fri, 3 Jul 2026 17:27:05 +0800 Subject: [PATCH 29/37] test(knowledge-vectorstore): rewrite KnowledgeVectorStoreService tests against a real temp-dir store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 22-test suite mocked all 10 collaborators (KnowledgeIndexStore, BetterSqlite3Driver, BetterSqlite3VectorIndex, schema.ts, indexMeta.ts, knowledgeItemService, pathStorage, fs, @logger, @main/core/lifecycle) plus hand-copied MOCK_SCHEMA_VERSION to mirror schema.ts's real constant — asserting against the mocks' own call counts rather than the real open/rebuild/evict logic. Rewrite against a real temp-dir SQLite store (mkdtempSync + real openBetterSqlite3IndexDriver, matching the pattern already used by BetterSqlite3Driver.test.ts), keeping only the app-DB and lifecycle boundary mocks (knowledgeItemService, pathStorage, @logger, @main/core/lifecycle). Stale/downgrade-version tests now stamp a real version via direct SQL (`UPDATE meta SET schema_version = ?`) rather than ensureIndexMeta, which hardcodes the current constant and can't write an arbitrary version. Failure-injection tests (evicts-a-failed-open, meta verification, schema creation) use targeted vi.spyOn on the real modules instead of blanket vi.mock. The "single-flight" test is retitled to reflect what getIndexStore's own doc comment already states: the method has no `await` before caching, so a second concurrent call can't observe a genuinely in-flight open — there's no async dedup mechanism to test. Co-Authored-By: Claude Sonnet 5 Signed-off-by: eeee0717 --- .../KnowledgeVectorStoreService.test.ts | 337 ++++++++---------- 1 file changed, 145 insertions(+), 192 deletions(-) diff --git a/src/main/features/knowledge/pipeline/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts b/src/main/features/knowledge/pipeline/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts index fb12ba1e868..6a1021cbd06 100644 --- a/src/main/features/knowledge/pipeline/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts @@ -1,51 +1,31 @@ +import fs, { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + import type * as LifecycleModule from '@main/core/lifecycle' import { DEFAULT_KNOWLEDGE_BASE_CHUNK_OVERLAP, DEFAULT_KNOWLEDGE_BASE_CHUNK_SIZE, type KnowledgeBase } from '@shared/data/types/knowledge' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { loggerDebugMock, loggerErrorMock, loggerInfoMock, loggerWarnMock, - openDriverMock, - createSchemaMock, - resetSchemaMock, - ensureIndexMetaMock, - readIndexSchemaVersionMock, - hasAnyMaterialMock, getItemsByBaseIdMock, - indexStoreCtorMock, getPathSyncMock, - deleteDirMock, - statMock + deleteDirMock } = vi.hoisted(() => ({ loggerDebugMock: vi.fn(), loggerErrorMock: vi.fn(), loggerInfoMock: vi.fn(), loggerWarnMock: vi.fn(), - openDriverMock: vi.fn(), - createSchemaMock: vi.fn(), - resetSchemaMock: vi.fn(), - ensureIndexMetaMock: vi.fn(), - readIndexSchemaVersionMock: vi.fn(), - hasAnyMaterialMock: vi.fn(), getItemsByBaseIdMock: vi.fn(), - indexStoreCtorMock: vi.fn(), getPathSyncMock: vi.fn(), - deleteDirMock: vi.fn(), - statMock: vi.fn() -})) - -/** Mirrors KNOWLEDGE_INDEX_SCHEMA_VERSION in schema.ts; the service test pins the open-time - * branching (rebuild vs create), while schema.test.ts pins the real constant's value. */ -const MOCK_SCHEMA_VERSION = 2 - -vi.mock('node:fs', () => ({ - default: { promises: { stat: statMock } } + deleteDirMock: vi.fn() })) vi.mock('@main/core/lifecycle', async (importOriginal) => { @@ -70,30 +50,11 @@ vi.mock('@logger', () => ({ } })) -vi.mock('../indexStore/KnowledgeIndexStore', () => ({ - KnowledgeIndexStore: indexStoreCtorMock -})) - -vi.mock('../indexStore/BetterSqlite3Driver', () => ({ - openBetterSqlite3IndexDriver: openDriverMock -})) - -vi.mock('../indexStore/BetterSqlite3VectorIndex', () => ({ - betterSqlite3VectorIndex: { kind: 'better-sqlite3' } -})) - -vi.mock('../indexStore/schema', () => ({ - createKnowledgeIndexSchema: createSchemaMock, - resetKnowledgeIndexSchema: resetSchemaMock, - KNOWLEDGE_INDEX_SCHEMA_VERSION: MOCK_SCHEMA_VERSION -})) - -vi.mock('../indexStore/indexMeta', () => ({ - ensureIndexMeta: ensureIndexMetaMock, - readIndexSchemaVersion: readIndexSchemaVersionMock, - hasAnyMaterial: hasAnyMaterialMock -})) - +// The app-DB boundary: everything under indexStore/ (driver, schema, meta, store) runs +// for real against a temp-dir SQLite file, so these tests exercise the real DDL/meta +// contract instead of pinning mock call shapes. `pathStorage`'s path resolver is +// redirected into the temp dir; `deleteKnowledgeBaseDir` stays mocked (its own removeDir +// behavior is covered by pathStorage's own tests). vi.mock('@data/services/KnowledgeItemService', () => ({ knowledgeItemService: { getItemsByBaseId: getItemsByBaseIdMock } })) @@ -104,6 +65,11 @@ vi.mock('../../../pathStorage', () => ({ })) const { KnowledgeVectorStoreService } = await import('../KnowledgeVectorStoreService') +const { KnowledgeIndexStore } = await import('../indexStore/KnowledgeIndexStore') +const { BetterSqlite3Driver, openBetterSqlite3IndexDriver } = await import('../indexStore/BetterSqlite3Driver') +const schemaModule = await import('../indexStore/schema') +const indexMetaModule = await import('../indexStore/indexMeta') +const { KNOWLEDGE_INDEX_SCHEMA_VERSION } = schemaModule function createBase(id = 'kb-1'): KnowledgeBase { return { @@ -124,36 +90,46 @@ function createBase(id = 'kb-1'): KnowledgeBase { } } -/** The store instance built by the most recent `new KnowledgeIndexStore(...)` call. */ -function lastStore() { - const results = indexStoreCtorMock.mock.results - return results[results.length - 1]?.value as { close: ReturnType } +function createFailedBase(): KnowledgeBase { + return { + ...createBase(), + dimensions: null, + embeddingModelId: null, + status: 'failed', + error: 'missing_embedding_model' + } +} + +/** Opens (and immediately closes) a store for `base` so its file exists on disk at the current schema version. */ +async function primeStoreOnDisk(base: KnowledgeBase): Promise { + const store = await new KnowledgeVectorStoreService().getIndexStore(base) + store.close() +} + +/** Opens a fresh raw driver directly on `base`'s file (bypassing the service/cache) to inspect or mutate it. */ +function withRawDriver(base: KnowledgeBase, fn: (driver: InstanceType) => T): T { + const driver = openBetterSqlite3IndexDriver(getPathSyncMock(base.id)) + try { + return fn(driver) + } finally { + driver.close() + } } describe('KnowledgeVectorStoreService', () => { + let tempDir: string + beforeEach(() => { vi.clearAllMocks() - getPathSyncMock.mockImplementation((baseId: string) => `/tmp/${baseId}/index.sqlite`) - // Each open returns a fresh closeable driver so failure paths can assert close(). - // The real driver port is synchronous (see BetterSqlite3Driver) — these mocks - // must return plain values, not promises, or the store under test would try to - // call .execute() on a Promise object. - openDriverMock.mockImplementation(() => ({ - kind: 'driver', - close: vi.fn() - })) - createSchemaMock.mockReturnValue(undefined) - resetSchemaMock.mockReturnValue(undefined) - ensureIndexMetaMock.mockReturnValue(undefined) - // Default: a fresh/blank file has no stored version → the open path takes the normal - // create branch (no rebuild). Mismatch tests override this per-case. - readIndexSchemaVersionMock.mockReturnValue(null) - // A non-empty material probe keeps the invisible-contents diagnostic quiet - // unless a test opts in. - hasAnyMaterialMock.mockReturnValue(true) + tempDir = mkdtempSync(join(tmpdir(), 'cs-knowledge-vsstore-')) + getPathSyncMock.mockImplementation((baseId: string) => join(tempDir, `${baseId}.sqlite`)) getItemsByBaseIdMock.mockReturnValue([]) deleteDirMock.mockResolvedValue(undefined) - indexStoreCtorMock.mockImplementation(() => ({ close: vi.fn().mockResolvedValue(undefined) })) + }) + + afterEach(() => { + vi.restoreAllMocks() + rmSync(tempDir, { recursive: true, force: true }) }) it('opens an index store on first request and caches it per base', async () => { @@ -164,205 +140,212 @@ describe('KnowledgeVectorStoreService', () => { const second = await service.getIndexStore(base) expect(first).toBe(second) - expect(indexStoreCtorMock).toHaveBeenCalledTimes(1) - expect(openDriverMock).toHaveBeenCalledTimes(1) - expect(createSchemaMock).toHaveBeenCalledTimes(1) + expect(first).toBeInstanceOf(KnowledgeIndexStore) expect(loggerInfoMock).toHaveBeenCalledWith('Opened knowledge index store', { baseId: base.id, cacheSize: 1 }) expect(loggerDebugMock).toHaveBeenCalledWith('Reusing cached knowledge index store', { baseId: base.id }) }) - it('shares a single open across concurrent callers for the same base (single-flight)', async () => { + it('synchronous open makes a second concurrent call observe the already-cached store', async () => { + // Opening a store (better-sqlite3 connect + schema + meta) is fully synchronous, so + // the first call's cache write completes before the second call starts — there is no + // in-flight promise for the second call to join. Pinned via a call-count assertion so + // a future refactor that reintroduces an async open cannot silently double-open. const service = new KnowledgeVectorStoreService() const base = createBase() + const createSchemaSpy = vi.spyOn(schemaModule, 'createKnowledgeIndexSchema') - // Both calls are issued before the first open resolves; the second must join - // the first's in-flight open rather than starting its own (which would leak a - // store no one closes). const [first, second] = await Promise.all([service.getIndexStore(base), service.getIndexStore(base)]) expect(first).toBe(second) - expect(openDriverMock).toHaveBeenCalledTimes(1) - expect(indexStoreCtorMock).toHaveBeenCalledTimes(1) + expect(createSchemaSpy).toHaveBeenCalledTimes(1) }) it('evicts a failed open so a later call retries instead of re-awaiting the failure', async () => { const service = new KnowledgeVectorStoreService() const base = createBase() - openDriverMock.mockImplementationOnce(() => { - throw new Error('open failed') - }) + // A regular file where the index dir must go makes the real mkdirSync in + // openBetterSqlite3IndexDriver fail with ENOTDIR before any Database is opened. + const blockerPath = join(tempDir, 'blocked-open') + writeFileSync(blockerPath, 'not a directory') + getPathSyncMock.mockReturnValueOnce(join(blockerPath, 'index.sqlite')) - await expect(service.getIndexStore(base)).rejects.toThrow('open failed') + await expect(service.getIndexStore(base)).rejects.toThrow() const store = await service.getIndexStore(base) - expect(store).toBe(lastStore()) - expect(openDriverMock).toHaveBeenCalledTimes(2) - expect(indexStoreCtorMock).toHaveBeenCalledTimes(1) + expect(store).toBeInstanceOf(KnowledgeIndexStore) }) it('stamps and verifies the meta identity row before handing out the store', async () => { const service = new KnowledgeVectorStoreService() const base = createBase() + const ensureIndexMetaSpy = vi.spyOn(indexMetaModule, 'ensureIndexMeta') await service.getIndexStore(base) - expect(ensureIndexMetaMock).toHaveBeenCalledTimes(1) - expect(ensureIndexMetaMock).toHaveBeenCalledWith(expect.objectContaining({ kind: 'driver' }), { - baseId: base.id - }) + expect(ensureIndexMetaSpy).toHaveBeenCalledTimes(1) + expect(ensureIndexMetaSpy).toHaveBeenCalledWith(expect.anything(), { baseId: base.id }) }) it('creates the schema normally when the stored version matches (no rebuild)', async () => { - const service = new KnowledgeVectorStoreService() const base = createBase() - readIndexSchemaVersionMock.mockReturnValueOnce(MOCK_SCHEMA_VERSION) + await primeStoreOnDisk(base) + const createSchemaSpy = vi.spyOn(schemaModule, 'createKnowledgeIndexSchema') + const resetSchemaSpy = vi.spyOn(schemaModule, 'resetKnowledgeIndexSchema') + const service = new KnowledgeVectorStoreService() await service.getIndexStore(base) - expect(createSchemaMock).toHaveBeenCalledTimes(1) - expect(resetSchemaMock).not.toHaveBeenCalled() + expect(createSchemaSpy).toHaveBeenCalledTimes(1) + expect(resetSchemaSpy).not.toHaveBeenCalled() }) it('rebuilds the derived index when an existing index.sqlite is at a stale schema version', async () => { - const service = new KnowledgeVectorStoreService() const base = createBase() - readIndexSchemaVersionMock.mockReturnValueOnce(MOCK_SCHEMA_VERSION - 1) + await primeStoreOnDisk(base) + // Leave a marker row and stamp a stale version directly, so the rebuild below is + // verified against real DDL (dropped tables), not a mocked branch decision. + withRawDriver(base, (driver) => { + driver.execute('INSERT INTO content (content_hash, text, created_at) VALUES (?, ?, ?)', ['marker', 'x', 0]) + driver.execute('UPDATE meta SET schema_version = ?', [KNOWLEDGE_INDEX_SCHEMA_VERSION - 1]) + }) + const createSchemaSpy = vi.spyOn(schemaModule, 'createKnowledgeIndexSchema') + const resetSchemaSpy = vi.spyOn(schemaModule, 'resetKnowledgeIndexSchema') + const ensureIndexMetaSpy = vi.spyOn(indexMetaModule, 'ensureIndexMeta') + const service = new KnowledgeVectorStoreService() await service.getIndexStore(base) // Drop+recreate via resetKnowledgeIndexSchema, NOT the plain create — an old layout // (e.g. pre-fts_rowid) cannot be retrofitted by CREATE ... IF NOT EXISTS. - expect(resetSchemaMock).toHaveBeenCalledTimes(1) - expect(createSchemaMock).not.toHaveBeenCalled() + expect(resetSchemaSpy).toHaveBeenCalledTimes(1) + expect(createSchemaSpy).not.toHaveBeenCalled() // Warn so the wipe-and-reindex is diagnosable. expect(loggerWarnMock).toHaveBeenCalledWith( 'Knowledge index schema version mismatch — rebuilding the derived index', expect.objectContaining({ baseId: base.id, - storedVersion: MOCK_SCHEMA_VERSION - 1, - expectedVersion: MOCK_SCHEMA_VERSION + storedVersion: KNOWLEDGE_INDEX_SCHEMA_VERSION - 1, + expectedVersion: KNOWLEDGE_INDEX_SCHEMA_VERSION }) ) // The reset drops meta, so the open path still restamps it afterwards. - expect(ensureIndexMetaMock).toHaveBeenCalledTimes(1) + expect(ensureIndexMetaSpy).toHaveBeenCalledTimes(1) + // The rebuild really dropped and recreated the schema: the marker inserted before + // the stale-version stamp is gone, and the restamped version is current. + expect( + withRawDriver(base, (driver) => driver.execute('SELECT 1 FROM content WHERE content_hash = ?', ['marker']).rows) + ).toEqual([]) + expect(withRawDriver(base, (driver) => indexMetaModule.readIndexSchemaVersion(driver))).toBe( + KNOWLEDGE_INDEX_SCHEMA_VERSION + ) }) it('also rebuilds when the stored version is NEWER than this build (downgrade)', async () => { // The guard is `!==`, not `<` — a file written by a newer build (user downgraded the app) is // just as incompatible a layout and must be rebuilt, not opened as-is. Pins that semantics so a // future refactor to `<` does not silently start mounting newer files. - const service = new KnowledgeVectorStoreService() const base = createBase() - readIndexSchemaVersionMock.mockReturnValueOnce(MOCK_SCHEMA_VERSION + 1) + await primeStoreOnDisk(base) + withRawDriver(base, (driver) => { + driver.execute('UPDATE meta SET schema_version = ?', [KNOWLEDGE_INDEX_SCHEMA_VERSION + 1]) + }) + const createSchemaSpy = vi.spyOn(schemaModule, 'createKnowledgeIndexSchema') + const resetSchemaSpy = vi.spyOn(schemaModule, 'resetKnowledgeIndexSchema') + const service = new KnowledgeVectorStoreService() await service.getIndexStore(base) - expect(resetSchemaMock).toHaveBeenCalledTimes(1) - expect(createSchemaMock).not.toHaveBeenCalled() + expect(resetSchemaSpy).toHaveBeenCalledTimes(1) + expect(createSchemaSpy).not.toHaveBeenCalled() }) it('closes the driver and aborts the open when meta verification fails (wrong/corrupt base)', async () => { + const base = createBase('kb-shared') + const otherBase = createBase('kb-other') + // Both bases resolve to the same on-disk file, so the second open's meta + // identity check genuinely finds a mismatched base_id. + getPathSyncMock.mockImplementation(() => join(tempDir, 'shared.sqlite')) + await primeStoreOnDisk(base) + const closeSpy = vi.spyOn(BetterSqlite3Driver.prototype, 'close') const service = new KnowledgeVectorStoreService() - const base = createBase() - let openedDriver: { close: ReturnType } | undefined - openDriverMock.mockImplementationOnce(() => { - openedDriver = { kind: 'driver', close: vi.fn().mockResolvedValue(undefined) } as never - return openedDriver - }) - ensureIndexMetaMock.mockImplementationOnce(() => { - throw new Error('belongs to a different base') - }) - await expect(service.getIndexStore(base)).rejects.toThrow('belongs to a different base') + await expect(service.getIndexStore(otherBase)).rejects.toThrow('belongs to a different base') - expect(openedDriver?.close).toHaveBeenCalledTimes(1) - expect(indexStoreCtorMock).not.toHaveBeenCalled() + expect(closeSpy).toHaveBeenCalledTimes(1) }) it('closes the driver when schema creation fails so the file handle is not leaked', async () => { const service = new KnowledgeVectorStoreService() const base = createBase() - let openedDriver: { close: ReturnType } | undefined - openDriverMock.mockImplementationOnce(() => { - openedDriver = { kind: 'driver', close: vi.fn().mockResolvedValue(undefined) } as never - return openedDriver - }) - createSchemaMock.mockImplementationOnce(() => { + const closeSpy = vi.spyOn(BetterSqlite3Driver.prototype, 'close') + vi.spyOn(schemaModule, 'createKnowledgeIndexSchema').mockImplementationOnce(() => { throw new Error('disk full') }) await expect(service.getIndexStore(base)).rejects.toThrow('disk full') - expect(openedDriver?.close).toHaveBeenCalledTimes(1) - expect(indexStoreCtorMock).not.toHaveBeenCalled() + expect(closeSpy).toHaveBeenCalledTimes(1) }) it('returns undefined from getIndexStoreIfExists when no backing file exists', async () => { const service = new KnowledgeVectorStoreService() const base = createBase() - statMock.mockRejectedValueOnce(Object.assign(new Error('missing'), { code: 'ENOENT' })) await expect(service.getIndexStoreIfExists(base)).resolves.toBeUndefined() - expect(indexStoreCtorMock).not.toHaveBeenCalled() expect(loggerDebugMock).toHaveBeenCalledWith('Knowledge index store does not exist on disk', { baseId: base.id }) }) it('opens an existing store from disk when getIndexStoreIfExists detects a backing file', async () => { - const service = new KnowledgeVectorStoreService() const base = createBase() - statMock.mockResolvedValueOnce({ isFile: () => true }) + await primeStoreOnDisk(base) + const service = new KnowledgeVectorStoreService() const store = await service.getIndexStoreIfExists(base) - expect(store).toBe(lastStore()) - expect(indexStoreCtorMock).toHaveBeenCalledTimes(1) + expect(store).toBeInstanceOf(KnowledgeIndexStore) }) it('returns the cached store from getIndexStoreIfExists without probing disk', async () => { const service = new KnowledgeVectorStoreService() const base = createBase() - const created = await service.getIndexStore(base) + const statSpy = vi.spyOn(fs.promises, 'stat') await expect(service.getIndexStoreIfExists(base)).resolves.toBe(created) - expect(statMock).not.toHaveBeenCalled() + expect(statSpy).not.toHaveBeenCalled() }) it('closes the cached store and removes the base directory on deleteStore', async () => { const service = new KnowledgeVectorStoreService() const base = createBase() + const closeSpy = vi.spyOn(BetterSqlite3Driver.prototype, 'close') const store = await service.getIndexStore(base) await service.deleteStore(base.id) - expect(store.close).toHaveBeenCalledTimes(1) + expect(closeSpy).toHaveBeenCalledTimes(1) expect(deleteDirMock).toHaveBeenCalledWith(base.id) // Close must precede directory removal — on Windows a still-open sqlite // handle makes the directory deletion fail. - expect((store.close as unknown as ReturnType).mock.invocationCallOrder[0]).toBeLessThan( - deleteDirMock.mock.invocationCallOrder[0] - ) + expect(closeSpy.mock.invocationCallOrder[0]).toBeLessThan(deleteDirMock.mock.invocationCallOrder[0]) // Cache was evicted: the next open builds a fresh instance. const reopened = await service.getIndexStore(base) expect(reopened).not.toBe(store) - expect(indexStoreCtorMock).toHaveBeenCalledTimes(2) }) it('deleteStore removes the directory even when no store was ever opened for the base', async () => { // Opening a store (see openIndexStore) is fully synchronous — it either completes and - // caches a store, or throws before caching anything. There is no longer an in-flight - // state deleteStore could observe mid-open, so this covers the remaining "nothing - // cached" case: deleteStore must still close-if-present (a no-op here) and remove - // the directory. + // caches a store, or throws before caching anything. There is no in-flight state + // deleteStore could observe mid-open, so this covers the "nothing cached" case: + // deleteStore must still close-if-present (a no-op here) and remove the directory. const service = new KnowledgeVectorStoreService() const base = createBase() await service.deleteStore(base.id) expect(deleteDirMock).toHaveBeenCalledWith(base.id) - expect(indexStoreCtorMock).not.toHaveBeenCalled() }) it('evicts the cached store even when directory removal fails', async () => { @@ -375,7 +358,6 @@ describe('KnowledgeVectorStoreService', () => { const reopened = await service.getIndexStore(base) expect(reopened).not.toBe(store) - expect(indexStoreCtorMock).toHaveBeenCalledTimes(2) }) it('closes all cached stores during stop and continues when one close throws', async () => { @@ -384,14 +366,15 @@ describe('KnowledgeVectorStoreService', () => { const first = await service.getIndexStore(createBase('kb-1')) const second = await service.getIndexStore(createBase('kb-2')) const closeError = new Error('close failed') - ;(first.close as unknown as ReturnType).mockImplementationOnce(() => { + const firstCloseSpy = vi.spyOn(first, 'close').mockImplementationOnce(() => { throw closeError }) + const secondCloseSpy = vi.spyOn(second, 'close') - await expect((service as any).onStop()).resolves.toBeUndefined() + await expect((service as unknown as { onStop: () => Promise }).onStop()).resolves.toBeUndefined() - expect(first.close).toHaveBeenCalledTimes(1) - expect(second.close).toHaveBeenCalledTimes(1) + expect(firstCloseSpy).toHaveBeenCalledTimes(1) + expect(secondCloseSpy).toHaveBeenCalledTimes(1) expect(loggerErrorMock).toHaveBeenCalledWith('Failed to close knowledge index store', closeError, { baseId: 'kb-1' }) @@ -405,57 +388,34 @@ describe('KnowledgeVectorStoreService', () => { it('rejects bases that are not ready before touching disk', async () => { const service = new KnowledgeVectorStoreService() - const base = { - ...createBase(), - dimensions: null, - embeddingModelId: null, - status: 'failed', - error: 'missing_embedding_model' - } satisfies KnowledgeBase + const base = createFailedBase() + const statSpy = vi.spyOn(fs.promises, 'stat') await expect(service.getIndexStore(base)).rejects.toThrow('not ready for vector store operations') - expect(indexStoreCtorMock).not.toHaveBeenCalled() + expect(statSpy).not.toHaveBeenCalled() }) it('lets cleanup on a failed base proceed: getIndexStoreIfExists returns undefined instead of asserting', async () => { - const service = new KnowledgeVectorStoreService() - const base = { - ...createBase(), - dimensions: null, - embeddingModelId: null, - status: 'failed', - error: 'missing_embedding_model' - } satisfies KnowledgeBase // Failed bases never get a store file (the vector migrator skips them and // getIndexStore asserts), so the existence probe is the path cleanup takes. - statMock.mockRejectedValueOnce(Object.assign(new Error('missing'), { code: 'ENOENT' })) + const service = new KnowledgeVectorStoreService() + const base = createFailedBase() await expect(service.getIndexStoreIfExists(base)).resolves.toBeUndefined() - - expect(indexStoreCtorMock).not.toHaveBeenCalled() }) it('still asserts readiness when a failed base unexpectedly has a store file on disk', async () => { const service = new KnowledgeVectorStoreService() - const base = { - ...createBase(), - dimensions: null, - embeddingModelId: null, - status: 'failed', - error: 'missing_embedding_model' - } satisfies KnowledgeBase - statMock.mockResolvedValueOnce({ isFile: () => true }) + const base = createFailedBase() + writeFileSync(getPathSyncMock(base.id), '') await expect(service.getIndexStoreIfExists(base)).rejects.toThrow('not ready for vector store operations') - - expect(indexStoreCtorMock).not.toHaveBeenCalled() }) it('logs an error when an empty index mounts under a base with completed items', async () => { const service = new KnowledgeVectorStoreService() const base = createBase() - hasAnyMaterialMock.mockReturnValueOnce(false) getItemsByBaseIdMock.mockReturnValueOnce([ { id: 'item-1', type: 'directory', status: 'completed' }, { id: 'item-2', type: 'file', status: 'completed' } @@ -463,7 +423,7 @@ describe('KnowledgeVectorStoreService', () => { const store = await service.getIndexStore(base) - expect(store).toBe(lastStore()) + expect(store).toBeInstanceOf(KnowledgeIndexStore) expect(loggerErrorMock).toHaveBeenCalledWith( expect.stringContaining('zero materials while the base has completed items'), expect.objectContaining({ baseId: base.id }) @@ -473,7 +433,6 @@ describe('KnowledgeVectorStoreService', () => { it('stays quiet when an empty index mounts under a base with no completed indexable items', async () => { const service = new KnowledgeVectorStoreService() const base = createBase() - hasAnyMaterialMock.mockReturnValueOnce(false) // A completed empty directory is legitimate without materials; in-flight leaves are too. getItemsByBaseIdMock.mockReturnValueOnce([ { id: 'item-1', type: 'directory', status: 'completed' }, @@ -488,12 +447,7 @@ describe('KnowledgeVectorStoreService', () => { it('fails the open and closes the driver when the empty-index diagnostic cannot read the base items', async () => { const service = new KnowledgeVectorStoreService() const base = createBase() - let openedDriver: { close: ReturnType } | undefined - openDriverMock.mockImplementationOnce(() => { - openedDriver = { kind: 'driver', close: vi.fn().mockResolvedValue(undefined) } as never - return openedDriver - }) - hasAnyMaterialMock.mockReturnValueOnce(false) + const closeSpy = vi.spyOn(BetterSqlite3Driver.prototype, 'close') getItemsByBaseIdMock.mockImplementationOnce(() => { throw new Error('app database unavailable') }) @@ -503,7 +457,6 @@ describe('KnowledgeVectorStoreService', () => { // lookup's NOT_FOUND is what makes that loud instead of caching an empty store). await expect(service.getIndexStore(base)).rejects.toThrow('app database unavailable') - expect(openedDriver?.close).toHaveBeenCalledTimes(1) - expect(indexStoreCtorMock).not.toHaveBeenCalled() + expect(closeSpy).toHaveBeenCalledTimes(1) }) }) From e4b5c50d00ce3269b3e381d7750b9a2156092b14 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Sun, 5 Jul 2026 10:16:09 +0800 Subject: [PATCH 30/37] refactor(knowledge): adopt core KeyedMutex/job primitives, desync facades, drop idle status Wires the KB module onto the core/job capabilities the maintainer merged in response to issue #16738 (KeyedMutex, typed settled events, list() filters), and clears scaffolding that predated them: - delete KnowledgeLockManager in favor of the core KeyedMutex it duplicated - collapse statusCleanup's updateStatus-then-setSubtreeStatus double-write into a single setSubtreeStatus call (its CTE already guards deleting rows) - desync KnowledgeVectorStoreService's getIndexStore/getIndexStoreIfExists now that better-sqlite3 makes them synchronous under the hood - drop the unreachable idle KnowledgeItem status; add a never_indexed error code for the migration-mapping branch that used to fall back to it - swap KNOWLEDGE_ACTIVE_JOB_STATUSES for the shared ACTIVE_JOB_STATUSES - read onSettled's typed event.input directly instead of re-fetching the job - drop the redundant input.parentJobId payload field now that ctx.parentId carries the same value - push parentId/type filters into JobManager.list() instead of filtering results in JS This is the first-two-layers slice of a maintainer-invited follow-up checklist; enqueueTx adoption and the two 5000-job-limit loop-until-drained fixes are intentionally left for a later pass. Signed-off-by: eeee0717 --- .../0018_easy_gertrude_yorkes.sql | 39 + .../sqlite-drizzle/meta/0018_snapshot.json | 3866 +++++++++++++++++ migrations/sqlite-drizzle/meta/_journal.json | 7 + src/main/data/db/schemas/knowledge.ts | 8 +- .../__tests__/KnowledgeMigrator.test.ts | 43 +- .../migrators/mappings/KnowledgeMappings.ts | 16 +- .../__tests__/KnowledgeMappings.test.ts | 31 +- .../__tests__/KnowledgeItemService.test.ts | 11 +- .../features/knowledge/KnowledgeService.ts | 4 +- .../KnowledgeService.integration.test.ts | 6 +- .../__tests__/KnowledgeService.test.ts | 57 +- .../knowledge/__tests__/items.test.ts | 2 +- .../base/KnowledgeBaseAdminService.ts | 8 +- .../knowledge/base/KnowledgeLockManager.ts | 34 - .../__tests__/KnowledgeLockManager.test.ts | 100 - .../ingestion/KnowledgeIngestionService.ts | 13 +- .../statusCleanup.integration.test.ts | 108 +- .../knowledge/ingestion/statusCleanup.ts | 21 +- .../__tests__/KnowledgeReaderMetadata.test.ts | 10 +- .../readers/__tests__/ReaderFactory.test.ts | 8 +- .../sources/__tests__/directory.test.ts | 16 +- .../KnowledgeVectorStoreService.ts | 18 +- .../KnowledgeVectorStoreService.test.ts | 126 +- .../__tests__/vectorCleanup.test.ts | 12 +- .../indexStore/BetterSqlite3Driver.ts | 2 +- .../pipeline/vectorstore/indexStore/schema.ts | 2 +- .../pipeline/vectorstore/vectorCleanup.ts | 4 +- .../query/KnowledgeConceptService.ts | 4 +- .../knowledge/query/KnowledgeQueryService.ts | 4 +- ...heckFileProcessingResultJobHandler.test.ts | 84 +- .../__tests__/deleteSubtreeJobHandler.test.ts | 11 +- .../indexDocumentsJobHandler.test.ts | 73 +- .../tasks/__tests__/jobHandlerTestUtils.ts | 14 +- .../__tests__/prepareRootJobHandler.test.ts | 44 +- .../reindexSubtreeJobHandler.test.ts | 32 +- .../checkFileProcessingResultJobHandler.ts | 8 +- .../tasks/deleteSubtreeJobHandler.ts | 6 +- .../tasks/indexDocumentsJobHandler.ts | 16 +- src/main/features/knowledge/tasks/jobTypes.ts | 2 - .../knowledge/tasks/prepareRootJobHandler.ts | 12 +- .../tasks/reindexSubtreeJobHandler.ts | 51 +- .../tasks/utils/__tests__/cancel.test.ts | 11 +- .../tasks/utils/__tests__/jobInput.test.ts | 39 +- .../features/knowledge/tasks/utils/cancel.ts | 18 +- .../knowledge/tasks/utils/jobInput.ts | 20 +- .../features/knowledge/tasks/utils/settled.ts | 17 +- src/main/features/knowledge/types.ts | 1 - src/renderer/i18n/locales/en-us.json | 3 +- src/renderer/i18n/locales/zh-cn.json | 3 +- src/renderer/i18n/locales/zh-tw.json | 3 +- .../__tests__/KnowledgeItemRow.test.tsx | 17 +- .../panels/dataSource/utils/models.ts | 2 +- src/renderer/pages/knowledge/utils/error.ts | 3 + .../api/schemas/__tests__/knowledges.test.ts | 12 +- src/shared/data/types/knowledge.ts | 40 +- 55 files changed, 4346 insertions(+), 776 deletions(-) create mode 100644 migrations/sqlite-drizzle/0018_easy_gertrude_yorkes.sql create mode 100644 migrations/sqlite-drizzle/meta/0018_snapshot.json delete mode 100644 src/main/features/knowledge/base/KnowledgeLockManager.ts delete mode 100644 src/main/features/knowledge/base/__tests__/KnowledgeLockManager.test.ts diff --git a/migrations/sqlite-drizzle/0018_easy_gertrude_yorkes.sql b/migrations/sqlite-drizzle/0018_easy_gertrude_yorkes.sql new file mode 100644 index 00000000000..560a4cc0750 --- /dev/null +++ b/migrations/sqlite-drizzle/0018_easy_gertrude_yorkes.sql @@ -0,0 +1,39 @@ +PRAGMA foreign_keys=OFF;--> statement-breakpoint +CREATE TABLE `__new_knowledge_item` ( + `id` text PRIMARY KEY NOT NULL, + `base_id` text NOT NULL, + `group_id` text, + `type` text NOT NULL, + `data` text NOT NULL, + `status` text NOT NULL, + `error` text, + `created_at` integer NOT NULL, + `updated_at` integer NOT NULL, + FOREIGN KEY (`base_id`) REFERENCES `knowledge_base`(`id`) ON UPDATE no action ON DELETE cascade, + FOREIGN KEY (`base_id`,`group_id`) REFERENCES `knowledge_item`(`base_id`,`id`) ON UPDATE no action ON DELETE cascade, + CONSTRAINT "knowledge_item_type_check" CHECK("__new_knowledge_item"."type" IN ('file', 'url', 'note', 'directory')), + CONSTRAINT "knowledge_item_status_check" CHECK("__new_knowledge_item"."status" IN ('preparing', 'processing', 'reading', 'embedding', 'completed', 'failed', 'deleting')), + CONSTRAINT "knowledge_item_type_status_check" CHECK( + ("__new_knowledge_item"."type" IN ('file', 'url', 'note') AND "__new_knowledge_item"."status" IN ('processing', 'reading', 'embedding', 'completed', 'failed', 'deleting')) + OR ("__new_knowledge_item"."type" = 'directory' AND "__new_knowledge_item"."status" IN ('preparing', 'processing', 'completed', 'failed', 'deleting')) + ), + CONSTRAINT "knowledge_item_status_error_check" CHECK( + ( + "__new_knowledge_item"."status" IN ('preparing', 'processing', 'reading', 'embedding', 'completed', 'deleting') + AND "__new_knowledge_item"."error" IS NULL + ) + OR ( + "__new_knowledge_item"."status" = 'failed' + AND "__new_knowledge_item"."error" IS NOT NULL + AND length(trim("__new_knowledge_item"."error")) > 0 + ) + ) +); +--> statement-breakpoint +INSERT INTO `__new_knowledge_item`("id", "base_id", "group_id", "type", "data", "status", "error", "created_at", "updated_at") SELECT "id", "base_id", "group_id", "type", "data", "status", "error", "created_at", "updated_at" FROM `knowledge_item`;--> statement-breakpoint +DROP TABLE `knowledge_item`;--> statement-breakpoint +ALTER TABLE `__new_knowledge_item` RENAME TO `knowledge_item`;--> statement-breakpoint +PRAGMA foreign_keys=ON;--> statement-breakpoint +CREATE INDEX `knowledge_item_base_type_created_idx` ON `knowledge_item` (`base_id`,`type`,`created_at`);--> statement-breakpoint +CREATE INDEX `knowledge_item_base_group_created_idx` ON `knowledge_item` (`base_id`,`group_id`,`created_at`);--> statement-breakpoint +CREATE UNIQUE INDEX `knowledge_item_baseId_id_unique` ON `knowledge_item` (`base_id`,`id`); \ No newline at end of file diff --git a/migrations/sqlite-drizzle/meta/0018_snapshot.json b/migrations/sqlite-drizzle/meta/0018_snapshot.json new file mode 100644 index 00000000000..315ee23507c --- /dev/null +++ b/migrations/sqlite-drizzle/meta/0018_snapshot.json @@ -0,0 +1,3866 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "c7a83c70-3fe5-4b08-b56a-db4a64463c93", + "prevId": "53ba65df-7a61-4a0c-a563-3e32dfba69fa", + "tables": { + "agent": { + "name": "agent", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "instructions": { + "name": "instructions", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "plan_model": { + "name": "plan_model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "small_model": { + "name": "small_model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disabled_tools": { + "name": "disabled_tools", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "configuration": { + "name": "configuration", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "agent_name_idx": { + "name": "agent_name_idx", + "columns": ["name"], + "isUnique": false + }, + "agent_type_idx": { + "name": "agent_type_idx", + "columns": ["type"], + "isUnique": false + }, + "agent_order_key_idx": { + "name": "agent_order_key_idx", + "columns": ["order_key"], + "isUnique": false + } + }, + "foreignKeys": { + "agent_model_user_model_id_fk": { + "name": "agent_model_user_model_id_fk", + "tableFrom": "agent", + "tableTo": "user_model", + "columnsFrom": ["model"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "agent_plan_model_user_model_id_fk": { + "name": "agent_plan_model_user_model_id_fk", + "tableFrom": "agent", + "tableTo": "user_model", + "columnsFrom": ["plan_model"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "agent_small_model_user_model_id_fk": { + "name": "agent_small_model_user_model_id_fk", + "tableFrom": "agent", + "tableTo": "user_model", + "columnsFrom": ["small_model"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "agent_channel": { + "name": "agent_channel", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "workspace": { + "name": "workspace", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "config": { + "name": "config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_active": { + "name": "is_active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "active_chat_ids": { + "name": "active_chat_ids", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "permission_mode": { + "name": "permission_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "agent_channel_agent_id_idx": { + "name": "agent_channel_agent_id_idx", + "columns": ["agent_id"], + "isUnique": false + }, + "agent_channel_type_idx": { + "name": "agent_channel_type_idx", + "columns": ["type"], + "isUnique": false + }, + "agent_channel_session_id_idx": { + "name": "agent_channel_session_id_idx", + "columns": ["session_id"], + "isUnique": false + } + }, + "foreignKeys": { + "agent_channel_agent_id_agent_id_fk": { + "name": "agent_channel_agent_id_agent_id_fk", + "tableFrom": "agent_channel", + "tableTo": "agent", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "agent_channel_session_id_agent_session_id_fk": { + "name": "agent_channel_session_id_agent_session_id_fk", + "tableFrom": "agent_channel", + "tableTo": "agent_session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "agent_channel_type_check": { + "name": "agent_channel_type_check", + "value": "\"agent_channel\".\"type\" IN ('telegram', 'feishu', 'qq', 'wechat', 'discord', 'slack')" + }, + "agent_channel_permission_mode_check": { + "name": "agent_channel_permission_mode_check", + "value": "\"agent_channel\".\"permission_mode\" IS NULL OR \"agent_channel\".\"permission_mode\" IN ('default', 'acceptEdits', 'bypassPermissions', 'plan')" + } + } + }, + "agent_channel_task": { + "name": "agent_channel_task", + "columns": { + "channel_id": { + "name": "channel_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "task_id": { + "name": "task_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "agent_channel_task_channel_id_idx": { + "name": "agent_channel_task_channel_id_idx", + "columns": ["channel_id"], + "isUnique": false + }, + "agent_channel_task_task_id_idx": { + "name": "agent_channel_task_task_id_idx", + "columns": ["task_id"], + "isUnique": false + } + }, + "foreignKeys": { + "agent_channel_task_channel_id_agent_channel_id_fk": { + "name": "agent_channel_task_channel_id_agent_channel_id_fk", + "tableFrom": "agent_channel_task", + "tableTo": "agent_channel", + "columnsFrom": ["channel_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_channel_task_task_id_job_schedule_id_fk": { + "name": "agent_channel_task_task_id_job_schedule_id_fk", + "tableFrom": "agent_channel_task", + "tableTo": "job_schedule", + "columnsFrom": ["task_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_channel_task_channel_id_task_id_pk": { + "columns": ["channel_id", "task_id"], + "name": "agent_channel_task_channel_id_task_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "agent_global_skill": { + "name": "agent_global_skill", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "folder_name": { + "name": "folder_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_enabled": { + "name": "is_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "agent_global_skill_folder_name_unique": { + "name": "agent_global_skill_folder_name_unique", + "columns": ["folder_name"], + "isUnique": true + }, + "agent_global_skill_source_idx": { + "name": "agent_global_skill_source_idx", + "columns": ["source"], + "isUnique": false + }, + "agent_global_skill_is_enabled_idx": { + "name": "agent_global_skill_is_enabled_idx", + "columns": ["is_enabled"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "agent_session": { + "name": "agent_session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_name_manually_edited": { + "name": "is_name_manually_edited", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "trace_id": { + "name": "trace_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "agent_session_order_key_idx": { + "name": "agent_session_order_key_idx", + "columns": ["order_key"], + "isUnique": false + } + }, + "foreignKeys": { + "agent_session_agent_id_agent_id_fk": { + "name": "agent_session_agent_id_agent_id_fk", + "tableFrom": "agent_session", + "tableTo": "agent", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "agent_session_workspace_id_agent_workspace_id_fk": { + "name": "agent_session_workspace_id_agent_workspace_id_fk", + "tableFrom": "agent_session", + "tableTo": "agent_workspace", + "columnsFrom": ["workspace_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "agent_session_message": { + "name": "agent_session_message", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "searchable_text": { + "name": "searchable_text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model_snapshot": { + "name": "model_snapshot", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stats": { + "name": "stats", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "runtime_resume_token": { + "name": "runtime_resume_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fts_rowid": { + "name": "fts_rowid", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "agent_session_message_session_created_id_idx": { + "name": "agent_session_message_session_created_id_idx", + "columns": ["session_id", "created_at", "id"], + "isUnique": false + }, + "agent_session_message_status_idx": { + "name": "agent_session_message_status_idx", + "columns": ["status"], + "isUnique": false + }, + "agent_session_message_fts_rowid_uniq": { + "name": "agent_session_message_fts_rowid_uniq", + "columns": ["fts_rowid"], + "isUnique": true + } + }, + "foreignKeys": { + "agent_session_message_session_id_agent_session_id_fk": { + "name": "agent_session_message_session_id_agent_session_id_fk", + "tableFrom": "agent_session_message", + "tableTo": "agent_session", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_session_message_model_id_user_model_id_fk": { + "name": "agent_session_message_model_id_user_model_id_fk", + "tableFrom": "agent_session_message", + "tableTo": "user_model", + "columnsFrom": ["model_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "agent_session_message_role_check": { + "name": "agent_session_message_role_check", + "value": "\"agent_session_message\".\"role\" IN ('user', 'assistant', 'system')" + }, + "agent_session_message_status_check": { + "name": "agent_session_message_status_check", + "value": "\"agent_session_message\".\"status\" IN ('pending', 'success', 'error', 'paused')" + } + } + }, + "agent_skill": { + "name": "agent_skill", + "columns": { + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "skill_id": { + "name": "skill_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_enabled": { + "name": "is_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "agent_skill_agent_id_idx": { + "name": "agent_skill_agent_id_idx", + "columns": ["agent_id"], + "isUnique": false + }, + "agent_skill_skill_id_idx": { + "name": "agent_skill_skill_id_idx", + "columns": ["skill_id"], + "isUnique": false + } + }, + "foreignKeys": { + "agent_skill_agent_id_agent_id_fk": { + "name": "agent_skill_agent_id_agent_id_fk", + "tableFrom": "agent_skill", + "tableTo": "agent", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_skill_skill_id_agent_global_skill_id_fk": { + "name": "agent_skill_skill_id_agent_global_skill_id_fk", + "tableFrom": "agent_skill", + "tableTo": "agent_global_skill", + "columnsFrom": ["skill_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_skill_agent_id_skill_id_pk": { + "columns": ["agent_id", "skill_id"], + "name": "agent_skill_agent_id_skill_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "agent_workspace": { + "name": "agent_workspace", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'user'" + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "agent_workspace_path_unique_idx": { + "name": "agent_workspace_path_unique_idx", + "columns": ["path"], + "isUnique": true + }, + "agent_workspace_order_key_idx": { + "name": "agent_workspace_order_key_idx", + "columns": ["order_key"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "agent_workspace_type_check": { + "name": "agent_workspace_type_check", + "value": "\"agent_workspace\".\"type\" IN ('user', 'system')" + } + } + }, + "app_state": { + "name": "app_state", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "assistant": { + "name": "assistant", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "emoji": { + "name": "emoji", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "settings": { + "name": "settings", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "assistant_created_at_idx": { + "name": "assistant_created_at_idx", + "columns": ["created_at"], + "isUnique": false + }, + "assistant_order_key_idx": { + "name": "assistant_order_key_idx", + "columns": ["order_key"], + "isUnique": false + } + }, + "foreignKeys": { + "assistant_model_id_user_model_id_fk": { + "name": "assistant_model_id_user_model_id_fk", + "tableFrom": "assistant", + "tableTo": "user_model", + "columnsFrom": ["model_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "agent_mcp_server": { + "name": "agent_mcp_server", + "columns": { + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mcp_server_id": { + "name": "mcp_server_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "agent_mcp_server_agent_id_agent_id_fk": { + "name": "agent_mcp_server_agent_id_agent_id_fk", + "tableFrom": "agent_mcp_server", + "tableTo": "agent", + "columnsFrom": ["agent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "agent_mcp_server_mcp_server_id_mcp_server_id_fk": { + "name": "agent_mcp_server_mcp_server_id_mcp_server_id_fk", + "tableFrom": "agent_mcp_server", + "tableTo": "mcp_server", + "columnsFrom": ["mcp_server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "agent_mcp_server_agent_id_mcp_server_id_pk": { + "columns": ["agent_id", "mcp_server_id"], + "name": "agent_mcp_server_agent_id_mcp_server_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "assistant_knowledge_base": { + "name": "assistant_knowledge_base", + "columns": { + "assistant_id": { + "name": "assistant_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "knowledge_base_id": { + "name": "knowledge_base_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "assistant_knowledge_base_assistant_id_assistant_id_fk": { + "name": "assistant_knowledge_base_assistant_id_assistant_id_fk", + "tableFrom": "assistant_knowledge_base", + "tableTo": "assistant", + "columnsFrom": ["assistant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "assistant_knowledge_base_knowledge_base_id_knowledge_base_id_fk": { + "name": "assistant_knowledge_base_knowledge_base_id_knowledge_base_id_fk", + "tableFrom": "assistant_knowledge_base", + "tableTo": "knowledge_base", + "columnsFrom": ["knowledge_base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "assistant_knowledge_base_assistant_id_knowledge_base_id_pk": { + "columns": ["assistant_id", "knowledge_base_id"], + "name": "assistant_knowledge_base_assistant_id_knowledge_base_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "assistant_mcp_server": { + "name": "assistant_mcp_server", + "columns": { + "assistant_id": { + "name": "assistant_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mcp_server_id": { + "name": "mcp_server_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "assistant_mcp_server_assistant_id_assistant_id_fk": { + "name": "assistant_mcp_server_assistant_id_assistant_id_fk", + "tableFrom": "assistant_mcp_server", + "tableTo": "assistant", + "columnsFrom": ["assistant_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "assistant_mcp_server_mcp_server_id_mcp_server_id_fk": { + "name": "assistant_mcp_server_mcp_server_id_mcp_server_id_fk", + "tableFrom": "assistant_mcp_server", + "tableTo": "mcp_server", + "columnsFrom": ["mcp_server_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "assistant_mcp_server_assistant_id_mcp_server_id_pk": { + "columns": ["assistant_id", "mcp_server_id"], + "name": "assistant_mcp_server_assistant_id_mcp_server_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "file_entry": { + "name": "file_entry", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ext": { + "name": "ext", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "external_path": { + "name": "external_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "fe_deleted_at_idx": { + "name": "fe_deleted_at_idx", + "columns": ["deleted_at"], + "isUnique": false + }, + "fe_created_at_idx": { + "name": "fe_created_at_idx", + "columns": ["created_at"], + "isUnique": false + }, + "fe_external_path_lower_unique_idx": { + "name": "fe_external_path_lower_unique_idx", + "columns": ["lower(\"external_path\")"], + "isUnique": true + }, + "fe_external_path_idx": { + "name": "fe_external_path_idx", + "columns": ["external_path"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "fe_origin_check": { + "name": "fe_origin_check", + "value": "\"file_entry\".\"origin\" IN ('internal', 'external')" + }, + "fe_origin_consistency": { + "name": "fe_origin_consistency", + "value": "(\"file_entry\".\"origin\" = 'internal' AND \"file_entry\".\"external_path\" IS NULL) OR (\"file_entry\".\"origin\" = 'external' AND \"file_entry\".\"external_path\" IS NOT NULL)" + }, + "fe_external_no_delete": { + "name": "fe_external_no_delete", + "value": "\"file_entry\".\"origin\" != 'external' OR \"file_entry\".\"deleted_at\" IS NULL" + }, + "fe_size_internal_only": { + "name": "fe_size_internal_only", + "value": "(\"file_entry\".\"origin\" = 'internal' AND \"file_entry\".\"size\" IS NOT NULL AND \"file_entry\".\"size\" >= 0) OR (\"file_entry\".\"origin\" = 'external' AND \"file_entry\".\"size\" IS NULL)" + } + } + }, + "chat_message_file_ref": { + "name": "chat_message_file_ref", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "file_entry_id": { + "name": "file_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "cmfr_entry_id_idx": { + "name": "cmfr_entry_id_idx", + "columns": ["file_entry_id"], + "isUnique": false + }, + "cmfr_source_id_idx": { + "name": "cmfr_source_id_idx", + "columns": ["source_id"], + "isUnique": false + }, + "cmfr_unique_idx": { + "name": "cmfr_unique_idx", + "columns": ["file_entry_id", "source_id", "role"], + "isUnique": true + } + }, + "foreignKeys": { + "chat_message_file_ref_file_entry_id_file_entry_id_fk": { + "name": "chat_message_file_ref_file_entry_id_file_entry_id_fk", + "tableFrom": "chat_message_file_ref", + "tableTo": "file_entry", + "columnsFrom": ["file_entry_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "chat_message_file_ref_source_id_message_id_fk": { + "name": "chat_message_file_ref_source_id_message_id_fk", + "tableFrom": "chat_message_file_ref", + "tableTo": "message", + "columnsFrom": ["source_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "cmfr_role_check": { + "name": "cmfr_role_check", + "value": "\"chat_message_file_ref\".\"role\" IN ('attachment')" + } + } + }, + "painting_file_ref": { + "name": "painting_file_ref", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "file_entry_id": { + "name": "file_entry_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "pfr_entry_id_idx": { + "name": "pfr_entry_id_idx", + "columns": ["file_entry_id"], + "isUnique": false + }, + "pfr_source_id_idx": { + "name": "pfr_source_id_idx", + "columns": ["source_id"], + "isUnique": false + }, + "pfr_unique_idx": { + "name": "pfr_unique_idx", + "columns": ["file_entry_id", "source_id", "role"], + "isUnique": true + } + }, + "foreignKeys": { + "painting_file_ref_file_entry_id_file_entry_id_fk": { + "name": "painting_file_ref_file_entry_id_file_entry_id_fk", + "tableFrom": "painting_file_ref", + "tableTo": "file_entry", + "columnsFrom": ["file_entry_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "painting_file_ref_source_id_painting_id_fk": { + "name": "painting_file_ref_source_id_painting_id_fk", + "tableFrom": "painting_file_ref", + "tableTo": "painting", + "columnsFrom": ["source_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "pfr_role_check": { + "name": "pfr_role_check", + "value": "\"painting_file_ref\".\"role\" IN ('output', 'input')" + } + } + }, + "group": { + "name": "group", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "entity_type": { + "name": "entity_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "group_entity_type_order_key_idx": { + "name": "group_entity_type_order_key_idx", + "columns": ["entity_type", "order_key"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "job_schedule": { + "name": "job_schedule", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "job_input_template": { + "name": "job_input_template", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "next_run": { + "name": "next_run", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_run": { + "name": "last_run", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "catch_up_policy": { + "name": "catch_up_policy", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "job_schedule_type_name_uq": { + "name": "job_schedule_type_name_uq", + "columns": ["type", "name"], + "isUnique": true + }, + "job_schedule_enabled_next_run_idx": { + "name": "job_schedule_enabled_next_run_idx", + "columns": ["enabled", "next_run"], + "isUnique": false + }, + "job_schedule_type_idx": { + "name": "job_schedule_type_idx", + "columns": ["type"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "job": { + "name": "job", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "queue": { + "name": "queue", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "schedule_id": { + "name": "schedule_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scheduled_at": { + "name": "scheduled_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 3 + }, + "input": { + "name": "input", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "output": { + "name": "output", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parent_id": { + "name": "parent_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cancel_requested": { + "name": "cancel_requested", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "timeout_ms": { + "name": "timeout_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "job_queue_status_scheduled_at_idx": { + "name": "job_queue_status_scheduled_at_idx", + "columns": ["queue", "status", "scheduled_at"], + "isUnique": false + }, + "job_status_idx": { + "name": "job_status_idx", + "columns": ["status"], + "isUnique": false + }, + "job_schedule_id_finished_at_idx": { + "name": "job_schedule_id_finished_at_idx", + "columns": ["schedule_id", "finished_at"], + "isUnique": false + }, + "job_parent_id_idx": { + "name": "job_parent_id_idx", + "columns": ["parent_id"], + "isUnique": false + }, + "job_idempotency_key_partial_uq": { + "name": "job_idempotency_key_partial_uq", + "columns": ["idempotency_key"], + "isUnique": true, + "where": "\"job\".\"idempotency_key\" IS NOT NULL AND \"job\".\"status\" NOT IN ('completed','failed','cancelled')" + } + }, + "foreignKeys": { + "job_schedule_id_job_schedule_id_fk": { + "name": "job_schedule_id_job_schedule_id_fk", + "tableFrom": "job", + "tableTo": "job_schedule", + "columnsFrom": ["schedule_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "job_parent_id_job_id_fk": { + "name": "job_parent_id_job_id_fk", + "tableFrom": "job", + "tableTo": "job", + "columnsFrom": ["parent_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "job_status_check": { + "name": "job_status_check", + "value": "\"job\".\"status\" IN ('pending','delayed','running','completed','failed','cancelled')" + } + } + }, + "knowledge_base": { + "name": "knowledge_base", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dimensions": { + "name": "dimensions", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "embedding_model_id": { + "name": "embedding_model_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rerank_model_id": { + "name": "rerank_model_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "file_processor_id": { + "name": "file_processor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "chunk_size": { + "name": "chunk_size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk_overlap": { + "name": "chunk_overlap", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk_strategy": { + "name": "chunk_strategy", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'structured'" + }, + "chunk_separator": { + "name": "chunk_separator", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'\\n\\n'" + }, + "threshold": { + "name": "threshold", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "document_count": { + "name": "document_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "search_mode": { + "name": "search_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hybrid_alpha": { + "name": "hybrid_alpha", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "knowledge_base_group_id_group_id_fk": { + "name": "knowledge_base_group_id_group_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "group", + "columnsFrom": ["group_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "knowledge_base_embedding_model_id_user_model_id_fk": { + "name": "knowledge_base_embedding_model_id_user_model_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "user_model", + "columnsFrom": ["embedding_model_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "knowledge_base_rerank_model_id_user_model_id_fk": { + "name": "knowledge_base_rerank_model_id_user_model_id_fk", + "tableFrom": "knowledge_base", + "tableTo": "user_model", + "columnsFrom": ["rerank_model_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "knowledge_base_search_mode_check": { + "name": "knowledge_base_search_mode_check", + "value": "\"knowledge_base\".\"search_mode\" IN ('vector', 'bm25', 'hybrid')" + }, + "knowledge_base_chunk_strategy_check": { + "name": "knowledge_base_chunk_strategy_check", + "value": "\"knowledge_base\".\"chunk_strategy\" IN ('structured', 'delimiter')" + }, + "knowledge_base_status_check": { + "name": "knowledge_base_status_check", + "value": "\"knowledge_base\".\"status\" IN ('completed', 'failed')" + }, + "knowledge_base_status_error_check": { + "name": "knowledge_base_status_error_check", + "value": "\n (\n \"knowledge_base\".\"status\" = 'completed'\n AND \"knowledge_base\".\"error\" IS NULL\n AND (\n (\n \"knowledge_base\".\"embedding_model_id\" IS NOT NULL\n AND \"knowledge_base\".\"dimensions\" IS NOT NULL\n AND \"knowledge_base\".\"dimensions\" > 0\n )\n OR (\n \"knowledge_base\".\"embedding_model_id\" IS NULL\n AND \"knowledge_base\".\"dimensions\" IS NULL\n AND \"knowledge_base\".\"search_mode\" = 'bm25'\n )\n )\n )\n OR (\n \"knowledge_base\".\"status\" = 'failed'\n AND \"knowledge_base\".\"error\" IS NOT NULL\n AND length(trim(\"knowledge_base\".\"error\")) > 0\n )\n " + } + } + }, + "knowledge_item": { + "name": "knowledge_item", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "base_id": { + "name": "base_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "knowledge_item_base_type_created_idx": { + "name": "knowledge_item_base_type_created_idx", + "columns": ["base_id", "type", "created_at"], + "isUnique": false + }, + "knowledge_item_base_group_created_idx": { + "name": "knowledge_item_base_group_created_idx", + "columns": ["base_id", "group_id", "created_at"], + "isUnique": false + }, + "knowledge_item_baseId_id_unique": { + "name": "knowledge_item_baseId_id_unique", + "columns": ["base_id", "id"], + "isUnique": true + } + }, + "foreignKeys": { + "knowledge_item_base_id_knowledge_base_id_fk": { + "name": "knowledge_item_base_id_knowledge_base_id_fk", + "tableFrom": "knowledge_item", + "tableTo": "knowledge_base", + "columnsFrom": ["base_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "knowledge_item_base_id_group_id_knowledge_item_base_id_id_fk": { + "name": "knowledge_item_base_id_group_id_knowledge_item_base_id_id_fk", + "tableFrom": "knowledge_item", + "tableTo": "knowledge_item", + "columnsFrom": ["base_id", "group_id"], + "columnsTo": ["base_id", "id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "knowledge_item_type_check": { + "name": "knowledge_item_type_check", + "value": "\"knowledge_item\".\"type\" IN ('file', 'url', 'note', 'directory')" + }, + "knowledge_item_status_check": { + "name": "knowledge_item_status_check", + "value": "\"knowledge_item\".\"status\" IN ('preparing', 'processing', 'reading', 'embedding', 'completed', 'failed', 'deleting')" + }, + "knowledge_item_type_status_check": { + "name": "knowledge_item_type_status_check", + "value": "\n (\"knowledge_item\".\"type\" IN ('file', 'url', 'note') AND \"knowledge_item\".\"status\" IN ('processing', 'reading', 'embedding', 'completed', 'failed', 'deleting'))\n OR (\"knowledge_item\".\"type\" = 'directory' AND \"knowledge_item\".\"status\" IN ('preparing', 'processing', 'completed', 'failed', 'deleting'))\n " + }, + "knowledge_item_status_error_check": { + "name": "knowledge_item_status_error_check", + "value": "\n (\n \"knowledge_item\".\"status\" IN ('preparing', 'processing', 'reading', 'embedding', 'completed', 'deleting')\n AND \"knowledge_item\".\"error\" IS NULL\n )\n OR (\n \"knowledge_item\".\"status\" = 'failed'\n AND \"knowledge_item\".\"error\" IS NOT NULL\n AND length(trim(\"knowledge_item\".\"error\")) > 0\n )\n " + } + } + }, + "mcp_server": { + "name": "mcp_server", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "command": { + "name": "command", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "registry_url": { + "name": "registry_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "args": { + "name": "args", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "env": { + "name": "env", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "headers": { + "name": "headers", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_url": { + "name": "provider_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "logo_url": { + "name": "logo_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "long_running": { + "name": "long_running", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "timeout": { + "name": "timeout", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dxt_version": { + "name": "dxt_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dxt_path": { + "name": "dxt_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reference": { + "name": "reference", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "search_key": { + "name": "search_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "config_sample": { + "name": "config_sample", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disabled_tools": { + "name": "disabled_tools", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "disabled_auto_approve_tools": { + "name": "disabled_auto_approve_tools", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "should_config": { + "name": "should_config", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "is_active": { + "name": "is_active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "install_source": { + "name": "install_source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_trusted": { + "name": "is_trusted", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "trusted_at": { + "name": "trusted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "installed_at": { + "name": "installed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "mcp_server_name_idx": { + "name": "mcp_server_name_idx", + "columns": ["name"], + "isUnique": false + }, + "mcp_server_is_active_idx": { + "name": "mcp_server_is_active_idx", + "columns": ["is_active"], + "isUnique": false + }, + "mcp_server_sort_order_idx": { + "name": "mcp_server_sort_order_idx", + "columns": ["sort_order"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "mcp_server_type_check": { + "name": "mcp_server_type_check", + "value": "\"mcp_server\".\"type\" IS NULL OR \"mcp_server\".\"type\" IN ('stdio', 'sse', 'streamableHttp', 'inMemory')" + }, + "mcp_server_install_source_check": { + "name": "mcp_server_install_source_check", + "value": "\"mcp_server\".\"install_source\" IS NULL OR \"mcp_server\".\"install_source\" IN ('builtin', 'manual', 'protocol', 'unknown')" + } + } + }, + "message": { + "name": "message", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "parent_id": { + "name": "parent_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "topic_id": { + "name": "topic_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "searchable_text": { + "name": "searchable_text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "siblings_group_id": { + "name": "siblings_group_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model_snapshot": { + "name": "model_snapshot", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stats": { + "name": "stats", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fts_rowid": { + "name": "fts_rowid", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "message_parent_id_idx": { + "name": "message_parent_id_idx", + "columns": ["parent_id"], + "isUnique": false + }, + "message_topic_created_idx": { + "name": "message_topic_created_idx", + "columns": ["topic_id", "created_at"], + "isUnique": false + }, + "message_status_idx": { + "name": "message_status_idx", + "columns": ["status"], + "isUnique": false + }, + "message_topic_root_uniq": { + "name": "message_topic_root_uniq", + "columns": ["topic_id"], + "isUnique": true, + "where": "\"message\".\"parent_id\" is null and \"message\".\"deleted_at\" is null" + }, + "message_fts_rowid_uniq": { + "name": "message_fts_rowid_uniq", + "columns": ["fts_rowid"], + "isUnique": true + } + }, + "foreignKeys": { + "message_topic_id_topic_id_fk": { + "name": "message_topic_id_topic_id_fk", + "tableFrom": "message", + "tableTo": "topic", + "columnsFrom": ["topic_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "message_model_id_user_model_id_fk": { + "name": "message_model_id_user_model_id_fk", + "tableFrom": "message", + "tableTo": "user_model", + "columnsFrom": ["model_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "message_parent_id_message_id_fk": { + "name": "message_parent_id_message_id_fk", + "tableFrom": "message", + "tableTo": "message", + "columnsFrom": ["parent_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "message_role_check": { + "name": "message_role_check", + "value": "\"message\".\"role\" IN ('user', 'assistant', 'system', 'root')" + }, + "message_status_check": { + "name": "message_status_check", + "value": "\"message\".\"status\" IN ('pending', 'success', 'error', 'paused')" + }, + "message_root_parent_check": { + "name": "message_root_parent_check", + "value": "(\"message\".\"role\" = 'root') = (\"message\".\"parent_id\" is null)" + } + } + }, + "mini_app": { + "name": "mini_app", + "columns": { + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "preset_mini_app_id": { + "name": "preset_mini_app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'enabled'" + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bordered": { + "name": "bordered", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "background": { + "name": "background", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "supported_regions": { + "name": "supported_regions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "configuration": { + "name": "configuration", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name_key": { + "name": "name_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "mini_app_status_order_key_idx": { + "name": "mini_app_status_order_key_idx", + "columns": ["status", "order_key"], + "isUnique": false + }, + "mini_app_preset_mini_app_id_idx": { + "name": "mini_app_preset_mini_app_id_idx", + "columns": ["preset_mini_app_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "mini_app_status_check": { + "name": "mini_app_status_check", + "value": "\"mini_app\".\"status\" IN ('enabled', 'disabled', 'pinned')" + } + } + }, + "note": { + "name": "note", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "root_path": { + "name": "root_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "is_starred": { + "name": "is_starred", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "is_expanded": { + "name": "is_expanded", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "note_root_path_path_unique_idx": { + "name": "note_root_path_path_unique_idx", + "columns": ["root_path", "path"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "note_has_state_check": { + "name": "note_has_state_check", + "value": "\"note\".\"is_starred\" = 1 OR \"note\".\"is_expanded\" = 1" + } + } + }, + "painting": { + "name": "painting", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "painting_order_key_idx": { + "name": "painting_order_key_idx", + "columns": ["order_key"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "pin": { + "name": "pin", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "entity_type": { + "name": "entity_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "entity_id": { + "name": "entity_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "pin_entity_type_entity_id_unique_idx": { + "name": "pin_entity_type_entity_id_unique_idx", + "columns": ["entity_type", "entity_id"], + "isUnique": true + }, + "pin_entity_type_order_key_idx": { + "name": "pin_entity_type_order_key_idx", + "columns": ["entity_type", "order_key"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "preference": { + "name": "preference", + "columns": { + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'default'" + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "preference_scope_key_pk": { + "columns": ["scope", "key"], + "name": "preference_scope_key_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "prompt": { + "name": "prompt", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "prompt_order_key_idx": { + "name": "prompt_order_key_idx", + "columns": ["order_key"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "entity_tag": { + "name": "entity_tag", + "columns": { + "entity_type": { + "name": "entity_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "entity_id": { + "name": "entity_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tag_id": { + "name": "tag_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "entity_tag_tag_id_idx": { + "name": "entity_tag_tag_id_idx", + "columns": ["tag_id"], + "isUnique": false + } + }, + "foreignKeys": { + "entity_tag_tag_id_tag_id_fk": { + "name": "entity_tag_tag_id_tag_id_fk", + "tableFrom": "entity_tag", + "tableTo": "tag", + "columnsFrom": ["tag_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "entity_tag_entity_type_entity_id_tag_id_pk": { + "columns": ["entity_type", "entity_id", "tag_id"], + "name": "entity_tag_entity_type_entity_id_tag_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tag": { + "name": "tag", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "tag_name_unique": { + "name": "tag_name_unique", + "columns": ["name"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "topic": { + "name": "topic", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "is_name_manually_edited": { + "name": "is_name_manually_edited", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "assistant_id": { + "name": "assistant_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "active_node_id": { + "name": "active_node_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "group_id": { + "name": "group_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "trace_id": { + "name": "trace_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "topic_group_updated_idx": { + "name": "topic_group_updated_idx", + "columns": ["group_id", "updated_at"], + "isUnique": false + }, + "topic_updated_at_idx": { + "name": "topic_updated_at_idx", + "columns": ["updated_at"], + "isUnique": false + }, + "topic_order_key_idx": { + "name": "topic_order_key_idx", + "columns": ["order_key"], + "isUnique": false + }, + "topic_assistant_id_idx": { + "name": "topic_assistant_id_idx", + "columns": ["assistant_id"], + "isUnique": false + } + }, + "foreignKeys": { + "topic_assistant_id_assistant_id_fk": { + "name": "topic_assistant_id_assistant_id_fk", + "tableFrom": "topic", + "tableTo": "assistant", + "columnsFrom": ["assistant_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "topic_group_id_group_id_fk": { + "name": "topic_group_id_group_id_fk", + "tableFrom": "topic", + "tableTo": "group", + "columnsFrom": ["group_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "translate_history": { + "name": "translate_history", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source_text": { + "name": "source_text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_text": { + "name": "target_text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_language": { + "name": "source_language", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_language": { + "name": "target_language", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "star": { + "name": "star", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "translate_history_created_at_idx": { + "name": "translate_history_created_at_idx", + "columns": ["created_at"], + "isUnique": false + }, + "translate_history_star_created_at_idx": { + "name": "translate_history_star_created_at_idx", + "columns": ["star", "created_at"], + "isUnique": false + } + }, + "foreignKeys": { + "translate_history_source_language_translate_language_lang_code_fk": { + "name": "translate_history_source_language_translate_language_lang_code_fk", + "tableFrom": "translate_history", + "tableTo": "translate_language", + "columnsFrom": ["source_language"], + "columnsTo": ["lang_code"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "translate_history_target_language_translate_language_lang_code_fk": { + "name": "translate_history_target_language_translate_language_lang_code_fk", + "tableFrom": "translate_history", + "tableTo": "translate_language", + "columnsFrom": ["target_language"], + "columnsTo": ["lang_code"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "translate_language": { + "name": "translate_language", + "columns": { + "lang_code": { + "name": "lang_code", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "emoji": { + "name": "emoji", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_model": { + "name": "user_model", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "preset_model_id": { + "name": "preset_model_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "group": { + "name": "group", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "capabilities": { + "name": "capabilities", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_modalities": { + "name": "input_modalities", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_modalities": { + "name": "output_modalities", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "endpoint_types": { + "name": "endpoint_types", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "custom_endpoint_url": { + "name": "custom_endpoint_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "context_window": { + "name": "context_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_input_tokens": { + "name": "max_input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_output_tokens": { + "name": "max_output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "supports_streaming": { + "name": "supports_streaming", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "reasoning": { + "name": "reasoning", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parameters": { + "name": "parameters", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pricing": { + "name": "pricing", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_enabled": { + "name": "is_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "is_hidden": { + "name": "is_hidden", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "is_deprecated": { + "name": "is_deprecated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_overrides": { + "name": "user_overrides", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "user_model_preset_idx": { + "name": "user_model_preset_idx", + "columns": ["preset_model_id"], + "isUnique": false + }, + "user_model_provider_enabled_idx": { + "name": "user_model_provider_enabled_idx", + "columns": ["provider_id", "is_enabled"], + "isUnique": false + }, + "user_model_provider_id_order_key_idx": { + "name": "user_model_provider_id_order_key_idx", + "columns": ["provider_id", "order_key"], + "isUnique": false + }, + "user_model_provider_model_unique": { + "name": "user_model_provider_model_unique", + "columns": ["provider_id", "model_id"], + "isUnique": true + } + }, + "foreignKeys": { + "user_model_provider_id_user_provider_provider_id_fk": { + "name": "user_model_provider_id_user_provider_provider_id_fk", + "tableFrom": "user_model", + "tableTo": "user_provider", + "columnsFrom": ["provider_id"], + "columnsTo": ["provider_id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user_provider": { + "name": "user_provider", + "columns": { + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "preset_provider_id": { + "name": "preset_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint_configs": { + "name": "endpoint_configs", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_chat_endpoint": { + "name": "default_chat_endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "api_keys": { + "name": "api_keys", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'[]'" + }, + "auth_config": { + "name": "auth_config", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "api_features": { + "name": "api_features", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_settings": { + "name": "provider_settings", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_enabled": { + "name": "is_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "order_key": { + "name": "order_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "user_provider_preset_idx": { + "name": "user_provider_preset_idx", + "columns": ["preset_provider_id"], + "isUnique": false + }, + "user_provider_enabled_idx": { + "name": "user_provider_enabled_idx", + "columns": ["is_enabled"], + "isUnique": false + }, + "user_provider_order_key_idx": { + "name": "user_provider_order_key_idx", + "columns": ["order_key"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": { + "fe_external_path_lower_unique_idx": { + "columns": { + "lower(\"external_path\")": { + "isExpression": true + } + } + } + } + } +} diff --git a/migrations/sqlite-drizzle/meta/_journal.json b/migrations/sqlite-drizzle/meta/_journal.json index 5bd210be24e..9da89e45795 100644 --- a/migrations/sqlite-drizzle/meta/_journal.json +++ b/migrations/sqlite-drizzle/meta/_journal.json @@ -126,6 +126,13 @@ "when": 1782817365873, "tag": "0017_minor_echo", "breakpoints": true + }, + { + "idx": 18, + "version": "6", + "when": 1783173408643, + "tag": "0018_easy_gertrude_yorkes", + "breakpoints": true } ], "version": "7" diff --git a/src/main/data/db/schemas/knowledge.ts b/src/main/data/db/schemas/knowledge.ts index d27fad0e2fe..7ffeaf9147b 100644 --- a/src/main/data/db/schemas/knowledge.ts +++ b/src/main/data/db/schemas/knowledge.ts @@ -102,20 +102,20 @@ export const knowledgeItemTable = sqliteTable( check('knowledge_item_type_check', sql`${t.type} IN ('file', 'url', 'note', 'directory')`), check( 'knowledge_item_status_check', - sql`${t.status} IN ('idle', 'preparing', 'processing', 'reading', 'embedding', 'completed', 'failed', 'deleting')` + sql`${t.status} IN ('preparing', 'processing', 'reading', 'embedding', 'completed', 'failed', 'deleting')` ), check( 'knowledge_item_type_status_check', sql` - (${t.type} IN ('file', 'url', 'note') AND ${t.status} IN ('idle', 'processing', 'reading', 'embedding', 'completed', 'failed', 'deleting')) - OR (${t.type} = 'directory' AND ${t.status} IN ('idle', 'preparing', 'processing', 'completed', 'failed', 'deleting')) + (${t.type} IN ('file', 'url', 'note') AND ${t.status} IN ('processing', 'reading', 'embedding', 'completed', 'failed', 'deleting')) + OR (${t.type} = 'directory' AND ${t.status} IN ('preparing', 'processing', 'completed', 'failed', 'deleting')) ` ), check( 'knowledge_item_status_error_check', sql` ( - ${t.status} IN ('idle', 'preparing', 'processing', 'reading', 'embedding', 'completed', 'deleting') + ${t.status} IN ('preparing', 'processing', 'reading', 'embedding', 'completed', 'deleting') AND ${t.error} IS NULL ) OR ( diff --git a/src/main/data/migration/v2/migrators/__tests__/KnowledgeMigrator.test.ts b/src/main/data/migration/v2/migrators/__tests__/KnowledgeMigrator.test.ts index e41d3c2fc63..6f44de1d104 100644 --- a/src/main/data/migration/v2/migrators/__tests__/KnowledgeMigrator.test.ts +++ b/src/main/data/migration/v2/migrators/__tests__/KnowledgeMigrator.test.ts @@ -3,7 +3,8 @@ import fs from 'node:fs' import { KNOWLEDGE_BASE_ERROR_MISSING_EMBEDDING_MODEL, KNOWLEDGE_BASE_ERROR_MISSING_VECTOR_STORE, - KNOWLEDGE_ITEM_ERROR_DIRECTORY_NOT_MIGRATED + KNOWLEDGE_ITEM_ERROR_DIRECTORY_NOT_MIGRATED, + KNOWLEDGE_ITEM_ERROR_NEVER_INDEXED } from '@shared/data/types/knowledge' import Database from 'better-sqlite3' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -856,9 +857,9 @@ describe('KnowledgeMigrator dimensions resolution', () => { ) expect(result.success).toBe(true) - expect(statusByLegacyId.get('i-no-unique-id')).toBe('idle') + expect(statusByLegacyId.get('i-no-unique-id')).toBe('failed') expect(statusByLegacyId.get('i-with-unique-id')).toBe('completed') - expect(statusByLegacyId.get('i-with-empty-unique-id')).toBe('idle') + expect(statusByLegacyId.get('i-with-empty-unique-id')).toBe('failed') expect(statusByLegacyId.get('i-processing-but-no-unique-id')).toBe('failed') expect(statusByLegacyId.get('i-failed-with-unique-id')).toBe('failed') }) @@ -1463,10 +1464,10 @@ describe('KnowledgeMigrator dimensions resolution', () => { ) }) - it('keeps an idle directory (loader ids but no completed marker) as idle without expanding it', async () => { + it('keeps a never-indexed directory (loader ids but no completed marker) as failed without expanding it', async () => { // Expansion keys off the `completed` marker (singular `uniqueId`), not the mere presence of - // child loader ids (plural `uniqueIds`). A folder with loader ids but no completed marker is - // `idle`, so the gate must NOT expand it even when the loader sources resolve. + // child loader ids (plural `uniqueIds`). A folder with loader ids but no completed marker never + // started indexing, so the gate must NOT expand it even when the loader sources resolve. const migrator = new KnowledgeMigrator() as any vi.spyOn(migrator, 'resolveDimensionsForBase').mockResolvedValue({ dimensions: 1024, reason: 'ok' }) vi.spyOn(migrator, 'loadLoaderSourceMap').mockResolvedValue({ @@ -1507,12 +1508,12 @@ describe('KnowledgeMigrator dimensions resolution', () => { expect(result.success).toBe(true) const migratedId = migrator.legacyItemIdRemap.get('item-directory') - // Not expanded: a single idle directory item, no synthesized children, no loader remap. + // Not expanded: a single never-indexed directory item, no synthesized children, no loader remap. expect(migrator.preparedItems.filter((i: any) => i.groupId === migratedId)).toHaveLength(0) expect(migrator.preparedItems.find((i: any) => i.id === migratedId)).toMatchObject({ type: 'directory', - status: 'idle', - error: null + status: 'failed', + error: KNOWLEDGE_ITEM_ERROR_NEVER_INDEXED }) expect(migrator.directoryChildLoaderRemap.size).toBe(0) }) @@ -1932,7 +1933,7 @@ describe('KnowledgeMigrator execute/validate paths', () => { groupId: null, type: 'note', data: { content: 'n1' }, - status: 'idle' + status: 'processing' }, { id: 'item-2', @@ -1940,7 +1941,7 @@ describe('KnowledgeMigrator execute/validate paths', () => { groupId: null, type: 'note', data: { content: 'n2' }, - status: 'idle' + status: 'processing' } ] @@ -2049,7 +2050,7 @@ describe('KnowledgeMigrator execute/validate paths', () => { groupId: null, type: 'note', data: { source: 'n1', content: 'n1' }, - status: 'idle', + status: 'processing', error: null } ] @@ -2140,7 +2141,7 @@ describe('KnowledgeMigrator execute/validate paths', () => { groupId: null, type: 'note', data: { source: 'note', content: 'note' }, - status: 'idle', + status: 'processing', error: null, createdAt: 1775114958369, updatedAt: 1775114958369 @@ -2175,7 +2176,7 @@ describe('KnowledgeMigrator execute/validate paths', () => { expect.objectContaining({ id: 'item-1', baseId: 'kb-missing-model', - status: 'idle' + status: 'processing' }) ] ]) @@ -2204,7 +2205,7 @@ describe('KnowledgeMigrator execute/validate paths', () => { groupId: null, type: 'note', data: { content: 'n1' }, - status: 'idle' + status: 'processing' }, { id: 'item-2', @@ -2212,7 +2213,7 @@ describe('KnowledgeMigrator execute/validate paths', () => { groupId: null, type: 'note', data: { content: 'n2' }, - status: 'idle' + status: 'processing' } ] @@ -2368,7 +2369,7 @@ describe('KnowledgeMigrator file item path storage', () => { groupId: null, type: 'file', data: { source: '/tmp/a.pdf', relativePath: 'a.pdf' }, - status: 'idle' + status: 'processing' }, { id: 'item-b', @@ -2376,7 +2377,7 @@ describe('KnowledgeMigrator file item path storage', () => { groupId: null, type: 'file', data: { source: '/tmp/b.pdf', relativePath: 'b.pdf', indexedRelativePath: 'b.md' }, - status: 'idle' + status: 'processing' }, { id: 'item-note', @@ -2384,7 +2385,7 @@ describe('KnowledgeMigrator file item path storage', () => { groupId: null, type: 'note', data: { source: 'some note', content: 'some note' }, - status: 'idle' + status: 'processing' } ] @@ -2419,7 +2420,7 @@ describe('KnowledgeMigrator file item path storage', () => { groupId: null, type: 'file', data: { source: '/tmp/ok.pdf', relativePath: 'ok.pdf' }, - status: 'idle' + status: 'processing' }, { id: 'item-skipped-file-entry', @@ -2427,7 +2428,7 @@ describe('KnowledgeMigrator file item path storage', () => { groupId: null, type: 'file', data: { source: '/tmp/bad.xyz', relativePath: 'bad.xyz' }, - status: 'idle' + status: 'processing' } ] migrator.legacyItemIdRemap = new Map([ diff --git a/src/main/data/migration/v2/migrators/mappings/KnowledgeMappings.ts b/src/main/data/migration/v2/migrators/mappings/KnowledgeMappings.ts index f3240862b28..6685c728f65 100644 --- a/src/main/data/migration/v2/migrators/mappings/KnowledgeMappings.ts +++ b/src/main/data/migration/v2/migrators/mappings/KnowledgeMappings.ts @@ -8,6 +8,7 @@ import { DEFAULT_KNOWLEDGE_SEARCH_MODE, KNOWLEDGE_BASE_ERROR_MISSING_EMBEDDING_MODEL, KNOWLEDGE_ITEM_ERROR_DIRECTORY_NOT_MIGRATED, + KNOWLEDGE_ITEM_ERROR_NEVER_INDEXED, KNOWLEDGE_NOTE_CONTENT_MAX, type KnowledgeItemData, type KnowledgeItemStatus @@ -143,7 +144,7 @@ export const inferKnowledgeItemStatus = ( return 'failed' } - return typeof item.uniqueId === 'string' && item.uniqueId.trim() !== '' ? 'completed' : 'idle' + return typeof item.uniqueId === 'string' && item.uniqueId.trim() !== '' ? 'completed' : 'failed' } const normalizeKnowledgeItemError = ( @@ -421,9 +422,14 @@ export const transformKnowledgeItem = ( // home), so letting the directory claim `completed` would leave an empty shell // that never re-indexes. Mark it `failed` with a code the UI renders as a // delete-and-re-upload prompt (it migrated as a record but its vectors were dropped). - // Interrupted (failed) and never-indexed (idle) directories keep their inferred status - // (only a `completed` directory is overridden to `failed`). + // Interrupted and never-indexed items keep their inferred `failed` status (only a + // `completed` directory is overridden to `failed`). const directoryIndexDropped = type === 'directory' && inferredStatus === 'completed' + const wasInterrupted = + item.processingStatus === 'failed' || item.processingStatus === 'processing' || item.processingStatus === 'pending' + // A v1 item that never started indexing (no uniqueId, not interrupted) has nothing to + // resume from; mark it with a code the UI renders as a re-index prompt. + const neverIndexed = !directoryIndexDropped && inferredStatus === 'failed' && !wasInterrupted const status = directoryIndexDropped ? 'failed' : inferredStatus return { @@ -439,7 +445,9 @@ export const transformKnowledgeItem = ( status, error: directoryIndexDropped ? KNOWLEDGE_ITEM_ERROR_DIRECTORY_NOT_MIGRATED - : normalizeKnowledgeItemError(status, item.processingStatus, item.processingError), + : neverIndexed + ? KNOWLEDGE_ITEM_ERROR_NEVER_INDEXED + : normalizeKnowledgeItemError(status, item.processingStatus, item.processingError), createdAt: toTimestamp(item.created_at), updatedAt: toTimestamp(item.updated_at) }, diff --git a/src/main/data/migration/v2/migrators/mappings/__tests__/KnowledgeMappings.test.ts b/src/main/data/migration/v2/migrators/mappings/__tests__/KnowledgeMappings.test.ts index e87dd7e7f53..27d498e31b2 100644 --- a/src/main/data/migration/v2/migrators/mappings/__tests__/KnowledgeMappings.test.ts +++ b/src/main/data/migration/v2/migrators/mappings/__tests__/KnowledgeMappings.test.ts @@ -2,6 +2,7 @@ import { FILE_TYPE } from '@shared/data/types/file' import { KNOWLEDGE_BASE_ERROR_MISSING_EMBEDDING_MODEL, KNOWLEDGE_ITEM_ERROR_DIRECTORY_NOT_MIGRATED, + KNOWLEDGE_ITEM_ERROR_NEVER_INDEXED, KNOWLEDGE_NOTE_CONTENT_MAX } from '@shared/data/types/knowledge' import { describe, expect, it } from 'vitest' @@ -42,13 +43,13 @@ describe('KnowledgeMappings', () => { expect(legacyModelToUniqueId({ id: 'silicon::BAAI/bge-m3', provider: 'silicon' })).toBe('silicon::BAAI/bge-m3') }) - it('inferKnowledgeItemStatus maps legacy transient states to failed', () => { + it('inferKnowledgeItemStatus maps legacy transient and never-indexed states to failed', () => { expect(inferKnowledgeItemStatus({ uniqueId: 'loader-1' } as any)).toBe('completed') - expect(inferKnowledgeItemStatus({ uniqueId: ' ' } as any)).toBe('idle') + expect(inferKnowledgeItemStatus({ uniqueId: ' ' } as any)).toBe('failed') expect(inferKnowledgeItemStatus({ processingStatus: 'pending' } as any)).toBe('failed') expect(inferKnowledgeItemStatus({ processingStatus: 'processing' } as any)).toBe('failed') expect(inferKnowledgeItemStatus({ processingStatus: 'failed', uniqueId: 'loader-1' } as any)).toBe('failed') - expect(inferKnowledgeItemStatus({} as any)).toBe('idle') + expect(inferKnowledgeItemStatus({} as any)).toBe('failed') }) it('transformKnowledgeBase marks knowledge bases without an embedding model as failed', () => { @@ -281,8 +282,8 @@ describe('KnowledgeMappings', () => { source: 'https://dexie.example.com', content: 'dexie-content' }, - status: 'idle', - error: null, + status: 'failed', + error: KNOWLEDGE_ITEM_ERROR_NEVER_INDEXED, createdAt: expect.any(Number), updatedAt: expect.any(Number) } @@ -514,13 +515,13 @@ describe('KnowledgeMappings', () => { expect(warnings[0]).toContain('blank v1 filename') }) - it('transformKnowledgeItem clears blank legacy processing errors for idle and completed items', () => { - const idleResult = transformKnowledgeItem( + it('transformKnowledgeItem marks a never-started item never_indexed (ignoring any stale processing error) and clears blank legacy processing errors for completed items', () => { + const neverIndexedResult = transformKnowledgeItem( 'kb-1', { - id: 'idle-note', + id: 'never-indexed-note', type: 'note', - content: 'idle note', + content: 'never indexed note', processingError: '' }, { @@ -543,11 +544,11 @@ describe('KnowledgeMappings', () => { } ) - expect(idleResult).toStrictEqual({ + expect(neverIndexedResult).toStrictEqual({ ok: true, value: expect.objectContaining({ - status: 'idle', - error: null + status: 'failed', + error: KNOWLEDGE_ITEM_ERROR_NEVER_INDEXED }) }) expect(completedResult).toStrictEqual({ @@ -651,7 +652,7 @@ describe('KnowledgeMappings', () => { }) }) - it('transformKnowledgeItem maps directory items to v2 directory node data', () => { + it('transformKnowledgeItem maps a never-indexed directory to v2 directory node data, marked never_indexed', () => { const result = transformKnowledgeItem( 'kb-1', { @@ -675,8 +676,8 @@ describe('KnowledgeMappings', () => { data: { source: '/tmp/docs' }, - status: 'idle', - error: null, + status: 'failed', + error: KNOWLEDGE_ITEM_ERROR_NEVER_INDEXED, createdAt: expect.any(Number), updatedAt: expect.any(Number) } diff --git a/src/main/data/services/__tests__/KnowledgeItemService.test.ts b/src/main/data/services/__tests__/KnowledgeItemService.test.ts index 5f8503fe86d..7467fb8715e 100644 --- a/src/main/data/services/__tests__/KnowledgeItemService.test.ts +++ b/src/main/data/services/__tests__/KnowledgeItemService.test.ts @@ -80,7 +80,7 @@ describe('KnowledgeItemService', () => { groupId: null, type: 'note', data: { source: 'seed-note', content: 'hello world' }, - status: 'idle', + status: 'processing', error: null, ...overrides } @@ -230,7 +230,7 @@ describe('KnowledgeItemService', () => { groupId: null, type: 'note', data: { source: ITEM_1_ID, content: 'item 1' }, - status: 'idle', + status: 'processing', error: null }) }) @@ -384,12 +384,11 @@ describe('KnowledgeItemService', () => { const READING_LEAF = itemId('7e02') const EMBEDDING_LEAF = itemId('7e03') const PROCESSING_LEAF = itemId('7e04') - const IDLE_LEAF = itemId('7e05') const COMPLETED_LEAF = itemId('7e06') const FAILED_LEAF = itemId('7e07') const DELETING_LEAF = itemId('7e08') - it('marks every in-flight item (leaves and containers) failed, leaving terminal/idle/deleting untouched', async () => { + it('marks every in-flight item (leaves and containers) failed, leaving terminal/deleting untouched', async () => { // In-flight: a container mid-expansion, a container with an in-flight child, and standalone leaves. await seedItem({ id: PREPARING_DIR, type: 'directory', data: { source: '/p' }, status: 'preparing' }) await seedItem({ id: PROCESSING_DIR, type: 'directory', data: { source: '/q' }, status: 'processing' }) @@ -402,8 +401,7 @@ describe('KnowledgeItemService', () => { await seedItem({ id: EMBEDDING_LEAF, data: { source: 'e', content: 'e' }, status: 'embedding' }) await seedItem({ id: PROCESSING_LEAF, data: { source: 'p2', content: 'p2' }, status: 'processing' }) - // Untouched: not started, already terminal, or being deleted. - await seedItem({ id: IDLE_LEAF, data: { source: 'i', content: 'i' }, status: 'idle' }) + // Untouched: already terminal, or being deleted. await seedItem({ id: COMPLETED_LEAF, data: { source: 'c', content: 'c' }, status: 'completed' }) await seedItem({ id: FAILED_LEAF, data: { source: 'f', content: 'f' }, status: 'failed', error: 'real failure' }) await seedItem({ id: DELETING_LEAF, data: { source: 'd', content: 'd' }, status: 'deleting' }) @@ -417,7 +415,6 @@ describe('KnowledgeItemService', () => { expect(item.error).toBe(INTERRUPTED) } - expect(service.getById(IDLE_LEAF).status).toBe('idle') expect(service.getById(COMPLETED_LEAF).status).toBe('completed') const realFailure = service.getById(FAILED_LEAF) expect(realFailure.status).toBe('failed') diff --git a/src/main/features/knowledge/KnowledgeService.ts b/src/main/features/knowledge/KnowledgeService.ts index c1eac65bf8c..f94bd43b403 100644 --- a/src/main/features/knowledge/KnowledgeService.ts +++ b/src/main/features/knowledge/KnowledgeService.ts @@ -1,4 +1,5 @@ import { application } from '@application' +import { KeyedMutex } from '@main/core/concurrency/KeyedMutex' import { BaseService, DependsOn, Injectable, Phase, ServicePhase } from '@main/core/lifecycle' import type { CreateKnowledgeBaseDto, @@ -14,7 +15,6 @@ import type { } from '@shared/data/types/knowledge' import { KnowledgeBaseAdminService } from './base/KnowledgeBaseAdminService' -import { KnowledgeLockManager } from './base/KnowledgeLockManager' import { KnowledgeIngestionService } from './ingestion/KnowledgeIngestionService' import type { KnowledgeConceptContent, @@ -40,7 +40,7 @@ import { createReindexSubtreeJobHandler } from './tasks/reindexSubtreeJobHandler @ServicePhase(Phase.WhenReady) @DependsOn(['KnowledgeVectorStoreService', 'JobManager', 'FileProcessingService']) export class KnowledgeService extends BaseService { - private readonly knowledgeLockManager = new KnowledgeLockManager() + private readonly knowledgeLockManager = new KeyedMutex() private readonly ingestionService = new KnowledgeIngestionService(this.knowledgeLockManager) private readonly baseAdmin = new KnowledgeBaseAdminService(this.knowledgeLockManager, this.ingestionService) private readonly queryService = new KnowledgeQueryService() diff --git a/src/main/features/knowledge/__tests__/KnowledgeService.integration.test.ts b/src/main/features/knowledge/__tests__/KnowledgeService.integration.test.ts index 4e06fc56df4..ebe4430c770 100644 --- a/src/main/features/knowledge/__tests__/KnowledgeService.integration.test.ts +++ b/src/main/features/knowledge/__tests__/KnowledgeService.integration.test.ts @@ -117,7 +117,7 @@ describe('KnowledgeService integration', () => { groupId: null, type: 'note', data: { source: 'source-root', content: 'root content' }, - status: 'idle', + status: 'processing', error: null }, { @@ -126,7 +126,7 @@ describe('KnowledgeService integration', () => { groupId: SOURCE_ROOT_ITEM_ID, type: 'note', data: { source: 'source-child', content: 'child content' }, - status: 'idle', + status: 'processing', error: null } ]) @@ -181,7 +181,7 @@ describe('KnowledgeService integration', () => { expect(enqueueMock).toHaveBeenCalledWith( 'knowledge.index-documents', - { baseId: restoredBase.id, itemId: restoredItems[0].id, parentJobId: null }, + { baseId: restoredBase.id, itemId: restoredItems[0].id }, { idempotencyKey: `knowledge:${restoredBase.id}:${restoredItems[0].id}:index`, queue: `base.${restoredBase.id}`, diff --git a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts index 940f0479356..f88545032ec 100644 --- a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts +++ b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts @@ -222,7 +222,7 @@ function createNoteItem( id = 'note-1', baseId = 'kb-1', groupId: string | null = null, - status: KnowledgeItemOf<'note'>['status'] = 'idle' + status: KnowledgeItemOf<'note'>['status'] = 'processing' ): KnowledgeItemOf<'note'> { const lifecycle = status === 'failed' ? ({ status, error: `failed ${id}` } as const) : ({ status, error: null } as const) @@ -242,7 +242,7 @@ function createNoteItem( function createDirectoryItem( id = 'dir-1', groupId: string | null = null, - status: KnowledgeItemOf<'directory'>['status'] = 'idle' + status: KnowledgeItemOf<'directory'>['status'] = 'processing' ): KnowledgeItemOf<'directory'> { const lifecycle = status === 'failed' ? ({ status, error: `failed ${id}` } as const) : ({ status, error: null } as const) @@ -263,7 +263,7 @@ function createFileItem( id = 'file-1', baseId = 'kb-1', source = '/docs/source.pdf', - status: KnowledgeItemOf<'file'>['status'] = 'idle' + status: KnowledgeItemOf<'file'>['status'] = 'processing' ): KnowledgeItemOf<'file'> { const lifecycle = status === 'failed' ? ({ status, error: `failed ${id}` } as const) : ({ status, error: null } as const) @@ -365,7 +365,7 @@ describe('KnowledgeService', () => { fileProcessingStartJobMock.mockResolvedValue({ id: 'fp-job-1', snapshot: {}, finished: Promise.resolve({}) }) getJobMock.mockResolvedValue(null) listMock.mockResolvedValue([]) - getIndexStoreMock.mockResolvedValue({ + getIndexStoreMock.mockReturnValue({ search: storeSearchMock, listMaterialUnits: listMaterialUnitsMock, getMaterialByRelativePath: getMaterialByRelativePathMock, @@ -531,7 +531,9 @@ describe('KnowledgeService', () => { ) expect(getIndexStoreMock).toHaveBeenCalledWith(base) - getIndexStoreMock.mockRejectedValueOnce(new Error('store failed')) + getIndexStoreMock.mockImplementationOnce(() => { + throw new Error('store failed') + }) await expect( service.createBase({ name: 'KB', dimensions: 3, embeddingModelId: 'provider::embed' }) ).rejects.toThrow('store failed') @@ -540,7 +542,9 @@ describe('KnowledgeService', () => { it('rollback removes the orphaned index dir and still surfaces the original error when cleanup itself fails', async () => { const service = new KnowledgeService() - getIndexStoreMock.mockRejectedValueOnce(new Error('store failed')) + getIndexStoreMock.mockImplementationOnce(() => { + throw new Error('store failed') + }) // Even if the orphan-dir cleanup throws, the caller must see the open error. deleteStoreMock.mockRejectedValueOnce(new Error('cleanup boom')) @@ -560,6 +564,13 @@ describe('KnowledgeService', () => { expect(listMock).toHaveBeenCalledWith({ queue: 'base.kb-1', status: ['pending', 'delayed', 'running'], + type: [ + 'knowledge.prepare-root', + 'knowledge.index-documents', + 'knowledge.check-file-processing-result', + 'knowledge.delete-subtree', + 'knowledge.reindex-subtree' + ], limit: 5000 }) expect(deleteStoreMock).toHaveBeenCalledWith('kb-1') @@ -581,8 +592,7 @@ describe('KnowledgeService', () => { itemId: 'file-1', fileProcessingJobId: 'fp-job-1', pollRound: 0, - firstScheduledAt: 1779811200000, - parentJobId: null + firstScheduledAt: 1779811200000 } } ]) @@ -1032,8 +1042,7 @@ describe('KnowledgeService', () => { itemId: 'file-1', fileProcessingJobId: 'fp-job-1', pollRound: 0, - firstScheduledAt: expect.any(Number), - parentJobId: 'fp-job-1' + firstScheduledAt: expect.any(Number) }, expect.objectContaining({ idempotencyKey: 'knowledge:kb-1:file-1:fp-check:fp-job-1:0', @@ -1306,8 +1315,7 @@ describe('KnowledgeService', () => { itemId: 'file-1', fileProcessingJobId: 'fp-job-1', pollRound: 0, - firstScheduledAt: expect.any(Number), - parentJobId: 'reindex-job' + firstScheduledAt: expect.any(Number) }, expect.objectContaining({ idempotencyKey: 'knowledge:kb-1:file-1:fp-check:fp-job-1:0', @@ -1380,7 +1388,7 @@ describe('KnowledgeService', () => { expect(enqueueMock).toHaveBeenCalledWith( 'knowledge.index-documents', - { baseId: 'kb-1', itemId: 'file-1', parentJobId: 'reindex-job' }, + { baseId: 'kb-1', itemId: 'file-1' }, { idempotencyKey: 'knowledge:kb-1:file-1:index:reindex-job', queue: 'base.kb-1', @@ -1419,8 +1427,7 @@ describe('KnowledgeService', () => { itemId: 'file-1', fileProcessingJobId: 'fp-job-1', pollRound: 1, - firstScheduledAt: Date.parse('2026-04-08T00:00:00.000Z'), - parentJobId: 'check-job-0' + firstScheduledAt: Date.parse('2026-04-08T00:00:00.000Z') }, expect.objectContaining({ idempotencyKey: 'knowledge:kb-1:file-1:fp-check:fp-job-1:1', @@ -1443,7 +1450,7 @@ describe('KnowledgeService', () => { expect(fileProcessingStartJobMock).not.toHaveBeenCalled() expect(enqueueMock).toHaveBeenCalledWith( 'knowledge.index-documents', - { baseId: 'kb-1', itemId: 'file-1', parentJobId: null }, + { baseId: 'kb-1', itemId: 'file-1' }, { idempotencyKey: 'knowledge:kb-1:file-1:index', queue: 'base.kb-1', @@ -1464,7 +1471,7 @@ describe('KnowledgeService', () => { expect(fileProcessingStartJobMock).not.toHaveBeenCalled() expect(enqueueMock).toHaveBeenCalledWith( 'knowledge.index-documents', - { baseId: 'kb-1', itemId: 'file-1', parentJobId: null }, + { baseId: 'kb-1', itemId: 'file-1' }, { idempotencyKey: 'knowledge:kb-1:file-1:index', queue: 'base.kb-1', @@ -1488,10 +1495,10 @@ describe('KnowledgeService', () => { ]) ).rejects.toThrow('enqueue failed') - expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith('note-2', 'failed', { + expect(knowledgeItemSetSubtreeStatusMock).toHaveBeenCalledWith('kb-1', ['note-2'], 'failed', { error: 'Failed to schedule knowledge item job: enqueue failed' }) - expect(knowledgeItemUpdateStatusMock).not.toHaveBeenCalledWith('note-1', 'failed', expect.anything()) + expect(knowledgeItemSetSubtreeStatusMock).not.toHaveBeenCalledWith('kb-1', ['note-1'], 'failed', expect.anything()) }) it('rolls back already-created addItems rows when a later create fails mid-batch', async () => { @@ -2094,11 +2101,11 @@ describe('KnowledgeService', () => { it('omits conceptId for a leaf that is not completed (not readable yet)', async () => { const service = new KnowledgeService() - knowledgeItemGetItemsByBaseIdMock.mockReturnValue([createFileItem('f1', 'kb-1', '/a.pdf', 'idle')]) + knowledgeItemGetItemsByBaseIdMock.mockReturnValue([createFileItem('f1', 'kb-1', '/a.pdf', 'processing')]) const tree = service.getOrganizationTree('kb-1') - expect(tree.nodes[0]).toMatchObject({ depth: 0, itemType: 'file', status: 'idle' }) + expect(tree.nodes[0]).toMatchObject({ depth: 0, itemType: 'file', status: 'processing' }) expect(tree.nodes[0].conceptId).toBeUndefined() }) @@ -2321,7 +2328,7 @@ describe('KnowledgeService', () => { it('translates a search failure into a defined error when the store was closed mid-flight', async () => { const service = new KnowledgeService() - getIndexStoreMock.mockResolvedValueOnce({ + getIndexStoreMock.mockReturnValueOnce({ search: vi.fn().mockRejectedValue(new Error('Knowledge index store driver is closed')), listMaterialUnits: listMaterialUnitsMock, isClosed: () => true @@ -2336,7 +2343,7 @@ describe('KnowledgeService', () => { it('rethrows a genuine search failure unchanged when the store is still open', async () => { const service = new KnowledgeService() const queryError = new Error('disk I/O error') - getIndexStoreMock.mockResolvedValueOnce({ + getIndexStoreMock.mockReturnValueOnce({ search: vi.fn().mockRejectedValue(queryError), listMaterialUnits: listMaterialUnitsMock, isClosed: () => false @@ -2405,7 +2412,7 @@ describe('KnowledgeService', () => { expect(listMaterialUnitsMock).not.toHaveBeenCalled() }) - it.each(['idle', 'processing', 'reading', 'embedding', 'failed', 'deleting'] as const)( + it.each(['processing', 'reading', 'embedding', 'failed', 'deleting'] as const)( 'rejects chunk operations for %s leaf items', async (status) => { const service = new KnowledgeService() @@ -2423,7 +2430,7 @@ describe('KnowledgeService', () => { it('translates a listItemChunks failure into a defined error when the store was closed mid-flight', async () => { const service = new KnowledgeService() knowledgeItemGetByIdMock.mockReturnValue(createNoteItem('note-1', 'kb-1', null, 'completed')) - getIndexStoreMock.mockResolvedValueOnce({ + getIndexStoreMock.mockReturnValueOnce({ search: storeSearchMock, listMaterialUnits: vi.fn().mockImplementation(() => { throw new Error('Knowledge index store driver is closed') diff --git a/src/main/features/knowledge/__tests__/items.test.ts b/src/main/features/knowledge/__tests__/items.test.ts index 93563ca992f..ffd5e90bcfb 100644 --- a/src/main/features/knowledge/__tests__/items.test.ts +++ b/src/main/features/knowledge/__tests__/items.test.ts @@ -26,7 +26,7 @@ function createItem(type: KnowledgeItem['type']): KnowledgeItem { id: `${type}-1`, baseId: 'kb-1', groupId: null, - status: 'idle', + status: 'processing', error: null, createdAt: '2026-04-08T00:00:00.000Z', updatedAt: '2026-04-08T00:00:00.000Z' diff --git a/src/main/features/knowledge/base/KnowledgeBaseAdminService.ts b/src/main/features/knowledge/base/KnowledgeBaseAdminService.ts index b5e99974879..339a84edf14 100644 --- a/src/main/features/knowledge/base/KnowledgeBaseAdminService.ts +++ b/src/main/features/knowledge/base/KnowledgeBaseAdminService.ts @@ -2,6 +2,7 @@ import { application } from '@application' import { knowledgeBaseService } from '@data/services/KnowledgeBaseService' import { knowledgeItemService } from '@data/services/KnowledgeItemService' import { loggerService } from '@logger' +import type { KeyedMutex } from '@main/core/concurrency/KeyedMutex' import { DataApiErrorFactory } from '@shared/data/api' import { KNOWLEDGE_BASES_MAX_LIMIT } from '@shared/data/api/schemas/knowledges' import { @@ -18,14 +19,13 @@ import type { KnowledgeIngestionService } from '../ingestion/KnowledgeIngestionS import { classifyKnowledgeItemSource } from '../items' import { getKnowledgeBaseFilePath } from '../pathStorage' import { cancelActiveKnowledgeJobs } from '../tasks/utils/cancel' -import type { KnowledgeLockManager } from './KnowledgeLockManager' const logger = loggerService.withContext('Knowledge:BaseAdmin') /** Knowledge base lifecycle: create (with rollback), delete, restore, and list — everything about the base row + its on-disk artifacts, not about items. */ export class KnowledgeBaseAdminService { constructor( - private readonly knowledgeLockManager: KnowledgeLockManager, + private readonly knowledgeLockManager: KeyedMutex, private readonly ingestionService: KnowledgeIngestionService ) {} @@ -34,7 +34,7 @@ export class KnowledgeBaseAdminService { const vectorStoreService = application.get('KnowledgeVectorStoreService') try { - await vectorStoreService.getIndexStore(base) + vectorStoreService.getIndexStore(base) } catch (error) { await this.rollbackFailedBaseCreation(base.id) throw error @@ -66,7 +66,7 @@ export class KnowledgeBaseAdminService { async deleteBase(baseId: string): Promise { await cancelActiveKnowledgeJobs(baseId, 'delete-base', { onCancelTimeout: 'proceed' }) - await this.knowledgeLockManager.withBaseMutationLock(baseId, async () => { + await this.knowledgeLockManager.runExclusive(baseId, async () => { try { const vectorStoreService = application.get('KnowledgeVectorStoreService') await vectorStoreService.deleteStore(baseId) diff --git a/src/main/features/knowledge/base/KnowledgeLockManager.ts b/src/main/features/knowledge/base/KnowledgeLockManager.ts deleted file mode 100644 index e247d3914fb..00000000000 --- a/src/main/features/knowledge/base/KnowledgeLockManager.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { Mutex } from 'async-mutex' - -export class KnowledgeLockManager { - private readonly baseMutexes = new Map() - - async withBaseMutationLock(baseId: string, task: () => T | Promise): Promise { - const mutex = this.getBaseMutex(baseId) - const release = await mutex.acquire() - - try { - return await task() - } finally { - release() - this.deleteIdleBaseMutex(baseId, mutex) - } - } - - private getBaseMutex(baseId: string): Mutex { - let mutex = this.baseMutexes.get(baseId) - if (!mutex) { - mutex = new Mutex() - this.baseMutexes.set(baseId, mutex) - } - return mutex - } - - private deleteIdleBaseMutex(baseId: string, mutex: Mutex): void { - // Only delete the exact mutex we released; a queued waiter may have already - // created a replacement for the same base after this task released. - if (!mutex.isLocked() && this.baseMutexes.get(baseId) === mutex) { - this.baseMutexes.delete(baseId) - } - } -} diff --git a/src/main/features/knowledge/base/__tests__/KnowledgeLockManager.test.ts b/src/main/features/knowledge/base/__tests__/KnowledgeLockManager.test.ts deleted file mode 100644 index 9c8cc884ac0..00000000000 --- a/src/main/features/knowledge/base/__tests__/KnowledgeLockManager.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { describe, expect, it } from 'vitest' - -import { KnowledgeLockManager } from '../KnowledgeLockManager' - -function createDeferred() { - let resolve!: (value: T | PromiseLike) => void - let reject!: (reason?: unknown) => void - const promise = new Promise((promiseResolve, promiseReject) => { - resolve = promiseResolve - reject = promiseReject - }) - return { promise, resolve, reject } -} - -function flushMicrotasks(): Promise { - return new Promise((resolve) => setTimeout(resolve, 0)) -} - -function getBaseMutexCount(lockManager: KnowledgeLockManager): number { - return ( - lockManager as unknown as { - baseMutexes: Map - } - ).baseMutexes.size -} - -describe('KnowledgeLockManager', () => { - it('serializes overlapping tasks for the same base', async () => { - const lockManager = new KnowledgeLockManager() - const releaseFirst = createDeferred() - const order: string[] = [] - - const firstTask = lockManager.withBaseMutationLock('kb-1', async () => { - order.push('first-start') - await releaseFirst.promise - order.push('first-end') - }) - const secondTask = lockManager.withBaseMutationLock('kb-1', async () => { - order.push('second-start') - }) - - await flushMicrotasks() - expect(order).toEqual(['first-start']) - - releaseFirst.resolve() - await Promise.all([firstTask, secondTask]) - - expect(order).toEqual(['first-start', 'first-end', 'second-start']) - }) - - it('runs tasks for different bases concurrently', async () => { - const lockManager = new KnowledgeLockManager() - const releaseFirst = createDeferred() - const order: string[] = [] - - const firstTask = lockManager.withBaseMutationLock('kb-1', async () => { - order.push('first-start') - await releaseFirst.promise - order.push('first-end') - }) - const secondTask = lockManager.withBaseMutationLock('kb-2', async () => { - order.push('second-start') - }) - - await flushMicrotasks() - expect(order).toEqual(['first-start', 'second-start']) - - releaseFirst.resolve() - await firstTask - await secondTask - }) - - it('releases a base lock when the task throws', async () => { - const lockManager = new KnowledgeLockManager() - const order: string[] = [] - - await expect( - lockManager.withBaseMutationLock('kb-1', async () => { - order.push('throwing') - throw new Error('boom') - }) - ).rejects.toThrow('boom') - - await lockManager.withBaseMutationLock('kb-1', async () => { - order.push('after-throw') - }) - - expect(order).toEqual(['throwing', 'after-throw']) - }) - - it('removes idle base mutex entries after tasks complete', async () => { - const lockManager = new KnowledgeLockManager() - - await lockManager.withBaseMutationLock('kb-1', async () => { - expect(getBaseMutexCount(lockManager)).toBe(1) - }) - - expect(getBaseMutexCount(lockManager)).toBe(0) - }) -}) diff --git a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts index 943a7fade7e..455078b3c4f 100644 --- a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts +++ b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts @@ -4,6 +4,7 @@ import { application } from '@application' import { knowledgeBaseService } from '@data/services/KnowledgeBaseService' import { knowledgeItemService } from '@data/services/KnowledgeItemService' import { loggerService } from '@logger' +import type { KeyedMutex } from '@main/core/concurrency/KeyedMutex' import { getFileExt } from '@main/utils/file' import { DataApiErrorFactory } from '@shared/data/api' import { FileProcessorIdSchema } from '@shared/data/presets/fileProcessing' @@ -21,7 +22,6 @@ import { import { knowledgeSupportedFileExts } from '@shared/utils/file' import { assertBaseCanRunRuntimeOperation } from '../base/baseGuards' -import type { KnowledgeLockManager } from '../base/KnowledgeLockManager' import { classifyKnowledgeItemSource } from '../items' import { assertKnowledgeFileTargetAvailable, @@ -78,7 +78,7 @@ export interface KnowledgeItemScheduler { /** Write-side orchestration: admission checks, item creation, conflict handling, and job enqueueing for the add/delete/reindex flows. */ export class KnowledgeIngestionService implements KnowledgeItemScheduler { - constructor(private readonly knowledgeLockManager: KnowledgeLockManager) {} + constructor(private readonly knowledgeLockManager: KeyedMutex) {} async addItems( baseId: string, @@ -122,7 +122,7 @@ export class KnowledgeIngestionService implements KnowledgeItemScheduler { const acceptedItems: KnowledgeItem[] = [] const copiedFileItems: Array> = [] - await this.knowledgeLockManager.withBaseMutationLock(base.id, async () => { + await this.knowledgeLockManager.runExclusive(base.id, async () => { try { if (conflictStrategy === 'replace') { // Purge the conflicting existing items synchronously inside the lock and @@ -202,7 +202,7 @@ export class KnowledgeIngestionService implements KnowledgeItemScheduler { knowledgeBaseService.getById(baseId) const knowledgeBaseId = toKnowledgeBaseId(baseId) const knowledgeRootItemIds = toKnowledgeItemIds(rootItemIds) - const markedIds = await this.knowledgeLockManager.withBaseMutationLock(baseId, () => + const markedIds = await this.knowledgeLockManager.runExclusive(baseId, () => knowledgeItemService.setSubtreeStatus(baseId, rootItemIds, 'deleting') ) try { @@ -347,8 +347,7 @@ export class KnowledgeIngestionService implements KnowledgeItemScheduler { itemId, fileProcessingJobId, pollRound, - firstScheduledAt, - parentJobId + firstScheduledAt }, { idempotencyKey: knowledgeFileProcessingCheckIdempotencyKey(baseId, itemId, fileProcessingJobId, pollRound), @@ -372,7 +371,7 @@ export class KnowledgeIngestionService implements KnowledgeItemScheduler { const jobManager = application.get('JobManager') jobManager.enqueue( 'knowledge.index-documents', - { baseId, itemId, parentJobId }, + { baseId, itemId }, { idempotencyKey: knowledgeIndexIdempotencyKey(baseId, itemId, parentJobId), queue: knowledgeQueueName(baseId), diff --git a/src/main/features/knowledge/ingestion/__tests__/statusCleanup.integration.test.ts b/src/main/features/knowledge/ingestion/__tests__/statusCleanup.integration.test.ts index 5ed491bae9b..bc9cf0b730d 100644 --- a/src/main/features/knowledge/ingestion/__tests__/statusCleanup.integration.test.ts +++ b/src/main/features/knowledge/ingestion/__tests__/statusCleanup.integration.test.ts @@ -87,43 +87,34 @@ describe('markUnscheduledKnowledgeItemsFailed integration', () => { ]) }) - it('falls back to subtree status without reviving deleting descendants', async () => { - const updateStatusSpy = vi.spyOn(knowledgeItemService, 'updateStatus') - updateStatusSpy.mockImplementationOnce(() => { - throw new Error('status busy') + it('does not revive deleting descendants when marking unscheduled items failed', async () => { + markUnscheduledKnowledgeItemsFailed({ + baseId: BASE_ID, + items: [ + { + id: ROOT_ID, + baseId: BASE_ID, + groupId: null, + type: 'directory', + data: { source: 'root' }, + status: 'processing', + error: null, + createdAt: '2026-04-08T00:00:00.000Z', + updatedAt: '2026-04-08T00:00:00.000Z' + } + ], + completedItemIds: new Set(), + errorMessage: 'enqueue failed', + failedStatusError: 'Failed to schedule knowledge child item job: enqueue failed', + logger: { + debug: vi.fn(), + error: vi.fn(), + info: vi.fn(), + warn: vi.fn() + } as unknown as LoggerService, + logMessage: 'Failed to mark unscheduled item' }) - try { - markUnscheduledKnowledgeItemsFailed({ - baseId: BASE_ID, - items: [ - { - id: ROOT_ID, - baseId: BASE_ID, - groupId: null, - type: 'directory', - data: { source: 'root' }, - status: 'processing', - error: null, - createdAt: '2026-04-08T00:00:00.000Z', - updatedAt: '2026-04-08T00:00:00.000Z' - } - ], - completedItemIds: new Set(), - errorMessage: 'enqueue failed', - failedStatusError: 'Failed to schedule knowledge child item job: enqueue failed', - logger: { - debug: vi.fn(), - error: vi.fn(), - info: vi.fn(), - warn: vi.fn() - } as unknown as LoggerService, - logMessage: 'Failed to mark unscheduled item' - }) - } finally { - updateStatusSpy.mockRestore() - } - const rows = await dbh.db.select().from(knowledgeItemTable).where(eq(knowledgeItemTable.baseId, BASE_ID)) expect(rows).toEqual( expect.arrayContaining([ @@ -140,4 +131,51 @@ describe('markUnscheduledKnowledgeItemsFailed integration', () => { ]) ) }) + + it('aggregates unrecovered item ids and throws when setSubtreeStatus fails', () => { + const setSubtreeStatusSpy = vi.spyOn(knowledgeItemService, 'setSubtreeStatus') + setSubtreeStatusSpy.mockImplementationOnce(() => { + throw new Error('status busy') + }) + const errorLogger = vi.fn() + + try { + expect(() => + markUnscheduledKnowledgeItemsFailed({ + baseId: BASE_ID, + items: [ + { + id: ROOT_ID, + baseId: BASE_ID, + groupId: null, + type: 'directory', + data: { source: 'root' }, + status: 'processing', + error: null, + createdAt: '2026-04-08T00:00:00.000Z', + updatedAt: '2026-04-08T00:00:00.000Z' + } + ], + completedItemIds: new Set(), + errorMessage: 'enqueue failed', + failedStatusError: 'Failed to schedule knowledge child item job: enqueue failed', + logger: { + debug: vi.fn(), + error: errorLogger, + info: vi.fn(), + warn: vi.fn() + } as unknown as LoggerService, + logMessage: 'Failed to mark unscheduled item' + }) + ).toThrow(`Failed to mark unscheduled knowledge items failed; unrecovered item ids: ${ROOT_ID}`) + } finally { + setSubtreeStatusSpy.mockRestore() + } + + expect(errorLogger).toHaveBeenCalledWith( + 'Failed to mark unscheduled item', + expect.any(Error), + expect.objectContaining({ baseId: BASE_ID, itemId: ROOT_ID }) + ) + }) }) diff --git a/src/main/features/knowledge/ingestion/statusCleanup.ts b/src/main/features/knowledge/ingestion/statusCleanup.ts index ea75eeec875..13386a89d57 100644 --- a/src/main/features/knowledge/ingestion/statusCleanup.ts +++ b/src/main/features/knowledge/ingestion/statusCleanup.ts @@ -21,11 +21,11 @@ export function markUnscheduledKnowledgeItemsFailed(input: MarkFailedInput): voi } try { - knowledgeItemService.updateStatus(item.id, 'failed', { + knowledgeItemService.setSubtreeStatus(input.baseId, [item.id], 'failed', { error: input.failedStatusError }) - continue } catch (cleanupError) { + unrecoveredItemIds.push(item.id) input.logger.error( input.logMessage, cleanupError instanceof Error ? cleanupError : new Error(String(cleanupError)), @@ -36,23 +36,6 @@ export function markUnscheduledKnowledgeItemsFailed(input: MarkFailedInput): voi } ) } - - try { - knowledgeItemService.setSubtreeStatus(input.baseId, [item.id], 'failed', { - error: input.failedStatusError - }) - } catch (fallbackError) { - unrecoveredItemIds.push(item.id) - input.logger.error( - 'Failed to mark unscheduled knowledge item through subtree fallback', - fallbackError instanceof Error ? fallbackError : new Error(String(fallbackError)), - { - baseId: input.baseId, - itemId: item.id, - scheduleError: input.errorMessage - } - ) - } } if (unrecoveredItemIds.length > 0) { diff --git a/src/main/features/knowledge/pipeline/readers/__tests__/KnowledgeReaderMetadata.test.ts b/src/main/features/knowledge/pipeline/readers/__tests__/KnowledgeReaderMetadata.test.ts index 2bc86693cfb..612e459c6e7 100644 --- a/src/main/features/knowledge/pipeline/readers/__tests__/KnowledgeReaderMetadata.test.ts +++ b/src/main/features/knowledge/pipeline/readers/__tests__/KnowledgeReaderMetadata.test.ts @@ -65,7 +65,7 @@ describe('knowledge reader metadata', () => { source: '/tmp/original.txt', relativePath: 'original.txt' }, - status: 'idle', + status: 'processing', error: null, createdAt: '2026-04-08T00:00:00.000Z', updatedAt: '2026-04-08T00:00:00.000Z' @@ -89,7 +89,7 @@ describe('knowledge reader metadata', () => { groupId: null, type: 'url', data: { source: 'https://example.com', url: 'https://example.com', relativePath: 'example.md' }, - status: 'idle', + status: 'processing', error: null, createdAt: '2026-04-08T00:00:00.000Z', updatedAt: '2026-04-08T00:00:00.000Z' @@ -114,7 +114,7 @@ describe('knowledge reader metadata', () => { groupId: null, type: 'url', data: { source: 'https://example.com', url: 'https://example.com' }, - status: 'idle', + status: 'processing', error: null, createdAt: '2026-04-08T00:00:00.000Z', updatedAt: '2026-04-08T00:00:00.000Z' @@ -134,7 +134,7 @@ describe('knowledge reader metadata', () => { groupId: null, type: 'note', data: { source: 'My note', content: '# Note title\n\nbody', relativePath: 'My note.md' }, - status: 'idle', + status: 'processing', error: null, createdAt: '2026-04-08T00:00:00.000Z', updatedAt: '2026-04-08T00:00:00.000Z' @@ -159,7 +159,7 @@ describe('knowledge reader metadata', () => { groupId: null, type: 'note', data: { source: 'My note', content: 'body' }, - status: 'idle', + status: 'processing', error: null, createdAt: '2026-04-08T00:00:00.000Z', updatedAt: '2026-04-08T00:00:00.000Z' diff --git a/src/main/features/knowledge/pipeline/readers/__tests__/ReaderFactory.test.ts b/src/main/features/knowledge/pipeline/readers/__tests__/ReaderFactory.test.ts index bf8bd86dadd..3666c27b1bc 100644 --- a/src/main/features/knowledge/pipeline/readers/__tests__/ReaderFactory.test.ts +++ b/src/main/features/knowledge/pipeline/readers/__tests__/ReaderFactory.test.ts @@ -109,7 +109,7 @@ function createFileItem(ext: string, sourcePath?: string): KnowledgeItemOf<'file baseId: 'base-1', groupId: null, type: 'file', - status: 'idle', + status: 'processing', error: null, createdAt: '2026-04-03T00:00:00.000Z', updatedAt: '2026-04-03T00:00:00.000Z', @@ -126,7 +126,7 @@ function createNoteItem(content: string, relativePath = 'note-1.md'): KnowledgeI baseId: 'base-1', groupId: null, type: 'note', - status: 'idle', + status: 'processing', error: null, createdAt: '2026-04-03T00:00:00.000Z', updatedAt: '2026-04-03T00:00:00.000Z', @@ -144,7 +144,7 @@ function createUrlItem(): KnowledgeItemOf<'url'> { baseId: 'base-1', groupId: null, type: 'url', - status: 'idle', + status: 'processing', error: null, createdAt: '2026-04-03T00:00:00.000Z', updatedAt: '2026-04-03T00:00:00.000Z', @@ -162,7 +162,7 @@ function createDirectoryItem(): KnowledgeItemOf<'directory'> { baseId: 'base-1', groupId: null, type: 'directory', - status: 'idle', + status: 'processing', error: null, createdAt: '2026-04-03T00:00:00.000Z', updatedAt: '2026-04-03T00:00:00.000Z', diff --git a/src/main/features/knowledge/pipeline/sources/__tests__/directory.test.ts b/src/main/features/knowledge/pipeline/sources/__tests__/directory.test.ts index 008cc48e2ed..4478011d346 100644 --- a/src/main/features/knowledge/pipeline/sources/__tests__/directory.test.ts +++ b/src/main/features/knowledge/pipeline/sources/__tests__/directory.test.ts @@ -71,7 +71,7 @@ describe('expandDirectoryOwnerToTree', () => { data: { source: rootDir }, - status: 'idle', + status: 'processing', error: null, createdAt: '2026-04-08T00:00:00.000Z', updatedAt: '2026-04-08T00:00:00.000Z' @@ -124,7 +124,7 @@ describe('expandDirectoryOwnerToTree', () => { data: { source: rootDir }, - status: 'idle', + status: 'processing', error: null, createdAt: '2026-04-08T00:00:00.000Z', updatedAt: '2026-04-08T00:00:00.000Z' @@ -189,7 +189,7 @@ describe('expandDirectoryOwnerToTree', () => { data: { source: rootDir }, - status: 'idle', + status: 'processing', error: null, createdAt: '2026-04-08T00:00:00.000Z', updatedAt: '2026-04-08T00:00:00.000Z' @@ -236,7 +236,7 @@ describe('expandDirectoryOwnerToTree', () => { data: { source: rootDir }, - status: 'idle', + status: 'processing', error: null, createdAt: '2026-04-08T00:00:00.000Z', updatedAt: '2026-04-08T00:00:00.000Z' @@ -269,7 +269,7 @@ describe('expandDirectoryOwnerToTree', () => { data: { source: rootDir }, - status: 'idle', + status: 'processing', error: null, createdAt: '2026-04-08T00:00:00.000Z', updatedAt: '2026-04-08T00:00:00.000Z' @@ -309,7 +309,7 @@ describe('expandDirectoryOwnerToTree', () => { data: { source: rootDir }, - status: 'idle', + status: 'processing', error: null, createdAt: '2026-04-08T00:00:00.000Z', updatedAt: '2026-04-08T00:00:00.000Z' @@ -358,7 +358,7 @@ describe('expandDirectoryOwnerToTree', () => { data: { source: rootDir }, - status: 'idle', + status: 'processing', error: null, createdAt: '2026-04-08T00:00:00.000Z', updatedAt: '2026-04-08T00:00:00.000Z' @@ -389,7 +389,7 @@ describe('expandDirectoryOwnerToTree', () => { data: { source: tempRoot }, - status: 'idle', + status: 'processing', error: null, createdAt: '2026-04-08T00:00:00.000Z', updatedAt: '2026-04-08T00:00:00.000Z' diff --git a/src/main/features/knowledge/pipeline/vectorstore/KnowledgeVectorStoreService.ts b/src/main/features/knowledge/pipeline/vectorstore/KnowledgeVectorStoreService.ts index 8a021a55f12..8aed736cc7f 100644 --- a/src/main/features/knowledge/pipeline/vectorstore/KnowledgeVectorStoreService.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/KnowledgeVectorStoreService.ts @@ -44,13 +44,14 @@ function assertVectorStoreReadyBase(base: KnowledgeBase): asserts base is Comple @ServicePhase(Phase.WhenReady) export class KnowledgeVectorStoreService extends BaseService { // Opening a store (better-sqlite3 connect + schema + meta) is fully synchronous - // (see openIndexStore), so it runs to completion in one JS turn — no concurrent - // getIndexStore call for the same base can ever observe an in-flight open, and a - // failed open never gets cached (the throw happens before .set() below runs). + // (see openIndexStore), so getIndexStore/getIndexStoreIfExists are plain sync + // methods that run to completion in one JS turn — no concurrent call for the + // same base can ever observe an in-flight open, and a failed open never gets + // cached (the throw happens before .set() below runs). private instanceCache = new Map() /** Open (or reuse) the base's index store, ensuring its schema exists. */ - async getIndexStore(base: KnowledgeBase): Promise { + getIndexStore(base: KnowledgeBase): KnowledgeIndexStore { assertVectorStoreReadyBase(base) const cached = this.instanceCache.get(base.id) @@ -66,7 +67,7 @@ export class KnowledgeVectorStoreService extends BaseService { } /** Reuse or open the store only if its file already exists on disk; used by cleanup paths that must not create one. */ - async getIndexStoreIfExists(base: KnowledgeBase): Promise { + getIndexStoreIfExists(base: KnowledgeBase): KnowledgeIndexStore | undefined { // No readiness assert here: cleanup must keep working on failed bases (see // operation-guards.md — deleteItems intentionally skips the guard, so its // delete-subtree job lands here for any base). A failed base never has a @@ -77,7 +78,7 @@ export class KnowledgeVectorStoreService extends BaseService { return cached } - if (!(await this.storeFileExists(base.id))) { + if (!this.storeFileExists(base.id)) { logger.debug('Knowledge index store does not exist on disk', { baseId: base.id }) return undefined } @@ -186,10 +187,9 @@ export class KnowledgeVectorStoreService extends BaseService { } } - private async storeFileExists(baseId: string): Promise { + private storeFileExists(baseId: string): boolean { try { - const stat = await fs.promises.stat(getKnowledgeVectorStoreFilePathSync(baseId)) - return stat.isFile() + return fs.statSync(getKnowledgeVectorStoreFilePathSync(baseId)).isFile() } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') { return false diff --git a/src/main/features/knowledge/pipeline/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts b/src/main/features/knowledge/pipeline/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts index 6a1021cbd06..5aeb1129e81 100644 --- a/src/main/features/knowledge/pipeline/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/__tests__/KnowledgeVectorStoreService.test.ts @@ -1,4 +1,4 @@ -import fs, { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -101,8 +101,8 @@ function createFailedBase(): KnowledgeBase { } /** Opens (and immediately closes) a store for `base` so its file exists on disk at the current schema version. */ -async function primeStoreOnDisk(base: KnowledgeBase): Promise { - const store = await new KnowledgeVectorStoreService().getIndexStore(base) +function primeStoreOnDisk(base: KnowledgeBase): void { + const store = new KnowledgeVectorStoreService().getIndexStore(base) store.close() } @@ -132,12 +132,12 @@ describe('KnowledgeVectorStoreService', () => { rmSync(tempDir, { recursive: true, force: true }) }) - it('opens an index store on first request and caches it per base', async () => { + it('opens an index store on first request and caches it per base', () => { const service = new KnowledgeVectorStoreService() const base = createBase() - const first = await service.getIndexStore(base) - const second = await service.getIndexStore(base) + const first = service.getIndexStore(base) + const second = service.getIndexStore(base) expect(first).toBe(second) expect(first).toBeInstanceOf(KnowledgeIndexStore) @@ -145,22 +145,24 @@ describe('KnowledgeVectorStoreService', () => { expect(loggerDebugMock).toHaveBeenCalledWith('Reusing cached knowledge index store', { baseId: base.id }) }) - it('synchronous open makes a second concurrent call observe the already-cached store', async () => { + it('synchronous open makes a second call observe the already-cached store', () => { // Opening a store (better-sqlite3 connect + schema + meta) is fully synchronous, so - // the first call's cache write completes before the second call starts — there is no - // in-flight promise for the second call to join. Pinned via a call-count assertion so - // a future refactor that reintroduces an async open cannot silently double-open. + // the first call's cache write completes before the second call runs — there is no + // in-flight state the second call could observe mid-open. Pinned via a call-count + // assertion so a future refactor that reintroduces an async open cannot silently + // double-open. const service = new KnowledgeVectorStoreService() const base = createBase() const createSchemaSpy = vi.spyOn(schemaModule, 'createKnowledgeIndexSchema') - const [first, second] = await Promise.all([service.getIndexStore(base), service.getIndexStore(base)]) + const first = service.getIndexStore(base) + const second = service.getIndexStore(base) expect(first).toBe(second) expect(createSchemaSpy).toHaveBeenCalledTimes(1) }) - it('evicts a failed open so a later call retries instead of re-awaiting the failure', async () => { + it('evicts a failed open so a later call retries instead of re-throwing the failure', () => { const service = new KnowledgeVectorStoreService() const base = createBase() // A regular file where the index dir must go makes the real mkdirSync in @@ -169,39 +171,39 @@ describe('KnowledgeVectorStoreService', () => { writeFileSync(blockerPath, 'not a directory') getPathSyncMock.mockReturnValueOnce(join(blockerPath, 'index.sqlite')) - await expect(service.getIndexStore(base)).rejects.toThrow() + expect(() => service.getIndexStore(base)).toThrow() - const store = await service.getIndexStore(base) + const store = service.getIndexStore(base) expect(store).toBeInstanceOf(KnowledgeIndexStore) }) - it('stamps and verifies the meta identity row before handing out the store', async () => { + it('stamps and verifies the meta identity row before handing out the store', () => { const service = new KnowledgeVectorStoreService() const base = createBase() const ensureIndexMetaSpy = vi.spyOn(indexMetaModule, 'ensureIndexMeta') - await service.getIndexStore(base) + service.getIndexStore(base) expect(ensureIndexMetaSpy).toHaveBeenCalledTimes(1) expect(ensureIndexMetaSpy).toHaveBeenCalledWith(expect.anything(), { baseId: base.id }) }) - it('creates the schema normally when the stored version matches (no rebuild)', async () => { + it('creates the schema normally when the stored version matches (no rebuild)', () => { const base = createBase() - await primeStoreOnDisk(base) + primeStoreOnDisk(base) const createSchemaSpy = vi.spyOn(schemaModule, 'createKnowledgeIndexSchema') const resetSchemaSpy = vi.spyOn(schemaModule, 'resetKnowledgeIndexSchema') const service = new KnowledgeVectorStoreService() - await service.getIndexStore(base) + service.getIndexStore(base) expect(createSchemaSpy).toHaveBeenCalledTimes(1) expect(resetSchemaSpy).not.toHaveBeenCalled() }) - it('rebuilds the derived index when an existing index.sqlite is at a stale schema version', async () => { + it('rebuilds the derived index when an existing index.sqlite is at a stale schema version', () => { const base = createBase() - await primeStoreOnDisk(base) + primeStoreOnDisk(base) // Leave a marker row and stamp a stale version directly, so the rebuild below is // verified against real DDL (dropped tables), not a mocked branch decision. withRawDriver(base, (driver) => { @@ -213,7 +215,7 @@ describe('KnowledgeVectorStoreService', () => { const ensureIndexMetaSpy = vi.spyOn(indexMetaModule, 'ensureIndexMeta') const service = new KnowledgeVectorStoreService() - await service.getIndexStore(base) + service.getIndexStore(base) // Drop+recreate via resetKnowledgeIndexSchema, NOT the plain create — an old layout // (e.g. pre-fts_rowid) cannot be retrofitted by CREATE ... IF NOT EXISTS. @@ -240,12 +242,12 @@ describe('KnowledgeVectorStoreService', () => { ) }) - it('also rebuilds when the stored version is NEWER than this build (downgrade)', async () => { + it('also rebuilds when the stored version is NEWER than this build (downgrade)', () => { // The guard is `!==`, not `<` — a file written by a newer build (user downgraded the app) is // just as incompatible a layout and must be rebuilt, not opened as-is. Pins that semantics so a // future refactor to `<` does not silently start mounting newer files. const base = createBase() - await primeStoreOnDisk(base) + primeStoreOnDisk(base) withRawDriver(base, (driver) => { driver.execute('UPDATE meta SET schema_version = ?', [KNOWLEDGE_INDEX_SCHEMA_VERSION + 1]) }) @@ -253,28 +255,28 @@ describe('KnowledgeVectorStoreService', () => { const resetSchemaSpy = vi.spyOn(schemaModule, 'resetKnowledgeIndexSchema') const service = new KnowledgeVectorStoreService() - await service.getIndexStore(base) + service.getIndexStore(base) expect(resetSchemaSpy).toHaveBeenCalledTimes(1) expect(createSchemaSpy).not.toHaveBeenCalled() }) - it('closes the driver and aborts the open when meta verification fails (wrong/corrupt base)', async () => { + it('closes the driver and aborts the open when meta verification fails (wrong/corrupt base)', () => { const base = createBase('kb-shared') const otherBase = createBase('kb-other') // Both bases resolve to the same on-disk file, so the second open's meta // identity check genuinely finds a mismatched base_id. getPathSyncMock.mockImplementation(() => join(tempDir, 'shared.sqlite')) - await primeStoreOnDisk(base) + primeStoreOnDisk(base) const closeSpy = vi.spyOn(BetterSqlite3Driver.prototype, 'close') const service = new KnowledgeVectorStoreService() - await expect(service.getIndexStore(otherBase)).rejects.toThrow('belongs to a different base') + expect(() => service.getIndexStore(otherBase)).toThrow('belongs to a different base') expect(closeSpy).toHaveBeenCalledTimes(1) }) - it('closes the driver when schema creation fails so the file handle is not leaked', async () => { + it('closes the driver when schema creation fails so the file handle is not leaked', () => { const service = new KnowledgeVectorStoreService() const base = createBase() const closeSpy = vi.spyOn(BetterSqlite3Driver.prototype, 'close') @@ -282,38 +284,40 @@ describe('KnowledgeVectorStoreService', () => { throw new Error('disk full') }) - await expect(service.getIndexStore(base)).rejects.toThrow('disk full') + expect(() => service.getIndexStore(base)).toThrow('disk full') expect(closeSpy).toHaveBeenCalledTimes(1) }) - it('returns undefined from getIndexStoreIfExists when no backing file exists', async () => { + it('returns undefined from getIndexStoreIfExists when no backing file exists', () => { const service = new KnowledgeVectorStoreService() const base = createBase() - await expect(service.getIndexStoreIfExists(base)).resolves.toBeUndefined() + expect(service.getIndexStoreIfExists(base)).toBeUndefined() expect(loggerDebugMock).toHaveBeenCalledWith('Knowledge index store does not exist on disk', { baseId: base.id }) }) - it('opens an existing store from disk when getIndexStoreIfExists detects a backing file', async () => { + it('opens an existing store from disk when getIndexStoreIfExists detects a backing file', () => { const base = createBase() - await primeStoreOnDisk(base) + primeStoreOnDisk(base) const service = new KnowledgeVectorStoreService() - const store = await service.getIndexStoreIfExists(base) + const store = service.getIndexStoreIfExists(base) expect(store).toBeInstanceOf(KnowledgeIndexStore) }) - it('returns the cached store from getIndexStoreIfExists without probing disk', async () => { + it('returns the cached store from getIndexStoreIfExists without probing disk', () => { const service = new KnowledgeVectorStoreService() const base = createBase() - const created = await service.getIndexStore(base) - const statSpy = vi.spyOn(fs.promises, 'stat') + const created = service.getIndexStore(base) + getPathSyncMock.mockClear() - await expect(service.getIndexStoreIfExists(base)).resolves.toBe(created) - expect(statSpy).not.toHaveBeenCalled() + expect(service.getIndexStoreIfExists(base)).toBe(created) + // storeFileExists resolves the store path before stat-ing it, so an + // untouched path resolver proves the disk probe never ran. + expect(getPathSyncMock).not.toHaveBeenCalled() }) it('closes the cached store and removes the base directory on deleteStore', async () => { @@ -321,7 +325,7 @@ describe('KnowledgeVectorStoreService', () => { const base = createBase() const closeSpy = vi.spyOn(BetterSqlite3Driver.prototype, 'close') - const store = await service.getIndexStore(base) + const store = service.getIndexStore(base) await service.deleteStore(base.id) expect(closeSpy).toHaveBeenCalledTimes(1) @@ -331,7 +335,7 @@ describe('KnowledgeVectorStoreService', () => { expect(closeSpy.mock.invocationCallOrder[0]).toBeLessThan(deleteDirMock.mock.invocationCallOrder[0]) // Cache was evicted: the next open builds a fresh instance. - const reopened = await service.getIndexStore(base) + const reopened = service.getIndexStore(base) expect(reopened).not.toBe(store) }) @@ -353,18 +357,18 @@ describe('KnowledgeVectorStoreService', () => { const base = createBase() deleteDirMock.mockRejectedValueOnce(new Error('delete failed')) - const store = await service.getIndexStore(base) + const store = service.getIndexStore(base) await expect(service.deleteStore(base.id)).rejects.toThrow('delete failed') - const reopened = await service.getIndexStore(base) + const reopened = service.getIndexStore(base) expect(reopened).not.toBe(store) }) it('closes all cached stores during stop and continues when one close throws', async () => { const service = new KnowledgeVectorStoreService() - const first = await service.getIndexStore(createBase('kb-1')) - const second = await service.getIndexStore(createBase('kb-2')) + const first = service.getIndexStore(createBase('kb-1')) + const second = service.getIndexStore(createBase('kb-2')) const closeError = new Error('close failed') const firstCloseSpy = vi.spyOn(first, 'close').mockImplementationOnce(() => { throw closeError @@ -382,38 +386,38 @@ describe('KnowledgeVectorStoreService', () => { expect(loggerInfoMock).toHaveBeenCalledWith('Stopped knowledge index stores', { storeCount: 2 }) // Cache cleared: reopening kb-2 builds a fresh instance. - const reopened = await service.getIndexStore(createBase('kb-2')) + const reopened = service.getIndexStore(createBase('kb-2')) expect(reopened).not.toBe(second) }) - it('rejects bases that are not ready before touching disk', async () => { + it('rejects bases that are not ready before touching disk', () => { const service = new KnowledgeVectorStoreService() const base = createFailedBase() - const statSpy = vi.spyOn(fs.promises, 'stat') - await expect(service.getIndexStore(base)).rejects.toThrow('not ready for vector store operations') + expect(() => service.getIndexStore(base)).toThrow('not ready for vector store operations') - expect(statSpy).not.toHaveBeenCalled() + // The readiness assert throws before openIndexStore ever resolves the store path. + expect(getPathSyncMock).not.toHaveBeenCalled() }) - it('lets cleanup on a failed base proceed: getIndexStoreIfExists returns undefined instead of asserting', async () => { + it('lets cleanup on a failed base proceed: getIndexStoreIfExists returns undefined instead of asserting', () => { // Failed bases never get a store file (the vector migrator skips them and // getIndexStore asserts), so the existence probe is the path cleanup takes. const service = new KnowledgeVectorStoreService() const base = createFailedBase() - await expect(service.getIndexStoreIfExists(base)).resolves.toBeUndefined() + expect(service.getIndexStoreIfExists(base)).toBeUndefined() }) - it('still asserts readiness when a failed base unexpectedly has a store file on disk', async () => { + it('still asserts readiness when a failed base unexpectedly has a store file on disk', () => { const service = new KnowledgeVectorStoreService() const base = createFailedBase() writeFileSync(getPathSyncMock(base.id), '') - await expect(service.getIndexStoreIfExists(base)).rejects.toThrow('not ready for vector store operations') + expect(() => service.getIndexStoreIfExists(base)).toThrow('not ready for vector store operations') }) - it('logs an error when an empty index mounts under a base with completed items', async () => { + it('logs an error when an empty index mounts under a base with completed items', () => { const service = new KnowledgeVectorStoreService() const base = createBase() getItemsByBaseIdMock.mockReturnValueOnce([ @@ -421,7 +425,7 @@ describe('KnowledgeVectorStoreService', () => { { id: 'item-2', type: 'file', status: 'completed' } ]) - const store = await service.getIndexStore(base) + const store = service.getIndexStore(base) expect(store).toBeInstanceOf(KnowledgeIndexStore) expect(loggerErrorMock).toHaveBeenCalledWith( @@ -430,7 +434,7 @@ describe('KnowledgeVectorStoreService', () => { ) }) - it('stays quiet when an empty index mounts under a base with no completed indexable items', async () => { + it('stays quiet when an empty index mounts under a base with no completed indexable items', () => { const service = new KnowledgeVectorStoreService() const base = createBase() // A completed empty directory is legitimate without materials; in-flight leaves are too. @@ -439,12 +443,12 @@ describe('KnowledgeVectorStoreService', () => { { id: 'item-2', type: 'file', status: 'processing' } ]) - await service.getIndexStore(base) + service.getIndexStore(base) expect(loggerErrorMock).not.toHaveBeenCalled() }) - it('fails the open and closes the driver when the empty-index diagnostic cannot read the base items', async () => { + it('fails the open and closes the driver when the empty-index diagnostic cannot read the base items', () => { const service = new KnowledgeVectorStoreService() const base = createBase() const closeSpy = vi.spyOn(BetterSqlite3Driver.prototype, 'close') @@ -455,7 +459,7 @@ describe('KnowledgeVectorStoreService', () => { // Deliberate fail-loud: swallowing the lookup failure would re-silence the // deleted-base race (open racing deleteBase recreates an empty file, and the // lookup's NOT_FOUND is what makes that loud instead of caching an empty store). - await expect(service.getIndexStore(base)).rejects.toThrow('app database unavailable') + expect(() => service.getIndexStore(base)).toThrow('app database unavailable') expect(closeSpy).toHaveBeenCalledTimes(1) }) diff --git a/src/main/features/knowledge/pipeline/vectorstore/__tests__/vectorCleanup.test.ts b/src/main/features/knowledge/pipeline/vectorstore/__tests__/vectorCleanup.test.ts index 1026bf6b3f6..914cde626c9 100644 --- a/src/main/features/knowledge/pipeline/vectorstore/__tests__/vectorCleanup.test.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/__tests__/vectorCleanup.test.ts @@ -60,7 +60,7 @@ function createBase(): KnowledgeBase { describe('deleteKnowledgeItemVectors', () => { beforeEach(() => { vi.clearAllMocks() - getIndexStoreIfExistsMock.mockResolvedValue({ + getIndexStoreIfExistsMock.mockReturnValue({ deleteMaterials: deleteMaterialsMock, reclaimSpace: reclaimSpaceMock }) @@ -69,7 +69,7 @@ describe('deleteKnowledgeItemVectors', () => { }) it('skips cleanup when no vector store exists', async () => { - getIndexStoreIfExistsMock.mockResolvedValueOnce(undefined) + getIndexStoreIfExistsMock.mockReturnValueOnce(undefined) await deleteKnowledgeItemVectors(createBase(), ['note-1']) @@ -97,7 +97,7 @@ describe('deleteKnowledgeItemVectors', () => { describe('reclaimKnowledgeIndexSpace', () => { beforeEach(() => { vi.clearAllMocks() - getIndexStoreIfExistsMock.mockResolvedValue({ + getIndexStoreIfExistsMock.mockReturnValue({ deleteMaterials: deleteMaterialsMock, reclaimSpace: reclaimSpaceMock }) @@ -105,7 +105,7 @@ describe('reclaimKnowledgeIndexSpace', () => { }) it('skips reclaim when no vector store exists', async () => { - getIndexStoreIfExistsMock.mockResolvedValueOnce(undefined) + getIndexStoreIfExistsMock.mockReturnValueOnce(undefined) await reclaimKnowledgeIndexSpace(createBase()) @@ -152,7 +152,9 @@ describe('reclaimKnowledgeIndexSpace', () => { // The open itself can throw (corrupt index, readiness/base_id mismatch, schema open failure). // It is inside the same best-effort try, so an open failure must not fail the delete job whose // rows and vectors are already gone. - getIndexStoreIfExistsMock.mockRejectedValueOnce(new Error('index store failed to open')) + getIndexStoreIfExistsMock.mockImplementationOnce(() => { + throw new Error('index store failed to open') + }) await expect(reclaimKnowledgeIndexSpace(createBase())).resolves.toBeUndefined() expect(reclaimSpaceMock).not.toHaveBeenCalled() diff --git a/src/main/features/knowledge/pipeline/vectorstore/indexStore/BetterSqlite3Driver.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/BetterSqlite3Driver.ts index 783e90e4c78..ec99188ad4f 100644 --- a/src/main/features/knowledge/pipeline/vectorstore/indexStore/BetterSqlite3Driver.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/indexStore/BetterSqlite3Driver.ts @@ -39,7 +39,7 @@ function toBindable(value: SqlValue): Bindable { * SqliteDriver backed by a synchronous better-sqlite3 connection (the only engine). * * No internal write mutex: every writer (rebuildMaterial / deleteMaterials / - * reclaimSpace) runs under `KnowledgeLockManager.withBaseMutationLock(baseId)`, a + * reclaimSpace) runs under `KeyedMutex.runExclusive(baseId, ...)`, a * per-base async-mutex that already serializes all writes to a base's index — so the * driver never self-serializes. better-sqlite3's single connection makes * `transaction` inherently atomic; reads run on the same connection and ride WAL / diff --git a/src/main/features/knowledge/pipeline/vectorstore/indexStore/schema.ts b/src/main/features/knowledge/pipeline/vectorstore/indexStore/schema.ts index ebc790bf145..31be4bd1673 100644 --- a/src/main/features/knowledge/pipeline/vectorstore/indexStore/schema.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/indexStore/schema.ts @@ -159,7 +159,7 @@ export const KNOWLEDGE_INDEX_SCHEMA_STATEMENTS: readonly string[] = [ `CREATE INDEX IF NOT EXISTS search_text_kind_idx ON search_text(kind)`, // UNIQUE so its backing index makes the trigger's per-row MAX(fts_rowid)+1 an O(log N) lookup and // rejects any duplicate loudly (the assignment is race-free only because all writes to a base's - // index are serialized upstream by KnowledgeLockManager.withBaseMutationLock — see BetterSqlite3Driver). + // index are serialized upstream by KeyedMutex.runExclusive (per knowledge base) — see BetterSqlite3Driver). `CREATE UNIQUE INDEX IF NOT EXISTS search_text_fts_rowid_uniq ON search_text(fts_rowid)`, // embedding — current embedding vector keyed by embedding-text hash. diff --git a/src/main/features/knowledge/pipeline/vectorstore/vectorCleanup.ts b/src/main/features/knowledge/pipeline/vectorstore/vectorCleanup.ts index 9396308f6db..507444bb088 100644 --- a/src/main/features/knowledge/pipeline/vectorstore/vectorCleanup.ts +++ b/src/main/features/knowledge/pipeline/vectorstore/vectorCleanup.ts @@ -11,7 +11,7 @@ export async function deleteKnowledgeItemVectors(base: KnowledgeBase, itemIds: s } const vectorStoreService = application.get('KnowledgeVectorStoreService') - const store = await vectorStoreService.getIndexStoreIfExists(base) + const store = vectorStoreService.getIndexStoreIfExists(base) if (!store) { return } @@ -44,7 +44,7 @@ export async function reclaimKnowledgeIndexSpace(base: KnowledgeBase): Promise; notFound: string[] }> { const base = knowledgeBaseService.getById(baseId) const vectorStoreService = application.get('KnowledgeVectorStoreService') - const store = await vectorStoreService.getIndexStore(base) + const store = vectorStoreService.getIndexStore(base) const found: Array<{ conceptId: string; itemId: string }> = [] const notFound: string[] = [] @@ -404,7 +404,7 @@ export class KnowledgeConceptService { const base = assertBaseCanRunRuntimeOperation(baseId, operation) const vectorStoreService = application.get('KnowledgeVectorStoreService') - const store = await vectorStoreService.getIndexStore(base) + const store = vectorStoreService.getIndexStore(base) const ref = await runStoreOperation(store, baseId, operation, () => store.getMaterialByRelativePath(conceptId)) if (!ref) { diff --git a/src/main/features/knowledge/query/KnowledgeQueryService.ts b/src/main/features/knowledge/query/KnowledgeQueryService.ts index af47218a349..557582c845a 100644 --- a/src/main/features/knowledge/query/KnowledgeQueryService.ts +++ b/src/main/features/knowledge/query/KnowledgeQueryService.ts @@ -54,7 +54,7 @@ export class KnowledgeQueryService { const candidateLimit = Math.min(resolvedTopK * KNOWLEDGE_SEARCH_OVERFETCH_FACTOR, KNOWLEDGE_SEARCH_CANDIDATE_CAP) const vectorStoreService = application.get('KnowledgeVectorStoreService') - const store = await vectorStoreService.getIndexStore(base) + const store = vectorStoreService.getIndexStore(base) const matches = await runStoreOperation(store, baseId, 'search', () => store.search({ queryText: query, @@ -90,7 +90,7 @@ export class KnowledgeQueryService { } const vectorStoreService = application.get('KnowledgeVectorStoreService') - const store = await vectorStoreService.getIndexStore(base) + const store = vectorStoreService.getIndexStore(base) const chunkGroups = await runStoreOperation(store, knowledgeBaseId, 'listItemChunks', () => leafItems.map((leafItem) => { const units = store.listMaterialUnits(leafItem.id) diff --git a/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts index ba0f6ed5b23..4fe06b4c08c 100644 --- a/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts @@ -48,7 +48,6 @@ function createCheckPayload( fileProcessingJobId: 'fp-job-1', pollRound: 0, firstScheduledAt: Date.now(), - parentJobId: null, ...overrides } } @@ -101,10 +100,10 @@ describe('check-file-processing-result job handler', () => { createCtx( createCheckPayload({ pollRound: 2, - firstScheduledAt, - parentJobId: 'reindex-job' + firstScheduledAt }), - 'check-job-2' + 'check-job-2', + 'reindex-job' ) ) @@ -198,13 +197,7 @@ describe('check-file-processing-result job handler', () => { }) ) - await handler.execute( - createCtx( - createCheckPayload({ - parentJobId: 'reindex-job' - }) - ) - ) + await handler.execute(createCtx(createCheckPayload(), 'job-1', 'reindex-job')) expect(ingestionService.scheduleIndexing).toHaveBeenCalledWith('kb-1', FILE_ITEM_ID, 'reindex-job') }) @@ -344,29 +337,6 @@ describe('check-file-processing-result job handler', () => { it('onSettled marks active items failed when the check job fails', async () => { const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) - getJobMock.mockResolvedValue({ - id: 'job-1', - status: 'failed', - priority: 0, - queue: 'base.kb-1', - idempotencyKey: null, - scheduleId: null, - scheduledAt: '2026-04-08T00:00:00.000Z', - startedAt: '2026-04-08T00:00:00.000Z', - finishedAt: null, - attempt: 1, - maxAttempts: 3, - output: null, - error: null, - parentId: null, - cancelRequested: false, - metadata: {}, - timeoutMs: null, - createdAt: '2026-04-08T00:00:00.000Z', - updatedAt: '2026-04-08T00:00:00.000Z', - type: 'knowledge.check-file-processing-result', - input: createCheckPayload() - }) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) await handler.onSettled?.({ @@ -386,29 +356,6 @@ describe('check-file-processing-result job handler', () => { it('onSettled falls back to the terminal status when a failed job has no error message', async () => { const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) - getJobMock.mockResolvedValue({ - id: 'job-1', - status: 'failed', - priority: 0, - queue: 'base.kb-1', - idempotencyKey: null, - scheduleId: null, - scheduledAt: '2026-04-08T00:00:00.000Z', - startedAt: '2026-04-08T00:00:00.000Z', - finishedAt: null, - attempt: 1, - maxAttempts: 3, - output: null, - error: null, - parentId: null, - cancelRequested: false, - metadata: {}, - timeoutMs: null, - createdAt: '2026-04-08T00:00:00.000Z', - updatedAt: '2026-04-08T00:00:00.000Z', - type: 'knowledge.check-file-processing-result', - input: createCheckPayload() - }) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) await handler.onSettled?.({ @@ -428,29 +375,6 @@ describe('check-file-processing-result job handler', () => { it('onSettled marks a cancelled (app-quit) item failed with the reindex-to-finish message', async () => { const handler = createCheckFileProcessingResultJobHandler(knowledgeLockManager as never, ingestionService) - getJobMock.mockResolvedValue({ - id: 'job-1', - status: 'cancelled', - priority: 0, - queue: 'base.kb-1', - idempotencyKey: null, - scheduleId: null, - scheduledAt: '2026-04-08T00:00:00.000Z', - startedAt: '2026-04-08T00:00:00.000Z', - finishedAt: null, - attempt: 1, - maxAttempts: 3, - output: null, - error: { code: 'CANCELLED', message: 'JobManager shutdown', retryable: false }, - parentId: null, - cancelRequested: false, - metadata: {}, - timeoutMs: null, - createdAt: '2026-04-08T00:00:00.000Z', - updatedAt: '2026-04-08T00:00:00.000Z', - type: 'knowledge.check-file-processing-result', - input: createCheckPayload() - }) knowledgeItemGetByIdMock.mockReturnValue(createFileItem()) await handler.onSettled?.({ diff --git a/src/main/features/knowledge/tasks/__tests__/deleteSubtreeJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/deleteSubtreeJobHandler.test.ts index 68243933c9b..dbe2e87d513 100644 --- a/src/main/features/knowledge/tasks/__tests__/deleteSubtreeJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/deleteSubtreeJobHandler.test.ts @@ -37,7 +37,7 @@ describe('delete-subtree job handler', () => { createJobSnapshot({ id: 'index-job', type: 'knowledge.index-documents', - input: { baseId: 'kb-1', itemId: 'note-1', parentJobId: null } + input: { baseId: 'kb-1', itemId: 'note-1' } }), createJobSnapshot({ id: 'check-job', @@ -47,14 +47,13 @@ describe('delete-subtree job handler', () => { itemId: 'note-1', fileProcessingJobId: 'fp-job-1', pollRound: 0, - firstScheduledAt: 1779811200000, - parentJobId: null + firstScheduledAt: 1779811200000 } }), createJobSnapshot({ id: 'unrelated-job', type: 'knowledge.index-documents', - input: { baseId: 'kb-1', itemId: 'other', parentJobId: null } + input: { baseId: 'kb-1', itemId: 'other' } }) ]) @@ -115,7 +114,7 @@ describe('delete-subtree job handler', () => { createJobSnapshot({ id: 'index-job', type: 'knowledge.index-documents', - input: { baseId: 'kb-1', itemId: 'note-1', parentJobId: null } + input: { baseId: 'kb-1', itemId: 'note-1' } }) ]) cancelMock.mockRejectedValue(new Error('cancel failed')) @@ -139,7 +138,7 @@ describe('delete-subtree job handler', () => { createJobSnapshot({ id: 'index-job', type: 'knowledge.index-documents', - input: { baseId: 'kb-1', itemId: 'note-1', parentJobId: null } + input: { baseId: 'kb-1', itemId: 'note-1' } }) ]) cancelMock.mockResolvedValue({ outcome: 'timed-out' }) diff --git a/src/main/features/knowledge/tasks/__tests__/indexDocumentsJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/indexDocumentsJobHandler.test.ts index c7b2b449e54..255b8984460 100644 --- a/src/main/features/knowledge/tasks/__tests__/indexDocumentsJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/indexDocumentsJobHandler.test.ts @@ -10,14 +10,12 @@ import { createCtx, createFileItem, createIndexDocumentsJobHandler, - createJobSnapshot, createNoteItem, createUrlItem, embedKnowledgeTextsMock, fakeEmbedVector, fetchKnowledgeWebPageMock, FILE_ITEM_ID, - getJobMock, knowledgeBaseGetByIdMock, knowledgeItemGetByIdMock, knowledgeItemUpdateSnapshotRelativePathMock, @@ -47,7 +45,7 @@ describe('index-documents job handler', () => { knowledgeItemGetByIdMock.mockReturnValue(createNoteItem(NOTE_ITEM_ID)) knowledgeItemUpdateStatusMock.mockReturnValue(createNoteItem(NOTE_ITEM_ID)) - await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID, parentJobId: null })) + await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID })) expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith(NOTE_ITEM_ID, 'reading') expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith(NOTE_ITEM_ID, 'embedding') @@ -60,11 +58,11 @@ describe('index-documents job handler', () => { }) ) expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith(NOTE_ITEM_ID, 'completed') - expect(handler.defaultQueue?.({ baseId: 'kb-1', itemId: NOTE_ITEM_ID, parentJobId: null })).toBe('base.kb-1') + expect(handler.defaultQueue?.({ baseId: 'kb-1', itemId: NOTE_ITEM_ID })).toBe('base.kb-1') // README's lock-boundary contract: embedding (paid API call) must run before the // mutation lock is taken, never inside it. expect(embedKnowledgeTextsMock.mock.invocationCallOrder[0]).toBeLessThan( - knowledgeLockManager.withBaseMutationLock.mock.invocationCallOrder[0] + knowledgeLockManager.runExclusive.mock.invocationCallOrder[0] ) }) @@ -74,7 +72,7 @@ describe('index-documents job handler', () => { knowledgeItemUpdateStatusMock.mockReturnValue(createNoteItem(NOTE_ITEM_ID)) loadKnowledgeItemDocumentsMock.mockResolvedValueOnce(distinctDocuments()) - await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID, parentJobId: null })) + await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID })) const input = lastRebuildInput() expect(input.embeddings.length).toBeGreaterThanOrEqual(3) @@ -102,7 +100,7 @@ describe('index-documents job handler', () => { const storedHash = hashEmbeddingText('bravo') listExistingEmbeddingHashesMock.mockReturnValueOnce(new Set([storedHash])) - await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID, parentJobId: null })) + await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID })) // The paid embed call received only the two missing bodies. const embeddedBodies = embedKnowledgeTextsMock.mock.calls[0][1] as string[] @@ -123,7 +121,7 @@ describe('index-documents job handler', () => { loadKnowledgeItemDocumentsMock.mockResolvedValueOnce(distinctDocuments()) listExistingEmbeddingHashesMock.mockReturnValueOnce(new Set(DISTINCT_DOCS.map(hashEmbeddingText))) - await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID, parentJobId: null })) + await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID })) // The paid embed seam receives zero bodies (embedKnowledgeTexts itself // short-circuits an empty input before AiService, pinned in embed.test.ts), @@ -147,7 +145,7 @@ describe('index-documents job handler', () => { knowledgeItemUpdateStatusMock.mockReturnValue(createNoteItem(NOTE_ITEM_ID)) loadKnowledgeItemDocumentsMock.mockResolvedValueOnce(distinctDocuments()) - await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID, parentJobId: null })) + await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID })) // No paid embed round-trip and no existing-hash lookup for a lexical base. expect(embedKnowledgeTextsMock).not.toHaveBeenCalled() @@ -166,7 +164,7 @@ describe('index-documents job handler', () => { knowledgeItemUpdateStatusMock.mockReturnValue(createNoteItem(NOTE_ITEM_ID)) loadKnowledgeItemDocumentsMock.mockResolvedValueOnce([]) - await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID, parentJobId: null })) + await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID })) // An image-only PDF or failed extraction must leave a diagnosable trace — // without the warn it would look indexed while matching nothing. @@ -185,7 +183,7 @@ describe('index-documents job handler', () => { knowledgeItemGetByIdMock.mockReturnValue(fileItem) knowledgeItemUpdateStatusMock.mockReturnValue(fileItem) - await handler.execute(createCtx({ baseId: 'kb-1', itemId: FILE_ITEM_ID, parentJobId: null })) + await handler.execute(createCtx({ baseId: 'kb-1', itemId: FILE_ITEM_ID })) expect(lastRebuildInput().material.relativePath).toBe('source.md') }) @@ -194,13 +192,7 @@ describe('index-documents job handler', () => { const handler = createIndexDocumentsJobHandler(knowledgeLockManager as never) knowledgeItemGetByIdMock.mockReturnValue(createFileItem(FILE_ITEM_ID)) - await handler.execute( - createCtx({ - baseId: 'kb-1', - itemId: FILE_ITEM_ID, - parentJobId: null - }) - ) + await handler.execute(createCtx({ baseId: 'kb-1', itemId: FILE_ITEM_ID })) expect(loadKnowledgeItemDocumentsMock).toHaveBeenCalledWith(expect.objectContaining({ id: FILE_ITEM_ID })) }) @@ -211,7 +203,7 @@ describe('index-documents job handler', () => { knowledgeItemUpdateStatusMock.mockReturnValue(createNoteItem(NOTE_ITEM_ID)) loadKnowledgeItemDocumentsMock.mockResolvedValueOnce([]) - await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID, parentJobId: null })) + await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID })) expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith(NOTE_ITEM_ID, 'reading') expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith(NOTE_ITEM_ID, 'embedding') @@ -228,7 +220,7 @@ describe('index-documents job handler', () => { .mockReturnValueOnce(createNoteItem(NOTE_ITEM_ID)) .mockReturnValueOnce(createNoteItem(NOTE_ITEM_ID, null, 'deleting')) - await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID, parentJobId: null })) + await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID })) expect(rebuildMaterialMock).not.toHaveBeenCalled() expect(knowledgeItemUpdateStatusMock).not.toHaveBeenCalledWith(NOTE_ITEM_ID, 'completed') @@ -241,9 +233,9 @@ describe('index-documents job handler', () => { throw new Error('vector write failed') }) - await expect( - handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID, parentJobId: null })) - ).rejects.toThrow('vector write failed') + await expect(handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID }))).rejects.toThrow( + 'vector write failed' + ) expect(knowledgeItemUpdateStatusMock).not.toHaveBeenCalledWith(NOTE_ITEM_ID, 'completed') }) @@ -251,9 +243,7 @@ describe('index-documents job handler', () => { it('stops before side effects when aborted before execution', async () => { const handler = createIndexDocumentsJobHandler(knowledgeLockManager as never) - await expect( - handler.execute(createAbortedCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID, parentJobId: null })) - ).rejects.toThrow() + await expect(handler.execute(createAbortedCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID }))).rejects.toThrow() expect(knowledgeBaseGetByIdMock).not.toHaveBeenCalled() expect(rebuildMaterialMock).not.toHaveBeenCalled() @@ -267,7 +257,7 @@ describe('index-documents job handler', () => { knowledgeItemGetByIdMock.mockReturnValue(createUrlItem('url-1')) captureUrlSnapshotFileMock.mockResolvedValue('example-page.md') - await handler.execute(createCtx({ baseId: 'kb-1', itemId: 'url-1', parentJobId: null })) + await handler.execute(createCtx({ baseId: 'kb-1', itemId: 'url-1' })) // Fetched exactly once, snapshot written, relativePath persisted. expect(fetchKnowledgeWebPageMock).toHaveBeenCalledTimes(1) @@ -290,7 +280,7 @@ describe('index-documents job handler', () => { // README's lock-boundary contract: the network fetch must complete before the // (first) mutation lock is taken, never inside it. expect(fetchKnowledgeWebPageMock.mock.invocationCallOrder[0]).toBeLessThan( - knowledgeLockManager.withBaseMutationLock.mock.invocationCallOrder[0] + knowledgeLockManager.runExclusive.mock.invocationCallOrder[0] ) }) @@ -298,7 +288,7 @@ describe('index-documents job handler', () => { const handler = createIndexDocumentsJobHandler(knowledgeLockManager as never) knowledgeItemGetByIdMock.mockReturnValue(createUrlItem('url-1', 'cached.md')) - await handler.execute(createCtx({ baseId: 'kb-1', itemId: 'url-1', parentJobId: null })) + await handler.execute(createCtx({ baseId: 'kb-1', itemId: 'url-1' })) expect(fetchKnowledgeWebPageMock).not.toHaveBeenCalled() expect(captureUrlSnapshotFileMock).not.toHaveBeenCalled() @@ -315,7 +305,7 @@ describe('index-documents job handler', () => { .mockReturnValueOnce(createUrlItem('url-1')) .mockReturnValueOnce(createUrlItem('url-1', 'raced.md')) - await handler.execute(createCtx({ baseId: 'kb-1', itemId: 'url-1', parentJobId: null })) + await handler.execute(createCtx({ baseId: 'kb-1', itemId: 'url-1' })) // Fetched before the lock, but the duplicate write/persist is skipped. expect(fetchKnowledgeWebPageMock).toHaveBeenCalledTimes(1) @@ -334,9 +324,7 @@ describe('index-documents job handler', () => { knowledgeItemGetByIdMock.mockReturnValue(createUrlItem('url-1')) fetchKnowledgeWebPageMock.mockResolvedValueOnce('') - await expect(handler.execute(createCtx({ baseId: 'kb-1', itemId: 'url-1', parentJobId: null }))).rejects.toThrow( - 'empty markdown' - ) + await expect(handler.execute(createCtx({ baseId: 'kb-1', itemId: 'url-1' }))).rejects.toThrow('empty markdown') expect(captureUrlSnapshotFileMock).not.toHaveBeenCalled() expect(knowledgeItemUpdateStatusMock).not.toHaveBeenCalledWith('url-1', 'completed') @@ -347,9 +335,7 @@ describe('index-documents job handler', () => { const emptyNote = { ...createNoteItem(NOTE_ITEM_ID), data: { source: 'My note', content: ' ' } } knowledgeItemGetByIdMock.mockReturnValue(emptyNote) - await expect( - handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID, parentJobId: null })) - ).rejects.toThrow('empty content') + await expect(handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID }))).rejects.toThrow('empty content') expect(captureNoteSnapshotFileMock).not.toHaveBeenCalled() expect(knowledgeItemUpdateStatusMock).not.toHaveBeenCalledWith(NOTE_ITEM_ID, 'completed') @@ -363,7 +349,7 @@ describe('index-documents job handler', () => { knowledgeItemGetByIdMock.mockReturnValue(noSnapshotNote) captureNoteSnapshotFileMock.mockResolvedValue('My note.md') - await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID, parentJobId: null })) + await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID })) // No network fetch; the in-hand content is written and the relativePath persisted. expect(fetchKnowledgeWebPageMock).not.toHaveBeenCalled() @@ -382,7 +368,7 @@ describe('index-documents job handler', () => { const handler = createIndexDocumentsJobHandler(knowledgeLockManager as never) knowledgeItemGetByIdMock.mockReturnValue(createNoteItem(NOTE_ITEM_ID, null, 'processing', 'cached-note.md')) - await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID, parentJobId: null })) + await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID })) expect(captureNoteSnapshotFileMock).not.toHaveBeenCalled() expect(knowledgeItemUpdateSnapshotRelativePathMock).not.toHaveBeenCalled() @@ -399,7 +385,7 @@ describe('index-documents job handler', () => { .mockReturnValueOnce(noSnapshotNote) .mockReturnValueOnce(createNoteItem(NOTE_ITEM_ID, null, 'processing', 'raced-note.md')) - await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID, parentJobId: null })) + await handler.execute(createCtx({ baseId: 'kb-1', itemId: NOTE_ITEM_ID })) expect(captureNoteSnapshotFileMock).not.toHaveBeenCalled() expect(knowledgeItemUpdateSnapshotRelativePathMock).not.toHaveBeenCalled() @@ -411,13 +397,6 @@ describe('index-documents job handler', () => { it('onSettled skips failed status when the item is deleting', async () => { const handler = createIndexDocumentsJobHandler(knowledgeLockManager as never) - getJobMock.mockResolvedValue( - createJobSnapshot({ - id: 'index-job', - type: 'knowledge.index-documents', - input: { baseId: 'kb-1', itemId: 'note-1', parentJobId: null } - }) - ) knowledgeItemGetByIdMock.mockReturnValue(createNoteItem('note-1', null, 'deleting')) await handler.onSettled?.({ @@ -426,7 +405,7 @@ describe('index-documents job handler', () => { scheduleId: null, parentId: null, status: 'failed', - input: { baseId: 'kb-1', itemId: 'note-1', parentJobId: null }, + input: { baseId: 'kb-1', itemId: 'note-1' }, error: { code: 'FAILED', message: 'cancelled', retryable: false }, attempt: 1, metadata: {} diff --git a/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts b/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts index 2c6ea969a42..1e015950506 100644 --- a/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts +++ b/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts @@ -281,12 +281,12 @@ export function createDirectoryItem( } } -export function createCtx(input: TInput, jobId = 'job-1'): JobContext { +export function createCtx(input: TInput, jobId = 'job-1', parentId: string | null = null): JobContext { return { jobId, input, attempt: 1, - parentId: null, + parentId, signal: new AbortController().signal, metadata: {}, patchMetadata: vi.fn().mockResolvedValue(undefined), @@ -335,7 +335,7 @@ export function createJobSnapshot(overrides: KnowledgeJobSnapshotInput): JobSnap } export const knowledgeLockManager = { - withBaseMutationLock: vi.fn(async (_baseId: string, task: () => Promise) => await task()) + runExclusive: vi.fn(async (_key: string, task: () => Promise) => await task()) } export const ingestionService: Mocked = { @@ -347,8 +347,8 @@ export const ingestionService: Mocked = { beforeEach(() => { vi.clearAllMocks() MockMainCacheServiceUtils.resetMocks() - knowledgeLockManager.withBaseMutationLock.mockImplementation( - async (_baseId: string, task: () => Promise) => await task() + knowledgeLockManager.runExclusive.mockImplementation( + async (_key: string, task: () => Promise) => await task() ) knowledgeBaseGetByIdMock.mockReturnValue(createBase()) knowledgeItemGetByIdMock.mockReturnValue(createNoteItem()) @@ -376,8 +376,8 @@ beforeEach(() => { reclaimSpace: reclaimSpaceMock, listExistingEmbeddingHashes: listExistingEmbeddingHashesMock } - getIndexStoreMock.mockResolvedValue(indexStore) - getIndexStoreIfExistsMock.mockResolvedValue(indexStore) + getIndexStoreMock.mockReturnValue(indexStore) + getIndexStoreIfExistsMock.mockReturnValue(indexStore) rebuildMaterialMock.mockReturnValue(undefined) deleteMaterialsMock.mockResolvedValue(undefined) reclaimSpaceMock.mockReturnValue({ vacuumed: false, reclaimedBytes: 0 }) diff --git a/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts index 2a8483fc8a5..89dd32e1f46 100644 --- a/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts @@ -3,13 +3,11 @@ import { describe, expect, it } from 'vitest' import { createCtx, createDirectoryItem, - createJobSnapshot, createNoteItem, createPrepareRootJobHandler, deleteItemsByIdsMock, deleteKnowledgeItemFilesBestEffortMock, deleteMaterialsMock, - getJobMock, ingestionService, knowledgeItemGetByIdMock, knowledgeItemGetSubtreeItemsMock, @@ -128,35 +126,11 @@ describe('prepare-root job handler', () => { expect(scheduleItemMock).toHaveBeenCalledWith('kb-1', 'leaf-1', 'prepare-job') expect(scheduleItemMock).toHaveBeenCalledWith('kb-1', 'leaf-2', 'prepare-job') expect(scheduleItemMock).not.toHaveBeenCalledWith('kb-1', 'leaf-3', 'prepare-job') - expect(knowledgeItemUpdateStatusMock).not.toHaveBeenCalledWith('leaf-1', 'failed', expect.anything()) - expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith('leaf-2', 'failed', { + expect(knowledgeItemSetSubtreeStatusMock).not.toHaveBeenCalledWith('kb-1', ['leaf-1'], 'failed', expect.anything()) + expect(knowledgeItemSetSubtreeStatusMock).toHaveBeenCalledWith('kb-1', ['leaf-2'], 'failed', { error: 'Failed to schedule knowledge child item job: enqueue failed' }) - expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith('leaf-3', 'failed', { - error: 'Failed to schedule knowledge child item job: enqueue failed' - }) - }) - - it('falls back to subtree failed status when marking an unscheduled leaf fails', async () => { - const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService) - const leaves = [createNoteItem('leaf-1', 'dir-1'), createNoteItem('leaf-2', 'dir-1')] - knowledgeItemGetByIdMock.mockReturnValue(createDirectoryItem()) - prepareKnowledgeItemMock.mockResolvedValue(leaves) - scheduleItemMock.mockRejectedValueOnce(new Error('enqueue failed')) - knowledgeItemUpdateStatusMock - .mockReturnValueOnce(createDirectoryItem('dir-1', 'processing')) - .mockImplementationOnce(() => { - throw new Error('status busy') - }) - - await expect(handler.execute(createCtx({ baseId: 'kb-1', itemId: 'dir-1' }, 'prepare-job'))).rejects.toThrow( - 'enqueue failed' - ) - - expect(knowledgeItemSetSubtreeStatusMock).toHaveBeenCalledWith('kb-1', ['leaf-1'], 'failed', { - error: 'Failed to schedule knowledge child item job: enqueue failed' - }) - expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith('leaf-2', 'failed', { + expect(knowledgeItemSetSubtreeStatusMock).toHaveBeenCalledWith('kb-1', ['leaf-3'], 'failed', { error: 'Failed to schedule knowledge child item job: enqueue failed' }) }) @@ -167,11 +141,6 @@ describe('prepare-root job handler', () => { knowledgeItemGetByIdMock.mockReturnValue(createDirectoryItem()) prepareKnowledgeItemMock.mockResolvedValue(leaves) scheduleItemMock.mockRejectedValueOnce(new Error('enqueue failed')) - knowledgeItemUpdateStatusMock - .mockReturnValueOnce(createDirectoryItem('dir-1', 'processing')) - .mockImplementationOnce(() => { - throw new Error('status busy') - }) knowledgeItemSetSubtreeStatusMock.mockImplementationOnce(() => { throw new Error('subtree busy') }) @@ -183,13 +152,6 @@ describe('prepare-root job handler', () => { it('onSettled skips failed status when the item is deleting', async () => { const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService) - getJobMock.mockResolvedValue( - createJobSnapshot({ - id: 'prepare-job', - type: 'knowledge.prepare-root', - input: { baseId: 'kb-1', itemId: 'dir-1' } - }) - ) knowledgeItemGetByIdMock.mockReturnValue(createDirectoryItem('dir-1', 'deleting')) await handler.onSettled?.({ diff --git a/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts index fe73f586520..100d28c3bb6 100644 --- a/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts @@ -13,7 +13,6 @@ import { deleteKnowledgeItemFilesBestEffortMock, deleteMaterialsMock, FILE_ITEM_ID, - getJobMock, ingestionService, knowledgeItemGetSubtreeItemsMock, knowledgeItemSetSubtreeStatusMock, @@ -286,13 +285,6 @@ describe('reindex-subtree job handler', () => { it('onSettled marks active roots without follow-up jobs failed', async () => { const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService) - getJobMock.mockResolvedValue( - createJobSnapshot({ - id: 'reindex-job', - type: 'knowledge.reindex-subtree', - input: { baseId: 'kb-1', rootItemIds: ['dir-1', 'note-1'] } - }) - ) listMock.mockResolvedValue([]) knowledgeItemGetSubtreeItemsMock.mockReturnValue([ createDirectoryItem('dir-1', 'preparing'), @@ -318,13 +310,6 @@ describe('reindex-subtree job handler', () => { it('onSettled stores the localized interruption code when the reindex job was cancelled', async () => { const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService) - getJobMock.mockResolvedValue( - createJobSnapshot({ - id: 'reindex-job', - type: 'knowledge.reindex-subtree', - input: { baseId: 'kb-1', rootItemIds: ['dir-1', 'note-1'] } - }) - ) listMock.mockResolvedValue([]) knowledgeItemGetSubtreeItemsMock.mockReturnValue([ createDirectoryItem('dir-1', 'preparing'), @@ -352,13 +337,6 @@ describe('reindex-subtree job handler', () => { it('onSettled skips deleting roots and roots with follow-up jobs', async () => { const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService) - getJobMock.mockResolvedValue( - createJobSnapshot({ - id: 'reindex-job', - type: 'knowledge.reindex-subtree', - input: { baseId: 'kb-1', rootItemIds: ['dir-1', 'note-1'] } - }) - ) listMock.mockResolvedValue([ createJobSnapshot({ id: 'prepare-dir-1', @@ -389,13 +367,6 @@ describe('reindex-subtree job handler', () => { it('onSettled treats file-processing check jobs as follow-up jobs', async () => { const handler = createReindexSubtreeJobHandler(knowledgeLockManager as never, ingestionService) - getJobMock.mockResolvedValue( - createJobSnapshot({ - id: 'reindex-job', - type: 'knowledge.reindex-subtree', - input: { baseId: 'kb-1', rootItemIds: [FILE_ITEM_ID] } - }) - ) listMock.mockResolvedValue([ createJobSnapshot({ id: 'check-file-1', @@ -406,8 +377,7 @@ describe('reindex-subtree job handler', () => { itemId: FILE_ITEM_ID, fileProcessingJobId: 'fp-job-1', pollRound: 0, - firstScheduledAt: 1779811200000, - parentJobId: 'reindex-job' + firstScheduledAt: 1779811200000 } }) ]) diff --git a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts index 743da2ee74a..5f85e6ee69f 100644 --- a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts +++ b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts @@ -3,6 +3,7 @@ import './jobTypes' import { application } from '@application' import { knowledgeItemService } from '@data/services/KnowledgeItemService' import { loggerService } from '@logger' +import type { KeyedMutex } from '@main/core/concurrency/KeyedMutex' import type { JobContext, JobHandler } from '@main/core/job/types' import { JOB_PROGRESS_KEY_PREFIX } from '@main/core/job/types' import { isTerminalStatus, type JobSnapshot } from '@shared/data/api/schemas/jobs' @@ -12,7 +13,6 @@ import { getFileProcessingMarkdownArtifactPath } from '../../fileProcessing/persistence/artifacts' import type { FileProcessingJobPayload } from '../../fileProcessing/tasks/shared' -import type { KnowledgeLockManager } from '../base/KnowledgeLockManager' import type { KnowledgeItemScheduler } from '../ingestion/KnowledgeIngestionService' import { toKnowledgeRelativePath } from '../pathStorage' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId, toKnowledgeItemId } from '../types' @@ -27,7 +27,7 @@ const FILE_PROCESSING_MAX_WAIT_MS = 30 * 60 * 1000 const FILE_PROCESSING_ITEM_UNAVAILABLE_CANCEL_REASON = 'knowledge-file-processing-item-unavailable' export function createCheckFileProcessingResultJobHandler( - knowledgeLockManager: KnowledgeLockManager, + knowledgeLockManager: KeyedMutex, ingestionService: KnowledgeItemScheduler ): JobHandler { return { @@ -47,7 +47,7 @@ export function createCheckFileProcessingResultJobHandler( async execute(ctx) { const { baseId, itemId, fileProcessingJobId } = ctx.input const firstScheduledAt = ctx.input.firstScheduledAt - const workflowParentJobId = ctx.input.parentJobId ?? ctx.jobId + const workflowParentJobId = ctx.parentId ?? ctx.jobId ctx.signal.throwIfAborted() if (shouldSkipMissingOrDeletingItem(baseId, itemId, ctx.jobId)) { @@ -109,7 +109,7 @@ export function createCheckFileProcessingResultJobHandler( return } - const canContinue = await knowledgeLockManager.withBaseMutationLock(baseId, async () => { + const canContinue = await knowledgeLockManager.runExclusive(baseId, async () => { if (shouldSkipMissingOrDeletingItem(baseId, itemId, ctx.jobId)) { return false } diff --git a/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts b/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts index 726feda4682..8e1781d89fd 100644 --- a/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts +++ b/src/main/features/knowledge/tasks/deleteSubtreeJobHandler.ts @@ -3,9 +3,9 @@ import './jobTypes' import { knowledgeBaseService } from '@data/services/KnowledgeBaseService' import { knowledgeItemService } from '@data/services/KnowledgeItemService' import { loggerService } from '@logger' +import type { KeyedMutex } from '@main/core/concurrency/KeyedMutex' import type { JobHandler } from '@main/core/job/types' -import type { KnowledgeLockManager } from '../base/KnowledgeLockManager' import { purgeKnowledgeSubtreeWithinLock } from '../ingestion/subtreePurge' import { reclaimKnowledgeIndexSpace } from '../pipeline/vectorstore/vectorCleanup' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId } from '../types' @@ -15,7 +15,7 @@ import { cancelActiveKnowledgeJobs } from './utils/cancel' const logger = loggerService.withContext('Knowledge:DeleteSubtreeJobHandler') export function createDeleteSubtreeJobHandler( - knowledgeLockManager: KnowledgeLockManager + knowledgeLockManager: KeyedMutex ): JobHandler { return { recovery: 'retry', @@ -51,7 +51,7 @@ export function createDeleteSubtreeJobHandler( }) // Cleanup is locked so no indexer can write vectors for rows being removed. - await knowledgeLockManager.withBaseMutationLock(baseId, async () => { + await knowledgeLockManager.runExclusive(baseId, async () => { const base = knowledgeBaseService.getById(baseId) const subtreeItems = knowledgeItemService .getSubtreeItems(baseId, rootItemIds, { includeRoots: true }) diff --git a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts index ad97c32c399..817f3071fd2 100644 --- a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts +++ b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts @@ -4,11 +4,11 @@ import { application } from '@application' import { knowledgeBaseService } from '@data/services/KnowledgeBaseService' import { knowledgeItemService } from '@data/services/KnowledgeItemService' import { loggerService } from '@logger' +import type { KeyedMutex } from '@main/core/concurrency/KeyedMutex' import type { JobContext, JobHandler } from '@main/core/job/types' import type { KnowledgeBase } from '@shared/data/types/knowledge' import { isCompletedVectorKnowledgeBase } from '@shared/data/types/knowledge' -import type { KnowledgeLockManager } from '../base/KnowledgeLockManager' import type { IndexableKnowledgeItem } from '../items' import { isIndexableKnowledgeItem } from '../items' import { collectKnowledgeReservedRelativePaths } from '../pathStorage' @@ -35,7 +35,7 @@ type LoadedIndexDocumentsInput = { type LoadedDocuments = Awaited> export function createIndexDocumentsJobHandler( - knowledgeLockManager: KnowledgeLockManager + knowledgeLockManager: KeyedMutex ): JobHandler { return { // Don't auto-resume on restart — a deliberate app quit must not re-spend the @@ -224,7 +224,7 @@ function resolveSnapshotCaptureSpec(item: IndexableKnowledgeItem): SnapshotCaptu async function ensureSnapshot( ctx: JobContext, item: IndexableKnowledgeItem, - knowledgeLockManager: KnowledgeLockManager + knowledgeLockManager: KeyedMutex ): Promise { const spec = resolveSnapshotCaptureSpec(item) if (!spec) { @@ -233,7 +233,7 @@ async function ensureSnapshot( const markdown = await spec.produce(ctx.signal) - return await knowledgeLockManager.withBaseMutationLock(ctx.input.baseId, async () => { + return await knowledgeLockManager.runExclusive(ctx.input.baseId, async () => { const latest = knowledgeItemService.getById(ctx.input.itemId) if (latest.type !== spec.type || latest.data.relativePath) { // Another job captured the snapshot (or the item changed) while we produced. @@ -273,7 +273,7 @@ async function buildRebuildMaterialInput( let embeddings: RebuildMaterialEmbeddingInput[] = [] if (usesEmbeddings) { const vectorStoreService = application.get('KnowledgeVectorStoreService') - const store = await vectorStoreService.getIndexStore(base) + const store = vectorStoreService.getIndexStore(base) const existingHashes = store.listExistingEmbeddingHashes([...bodyByHash.keys()]) const missing = [...bodyByHash.entries()].filter(([hash]) => !existingHashes.has(hash)) const vectors = await embedKnowledgeTexts( @@ -306,11 +306,11 @@ async function writeItemMaterial( ctx: JobContext, base: KnowledgeBase, input: RebuildMaterialInput, - knowledgeLockManager: KnowledgeLockManager + knowledgeLockManager: KeyedMutex ): Promise { const { baseId, itemId } = ctx.input - await knowledgeLockManager.withBaseMutationLock(baseId, async () => { + await knowledgeLockManager.runExclusive(baseId, async () => { ctx.signal.throwIfAborted() const result = resolveLiveKnowledgeItem(itemId) if ('skip' in result) { @@ -319,7 +319,7 @@ async function writeItemMaterial( } const vectorStoreService = application.get('KnowledgeVectorStoreService') - const store = await vectorStoreService.getIndexStore(base) + const store = vectorStoreService.getIndexStore(base) store.rebuildMaterial(itemId, input) knowledgeItemService.updateStatus(itemId, 'completed') }) diff --git a/src/main/features/knowledge/tasks/jobTypes.ts b/src/main/features/knowledge/tasks/jobTypes.ts index 9d537241df6..fbb44447325 100644 --- a/src/main/features/knowledge/tasks/jobTypes.ts +++ b/src/main/features/knowledge/tasks/jobTypes.ts @@ -15,7 +15,6 @@ declare module '@main/core/job/jobRegistry' { 'knowledge.index-documents': { baseId: string itemId: string - parentJobId: string | null } 'knowledge.check-file-processing-result': { baseId: string @@ -23,7 +22,6 @@ declare module '@main/core/job/jobRegistry' { fileProcessingJobId: string pollRound: number firstScheduledAt: number - parentJobId: string | null } 'knowledge.delete-subtree': { baseId: string diff --git a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts index f183d6cd048..5a90fbd118d 100644 --- a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts +++ b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts @@ -3,10 +3,10 @@ import './jobTypes' import { knowledgeBaseService } from '@data/services/KnowledgeBaseService' import { knowledgeItemService } from '@data/services/KnowledgeItemService' import { loggerService } from '@logger' +import type { KeyedMutex } from '@main/core/concurrency/KeyedMutex' import type { JobContext, JobHandler } from '@main/core/job/types' import type { KnowledgeItem } from '@shared/data/types/knowledge' -import type { KnowledgeLockManager } from '../base/KnowledgeLockManager' import type { KnowledgeItemScheduler } from '../ingestion/KnowledgeIngestionService' import { markUnscheduledKnowledgeItemsFailed } from '../ingestion/statusCleanup' import { purgeKnowledgeSubtreeWithinLock } from '../ingestion/subtreePurge' @@ -19,7 +19,7 @@ import { isDataApiNotFoundError, markKnowledgeItemFailedOnSettled } from './util const logger = loggerService.withContext('Knowledge:PrepareRootJobHandler') export function createPrepareRootJobHandler( - knowledgeLockManager: KnowledgeLockManager, + knowledgeLockManager: KeyedMutex, ingestionService: KnowledgeItemScheduler ): JobHandler { return { @@ -96,9 +96,9 @@ function loadPrepareRootItemOrSkip(ctx: JobContext) async function deletePreviousLeafExpansion( baseId: string, itemId: string, - knowledgeLockManager: KnowledgeLockManager + knowledgeLockManager: KeyedMutex ): Promise { - await knowledgeLockManager.withBaseMutationLock(baseId, async () => { + await knowledgeLockManager.runExclusive(baseId, async () => { const base = knowledgeBaseService.getById(baseId) const descendants = knowledgeItemService.getSubtreeItems(baseId, [itemId]) const removableDescendants = descendants.filter((item) => item.status !== 'deleting') @@ -108,11 +108,11 @@ async function deletePreviousLeafExpansion( async function scanRootItem( ctx: JobContext, - knowledgeLockManager: KnowledgeLockManager + knowledgeLockManager: KeyedMutex ): Promise { const { baseId, itemId } = ctx.input - return await knowledgeLockManager.withBaseMutationLock(baseId, async () => { + return await knowledgeLockManager.runExclusive(baseId, async () => { const result = resolveLiveKnowledgeItem(itemId) if ('skip' in result) { if (result.skip === 'deleting') { diff --git a/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts b/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts index 215b0225a91..252c75627c1 100644 --- a/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts +++ b/src/main/features/knowledge/tasks/reindexSubtreeJobHandler.ts @@ -4,23 +4,16 @@ import { application } from '@application' import { knowledgeBaseService } from '@data/services/KnowledgeBaseService' import { knowledgeItemService } from '@data/services/KnowledgeItemService' import { loggerService } from '@logger' +import type { KeyedMutex } from '@main/core/concurrency/KeyedMutex' import type { JobHandler, JobSettledEvent } from '@main/core/job/types' -import type { JobSnapshot } from '@shared/data/api/schemas/jobs' +import { ACTIVE_JOB_STATUSES, type JobSnapshot } from '@shared/data/api/schemas/jobs' import { KNOWLEDGE_ITEM_ERROR_INDEXING_INTERRUPTED, type KnowledgeItemStatus } from '@shared/data/types/knowledge' -import type { KnowledgeLockManager } from '../base/KnowledgeLockManager' import type { KnowledgeItemScheduler } from '../ingestion/KnowledgeIngestionService' import { canKnowledgeItemRebuildSource, isContainerKnowledgeItem } from '../items' import { deleteKnowledgeItemFilesBestEffort } from '../pathStorage' import { deleteKnowledgeItemVectors } from '../pipeline/vectorstore/vectorCleanup' -import { - KNOWLEDGE_ACTIVE_JOB_LIMIT, - KNOWLEDGE_ACTIVE_JOB_STATUSES, - knowledgeQueueName, - reportKnowledgeProgress, - toKnowledgeBaseId, - toKnowledgeItemId -} from '../types' +import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId, toKnowledgeItemId } from '../types' import type { KnowledgeReindexSubtreePayload } from './jobTypes' import { narrowKnowledgeJobInput } from './utils/jobInput' import { resolveLiveKnowledgeSubtree } from './utils/liveItem' @@ -29,7 +22,7 @@ const logger = loggerService.withContext('Knowledge:ReindexSubtreeJobHandler') const REINDEX_RECOVERY_ACTIVE_STATUSES = new Set(['preparing', 'processing']) export function createReindexSubtreeJobHandler( - knowledgeLockManager: KnowledgeLockManager, + knowledgeLockManager: KeyedMutex, ingestionService: KnowledgeItemScheduler ): JobHandler { return { @@ -59,7 +52,7 @@ export function createReindexSubtreeJobHandler( } // Reset vectors, expanded children, and root statuses as one base-level mutation. - const resetResult = await knowledgeLockManager.withBaseMutationLock(baseId, async () => { + const resetResult = await knowledgeLockManager.runExclusive(baseId, async () => { const base = knowledgeBaseService.getById(baseId) // Re-check under the mutation lock so reindex cannot turn a just-deleted // subtree back into preparing/processing during cleanup/reset. @@ -184,28 +177,28 @@ function shouldSkipDeletingSubtreeReindex(baseId: string, rootItemIds: string[], return isSkipped } -async function markReindexSubtreeFailedOnSettled(event: JobSettledEvent): Promise { +async function markReindexSubtreeFailedOnSettled( + event: JobSettledEvent +): Promise { if (event.status === 'completed') return - const jobManager = application.get('JobManager') - const snapshot = await jobManager.get(event.jobId) - const narrowed = snapshot ? narrowKnowledgeJobInput(snapshot) : null - if (!narrowed || !('rootItemIds' in narrowed.input) || narrowed.input.rootItemIds.length === 0) return - const { input } = narrowed + const { baseId, rootItemIds } = event.input + if (rootItemIds.length === 0) return const reason = event.error?.message?.trim() || `Job ${event.status}` try { + const jobManager = application.get('JobManager') const activeJobs = await jobManager.list({ - queue: knowledgeQueueName(toKnowledgeBaseId(input.baseId)), - status: [...KNOWLEDGE_ACTIVE_JOB_STATUSES], - limit: KNOWLEDGE_ACTIVE_JOB_LIMIT + queue: knowledgeQueueName(toKnowledgeBaseId(baseId)), + status: [...ACTIVE_JOB_STATUSES], + parentId: event.jobId }) - const rootsWithFollowUpJobs = getRootsWithFollowUpJobs(activeJobs, event.jobId, input.rootItemIds) - const rootItems = knowledgeItemService.getSubtreeItems(input.baseId, input.rootItemIds, { + const rootsWithFollowUpJobs = getRootsWithFollowUpJobs(activeJobs, rootItemIds) + const rootItems = knowledgeItemService.getSubtreeItems(baseId, rootItemIds, { includeRoots: true }) const rootsToFail = rootItems - .filter((item) => input.rootItemIds.includes(item.id)) + .filter((item) => rootItemIds.includes(item.id)) .filter((item) => REINDEX_RECOVERY_ACTIVE_STATUSES.has(item.status)) .filter((item) => !rootsWithFollowUpJobs.has(item.id)) .map((item) => item.id) @@ -220,26 +213,24 @@ async function markReindexSubtreeFailedOnSettled(event: JobSettledEvent): Promis event.status === 'cancelled' ? KNOWLEDGE_ITEM_ERROR_INDEXING_INTERRUPTED : `Reindex job ${event.status}: ${reason}` - knowledgeItemService.setSubtreeStatus(input.baseId, rootsToFail, 'failed', { error }) + knowledgeItemService.setSubtreeStatus(baseId, rootsToFail, 'failed', { error }) } catch (error) { logger.error( 'Failed to flip reindex-subtree targets to failed in onSettled', error instanceof Error ? error : new Error(String(error)), { jobId: event.jobId, - baseId: input.baseId, - rootItemIds: input.rootItemIds + baseId, + rootItemIds } ) } } -function getRootsWithFollowUpJobs(activeJobs: JobSnapshot[], reindexJobId: string, rootItemIds: string[]): Set { +function getRootsWithFollowUpJobs(activeJobs: JobSnapshot[], rootItemIds: string[]): Set { const rootItemIdSet = new Set(rootItemIds) const rootsWithFollowUpJobs = new Set() for (const job of activeJobs) { - if (job.parentId !== reindexJobId) continue - const narrowed = narrowKnowledgeJobInput(job) if (narrowed && 'itemId' in narrowed.input && rootItemIdSet.has(narrowed.input.itemId)) { rootsWithFollowUpJobs.add(narrowed.input.itemId) diff --git a/src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts b/src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts index cffa708083c..47ddf394750 100644 --- a/src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts +++ b/src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts @@ -89,8 +89,7 @@ describe('cancelActiveKnowledgeJobs', () => { itemId: 'file-1', fileProcessingJobId: 'fp-job-1', pollRound: 0, - firstScheduledAt: 1779811200000, - parentJobId: null + firstScheduledAt: 1779811200000 } }) ]) @@ -125,12 +124,12 @@ describe('cancelActiveKnowledgeJobs', () => { createJobSnapshot({ id: 'index-job', type: 'knowledge.index-documents', - input: { baseId: 'kb-1', itemId: 'note-1', parentJobId: null } + input: { baseId: 'kb-1', itemId: 'note-1' } }), createJobSnapshot({ id: 'unrelated-job', type: 'knowledge.index-documents', - input: { baseId: 'kb-1', itemId: 'other', parentJobId: null } + input: { baseId: 'kb-1', itemId: 'other' } }) ]) @@ -150,7 +149,7 @@ describe('cancelActiveKnowledgeJobs', () => { createJobSnapshot({ id: 'index-job', type: 'knowledge.index-documents', - input: { baseId: 'kb-1', itemId: 'note-1', parentJobId: null } + input: { baseId: 'kb-1', itemId: 'note-1' } }) ]) cancelMock.mockResolvedValue({ outcome: 'timed-out' }) @@ -168,7 +167,7 @@ describe('cancelActiveKnowledgeJobs', () => { createJobSnapshot({ id: 'index-job', type: 'knowledge.index-documents', - input: { baseId: 'kb-1', itemId: 'note-1', parentJobId: null } + input: { baseId: 'kb-1', itemId: 'note-1' } }) ]) cancelMock.mockResolvedValue({ outcome: 'timed-out' }) diff --git a/src/main/features/knowledge/tasks/utils/__tests__/jobInput.test.ts b/src/main/features/knowledge/tasks/utils/__tests__/jobInput.test.ts index 9d13fb5aa22..e0cd07f32db 100644 --- a/src/main/features/knowledge/tasks/utils/__tests__/jobInput.test.ts +++ b/src/main/features/knowledge/tasks/utils/__tests__/jobInput.test.ts @@ -7,34 +7,13 @@ describe('narrowKnowledgeJobInput', () => { expect( narrowKnowledgeJobInput({ type: 'knowledge.index-documents', - input: { baseId: 'kb-1', itemId: 'note-1', parentJobId: null } + input: { baseId: 'kb-1', itemId: 'note-1' } }) ).toEqual({ type: 'knowledge.index-documents', input: { baseId: 'kb-1', - itemId: 'note-1', - parentJobId: null - } - }) - }) - - it('preserves index-documents payload fields', () => { - expect( - narrowKnowledgeJobInput({ - type: 'knowledge.index-documents', - input: { - baseId: 'kb-1', - itemId: 'file-1', - parentJobId: 'reindex-job' - } - }) - ).toEqual({ - type: 'knowledge.index-documents', - input: { - baseId: 'kb-1', - itemId: 'file-1', - parentJobId: 'reindex-job' + itemId: 'note-1' } }) }) @@ -48,8 +27,7 @@ describe('narrowKnowledgeJobInput', () => { itemId: 'file-1', fileProcessingJobId: 'fp-job-1', pollRound: 2, - firstScheduledAt: 1779811200000, - parentJobId: 'reindex-job' + firstScheduledAt: 1779811200000 } }) ).toEqual({ @@ -59,8 +37,7 @@ describe('narrowKnowledgeJobInput', () => { itemId: 'file-1', fileProcessingJobId: 'fp-job-1', pollRound: 2, - firstScheduledAt: 1779811200000, - parentJobId: 'reindex-job' + firstScheduledAt: 1779811200000 } }) }) @@ -100,13 +77,7 @@ describe('narrowKnowledgeJobInput', () => { expect( narrowKnowledgeJobInput({ type: 'knowledge.index-documents', - input: { baseId: 'kb-1', itemId: 'file-1' } - }) - ).toBeNull() - expect( - narrowKnowledgeJobInput({ - type: 'knowledge.index-documents', - input: { baseId: 'kb-1', itemId: 'file-1', parentJobId: 1 } + input: { baseId: 'kb-1' } }) ).toBeNull() }) diff --git a/src/main/features/knowledge/tasks/utils/cancel.ts b/src/main/features/knowledge/tasks/utils/cancel.ts index 66117ac8ff3..ad7902e34c4 100644 --- a/src/main/features/knowledge/tasks/utils/cancel.ts +++ b/src/main/features/knowledge/tasks/utils/cancel.ts @@ -1,17 +1,10 @@ import { application } from '@application' import { knowledgeItemService } from '@data/services/KnowledgeItemService' +import { ACTIVE_JOB_STATUSES } from '@shared/data/api/schemas/jobs' -import { - KNOWLEDGE_ACTIVE_JOB_LIMIT, - KNOWLEDGE_ACTIVE_JOB_STATUSES, - KNOWLEDGE_JOB_TYPES, - knowledgeQueueName, - toKnowledgeBaseId -} from '../../types' +import { KNOWLEDGE_ACTIVE_JOB_LIMIT, KNOWLEDGE_JOB_TYPES, knowledgeQueueName, toKnowledgeBaseId } from '../../types' import { narrowKnowledgeJobInput } from './jobInput' -const KNOWLEDGE_JOB_TYPE_SET = new Set(KNOWLEDGE_JOB_TYPES) - export async function cancelJobOrThrow(jobId: string, reason: string): Promise { const result = await application.get('JobManager').cancel(jobId, reason) if (result.outcome === 'timed-out') { @@ -38,7 +31,7 @@ export interface CancelActiveKnowledgeJobsOptions { * Cancel any in-flight knowledge job in `baseId`'s queue (optionally scoped to a * subtree). MUST run OUTSIDE the base mutation lock: `cancel` awaits each running * handler's settlement, and knowledge index/prepare/reindex handlers themselves - * acquire `withBaseMutationLock`, so cancelling while holding the lock would + * acquire the per-base `KeyedMutex.runExclusive` lock, so cancelling while holding the lock would * deadlock (the handler can never reach its abort check). This is why every * caller (delete-base, delete-subtree, replace-on-add) cancels first, then * purges/locks. @@ -62,13 +55,14 @@ export async function cancelActiveKnowledgeJobs( const jobManager = application.get('JobManager') const activeJobs = await jobManager.list({ queue: knowledgeQueueName(toKnowledgeBaseId(baseId)), - status: [...KNOWLEDGE_ACTIVE_JOB_STATUSES], + status: [...ACTIVE_JOB_STATUSES], + type: [...KNOWLEDGE_JOB_TYPES], limit: KNOWLEDGE_ACTIVE_JOB_LIMIT }) const jobIds = activeJobs .filter((job) => job.id !== excludeJobId) - .filter((job) => (subtreeItemIds ? jobTouchesSubtree(job, subtreeItemIds) : KNOWLEDGE_JOB_TYPE_SET.has(job.type))) + .filter((job) => !subtreeItemIds || jobTouchesSubtree(job, subtreeItemIds)) .map((job) => job.id) const fileProcessingJobIds = activeJobs.flatMap((job) => getLinkedFileProcessingJobIds(job, subtreeItemIds)) diff --git a/src/main/features/knowledge/tasks/utils/jobInput.ts b/src/main/features/knowledge/tasks/utils/jobInput.ts index f77f9c0c860..4ed36829ead 100644 --- a/src/main/features/knowledge/tasks/utils/jobInput.ts +++ b/src/main/features/knowledge/tasks/utils/jobInput.ts @@ -67,17 +67,13 @@ function narrowFileProcessingCheckJobPayload( if (typeof input.fileProcessingJobId !== 'string') return null if (typeof input.pollRound !== 'number') return null if (typeof input.firstScheduledAt !== 'number') return null - if (!('parentJobId' in input)) return null - const parentJobId = input.parentJobId - if (parentJobId !== null && typeof parentJobId !== 'string') return null return { baseId: basePayload.baseId, itemId: basePayload.itemId, fileProcessingJobId: input.fileProcessingJobId, pollRound: input.pollRound, - firstScheduledAt: input.firstScheduledAt, - parentJobId + firstScheduledAt: input.firstScheduledAt } } @@ -96,19 +92,7 @@ function narrowItemJobPayload( } function narrowIndexDocumentsJobPayload(rawInput: JobSnapshot['input']): KnowledgeIndexDocumentsPayload | null { - const input = narrowJobInputObject(rawInput) - if (!input) return null - const basePayload = narrowItemJobPayload(rawInput) - if (!basePayload) return null - if (!('parentJobId' in input)) return null - const parentJobId = input.parentJobId - if (parentJobId !== null && typeof parentJobId !== 'string') return null - - return { - baseId: basePayload.baseId, - itemId: basePayload.itemId, - parentJobId - } + return narrowItemJobPayload(rawInput) } function narrowSubtreeJobPayload( diff --git a/src/main/features/knowledge/tasks/utils/settled.ts b/src/main/features/knowledge/tasks/utils/settled.ts index ea0b34d0fe5..cebf774fdcc 100644 --- a/src/main/features/knowledge/tasks/utils/settled.ts +++ b/src/main/features/knowledge/tasks/utils/settled.ts @@ -1,28 +1,21 @@ -import { application } from '@application' import { knowledgeItemService } from '@data/services/KnowledgeItemService' import type { JobSettledEvent } from '@main/core/job/types' import type { LoggerService } from '@main/core/logger/LoggerService' import { ErrorCode, isDataApiError } from '@shared/data/api' import { KNOWLEDGE_ITEM_ERROR_INDEXING_INTERRUPTED } from '@shared/data/types/knowledge' -import { narrowKnowledgeJobInput } from './jobInput' - export function isDataApiNotFoundError(error: unknown): boolean { return isDataApiError(error) && error.code === ErrorCode.NOT_FOUND } export async function markKnowledgeItemFailedOnSettled( - event: JobSettledEvent, + event: JobSettledEvent<{ itemId: string }>, logger: LoggerService, logMessage: string ): Promise { if (event.status === 'completed') return - const jobManager = application.get('JobManager') - const snapshot = await jobManager.get(event.jobId) - const narrowed = snapshot ? narrowKnowledgeJobInput(snapshot) : null - if (!narrowed || !('itemId' in narrowed.input)) return - const { input } = narrowed + const { itemId } = event.input // A cancelled indexing job for a non-deleting item means it was aborted by an // app quit / restart (knowledge has no per-item user cancel), so store the @@ -33,15 +26,15 @@ export async function markKnowledgeItemFailedOnSettled( ? KNOWLEDGE_ITEM_ERROR_INDEXING_INTERRUPTED : event.error?.message?.trim() || `Job ${event.status}` try { - const item = knowledgeItemService.getById(input.itemId) + const item = knowledgeItemService.getById(itemId) if (item.status === 'deleting') return - knowledgeItemService.updateStatus(input.itemId, 'failed', { error: reason }) + knowledgeItemService.updateStatus(itemId, 'failed', { error: reason }) } catch (error) { if (isDataApiNotFoundError(error)) return logger.error(logMessage, error instanceof Error ? error : new Error(String(error)), { jobId: event.jobId, - itemId: input.itemId + itemId }) } } diff --git a/src/main/features/knowledge/types.ts b/src/main/features/knowledge/types.ts index 9c7acdb01b3..045c73cd298 100644 --- a/src/main/features/knowledge/types.ts +++ b/src/main/features/knowledge/types.ts @@ -45,7 +45,6 @@ export type KnowledgeProgressDetail = stage: 'failed' } -export const KNOWLEDGE_ACTIVE_JOB_STATUSES = ['pending', 'delayed', 'running'] as const export const KNOWLEDGE_ACTIVE_JOB_LIMIT = 5000 export function toKnowledgeBaseId(baseId: string): KnowledgeBaseId { diff --git a/src/renderer/i18n/locales/en-us.json b/src/renderer/i18n/locales/en-us.json index 735d137482c..d8a1bd8f57a 100644 --- a/src/renderer/i18n/locales/en-us.json +++ b/src/renderer/i18n/locales/en-us.json @@ -2743,7 +2743,8 @@ "indexing_interrupted": "Indexing was interrupted because the app closed. Reindex this item to finish.", "missing_embedding_model": "The embedding model used by this knowledge base was not found during migration. Rebuild the knowledge base and choose a new embedding model.", "missing_vector_store": "This knowledge base's vector store could not be read during migration (missing, empty, or locked). The knowledge base was kept; re-index it to recover.", - "model_invalid": "No model selected" + "model_invalid": "No model selected", + "never_indexed": "This item never finished indexing during migration. Reindex it to complete." }, "groups": { "add": "New Group", diff --git a/src/renderer/i18n/locales/zh-cn.json b/src/renderer/i18n/locales/zh-cn.json index 1175bc96d61..9819126ea96 100644 --- a/src/renderer/i18n/locales/zh-cn.json +++ b/src/renderer/i18n/locales/zh-cn.json @@ -2743,7 +2743,8 @@ "indexing_interrupted": "索引因应用关闭而中断，请重新索引以完成。", "missing_embedding_model": "迁移时未找到原知识库使用的嵌入模型，请重建知识库并选择新的嵌入模型。", "missing_vector_store": "迁移时未能读取该知识库的向量库（缺失、为空或被占用），知识库已保留，请重新索引以恢复。", - "model_invalid": "未选择模型" + "model_invalid": "未选择模型", + "never_indexed": "该项在迁移时从未开始索引，请重新索引以完成。" }, "groups": { "add": "新建分组", diff --git a/src/renderer/i18n/locales/zh-tw.json b/src/renderer/i18n/locales/zh-tw.json index dce9ffc6cb5..9575156a598 100644 --- a/src/renderer/i18n/locales/zh-tw.json +++ b/src/renderer/i18n/locales/zh-tw.json @@ -2743,7 +2743,8 @@ "indexing_interrupted": "索引因應用程式關閉而中斷，請重新索引以完成。", "missing_embedding_model": "遷移時未找到原知識庫使用的嵌入模型，請重建知識庫並選擇新的嵌入模型。", "missing_vector_store": "遷移時未能讀取該知識庫的向量庫（缺失、為空或被占用），知識庫已保留，請重新索引以恢復。", - "model_invalid": "未選擇模型" + "model_invalid": "未選擇模型", + "never_indexed": "該項在遷移時從未開始索引，請重新索引以完成。" }, "groups": { "add": "新建分组", diff --git a/src/renderer/pages/knowledge/panels/dataSource/__tests__/KnowledgeItemRow.test.tsx b/src/renderer/pages/knowledge/panels/dataSource/__tests__/KnowledgeItemRow.test.tsx index ad4565d209b..aaba64ddc21 100644 --- a/src/renderer/pages/knowledge/panels/dataSource/__tests__/KnowledgeItemRow.test.tsx +++ b/src/renderer/pages/knowledge/panels/dataSource/__tests__/KnowledgeItemRow.test.tsx @@ -449,7 +449,7 @@ describe('KnowledgeItemRow', () => { expect(handleClick).not.toHaveBeenCalled() }) - it.each(['idle', 'processing', 'reading', 'embedding', 'failed', 'deleting'] as const)( + it.each(['processing', 'reading', 'embedding', 'failed', 'deleting'] as const)( 'hides view chunks for %s leaf items', (status) => { render() @@ -551,15 +551,12 @@ describe('KnowledgeItemRow', () => { expect(screen.getByRole('button', { name: '重新索引' })).toBeInTheDocument() }) - it.each(['idle', 'processing', 'reading', 'embedding', 'deleting'] as const)( - 'hides reindex for %s leaf items', - (status) => { - render() + it.each(['processing', 'reading', 'embedding', 'deleting'] as const)('hides reindex for %s leaf items', (status) => { + render() - fireEvent.contextMenu(screen.getByRole('row')) + fireEvent.contextMenu(screen.getByRole('row')) - expect(screen.queryByRole('button', { name: '重新索引' })).not.toBeInTheDocument() - expect(screen.getByRole('button', { name: '删除' })).toBeInTheDocument() - } - ) + expect(screen.queryByRole('button', { name: '重新索引' })).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: '删除' })).toBeInTheDocument() + }) }) diff --git a/src/renderer/pages/knowledge/panels/dataSource/utils/models.ts b/src/renderer/pages/knowledge/panels/dataSource/utils/models.ts index db46ea6235f..262abcac4a6 100644 --- a/src/renderer/pages/knowledge/panels/dataSource/utils/models.ts +++ b/src/renderer/pages/knowledge/panels/dataSource/utils/models.ts @@ -105,7 +105,7 @@ export const resolveDataSourceStatusViewModel = (status: KnowledgeItemStatus): D } } - if (status === 'idle' || status === 'preparing') { + if (status === 'preparing') { return { kind: 'processing', labelKey: 'knowledge.data_source.status.pending', diff --git a/src/renderer/pages/knowledge/utils/error.ts b/src/renderer/pages/knowledge/utils/error.ts index 9184166d30d..7a3759f461a 100644 --- a/src/renderer/pages/knowledge/utils/error.ts +++ b/src/renderer/pages/knowledge/utils/error.ts @@ -14,6 +14,7 @@ type KnowledgeErrorTranslator = ( | 'knowledge.error.missing_vector_store' | 'knowledge.error.directory_not_migrated' | 'knowledge.error.indexing_interrupted' + | 'knowledge.error.never_indexed' ) => string /** Localized copy for a known base error code. Exhaustive over `KnowledgeBaseErrorCode`. */ @@ -50,6 +51,8 @@ const translateKnowledgeItemErrorCode = (code: KnowledgeItemErrorCode, t: Knowle return t('knowledge.error.directory_not_migrated') case 'indexing_interrupted': return t('knowledge.error.indexing_interrupted') + case 'never_indexed': + return t('knowledge.error.never_indexed') default: return code satisfies never } diff --git a/src/shared/data/api/schemas/__tests__/knowledges.test.ts b/src/shared/data/api/schemas/__tests__/knowledges.test.ts index cef118ebc5d..0514b0dd009 100644 --- a/src/shared/data/api/schemas/__tests__/knowledges.test.ts +++ b/src/shared/data/api/schemas/__tests__/knowledges.test.ts @@ -435,7 +435,7 @@ describe('Knowledge base schemas', () => { groupId: null, type: 'note', data: { source: 'hello', content: 'hello' }, - status: 'idle', + status: 'processing', error: null, createdAt: '2026-04-10T00:00:00.000Z', updatedAt: '2026-04-10T00:00:00.000Z' @@ -449,7 +449,7 @@ describe('Knowledge base schemas', () => { groupId: null, type: 'note', data: { source: 'hello', content: 'hello' }, - status: 'idle', + status: 'processing', createdAt: '2026-04-10T00:00:00.000Z', updatedAt: '2026-04-10T00:00:00.000Z' }).success @@ -497,7 +497,6 @@ describe('Knowledge base schemas', () => { updatedAt: '2026-04-10T00:00:00.000Z' } - expect(KnowledgeItemSchema.safeParse({ ...validItem, status: 'idle', error: null }).success).toBe(true) expect(KnowledgeItemSchema.safeParse({ ...validItem, status: 'completed', error: null }).success).toBe(true) expect(KnowledgeItemSchema.safeParse({ ...validItem, status: 'processing', error: null }).success).toBe(true) expect(KnowledgeItemSchema.safeParse({ ...validItem, status: 'reading', error: null }).success).toBe(true) @@ -505,9 +504,10 @@ describe('Knowledge base schemas', () => { expect(KnowledgeItemSchema.safeParse({ ...validItem, status: 'failed', error: 'read failed' }).success).toBe(true) expect(KnowledgeItemSchema.safeParse({ ...validItem, status: 'deleting', error: null }).success).toBe(true) - expect(KnowledgeItemSchema.safeParse({ ...validItem, status: 'idle', phase: 'reading', error: null }).success).toBe( - false - ) + expect(KnowledgeItemSchema.safeParse({ ...validItem, status: 'idle', error: null }).success).toBe(false) + expect( + KnowledgeItemSchema.safeParse({ ...validItem, status: 'processing', phase: 'reading', error: null }).success + ).toBe(false) expect(KnowledgeItemSchema.safeParse({ ...validItem, status: 'completed', error: 'stale' }).success).toBe(false) expect(KnowledgeItemSchema.safeParse({ ...validItem, status: 'processing', error: 'stale' }).success).toBe(false) expect( diff --git a/src/shared/data/types/knowledge.ts b/src/shared/data/types/knowledge.ts index 9dab17bd6fd..ba12354c275 100644 --- a/src/shared/data/types/knowledge.ts +++ b/src/shared/data/types/knowledge.ts @@ -26,19 +26,18 @@ export type KnowledgeItemType = z.infer * * ```text * file/url/note: - * idle -> processing -> reading -> embedding -> completed - * \ \ \ \ - * +--------------------+-------------+-----------> failed - * \---------------------------------------------> deleting + * processing -> reading -> embedding -> completed + * \ \ \ + * +-------------+-----------> failed + * \---------------------------------------------> deleting * * directory: - * idle -> preparing -> processing -> completed - * \ \ \ \ - * +--------+-------------+-----------> failed - * \---------------------------------> deleting + * preparing -> processing -> completed + * \ \ \ + * +-------------+-----------> failed + * \---------------------------------> deleting * ``` * - * - `idle`: item row exists but indexing has not started. * - `preparing`: container expansion is running; only `directory` items may use it. * - `processing`: work has been queued or is running before a more specific phase is known. * - `reading`: leaf source documents are being read; only `file` / `url` / `note` items may use it. @@ -50,7 +49,6 @@ export type KnowledgeItemType = z.infer * - `deleting`: delete cleanup is in progress; default list/search/RAG reads hide the item. */ export const KNOWLEDGE_ITEM_STATUSES = [ - 'idle', 'preparing', 'processing', 'reading', @@ -88,20 +86,23 @@ export const KNOWLEDGE_BASE_ERROR_MISSING_EMBEDDING_MODEL: KnowledgeBaseErrorCod export const KNOWLEDGE_BASE_ERROR_MISSING_VECTOR_STORE: KnowledgeBaseErrorCode = 'missing_vector_store' /** - * Item-level error codes stored on `knowledge_item.error`. Two are set today: + * Item-level error codes stored on `knowledge_item.error`. Three are set today: * - `directory_not_migrated`: a v1-indexed `directory` whose container-level vectors could not * be re-attributed to per-file children (unreadable legacy sources, or no migratable vectors). * - `indexing_interrupted`: an indexing job was abandoned by an app quit / restart, so the item * was parked at `failed` instead of silently resumed (see KnowledgeService.recoverInterruptedItems). + * - `never_indexed`: a migrated v1 item never started indexing (no `uniqueId`), so there is + * nothing to resume from and it is parked at `failed` instead of a synthetic in-progress state. * Modeled as a zod enum (the same shape as the base error codes above) so the renderer's * code → i18n switch in `error.ts` stays exhaustive-checkable and the code ↔ translator-key * triple is tied together. Codes are localized by the UI; any other value is a free-form message. */ -export const KNOWLEDGE_ITEM_ERROR_CODES = ['directory_not_migrated', 'indexing_interrupted'] as const +export const KNOWLEDGE_ITEM_ERROR_CODES = ['directory_not_migrated', 'indexing_interrupted', 'never_indexed'] as const export const KnowledgeItemErrorCodeSchema = z.enum(KNOWLEDGE_ITEM_ERROR_CODES) export type KnowledgeItemErrorCode = z.infer export const KNOWLEDGE_ITEM_ERROR_DIRECTORY_NOT_MIGRATED: KnowledgeItemErrorCode = 'directory_not_migrated' export const KNOWLEDGE_ITEM_ERROR_INDEXING_INTERRUPTED: KnowledgeItemErrorCode = 'indexing_interrupted' +export const KNOWLEDGE_ITEM_ERROR_NEVER_INDEXED: KnowledgeItemErrorCode = 'never_indexed' export const KnowledgeChunkSizeSchema = z.number().int().positive() export const KnowledgeChunkOverlapSchema = z.number().int().min(0) @@ -361,11 +362,6 @@ const KnowledgeItemEntityBaseSchema = z.strictObject({ updatedAt: z.iso.datetime().describe('ISO timestamp when the item row was last updated.') }) -const IdleKnowledgeItemLifecycleSchema = { - status: z.literal('idle').describe('Item row exists but indexing has not started.'), - error: z.null().describe('No error is stored for non-failed lifecycle states.') -} as const - const PreparingKnowledgeItemLifecycleSchema = { status: z.literal('preparing').describe('Container expansion is running; only directory items use it.'), error: z.null().describe('No error is stored for non-failed lifecycle states.') @@ -408,11 +404,6 @@ const createLeafKnowledgeItemEntitySchemas = [ - KnowledgeItemEntityBaseSchema.extend({ - type: z.literal(type), - data, - ...IdleKnowledgeItemLifecycleSchema - }), KnowledgeItemEntityBaseSchema.extend({ type: z.literal(type), data, @@ -450,11 +441,6 @@ const createContainerKnowledgeItemEntitySchemas = [ - KnowledgeItemEntityBaseSchema.extend({ - type: z.literal(type), - data, - ...IdleKnowledgeItemLifecycleSchema - }), KnowledgeItemEntityBaseSchema.extend({ type: z.literal(type), data, From a1408a55689cbc906f1d7f9477aef355d6422e20 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Sun, 5 Jul 2026 10:57:13 +0800 Subject: [PATCH 31/37] refactor(knowledge): close delete-marking/enqueue atomicity gap, drain cancel beyond the page limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deleteItems previously set items to 'deleting' inside the base lock, then enqueued the delete-subtree job in a separate step with no rollback on failure — a failed enqueue left items stuck in 'deleting' forever with no job to clean them up. Split KnowledgeItemService.setSubtreeStatus into a Tx variant and run the status update + JobManager.enqueueTx together in one write transaction, so either both commit or both roll back. cancelActiveKnowledgeJobs capped its job listing at KNOWLEDGE_ACTIVE_JOB_LIMIT (5000), silently leaving any excess active jobs uncancelled. Reinterpret the constant as a page size and loop until a page comes back short, re-querying from the top each round since cancelling shrinks the active-job set between rounds and list()'s createdAt-only ordering has no stable offset. Co-Authored-By: Claude Sonnet 5 Signed-off-by: eeee0717 --- .../data/services/KnowledgeItemService.ts | 26 +++++++++++-- .../__tests__/KnowledgeItemService.test.ts | 14 +++++++ .../KnowledgeService.integration.test.ts | 34 +++++++++++++---- .../__tests__/KnowledgeService.test.ts | 37 +++++++++++++------ .../ingestion/KnowledgeIngestionService.ts | 33 +++++++---------- .../tasks/utils/__tests__/cancel.test.ts | 27 ++++++++++++++ .../features/knowledge/tasks/utils/cancel.ts | 36 +++++++++++------- 7 files changed, 151 insertions(+), 56 deletions(-) diff --git a/src/main/data/services/KnowledgeItemService.ts b/src/main/data/services/KnowledgeItemService.ts index e7349399c7d..2c080f9206e 100644 --- a/src/main/data/services/KnowledgeItemService.ts +++ b/src/main/data/services/KnowledgeItemService.ts @@ -7,7 +7,7 @@ import { application } from '@application' import { knowledgeItemTable } from '@data/db/schemas/knowledge' import { type SqliteErrorHandlers, withSqliteErrors } from '@data/db/sqliteErrors' -import type { DbType } from '@data/db/types' +import type { DbOrTx, DbType } from '@data/db/types' import { loggerService } from '@logger' import { DataApiErrorFactory } from '@shared/data/api' import type { KnowledgeItemListResponse, ListKnowledgeItemsQuery } from '@shared/data/api/schemas/knowledges' @@ -358,6 +358,27 @@ export class KnowledgeItemService { rootIds: string[], status: 'deleting' | 'failed', update: FailedKnowledgeItemStatusUpdate | undefined = undefined + ): string[] { + if (status === 'failed') { + return this.setSubtreeStatusTx(this.db, baseId, rootIds, status, update as FailedKnowledgeItemStatusUpdate) + } + return this.setSubtreeStatusTx(this.db, baseId, rootIds, status) + } + + setSubtreeStatusTx(tx: DbOrTx, baseId: string, rootIds: string[], status: 'deleting', update?: never): string[] + setSubtreeStatusTx( + tx: DbOrTx, + baseId: string, + rootIds: string[], + status: 'failed', + update: FailedKnowledgeItemStatusUpdate + ): string[] + setSubtreeStatusTx( + tx: DbOrTx, + baseId: string, + rootIds: string[], + status: 'deleting' | 'failed', + update: FailedKnowledgeItemStatusUpdate | undefined = undefined ): string[] { const error = status === 'failed' ? update?.error.trim() : null @@ -372,8 +393,7 @@ export class KnowledgeItemService { return [] } - const db = application.get('DbService').getDb() - const updatedRows = db.all<{ id: string; groupId: string | null }>(sql` + const updatedRows = tx.all<{ id: string; groupId: string | null }>(sql` WITH RECURSIVE subtree AS ( SELECT id FROM knowledge_item diff --git a/src/main/data/services/__tests__/KnowledgeItemService.test.ts b/src/main/data/services/__tests__/KnowledgeItemService.test.ts index 7467fb8715e..57c6b1519e6 100644 --- a/src/main/data/services/__tests__/KnowledgeItemService.test.ts +++ b/src/main/data/services/__tests__/KnowledgeItemService.test.ts @@ -896,6 +896,20 @@ describe('KnowledgeItemService', () => { error: 'One or more child items failed' }) }) + + it('marks a subtree deleting', async () => { + await seedItem({ id: NOTE_1_ID, data: { source: 'note', content: 'note' }, status: 'processing' }) + + expect(service.setSubtreeStatus(KNOWLEDGE_BASE_ID, [NOTE_1_ID], 'deleting')).toEqual([NOTE_1_ID]) + await expect(getItemRow(NOTE_1_ID)).resolves.toMatchObject({ status: 'deleting', error: null }) + }) + + it('is a no-op to mark an already-deleting subtree deleting again', async () => { + await seedItem({ id: NOTE_1_ID, data: { source: 'note', content: 'note' }, status: 'deleting' }) + + expect(service.setSubtreeStatus(KNOWLEDGE_BASE_ID, [NOTE_1_ID], 'deleting')).toEqual([NOTE_1_ID]) + await expect(getItemRow(NOTE_1_ID)).resolves.toMatchObject({ status: 'deleting', error: null }) + }) }) describe('updateStatus', () => { diff --git a/src/main/features/knowledge/__tests__/KnowledgeService.integration.test.ts b/src/main/features/knowledge/__tests__/KnowledgeService.integration.test.ts index ebe4430c770..3b33f085992 100644 --- a/src/main/features/knowledge/__tests__/KnowledgeService.integration.test.ts +++ b/src/main/features/knowledge/__tests__/KnowledgeService.integration.test.ts @@ -15,13 +15,16 @@ import { setupTestDatabase } from '@test-helpers/db' import { eq, isNull } from 'drizzle-orm' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { getIndexStoreMock, deleteStoreMock, enqueueMock, listMock, registerHandlerMock } = vi.hoisted(() => ({ - getIndexStoreMock: vi.fn(), - deleteStoreMock: vi.fn(), - enqueueMock: vi.fn(), - listMock: vi.fn(), - registerHandlerMock: vi.fn() -})) +const { getIndexStoreMock, deleteStoreMock, enqueueMock, enqueueTxMock, listMock, registerHandlerMock } = vi.hoisted( + () => ({ + getIndexStoreMock: vi.fn(), + deleteStoreMock: vi.fn(), + enqueueMock: vi.fn(), + enqueueTxMock: vi.fn(), + listMock: vi.fn(), + registerHandlerMock: vi.fn() + }) +) vi.mock('@application', async () => { const { mockApplicationFactory } = await import('@test-mocks/main/application') @@ -30,6 +33,7 @@ vi.mock('@application', async () => { cancel: vi.fn(), cancelMany: vi.fn(), enqueue: enqueueMock, + enqueueTx: enqueueTxMock, list: listMock, registerHandler: registerHandlerMock }, @@ -70,6 +74,7 @@ describe('KnowledgeService integration', () => { getIndexStoreMock.mockResolvedValue({}) deleteStoreMock.mockResolvedValue(undefined) enqueueMock.mockResolvedValue({ id: 'job-1', snapshot: {}, finished: Promise.resolve({}) }) + enqueueTxMock.mockReturnValue({ id: 'job-1', snapshot: {}, finished: Promise.resolve({}) }) listMock.mockResolvedValue([]) const [providerOrderKey, embeddingModelOrderKey] = generateOrderKeySequence(2) @@ -207,6 +212,21 @@ describe('KnowledgeService integration', () => { expect(ungroupedRestoredItems.some((item) => item.baseId === restoredBase.id)).toBe(true) }) + it('rolls back the deleting status when enqueueTx fails inside deleteItems', async () => { + const service = new KnowledgeService() + enqueueTxMock.mockImplementationOnce(() => { + throw new Error('enqueue failed') + }) + + await expect(service.deleteItems(SOURCE_BASE_ID, [SOURCE_ROOT_ITEM_ID])).rejects.toThrow('enqueue failed') + + const [rootRow] = await dbh.db + .select() + .from(knowledgeItemTable) + .where(eq(knowledgeItemTable.id, SOURCE_ROOT_ITEM_ID)) + expect(rootRow.status).toBe('processing') + }) + describe('addItems conflict resolution', () => { const COMPLETED_BASE_ID = '33333333-3333-4333-8333-333333333333' const EXISTING_NOTE_ID = '0198f3f2-7d2a-7abc-8def-123456789abc' diff --git a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts index f88545032ec..bd13e65dc6a 100644 --- a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts +++ b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts @@ -18,6 +18,7 @@ const { getIndexStoreMock, deleteStoreMock, enqueueMock, + enqueueTxMock, fileProcessingStartJobMock, getJobMock, aiEmbedManyMock, @@ -35,6 +36,7 @@ const { knowledgeItemGetRootItemsByBaseIdMock, knowledgeItemGetSubtreeItemsMock, knowledgeItemSetSubtreeStatusMock, + knowledgeItemSetSubtreeStatusTxMock, knowledgeItemUpdateStatusMock, listMock, registerHandlerMock, @@ -55,6 +57,7 @@ const { getIndexStoreMock: vi.fn(), deleteStoreMock: vi.fn(), enqueueMock: vi.fn(), + enqueueTxMock: vi.fn(), fileProcessingStartJobMock: vi.fn(), getJobMock: vi.fn(), aiEmbedManyMock: vi.fn(), @@ -72,6 +75,7 @@ const { knowledgeItemGetRootItemsByBaseIdMock: vi.fn(), knowledgeItemGetSubtreeItemsMock: vi.fn(), knowledgeItemSetSubtreeStatusMock: vi.fn(), + knowledgeItemSetSubtreeStatusTxMock: vi.fn(), knowledgeItemUpdateStatusMock: vi.fn(), listMock: vi.fn(), registerHandlerMock: vi.fn(), @@ -98,6 +102,7 @@ vi.mock('@application', async () => { cancel: cancelMock, cancelMany: cancelManyMock, enqueue: enqueueMock, + enqueueTx: enqueueTxMock, get: getJobMock, list: listMock, registerHandler: registerHandlerMock @@ -166,6 +171,7 @@ vi.mock('@data/services/KnowledgeItemService', () => ({ getOutermostSelectedItemIds: knowledgeItemGetOutermostSelectedItemIdsMock, getRootItemsByBaseId: knowledgeItemGetRootItemsByBaseIdMock, setSubtreeStatus: knowledgeItemSetSubtreeStatusMock, + setSubtreeStatusTx: knowledgeItemSetSubtreeStatusTxMock, updateStatus: knowledgeItemUpdateStatusMock } })) @@ -358,10 +364,12 @@ describe('KnowledgeService', () => { } ) knowledgeItemSetSubtreeStatusMock.mockReturnValue(['note-1']) + knowledgeItemSetSubtreeStatusTxMock.mockReturnValue(['note-1']) knowledgeItemUpdateStatusMock.mockImplementation((id: string, status: KnowledgeItemOf<'note'>['status']) => { return createNoteItem(id, createdItemBaseIds.get(id) ?? 'kb-1', null, status) }) enqueueMock.mockReturnValue({ id: 'job-1', snapshot: {}, finished: Promise.resolve({}) }) + enqueueTxMock.mockReturnValue({ id: 'job-1', snapshot: {}, finished: Promise.resolve({}) }) fileProcessingStartJobMock.mockResolvedValue({ id: 'fp-job-1', snapshot: {}, finished: Promise.resolve({}) }) getJobMock.mockResolvedValue(null) listMock.mockResolvedValue([]) @@ -1006,10 +1014,10 @@ describe('KnowledgeService', () => { expect(enqueueMock.mock.calls.map((call) => call[0])).toEqual([ 'knowledge.index-documents', - 'knowledge.delete-subtree', 'knowledge.reindex-subtree' ]) - expect(knowledgeItemSetSubtreeStatusMock).toHaveBeenCalledWith('kb-1', ['note-1'], 'deleting') + expect(enqueueTxMock.mock.calls.map((call) => call[1])).toEqual(['knowledge.delete-subtree']) + expect(knowledgeItemSetSubtreeStatusTxMock).toHaveBeenCalledWith(expect.anything(), 'kb-1', ['note-1'], 'deleting') }) it('starts file processing and schedules a check job for supported document files when the base has a processor', async () => { @@ -1537,18 +1545,17 @@ describe('KnowledgeService', () => { expect(deleteKnowledgeItemFilesBestEffortMock).toHaveBeenCalledTimes(1) }) - it('keeps items deleting when delete cleanup enqueue fails', async () => { + it('runs delete marking and enqueue inside one transaction', async () => { const service = new KnowledgeService() knowledgeItemGetByIdMock.mockReturnValue(createNoteItem('note-1')) - enqueueMock.mockImplementationOnce(() => { + enqueueTxMock.mockImplementationOnce(() => { throw new Error('enqueue failed') }) await expect(service.deleteItems('kb-1', ['note-1'])).rejects.toThrow('enqueue failed') - expect(knowledgeItemSetSubtreeStatusMock).toHaveBeenCalledWith('kb-1', ['note-1'], 'deleting') - expect(knowledgeItemSetSubtreeStatusMock).not.toHaveBeenCalledWith('kb-1', ['note-1'], 'failed', expect.anything()) - expect(enqueueMock).toHaveBeenCalledTimes(1) + expect(knowledgeItemSetSubtreeStatusTxMock).toHaveBeenCalledWith(expect.anything(), 'kb-1', ['note-1'], 'deleting') + expect(enqueueTxMock).toHaveBeenCalledTimes(1) }) it('collapses nested delete and reindex inputs to top-level roots', async () => { @@ -1562,14 +1569,15 @@ describe('KnowledgeService', () => { expect(knowledgeItemGetOutermostSelectedItemIdsMock).toHaveBeenNthCalledWith(1, 'kb-1', ['dir-1', 'note-1']) expect(knowledgeItemGetOutermostSelectedItemIdsMock).toHaveBeenNthCalledWith(2, 'kb-1', ['dir-1', 'note-1']) - expect(enqueueMock).toHaveBeenNthCalledWith( + expect(enqueueTxMock).toHaveBeenNthCalledWith( 1, + expect.anything(), 'knowledge.delete-subtree', { baseId: 'kb-1', rootItemIds: ['dir-1'] }, expect.any(Object) ) expect(enqueueMock).toHaveBeenNthCalledWith( - 2, + 1, 'knowledge.reindex-subtree', { baseId: 'kb-1', rootItemIds: ['dir-1'] }, expect.any(Object) @@ -2161,7 +2169,12 @@ describe('KnowledgeService', () => { expect(getMaterialByRelativePathMock).toHaveBeenCalledWith(CONCEPT_ID) expect(knowledgeItemGetOutermostSelectedItemIdsMock).toHaveBeenCalledWith('kb-1', [NOTE_ITEM_ID]) - expect(knowledgeItemSetSubtreeStatusMock).toHaveBeenCalledWith('kb-1', [NOTE_ITEM_ID], 'deleting') + expect(knowledgeItemSetSubtreeStatusTxMock).toHaveBeenCalledWith( + expect.anything(), + 'kb-1', + [NOTE_ITEM_ID], + 'deleting' + ) expect(result).toEqual({ applied: [CONCEPT_ID], notFound: [] }) }) @@ -2181,7 +2194,7 @@ describe('KnowledgeService', () => { const result = await service.deleteConcepts('kb-1', [CONCEPT_ID]) expect(result).toEqual({ applied: [], notFound: [CONCEPT_ID] }) - expect(knowledgeItemSetSubtreeStatusMock).not.toHaveBeenCalled() + expect(knowledgeItemSetSubtreeStatusTxMock).not.toHaveBeenCalled() }) it('collapses duplicate Concept IDs to a single resolution', async () => { @@ -2200,7 +2213,7 @@ describe('KnowledgeService', () => { const result = await service.deleteConcepts('kb-1', ['docs/missing.md']) expect(result).toEqual({ applied: [], notFound: ['docs/missing.md'] }) - expect(enqueueMock).not.toHaveBeenCalled() + expect(enqueueTxMock).not.toHaveBeenCalled() }) }) diff --git a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts index 455078b3c4f..5b5aed668a7 100644 --- a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts +++ b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts @@ -202,27 +202,20 @@ export class KnowledgeIngestionService implements KnowledgeItemScheduler { knowledgeBaseService.getById(baseId) const knowledgeBaseId = toKnowledgeBaseId(baseId) const knowledgeRootItemIds = toKnowledgeItemIds(rootItemIds) - const markedIds = await this.knowledgeLockManager.runExclusive(baseId, () => - knowledgeItemService.setSubtreeStatus(baseId, rootItemIds, 'deleting') - ) - try { - const jobManager = application.get('JobManager') - jobManager.enqueue( - 'knowledge.delete-subtree', - { baseId, rootItemIds }, - { - idempotencyKey: knowledgeDeleteSubtreeIdempotencyKey(knowledgeBaseId, knowledgeRootItemIds), - queue: knowledgeQueueName(knowledgeBaseId) - } - ) - } catch (error) { - logger.error('Failed to enqueue knowledge delete cleanup after marking items deleting', error as Error, { - baseId, - rootItemIds, - markedIds + await this.knowledgeLockManager.runExclusive(baseId, () => + application.get('DbService').withWriteTx((tx) => { + knowledgeItemService.setSubtreeStatusTx(tx, baseId, rootItemIds, 'deleting') + application.get('JobManager').enqueueTx( + tx, + 'knowledge.delete-subtree', + { baseId, rootItemIds }, + { + idempotencyKey: knowledgeDeleteSubtreeIdempotencyKey(knowledgeBaseId, knowledgeRootItemIds), + queue: knowledgeQueueName(knowledgeBaseId) + } + ) }) - throw error - } + ) } async reindexItems(baseId: string, itemIds: string[]): Promise { diff --git a/src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts b/src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts index 47ddf394750..015be755b53 100644 --- a/src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts +++ b/src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts @@ -2,6 +2,8 @@ import type { JobSnapshot } from '@shared/data/api/schemas/jobs' import type { KnowledgeItem } from '@shared/data/types/knowledge' import { beforeEach, describe, expect, it, vi } from 'vitest' +import { KNOWLEDGE_ACTIVE_JOB_LIMIT } from '../../../types' + const { cancelMock, listMock, knowledgeItemGetSubtreeItemsMock } = vi.hoisted(() => ({ cancelMock: vi.fn(), listMock: vi.fn(), @@ -176,4 +178,29 @@ describe('cancelActiveKnowledgeJobs', () => { cancelActiveKnowledgeJobs('kb-1', 'delete-base', { onCancelTimeout: 'proceed' }) ).resolves.toBeUndefined() }) + + it('drains a full page and re-queries until a page comes back short', async () => { + const fullPage = Array.from({ length: KNOWLEDGE_ACTIVE_JOB_LIMIT }, (_, index) => + createJobSnapshot({ + id: `index-job-${index}`, + type: 'knowledge.index-documents', + input: { baseId: 'kb-1', itemId: `note-${index}` } + }) + ) + listMock.mockResolvedValueOnce(fullPage).mockResolvedValueOnce([]) + + await cancelActiveKnowledgeJobs('kb-1', 'delete-base', { onCancelTimeout: 'proceed' }) + + expect(listMock).toHaveBeenCalledTimes(2) + expect(cancelMock).toHaveBeenCalledTimes(KNOWLEDGE_ACTIVE_JOB_LIMIT) + }) + + it('stops after a single empty page without cancelling anything', async () => { + listMock.mockResolvedValueOnce([]) + + await cancelActiveKnowledgeJobs('kb-1', 'delete-base', { onCancelTimeout: 'proceed' }) + + expect(listMock).toHaveBeenCalledTimes(1) + expect(cancelMock).not.toHaveBeenCalled() + }) }) diff --git a/src/main/features/knowledge/tasks/utils/cancel.ts b/src/main/features/knowledge/tasks/utils/cancel.ts index ad7902e34c4..30fe6d5fbbb 100644 --- a/src/main/features/knowledge/tasks/utils/cancel.ts +++ b/src/main/features/knowledge/tasks/utils/cancel.ts @@ -53,25 +53,33 @@ export async function cancelActiveKnowledgeJobs( } const jobManager = application.get('JobManager') - const activeJobs = await jobManager.list({ - queue: knowledgeQueueName(toKnowledgeBaseId(baseId)), - status: [...ACTIVE_JOB_STATUSES], - type: [...KNOWLEDGE_JOB_TYPES], - limit: KNOWLEDGE_ACTIVE_JOB_LIMIT - }) - - const jobIds = activeJobs - .filter((job) => job.id !== excludeJobId) - .filter((job) => !subtreeItemIds || jobTouchesSubtree(job, subtreeItemIds)) - .map((job) => job.id) - const fileProcessingJobIds = activeJobs.flatMap((job) => getLinkedFileProcessingJobIds(job, subtreeItemIds)) - + const queue = knowledgeQueueName(toKnowledgeBaseId(baseId)) const cancelOne: (jobId: string) => Promise = onCancelTimeout === 'throw' ? (jobId) => cancelJobOrThrow(jobId, reason) : (jobId) => jobManager.cancel(jobId, reason) - await Promise.all([...jobIds, ...fileProcessingJobIds].map(cancelOne)) + // Re-query from the top each round (no offset): cancelling shrinks the active-job + // set between rounds, so offset-based paging could skip rows as the window shifts, + // and `list()`'s createdAt-only ordering has no tie-breaker to make offsets stable. + while (true) { + const activeJobs = await jobManager.list({ + queue, + status: [...ACTIVE_JOB_STATUSES], + type: [...KNOWLEDGE_JOB_TYPES], + limit: KNOWLEDGE_ACTIVE_JOB_LIMIT + }) + + const jobIds = activeJobs + .filter((job) => job.id !== excludeJobId) + .filter((job) => !subtreeItemIds || jobTouchesSubtree(job, subtreeItemIds)) + .map((job) => job.id) + const fileProcessingJobIds = activeJobs.flatMap((job) => getLinkedFileProcessingJobIds(job, subtreeItemIds)) + + await Promise.all([...jobIds, ...fileProcessingJobIds].map(cancelOne)) + + if (activeJobs.length < KNOWLEDGE_ACTIVE_JOB_LIMIT) break + } } function jobTouchesSubtree(job: { type: string; input: unknown }, subtreeIds: Set): boolean { From 090817189cbf9e168322aa79f906a3db6cfc98f3 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Sun, 5 Jul 2026 11:49:28 +0800 Subject: [PATCH 32/37] refactor(knowledge-data): converge KnowledgeBase row invariants into one shared schema create()/update() each hand-rolled their own subset of the cross-field rules already expressed in KnowledgeBaseSchema's superRefine, so the embedding/dimensions pairing, no-model-bm25, and chunk-overlap checks were defined 2-3 times with drifting wording, and no boundary validated the delimiter/separator pairing at all. Extract the refine callback into refineKnowledgeBaseInvariants, reuse it for a new KnowledgeBaseWriteSchema covering the pre-write candidate row, and have create()/ update() validate their about-to-be-written row against it via safeParse + toDataApiError instead of their own DataApiErrorFactory.validation calls. Add the missing delimiter/separator check, and align update()'s two status-gated rules (embedding pairing and no-model-bm25) to both fire only when completed. Co-Authored-By: Claude Sonnet 5 Signed-off-by: eeee0717 --- .../data/services/KnowledgeBaseService.ts | 122 ++++++------------ .../__tests__/KnowledgeBaseService.test.ts | 38 +++++- src/shared/data/types/knowledge.ts | 32 ++++- 3 files changed, 101 insertions(+), 91 deletions(-) diff --git a/src/main/data/services/KnowledgeBaseService.ts b/src/main/data/services/KnowledgeBaseService.ts index 5529eded56d..857e35591a5 100644 --- a/src/main/data/services/KnowledgeBaseService.ts +++ b/src/main/data/services/KnowledgeBaseService.ts @@ -7,7 +7,7 @@ import { application } from '@application' import { knowledgeBaseTable, knowledgeItemTable } from '@data/db/schemas/knowledge' import { loggerService } from '@logger' -import { DataApiErrorFactory } from '@shared/data/api' +import { DataApiErrorFactory, toDataApiError } from '@shared/data/api' import type { OffsetPaginationResponse } from '@shared/data/api/apiTypes' import type { KnowledgeBaseListItem, @@ -24,7 +24,8 @@ import { DEFAULT_KNOWLEDGE_CHUNK_STRATEGY, DEFAULT_KNOWLEDGE_SEARCH_MODE, type KnowledgeBase, - KnowledgeBaseSchema + KnowledgeBaseSchema, + KnowledgeBaseWriteSchema } from '@shared/data/types/knowledge' import { and, asc, count as sqlCount, desc, eq, gte, ne, type SQL, sql } from 'drizzle-orm' @@ -35,63 +36,6 @@ const logger = loggerService.withContext('DataApi:KnowledgeBaseService') type KnowledgeBaseRow = typeof knowledgeBaseTable.$inferSelect type KnowledgeBaseEntitySearchItem = Extract -function validateKnowledgeBaseConfig(config: { - chunkSize: number - chunkOverlap: number - chunkStrategy?: string | null - chunkSeparator?: string | null - searchMode?: string | null - hybridAlpha?: number | null -}): Record { - const fieldErrors: Record = {} - - if (config.chunkOverlap >= config.chunkSize) { - fieldErrors.chunkOverlap = ['Chunk overlap must be smaller than chunk size'] - } - - if (config.chunkStrategy === 'delimiter' && !config.chunkSeparator) { - fieldErrors.chunkSeparator = ['Separator is required when chunk strategy is delimiter'] - } - - if (config.hybridAlpha != null && config.searchMode !== 'hybrid') { - fieldErrors.hybridAlpha = ['Hybrid alpha requires hybrid search mode'] - } - - return fieldErrors -} - -// Vector and hybrid retrieval need an embedding model; without one a base is -// BM25-only and cannot run a non-bm25 search mode. Mirrors the `completed`-only -// gate in `KnowledgeBaseSchema.superRefine`: a failed base's leftover searchMode -// isn't governed by this invariant until it goes through restore, so callers -// must only apply it to a base that is (or will become) completed. Only -// update() calls this: create() always coerces searchMode to 'bm25' up front -// when there is no model, so the invariant already holds by construction there. -function validateSearchModeNeedsEmbedding( - embeddingModelId: string | null, - searchMode: string | null | undefined -): Record { - if (embeddingModelId == null && searchMode != null && searchMode !== 'bm25') { - return { searchMode: ['A knowledge base without an embedding model can only use bm25 search'] } - } - return {} -} - -// The vector arm of the DB CHECK requires a positive dimensions alongside the model; -// a no-model base always persists a null dimensions regardless of what is passed. The -// IPC boundary already rejects a model without dimensions via CreateKnowledgeBaseSchema's -// refine, so this guards internal callers (e.g. restoreBase) that build a DTO directly, -// before the write reaches the DB CHECK as an untranslated constraint violation. -function validateDimensionsForEmbeddingModel( - embeddingModelId: string | null, - dimensions: number | null | undefined -): Record { - if (embeddingModelId != null && !(typeof dimensions === 'number' && Number.isInteger(dimensions) && dimensions > 0)) { - return { dimensions: ['A knowledge base with an embedding model requires positive dimensions'] } - } - return {} -} - function rowToKnowledgeBase(row: KnowledgeBaseRow): KnowledgeBase { const clean = nullsToUndefined(row) return KnowledgeBaseSchema.parse({ @@ -223,17 +167,6 @@ export class KnowledgeBaseService { searchMode: usesEmbeddings ? (dto.searchMode ?? DEFAULT_KNOWLEDGE_SEARCH_MODE) : 'bm25', hybridAlpha: usesEmbeddings ? dto.hybridAlpha : undefined } - const createFieldErrors = { - // Validated against the raw dto.hybridAlpha, not the coerced createConfig value - // below, so an explicit hybridAlpha on a no-model base is rejected instead of - // silently discarded — create() and update() reject the same input shape. - ...validateKnowledgeBaseConfig({ ...createConfig, hybridAlpha: dto.hybridAlpha }), - ...validateDimensionsForEmbeddingModel(embeddingModelId, dto.dimensions) - } - if (Object.keys(createFieldErrors).length > 0) { - throw DataApiErrorFactory.validation(createFieldErrors) - } - const createValues: Omit = { name: dto.name.trim(), groupId: dto.groupId ?? null, @@ -253,6 +186,20 @@ export class KnowledgeBaseService { hybridAlpha: createConfig.hybridAlpha ?? null } + // Validated against the raw dto.hybridAlpha, not the coerced createValues value, + // so an explicit hybridAlpha on a no-model base is rejected instead of silently + // discarded — create() and update() reject the same input shape. + const createCandidate = { + ...createValues, + hybridAlpha: dto.hybridAlpha ?? undefined, + threshold: createValues.threshold ?? undefined, + documentCount: createValues.documentCount ?? undefined + } + const createValidation = KnowledgeBaseWriteSchema.safeParse(createCandidate) + if (!createValidation.success) { + throw toDataApiError(createValidation.error, 'create knowledge base') + } + const db = application.get('DbService').getDb() const [row] = db.insert(knowledgeBaseTable).values(createValues).returning().all() @@ -307,19 +254,30 @@ export class KnowledgeBaseService { nextConfig.hybridAlpha = null } - // Only a completed base is governed by the no-model=>bm25 invariant (mirrors - // KnowledgeBaseSchema.superRefine's own completed-only gate): a failed base - // may carry a leftover incompatible searchMode from before it failed/migrated, - // and metadata-only updates (rename, move group) must not be blocked by it. - const updateFieldErrors = { - ...validateKnowledgeBaseConfig(nextConfig), - ...validateDimensionsForEmbeddingModel(nextEmbeddingModelId, nextDimensions), - ...(existing.status === 'completed' - ? validateSearchModeNeedsEmbedding(nextEmbeddingModelId, nextConfig.searchMode) - : {}) + // Validate the merged next-state (existing row + this PATCH) against the same + // invariants as the read schema — a failed base's leftover-incompatible pairing + // or searchMode isn't governed by these invariants until it goes through + // restore, so metadata-only updates (rename, move group) must not be blocked by + // them; that gating lives inside `refineKnowledgeBaseInvariants` itself (only + // enforced when `status === 'completed'`). + const { id: _id, createdAt: _createdAt, updatedAt: _updatedAt, ...existingConfig } = existing + void _id // Intentionally unused - excluding id/createdAt/updatedAt from the write candidate + void _createdAt + void _updatedAt + const updateCandidate = { + ...existingConfig, + embeddingModelId: nextEmbeddingModelId, + dimensions: nextDimensions, + chunkSize: nextConfig.chunkSize, + chunkOverlap: nextConfig.chunkOverlap, + chunkStrategy: nextConfig.chunkStrategy, + chunkSeparator: nextConfig.chunkSeparator, + searchMode: nextConfig.searchMode, + hybridAlpha: nextConfig.hybridAlpha ?? undefined } - if (Object.keys(updateFieldErrors).length > 0) { - throw DataApiErrorFactory.validation(updateFieldErrors) + const updateValidation = KnowledgeBaseWriteSchema.safeParse(updateCandidate) + if (!updateValidation.success) { + throw toDataApiError(updateValidation.error, 'update knowledge base') } const updates: Partial = {} diff --git a/src/main/data/services/__tests__/KnowledgeBaseService.test.ts b/src/main/data/services/__tests__/KnowledgeBaseService.test.ts index c0bbdba5932..9998eea3cdb 100644 --- a/src/main/data/services/__tests__/KnowledgeBaseService.test.ts +++ b/src/main/data/services/__tests__/KnowledgeBaseService.test.ts @@ -345,6 +345,12 @@ describe('KnowledgeBaseService', () => { expect(() => service.getById(KNOWLEDGE_BASE_ID)).toThrow('Chunk overlap must be smaller than chunk size') }) + + it('should reject an invalid persisted separator at the read boundary', async () => { + await seedKnowledgeBase({ chunkStrategy: 'delimiter', chunkSeparator: '' }) + + expect(() => service.getById(KNOWLEDGE_BASE_ID)).toThrow('Separator is required when chunk strategy is delimiter') + }) }) describe('create', () => { @@ -453,17 +459,17 @@ describe('KnowledgeBaseService', () => { code: ErrorCode.VALIDATION_ERROR, details: { fieldErrors: { - dimensions: ['A knowledge base with an embedding model requires positive dimensions'] + dimensions: ['Embedding model and dimensions must be set together'] } } }) }) it.each([ - ['zero', 0], - ['negative', -1], - ['non-integer', 1.5] - ])('rejects an embedding model with %s dimensions', (_label, dimensions) => { + ['zero', 0, 'Too small: expected number to be >0'], + ['negative', -1, 'Too small: expected number to be >0'], + ['non-integer', 1.5, 'Invalid input: expected int, received number'] + ])('rejects an embedding model with %s dimensions', (_label, dimensions, message) => { let err: unknown try { service.create({ @@ -478,7 +484,7 @@ describe('KnowledgeBaseService', () => { code: ErrorCode.VALIDATION_ERROR, details: { fieldErrors: { - dimensions: ['A knowledge base with an embedding model requires positive dimensions'] + dimensions: [message] } } }) @@ -769,7 +775,7 @@ describe('KnowledgeBaseService', () => { code: ErrorCode.VALIDATION_ERROR, details: { fieldErrors: { - searchMode: ['A knowledge base without an embedding model can only use bm25 search'] + searchMode: ['A knowledge base without an embedding model must use bm25 search mode'] } } }) @@ -798,6 +804,24 @@ describe('KnowledgeBaseService', () => { expect(result.searchMode).toBe('hybrid') }) + it('allows renaming or moving a recoverable failed base despite a leftover mismatched embedding/dimensions pair', async () => { + // A migration-failed base can carry a leftover embeddingModelId with null + // dimensions (or vice versa); the DB CHECK only constrains this pairing for + // completed bases, so a plain metadata PATCH must not resurrect that check + // (the pairing invariant is still enforced for completed bases — see "rejects + // an embedding model without positive dimensions" above). + await seedKnowledgeBase({ + embeddingModelId: createUniqueModelId('openai', 'embed-model'), + dimensions: null, + status: 'failed', + error: KNOWLEDGE_BASE_ERROR_MISSING_EMBEDDING_MODEL + }) + + const result = service.update(KNOWLEDGE_BASE_ID, { name: 'Renamed while failed' }) + expect(result.name).toBe('Renamed while failed') + expect(result.dimensions).toBeNull() + }) + it('should reject shrinking chunkSize when the existing chunkOverlap no longer fits', async () => { await seedKnowledgeBase({ chunkSize: 256, chunkOverlap: 120 }) diff --git a/src/shared/data/types/knowledge.ts b/src/shared/data/types/knowledge.ts index ba12354c275..59326c96fe9 100644 --- a/src/shared/data/types/knowledge.ts +++ b/src/shared/data/types/knowledge.ts @@ -153,7 +153,14 @@ export const KnowledgeBaseEntitySchema = z.strictObject({ updatedAt: z.iso.datetime() }) -export const KnowledgeBaseSchema = KnowledgeBaseEntitySchema.superRefine((value, ctx) => { +/** + * Cross-field invariants for a knowledge base row, shared by the read-side entity + * schema and the pre-write candidate schema so a rule is defined exactly once. + */ +function refineKnowledgeBaseInvariants( + value: Omit, 'id' | 'createdAt' | 'updatedAt'>, + ctx: z.RefinementCtx +): void { if (value.status === 'completed') { if (value.error !== null) { ctx.addIssue({ @@ -207,9 +214,30 @@ export const KnowledgeBaseSchema = KnowledgeBaseEntitySchema.superRefine((value, message: 'Hybrid alpha requires hybrid search mode' }) } -}) + + if (value.chunkStrategy === 'delimiter' && !value.chunkSeparator) { + ctx.addIssue({ + code: 'custom', + path: ['chunkSeparator'], + message: 'Separator is required when chunk strategy is delimiter' + }) + } +} + +export const KnowledgeBaseSchema = KnowledgeBaseEntitySchema.superRefine(refineKnowledgeBaseInvariants) export type KnowledgeBase = z.infer +/** + * The full row about to be inserted/updated, validated against the same + * invariants as the read-side schema before it ever reaches the DB CHECK + * constraints — `id`/`createdAt`/`updatedAt` don't exist yet at write time. + */ +export const KnowledgeBaseWriteSchema = KnowledgeBaseEntitySchema.omit({ + id: true, + createdAt: true, + updatedAt: true +}).superRefine(refineKnowledgeBaseInvariants) + /** * A knowledge base that has finished setup and is ready for runtime operations * (opening its index store, BM25 search, indexing). Covers both a vector base From f8c6a6f8e885fafe01a0b014ac7ed04818aeb02b Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Sun, 5 Jul 2026 13:16:59 +0800 Subject: [PATCH 33/37] refactor(knowledge): hoist material-path derivation into items.ts, thread processedRelativePath through job payload Eliminates a query->pipeline reverse import (toMaterialRelativePath now lives in the shared items.ts) and removes the fp-check job's absolute-path rederivation in favor of passing the already-known processedRelativePath through the job payload, deleting the now-dead toKnowledgeRelativePath/ isPathInsideBase helpers. Signed-off-by: eeee0717 --- .../v2/migrators/KnowledgeVectorMigrator.ts | 5 +- .../__tests__/KnowledgeService.test.ts | 22 +++++-- .../knowledge/__tests__/items.test.ts | 62 ++++++++++++++++++- .../knowledge/__tests__/pathStorage.test.ts | 22 +------ .../ingestion/KnowledgeIngestionService.ts | 12 ++-- src/main/features/knowledge/items.ts | 32 +++++++++- src/main/features/knowledge/pathStorage.ts | 15 ----- .../indexing/__tests__/materialFields.test.ts | 59 ------------------ .../pipeline/indexing/materialFields.ts | 31 ---------- .../pipeline/readers/KnowledgeFileReader.ts | 7 +-- .../features/knowledge/query/visibility.ts | 2 +- ...heckFileProcessingResultJobHandler.test.ts | 7 ++- .../__tests__/deleteSubtreeJobHandler.test.ts | 3 +- .../reindexSubtreeJobHandler.test.ts | 3 +- .../checkFileProcessingResultJobHandler.ts | 20 +++--- .../tasks/indexDocumentsJobHandler.ts | 3 +- src/main/features/knowledge/tasks/jobTypes.ts | 1 + .../tasks/utils/__tests__/cancel.test.ts | 3 +- .../tasks/utils/__tests__/jobInput.test.ts | 6 +- .../knowledge/tasks/utils/jobInput.ts | 4 +- 20 files changed, 153 insertions(+), 166 deletions(-) delete mode 100644 src/main/features/knowledge/pipeline/indexing/__tests__/materialFields.test.ts delete mode 100644 src/main/features/knowledge/pipeline/indexing/materialFields.ts diff --git a/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts b/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts index 8d0d7fe3fda..03b65af9b5e 100644 --- a/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts +++ b/src/main/data/migration/v2/migrators/KnowledgeVectorMigrator.ts @@ -3,16 +3,13 @@ import path from 'node:path' import { knowledgeBaseTable, knowledgeItemTable } from '@data/db/schemas/knowledge' import { loggerService } from '@logger' +import { type MaterialFieldSource, toMaterialRelativePath } from '@main/features/knowledge/items' import { assertSafeKnowledgeRelativePath, collectKnowledgeReservedRelativePaths, reserveImportedFileRelativePath } from '@main/features/knowledge/pathStorage' import { DOCUMENT_SEPARATOR } from '@main/features/knowledge/pipeline/indexing/chunk' -import { - type MaterialFieldSource, - toMaterialRelativePath -} from '@main/features/knowledge/pipeline/indexing/materialFields' import { deriveNoteSnapshotSlug } from '@main/features/knowledge/pipeline/sources/noteSnapshot' import { serializeOkfFrontmatter } from '@main/features/knowledge/pipeline/sources/okfFrontmatter' import { deriveUrlSnapshotSlug, deriveUrlSnapshotTitle } from '@main/features/knowledge/pipeline/sources/urlSnapshot' diff --git a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts index bd13e65dc6a..33df0e3229c 100644 --- a/src/main/features/knowledge/__tests__/KnowledgeService.test.ts +++ b/src/main/features/knowledge/__tests__/KnowledgeService.test.ts @@ -600,7 +600,8 @@ describe('KnowledgeService', () => { itemId: 'file-1', fileProcessingJobId: 'fp-job-1', pollRound: 0, - firstScheduledAt: 1779811200000 + firstScheduledAt: 1779811200000, + processedRelativePath: 'source.md' } } ]) @@ -1050,7 +1051,8 @@ describe('KnowledgeService', () => { itemId: 'file-1', fileProcessingJobId: 'fp-job-1', pollRound: 0, - firstScheduledAt: expect.any(Number) + firstScheduledAt: expect.any(Number), + processedRelativePath: 'source.md' }, expect.objectContaining({ idempotencyKey: 'knowledge:kb-1:file-1:fp-check:fp-job-1:0', @@ -1323,7 +1325,8 @@ describe('KnowledgeService', () => { itemId: 'file-1', fileProcessingJobId: 'fp-job-1', pollRound: 0, - firstScheduledAt: expect.any(Number) + firstScheduledAt: expect.any(Number), + processedRelativePath: 'source.md' }, expect.objectContaining({ idempotencyKey: 'knowledge:kb-1:file-1:fp-check:fp-job-1:0', @@ -1416,7 +1419,12 @@ describe('KnowledgeService', () => { baseId: string, itemId: string, fileProcessingJobId: string, - options: { pollRound: number; firstScheduledAt: number; parentJobId: string | null } + options: { + pollRound: number + firstScheduledAt: number + parentJobId: string | null + processedRelativePath: string + } ): Promise } } @@ -1425,7 +1433,8 @@ describe('KnowledgeService', () => { await ingestionService.scheduleFileProcessingCheck('kb-1', 'file-1', 'fp-job-1', { pollRound: 1, firstScheduledAt: Date.parse('2026-04-08T00:00:00.000Z'), - parentJobId: 'check-job-0' + parentJobId: 'check-job-0', + processedRelativePath: 'source.md' }) expect(enqueueMock).toHaveBeenCalledWith( @@ -1435,7 +1444,8 @@ describe('KnowledgeService', () => { itemId: 'file-1', fileProcessingJobId: 'fp-job-1', pollRound: 1, - firstScheduledAt: Date.parse('2026-04-08T00:00:00.000Z') + firstScheduledAt: Date.parse('2026-04-08T00:00:00.000Z'), + processedRelativePath: 'source.md' }, expect.objectContaining({ idempotencyKey: 'knowledge:kb-1:file-1:fp-check:fp-job-1:1', diff --git a/src/main/features/knowledge/__tests__/items.test.ts b/src/main/features/knowledge/__tests__/items.test.ts index ffd5e90bcfb..be806a31fc4 100644 --- a/src/main/features/knowledge/__tests__/items.test.ts +++ b/src/main/features/knowledge/__tests__/items.test.ts @@ -1,6 +1,7 @@ import type { KnowledgeItem, KnowledgeItemOf } from '@shared/data/types/knowledge' import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { MaterialFieldSource } from '../items' import type * as PathStorage from '../pathStorage' const { probeKnowledgeFileMock, probeKnowledgeSourcePathMock } = vi.hoisted(() => ({ @@ -17,9 +18,8 @@ vi.mock('../pathStorage', async () => { } }) -const { canKnowledgeItemRebuildSource, classifyKnowledgeItemSource, isIndexableKnowledgeItem } = await import( - '../items' -) +const { canKnowledgeItemRebuildSource, classifyKnowledgeItemSource, isIndexableKnowledgeItem, toMaterialRelativePath } = + await import('../items') function createItem(type: KnowledgeItem['type']): KnowledgeItem { const base = { @@ -115,3 +115,59 @@ describe('canKnowledgeItemRebuildSource', () => { await expect(canKnowledgeItemRebuildSource('kb-1', createItem('directory'))).resolves.toBe(false) }) }) + +describe('toMaterialRelativePath', () => { + it('uses a file’s stored relativePath when there is no processed artifact', () => { + const file: MaterialFieldSource = { + id: 'file-1', + type: 'file', + data: { source: '/docs/report.pdf', relativePath: 'report.pdf' } + } + expect(toMaterialRelativePath(file)).toBe('report.pdf') + }) + + it('prefers a file’s processed-artifact path (indexedRelativePath) over the source path', () => { + const file: MaterialFieldSource = { + id: 'file-2', + type: 'file', + data: { source: '/docs/report.pdf', relativePath: 'report.pdf', indexedRelativePath: 'report.md' } + } + expect(toMaterialRelativePath(file)).toBe('report.md') + }) + + it('uses a url’s captured snapshot path once it has one (the real raw/ file, matching the migrator)', () => { + const url: MaterialFieldSource = { + id: 'url-1', + type: 'url', + data: { source: 'https://example.com', url: 'https://example.com', relativePath: 'example-page.md' } + } + expect(toMaterialRelativePath(url)).toBe('example-page.md') + }) + + it('uses a note’s captured snapshot path once it has one (the real raw/ file, matching the migrator)', () => { + const note: MaterialFieldSource = { + id: 'note-1', + type: 'note', + data: { source: 'My note', content: 'hello', relativePath: 'My note.md' } + } + expect(toMaterialRelativePath(note)).toBe('My note.md') + }) + + it('throws for a url that has not been captured yet — a snapshot is always materialized first', () => { + const url: MaterialFieldSource = { + id: 'url-2', + type: 'url', + data: { source: 'https://example.com', url: 'https://example.com' } + } + expect(() => toMaterialRelativePath(url)).toThrow('has no captured snapshot relativePath') + }) + + it('throws for a note that has not been captured yet — a snapshot is always materialized first', () => { + const note: MaterialFieldSource = { + id: 'note-2', + type: 'note', + data: { source: 'My note', content: 'hello' } + } + expect(() => toMaterialRelativePath(note)).toThrow('has no captured snapshot relativePath') + }) +}) diff --git a/src/main/features/knowledge/__tests__/pathStorage.test.ts b/src/main/features/knowledge/__tests__/pathStorage.test.ts index a255e288931..cede79363b6 100644 --- a/src/main/features/knowledge/__tests__/pathStorage.test.ts +++ b/src/main/features/knowledge/__tests__/pathStorage.test.ts @@ -1,7 +1,7 @@ // Path-safety tests for the knowledge "store by relative path" core. The reject -// branches of `assertSafeKnowledgeRelativePath` / `isPathInsideBase` are the -// security boundary for every copied source file, so they are exercised here -// through the exported helpers (the guards themselves are private). +// branches of `assertSafeKnowledgeRelativePath` are the security boundary for +// every copied source file, so they are exercised here through the exported +// helpers (the guard itself is private). import path from 'node:path' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -43,11 +43,9 @@ vi.mock('@main/utils/file/fs', () => ({ })) const { - getKnowledgeBaseDir, getKnowledgeMaterialDir, getKnowledgeBaseFilePath, getKnowledgeSourceRelativePath, - toKnowledgeRelativePath, getProcessedMarkdownRelativePath, reserveImportedFileRelativePath, needsProcessedArtifactReservation, @@ -59,7 +57,6 @@ const { } = await import('../pathStorage') const BASE_ID = 'kb-1' -const BASE_DIR = getKnowledgeBaseDir(BASE_ID) // Material bytes resolve under the base's `raw/` material root, not the base dir itself. const MATERIAL_DIR = getKnowledgeMaterialDir(BASE_ID) @@ -99,19 +96,6 @@ describe('pathStorage relative-path safety', () => { }) }) - describe('toKnowledgeRelativePath', () => { - it('returns a POSIX relative path for a path inside the base', () => { - expect(toKnowledgeRelativePath(BASE_ID, path.join(MATERIAL_DIR, 'a', 'b.md'))).toBe('a/b.md') - }) - - it.each([ - ['an unrelated absolute path', '/etc/passwd'], - ['a sibling base', path.join(path.dirname(BASE_DIR), 'kb-2', 'x.md')] - ])('rejects %s', (_label, absolutePath) => { - expect(() => toKnowledgeRelativePath(BASE_ID, absolutePath)).toThrow() - }) - }) - describe('getProcessedMarkdownRelativePath', () => { it('swaps the extension to .md while keeping directories', () => { expect(getProcessedMarkdownRelativePath('sub/report.pdf')).toBe('sub/report.md') diff --git a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts index 5b5aed668a7..dc2e9311f55 100644 --- a/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts +++ b/src/main/features/knowledge/ingestion/KnowledgeIngestionService.ts @@ -72,7 +72,7 @@ export interface KnowledgeItemScheduler { baseId: KnowledgeBaseId, itemId: KnowledgeItemId, fileProcessingJobId: string, - options: { pollRound: number; firstScheduledAt: number; parentJobId: string | null } + options: { pollRound: number; firstScheduledAt: number; parentJobId: string | null; processedRelativePath: string } ): Promise } @@ -306,7 +306,8 @@ export class KnowledgeIngestionService implements KnowledgeItemScheduler { firstScheduledAt: Date.now(), // Use the file-processing job as workflow parent when this is a direct add flow, // so retries keep a stable index idempotency key across poll rounds. - parentJobId: parentJobId ?? fileProcessingJob.id + parentJobId: parentJobId ?? fileProcessingJob.id, + processedRelativePath }) } catch (error) { try { @@ -329,9 +330,9 @@ export class KnowledgeIngestionService implements KnowledgeItemScheduler { baseId: KnowledgeBaseId, itemId: KnowledgeItemId, fileProcessingJobId: string, - options: { pollRound: number; firstScheduledAt: number; parentJobId: string | null } + options: { pollRound: number; firstScheduledAt: number; parentJobId: string | null; processedRelativePath: string } ): Promise { - const { pollRound, firstScheduledAt, parentJobId } = options + const { pollRound, firstScheduledAt, parentJobId, processedRelativePath } = options const jobManager = application.get('JobManager') jobManager.enqueue( 'knowledge.check-file-processing-result', @@ -340,7 +341,8 @@ export class KnowledgeIngestionService implements KnowledgeItemScheduler { itemId, fileProcessingJobId, pollRound, - firstScheduledAt + firstScheduledAt, + processedRelativePath }, { idempotencyKey: knowledgeFileProcessingCheckIdempotencyKey(baseId, itemId, fileProcessingJobId, pollRound), diff --git a/src/main/features/knowledge/items.ts b/src/main/features/knowledge/items.ts index 494cc25897d..c3617e312ae 100644 --- a/src/main/features/knowledge/items.ts +++ b/src/main/features/knowledge/items.ts @@ -15,6 +15,36 @@ export function isContainerKnowledgeItem(item: KnowledgeItem): item is Container return item.type === 'directory' } +/** + * The subset of an indexable knowledge item needed to derive its index-store + * material fields. The `Pick` is distributed per member so `type` and `data` + * stay correlated (a single `Pick` over the union would collapse `data` to a bare + * union and lose the file-only `relativePath` / `indexedRelativePath`). Shared by + * the indexing job and the v1→v2 vector migrator so both stamp the material + * identically (knowledge-technical-design.md §4.2). + */ +export type MaterialFieldSource = + | Pick, 'id' | 'type' | 'data'> + | Pick, 'id' | 'type' | 'data'> + | Pick, 'id' | 'type' | 'data'> + +/** + * A material's stable relative path. A file uses its stored path (the processed + * artifact when present). A url or note uses its captured snapshot path — a real + * base file under `raw/`, materialized before the material is stamped (the index + * job's ensure-snapshot step, or the vector migrator), so it is always present + * here; a missing one is an invariant violation, not a fallback case. + */ +export function toMaterialRelativePath(item: MaterialFieldSource): string { + if (item.type === 'file') { + return item.data.indexedRelativePath ?? item.data.relativePath + } + if (!item.data.relativePath) { + throw new Error(`Knowledge ${item.type} item ${item.id} has no captured snapshot relativePath for its material`) + } + return item.data.relativePath +} + /** Whether a knowledge item's rebuild source is present, genuinely gone, or merely unverifiable. */ export type KnowledgeItemSourceState = 'rebuildable' | 'missing' | 'unverifiable' @@ -37,7 +67,7 @@ export async function classifyKnowledgeItemSource( return toSourceState(await probeKnowledgeSourcePath(item.data.source)) } if (item.type === 'file') { - return toSourceState(await probeKnowledgeFile(baseId, item.data.indexedRelativePath ?? item.data.relativePath)) + return toSourceState(await probeKnowledgeFile(baseId, toMaterialRelativePath(item))) } return 'rebuildable' } diff --git a/src/main/features/knowledge/pathStorage.ts b/src/main/features/knowledge/pathStorage.ts index d447f2222ab..1bac3a2aa1a 100644 --- a/src/main/features/knowledge/pathStorage.ts +++ b/src/main/features/knowledge/pathStorage.ts @@ -84,16 +84,6 @@ export function getKnowledgeSourceRelativePath(sourcePath: string): string { return fileName } -export function toKnowledgeRelativePath(baseId: string, absolutePath: string): string { - const materialDir = getKnowledgeMaterialDir(baseId) - const relativePath = path.relative(materialDir, absolutePath) - assertSafeKnowledgeRelativePath(relativePath) - if (!isPathInsideBase(materialDir, absolutePath)) { - throw new Error(`Path is outside knowledge base material root '${baseId}': ${absolutePath}`) - } - return normalizeRelativePath(relativePath) -} - export function getProcessedMarkdownRelativePath(relativePath: string): string { assertSafeKnowledgeRelativePath(relativePath) const parsed = path.parse(relativePath) @@ -319,11 +309,6 @@ function normalizeRelativePath(relativePath: string): string { return path.normalize(relativePath).replace(/\\/g, '/') } -function isPathInsideBase(baseDir: string, candidatePath: string): boolean { - const relativePath = path.relative(baseDir, candidatePath) - return Boolean(relativePath) && !relativePath.startsWith('..') && !path.isAbsolute(relativePath) -} - async function assertTargetAvailable(destPath: FilePath): Promise { try { await fs.lstat(destPath) diff --git a/src/main/features/knowledge/pipeline/indexing/__tests__/materialFields.test.ts b/src/main/features/knowledge/pipeline/indexing/__tests__/materialFields.test.ts deleted file mode 100644 index 31208bbd362..00000000000 --- a/src/main/features/knowledge/pipeline/indexing/__tests__/materialFields.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { describe, expect, it } from 'vitest' - -import { type MaterialFieldSource, toMaterialRelativePath } from '../materialFields' - -describe('toMaterialRelativePath', () => { - it('uses a file’s stored relativePath when there is no processed artifact', () => { - const file: MaterialFieldSource = { - id: 'file-1', - type: 'file', - data: { source: '/docs/report.pdf', relativePath: 'report.pdf' } - } - expect(toMaterialRelativePath(file)).toBe('report.pdf') - }) - - it('prefers a file’s processed-artifact path (indexedRelativePath) over the source path', () => { - const file: MaterialFieldSource = { - id: 'file-2', - type: 'file', - data: { source: '/docs/report.pdf', relativePath: 'report.pdf', indexedRelativePath: 'report.md' } - } - expect(toMaterialRelativePath(file)).toBe('report.md') - }) - - it('uses a url’s captured snapshot path once it has one (the real raw/ file, matching the migrator)', () => { - const url: MaterialFieldSource = { - id: 'url-1', - type: 'url', - data: { source: 'https://example.com', url: 'https://example.com', relativePath: 'example-page.md' } - } - expect(toMaterialRelativePath(url)).toBe('example-page.md') - }) - - it('uses a note’s captured snapshot path once it has one (the real raw/ file, matching the migrator)', () => { - const note: MaterialFieldSource = { - id: 'note-1', - type: 'note', - data: { source: 'My note', content: 'hello', relativePath: 'My note.md' } - } - expect(toMaterialRelativePath(note)).toBe('My note.md') - }) - - it('throws for a url that has not been captured yet — a snapshot is always materialized first', () => { - const url: MaterialFieldSource = { - id: 'url-2', - type: 'url', - data: { source: 'https://example.com', url: 'https://example.com' } - } - expect(() => toMaterialRelativePath(url)).toThrow('has no captured snapshot relativePath') - }) - - it('throws for a note that has not been captured yet — a snapshot is always materialized first', () => { - const note: MaterialFieldSource = { - id: 'note-2', - type: 'note', - data: { source: 'My note', content: 'hello' } - } - expect(() => toMaterialRelativePath(note)).toThrow('has no captured snapshot relativePath') - }) -}) diff --git a/src/main/features/knowledge/pipeline/indexing/materialFields.ts b/src/main/features/knowledge/pipeline/indexing/materialFields.ts deleted file mode 100644 index eddbfe65052..00000000000 --- a/src/main/features/knowledge/pipeline/indexing/materialFields.ts +++ /dev/null @@ -1,31 +0,0 @@ -import type { KnowledgeItemOf } from '@shared/data/types/knowledge' - -/** - * The subset of an indexable knowledge item needed to derive its index-store - * material fields. The `Pick` is distributed per member so `type` and `data` - * stay correlated (a single `Pick` over the union would collapse `data` to a bare - * union and lose the file-only `relativePath` / `indexedRelativePath`). Shared by - * the indexing job and the v1→v2 vector migrator so both stamp the material - * identically (knowledge-technical-design.md §4.2). - */ -export type MaterialFieldSource = - | Pick, 'id' | 'type' | 'data'> - | Pick, 'id' | 'type' | 'data'> - | Pick, 'id' | 'type' | 'data'> - -/** - * A material's stable relative path. A file uses its stored path (the processed - * artifact when present). A url or note uses its captured snapshot path — a real - * base file under `raw/`, materialized before the material is stamped (the index - * job's ensure-snapshot step, or the vector migrator), so it is always present - * here; a missing one is an invariant violation, not a fallback case. - */ -export function toMaterialRelativePath(item: MaterialFieldSource): string { - if (item.type === 'file') { - return item.data.indexedRelativePath ?? item.data.relativePath - } - if (!item.data.relativePath) { - throw new Error(`Knowledge ${item.type} item ${item.id} has no captured snapshot relativePath for its material`) - } - return item.data.relativePath -} diff --git a/src/main/features/knowledge/pipeline/readers/KnowledgeFileReader.ts b/src/main/features/knowledge/pipeline/readers/KnowledgeFileReader.ts index 8d718c5a09b..cbbd641ee3d 100644 --- a/src/main/features/knowledge/pipeline/readers/KnowledgeFileReader.ts +++ b/src/main/features/knowledge/pipeline/readers/KnowledgeFileReader.ts @@ -10,6 +10,7 @@ import { MarkdownReader } from '@vectorstores/readers/markdown' import { PDFReader } from '@vectorstores/readers/pdf' import { TextFileReader } from '@vectorstores/readers/text' +import { toMaterialRelativePath } from '../../items' import { getKnowledgeBaseFilePath } from '../../pathStorage' import { DraftsExportReader } from './files/DraftsExportReader' import { EpubReader } from './files/EpubReader' @@ -67,9 +68,5 @@ export async function loadDocumentsFromKnowledgeBaseFile( } export async function loadFileDocuments(item: KnowledgeItemOf<'file'>): Promise { - return loadDocumentsFromKnowledgeBaseFile( - item.baseId, - item.data.indexedRelativePath ?? item.data.relativePath, - item.data.source - ) + return loadDocumentsFromKnowledgeBaseFile(item.baseId, toMaterialRelativePath(item), item.data.source) } diff --git a/src/main/features/knowledge/query/visibility.ts b/src/main/features/knowledge/query/visibility.ts index 19aa98fc342..8764236c31f 100644 --- a/src/main/features/knowledge/query/visibility.ts +++ b/src/main/features/knowledge/query/visibility.ts @@ -2,7 +2,7 @@ import { knowledgeItemService } from '@data/services/KnowledgeItemService' import { loggerService } from '@logger' import type { KnowledgeItem } from '@shared/data/types/knowledge' -import { toMaterialRelativePath } from '../pipeline/indexing/materialFields' +import { toMaterialRelativePath } from '../items' import { isDataApiNotFoundError } from '../tasks/utils/settled' const logger = loggerService.withContext('Knowledge:Query') diff --git a/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts index 4fe06b4c08c..fa4f6c78f1c 100644 --- a/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/checkFileProcessingResultJobHandler.test.ts @@ -48,6 +48,7 @@ function createCheckPayload( fileProcessingJobId: 'fp-job-1', pollRound: 0, firstScheduledAt: Date.now(), + processedRelativePath: PROCESSED_RELATIVE_PATH, ...overrides } } @@ -84,7 +85,8 @@ describe('check-file-processing-result job handler', () => { expect(ingestionService.scheduleFileProcessingCheck).toHaveBeenCalledWith('kb-1', FILE_ITEM_ID, 'fp-job-1', { pollRound: 3, firstScheduledAt, - parentJobId: 'job-1' + parentJobId: 'job-1', + processedRelativePath: PROCESSED_RELATIVE_PATH }) expect(ingestionService.scheduleIndexing).not.toHaveBeenCalled() expect(ctx.reportProgress).toHaveBeenCalledWith(0, { stage: 'waiting', pollRound: 3 }) @@ -110,7 +112,8 @@ describe('check-file-processing-result job handler', () => { expect(ingestionService.scheduleFileProcessingCheck).toHaveBeenCalledWith('kb-1', FILE_ITEM_ID, 'fp-job-1', { pollRound: 3, firstScheduledAt, - parentJobId: 'reindex-job' + parentJobId: 'reindex-job', + processedRelativePath: PROCESSED_RELATIVE_PATH }) }) diff --git a/src/main/features/knowledge/tasks/__tests__/deleteSubtreeJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/deleteSubtreeJobHandler.test.ts index dbe2e87d513..b72f909f12a 100644 --- a/src/main/features/knowledge/tasks/__tests__/deleteSubtreeJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/deleteSubtreeJobHandler.test.ts @@ -47,7 +47,8 @@ describe('delete-subtree job handler', () => { itemId: 'note-1', fileProcessingJobId: 'fp-job-1', pollRound: 0, - firstScheduledAt: 1779811200000 + firstScheduledAt: 1779811200000, + processedRelativePath: 'source.md' } }), createJobSnapshot({ diff --git a/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts index 100d28c3bb6..00ead97f310 100644 --- a/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/reindexSubtreeJobHandler.test.ts @@ -377,7 +377,8 @@ describe('reindex-subtree job handler', () => { itemId: FILE_ITEM_ID, fileProcessingJobId: 'fp-job-1', pollRound: 0, - firstScheduledAt: 1779811200000 + firstScheduledAt: 1779811200000, + processedRelativePath: 'source.md' } }) ]) diff --git a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts index 5f85e6ee69f..fd312bed123 100644 --- a/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts +++ b/src/main/features/knowledge/tasks/checkFileProcessingResultJobHandler.ts @@ -14,7 +14,6 @@ import { } from '../../fileProcessing/persistence/artifacts' import type { FileProcessingJobPayload } from '../../fileProcessing/tasks/shared' import type { KnowledgeItemScheduler } from '../ingestion/KnowledgeIngestionService' -import { toKnowledgeRelativePath } from '../pathStorage' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId, toKnowledgeItemId } from '../types' import type { KnowledgeCheckFileProcessingResultPayload } from './jobTypes' import { cancelJobOrThrow } from './utils/cancel' @@ -86,7 +85,8 @@ export function createCheckFileProcessingResultJobHandler( { pollRound: nextPollRound, firstScheduledAt, - parentJobId: workflowParentJobId + parentJobId: workflowParentJobId, + processedRelativePath: ctx.input.processedRelativePath } ) reportWaitingProgress(ctx, fileProcessingJobId, nextPollRound) @@ -102,12 +102,12 @@ export function createCheckFileProcessingResultJobHandler( return } - const indexedRelativePath = parseMarkdownArtifactRelativePathOrNull(baseId, snapshot) - if (!indexedRelativePath) { + if (!isCompletedMarkdownArtifact(snapshot)) { markItemFailed(itemId, `Invalid file processing result for job ${fileProcessingJobId}`) reportKnowledgeProgress(ctx, 100, { stage: 'failed' }) return } + const indexedRelativePath = ctx.input.processedRelativePath const canContinue = await knowledgeLockManager.runExclusive(baseId, async () => { if (shouldSkipMissingOrDeletingItem(baseId, itemId, ctx.jobId)) { @@ -197,14 +197,20 @@ function markItemFailed(itemId: string, error: string): void { knowledgeItemService.updateStatus(itemId, 'failed', { error }) } -function parseMarkdownArtifactRelativePathOrNull(baseId: string, snapshot: JobSnapshot): string | null { +/** + * Validates that a completed file-processing job actually produced a markdown-file + * artifact (not a degraded inline-text result) — the item's `indexedRelativePath` + * is the job's own `processedRelativePath` input, not derived from this artifact. + */ +function isCompletedMarkdownArtifact(snapshot: JobSnapshot): boolean { try { - return toKnowledgeRelativePath(baseId, getFileProcessingMarkdownArtifactPath(snapshot)) + getFileProcessingMarkdownArtifactPath(snapshot) + return true } catch (error) { logger.warn('Invalid file-processing result for knowledge item', { jobId: snapshot.id, error: error instanceof Error ? error.message : String(error) }) - return null + return false } } diff --git a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts index 817f3071fd2..9e2d39b7d24 100644 --- a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts +++ b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts @@ -10,11 +10,10 @@ import type { KnowledgeBase } from '@shared/data/types/knowledge' import { isCompletedVectorKnowledgeBase } from '@shared/data/types/knowledge' import type { IndexableKnowledgeItem } from '../items' -import { isIndexableKnowledgeItem } from '../items' +import { isIndexableKnowledgeItem, toMaterialRelativePath } from '../items' import { collectKnowledgeReservedRelativePaths } from '../pathStorage' import { type ChunkedKnowledgeContent, chunkKnowledgeDocuments } from '../pipeline/indexing/chunk' import { embedKnowledgeTexts } from '../pipeline/indexing/embed' -import { toMaterialRelativePath } from '../pipeline/indexing/materialFields' import { loadKnowledgeItemDocuments } from '../pipeline/readers/KnowledgeReader' import { captureNoteSnapshotFile } from '../pipeline/sources/noteSnapshot' import { fetchKnowledgeWebPage } from '../pipeline/sources/url' diff --git a/src/main/features/knowledge/tasks/jobTypes.ts b/src/main/features/knowledge/tasks/jobTypes.ts index fbb44447325..c9dc88cfa6c 100644 --- a/src/main/features/knowledge/tasks/jobTypes.ts +++ b/src/main/features/knowledge/tasks/jobTypes.ts @@ -22,6 +22,7 @@ declare module '@main/core/job/jobRegistry' { fileProcessingJobId: string pollRound: number firstScheduledAt: number + processedRelativePath: string } 'knowledge.delete-subtree': { baseId: string diff --git a/src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts b/src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts index 015be755b53..880274201ce 100644 --- a/src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts +++ b/src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts @@ -91,7 +91,8 @@ describe('cancelActiveKnowledgeJobs', () => { itemId: 'file-1', fileProcessingJobId: 'fp-job-1', pollRound: 0, - firstScheduledAt: 1779811200000 + firstScheduledAt: 1779811200000, + processedRelativePath: 'source.md' } }) ]) diff --git a/src/main/features/knowledge/tasks/utils/__tests__/jobInput.test.ts b/src/main/features/knowledge/tasks/utils/__tests__/jobInput.test.ts index e0cd07f32db..adbf2b35d41 100644 --- a/src/main/features/knowledge/tasks/utils/__tests__/jobInput.test.ts +++ b/src/main/features/knowledge/tasks/utils/__tests__/jobInput.test.ts @@ -27,7 +27,8 @@ describe('narrowKnowledgeJobInput', () => { itemId: 'file-1', fileProcessingJobId: 'fp-job-1', pollRound: 2, - firstScheduledAt: 1779811200000 + firstScheduledAt: 1779811200000, + processedRelativePath: 'source.md' } }) ).toEqual({ @@ -37,7 +38,8 @@ describe('narrowKnowledgeJobInput', () => { itemId: 'file-1', fileProcessingJobId: 'fp-job-1', pollRound: 2, - firstScheduledAt: 1779811200000 + firstScheduledAt: 1779811200000, + processedRelativePath: 'source.md' } }) }) diff --git a/src/main/features/knowledge/tasks/utils/jobInput.ts b/src/main/features/knowledge/tasks/utils/jobInput.ts index 4ed36829ead..4ed1f3d669c 100644 --- a/src/main/features/knowledge/tasks/utils/jobInput.ts +++ b/src/main/features/knowledge/tasks/utils/jobInput.ts @@ -67,13 +67,15 @@ function narrowFileProcessingCheckJobPayload( if (typeof input.fileProcessingJobId !== 'string') return null if (typeof input.pollRound !== 'number') return null if (typeof input.firstScheduledAt !== 'number') return null + if (typeof input.processedRelativePath !== 'string') return null return { baseId: basePayload.baseId, itemId: basePayload.itemId, fileProcessingJobId: input.fileProcessingJobId, pollRound: input.pollRound, - firstScheduledAt: input.firstScheduledAt + firstScheduledAt: input.firstScheduledAt, + processedRelativePath: input.processedRelativePath } } From f325bb49ade3c749710698c49ca3413142e90c0b Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Sun, 5 Jul 2026 14:39:44 +0800 Subject: [PATCH 34/37] fix(knowledge): pin directory container prefix before byte copy to close crash-safety gap Directory expansion durably copied file bytes into raw// before pinning the container's relativePath or committing any child row. A crash/kill in that window left orphan bytes that no retry could reclaim: the retry's prefix selection only counts DB rows (orphan bytes contribute nothing), re-picks the same prefix, and assertTargetAvailable then hits "Knowledge file already exists" in a permanent failure loop. The retry purge walks getSubtreeItems, which excludes the container row, so the container's own raw/ shell was never swept. Design P (pin-first + explicit container-prefix reclaim on retry): - directory.ts: split the pure prefix chooser (chooseDirectoryPathPrefix) out of expandDirectoryOwnerToTree; the expander now takes pathPrefix and returns the node array. Drop the stale in-function try/catch cleanup. - prepareItem.ts: pin updateDirectoryRelativePath BEFORE any byte is copied. - prepareRootJobHandler.ts: after the descendant purge, removeDir the container's own raw/ shell (idempotent, same runExclusive lock). Invariant: the pin strictly precedes the first copy, so any window that holds orphan bytes has the container row already pinned to those bytes' prefix -> the retry removeDir deletes a superset of the orphans. happy-path behavior is unchanged; empty/failed directories now reserve their top-level name (a failure-path change we accept as more correct). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: eeee0717 --- .../sources/__tests__/directory.test.ts | 254 +++++------------- .../knowledge/pipeline/sources/directory.ts | 90 +++---- .../tasks/__tests__/jobHandlerTestUtils.ts | 18 +- .../tasks/__tests__/prepareItem.test.ts | 79 +++--- .../__tests__/prepareRootJobHandler.test.ts | 53 +++- .../features/knowledge/tasks/prepareItem.ts | 21 +- .../knowledge/tasks/prepareRootJobHandler.ts | 15 ++ 7 files changed, 247 insertions(+), 283 deletions(-) diff --git a/src/main/features/knowledge/pipeline/sources/__tests__/directory.test.ts b/src/main/features/knowledge/pipeline/sources/__tests__/directory.test.ts index 4478011d346..e79eb0303c6 100644 --- a/src/main/features/knowledge/pipeline/sources/__tests__/directory.test.ts +++ b/src/main/features/knowledge/pipeline/sources/__tests__/directory.test.ts @@ -2,7 +2,7 @@ import type * as NodeFs from 'node:fs' import type * as NodeOs from 'node:os' import path from 'node:path' -import type * as FsUtils from '@main/utils/file/fs' +import type { KnowledgeItem } from '@shared/data/types/knowledge' import { afterEach, describe, expect, it, vi } from 'vitest' import type * as PathStorage from '../../../pathStorage' @@ -10,19 +10,12 @@ import type * as PathStorage from '../../../pathStorage' const copyFileIntoKnowledgeBaseAtMock = vi.hoisted(() => vi.fn(async (_baseId: string, _externalPath: string, relativePath: string) => relativePath) ) -const removeDirMock = vi.hoisted(() => vi.fn()) vi.mock('@application', async () => { const { mockApplicationFactory } = await import('@test-mocks/main/application') return mockApplicationFactory() }) -vi.mock('@main/utils/file/fs', async () => { - const actual = await vi.importActual('@main/utils/file/fs') - removeDirMock.mockImplementation(actual.removeDir) - return { ...actual, removeDir: removeDirMock } -}) - vi.mock('../../../pathStorage', async () => { const actual = await vi.importActual('../../../pathStorage') return { @@ -31,8 +24,7 @@ vi.mock('../../../pathStorage', async () => { } }) -const { expandDirectoryOwnerToTree } = await import('../directory') -const { getKnowledgeBaseFilePath } = await import('../../../pathStorage') +const { chooseDirectoryPathPrefix, expandDirectoryOwnerToTree } = await import('../directory') const realFs = await vi.importActual('node:fs') const realOs = await vi.importActual('node:os') @@ -44,6 +36,45 @@ function createSignal() { return new AbortController().signal } +function createDirectoryOwner(source: string): KnowledgeItem { + return { + id: 'dir-owner-1', + baseId: 'kb-1', + groupId: null, + type: 'directory', + data: { source }, + status: 'processing', + error: null, + createdAt: '2026-04-08T00:00:00.000Z', + updatedAt: '2026-04-08T00:00:00.000Z' + } +} + +describe('chooseDirectoryPathPrefix', () => { + it('uses the directory basename when that top-level name is free', () => { + expect(chooseDirectoryPathPrefix(createDirectoryOwner('/some/path/anna'), new Set())).toBe('anna') + }) + + it('dedupes with a `_N` suffix when the top-level name is already taken', () => { + expect(chooseDirectoryPathPrefix(createDirectoryOwner('/some/path/project'), new Set(['project']))).toBe( + 'project_1' + ) + }) + + it('dedupes after the whole basename, not before a fake extension', () => { + // A folder literally named `report.v2`: the trailing `.v2` is part of the name, + // not a file extension, so the suffix must land after it (`report.v2_1`). + expect(chooseDirectoryPathPrefix(createDirectoryOwner('/some/path/report.v2'), new Set(['report.v2']))).toBe( + 'report.v2_1' + ) + }) + + it('rejects a non-directory owner', () => { + const fileOwner = { ...createDirectoryOwner('/some/path/x'), type: 'file' } as KnowledgeItem + expect(() => chooseDirectoryPathPrefix(fileOwner, new Set())).toThrow("must be type 'directory'") + }) +}) + describe('expandDirectoryOwnerToTree', () => { let tempRoot: string | undefined @@ -62,24 +93,9 @@ describe('expandDirectoryOwnerToTree', () => { realFs.writeFileSync(path.join(rootDir, '.dockerignore'), 'node_modules') realFs.writeFileSync(path.join(nestedDir, 'skill.md'), '# skill') - const { pathPrefix, children } = await expandDirectoryOwnerToTree( - { - id: 'dir-owner-1', - baseId: 'kb-1', - groupId: null, - type: 'directory', - data: { - source: rootDir - }, - status: 'processing', - error: null, - createdAt: '2026-04-08T00:00:00.000Z', - updatedAt: '2026-04-08T00:00:00.000Z' - }, - 'kb-1', - new Set(), - createSignal() - ) + const owner = createDirectoryOwner(rootDir) + const pathPrefix = chooseDirectoryPathPrefix(owner, new Set()) + const children = await expandDirectoryOwnerToTree(owner, 'kb-1', pathPrefix, createSignal()) expect(pathPrefix).toBe('anna') expect(children).toEqual([ @@ -115,22 +131,11 @@ describe('expandDirectoryOwnerToTree', () => { realFs.writeFileSync(path.join(rootDir, 'readme.md'), '# readme') realFs.writeFileSync(path.join(nestedDir, 'reference.md'), '# reference') - const { children } = await expandDirectoryOwnerToTree( - { - id: 'dir-owner-1', - baseId: 'kb-1', - groupId: null, - type: 'directory', - data: { - source: rootDir - }, - status: 'processing', - error: null, - createdAt: '2026-04-08T00:00:00.000Z', - updatedAt: '2026-04-08T00:00:00.000Z' - }, + const owner = createDirectoryOwner(rootDir) + const children = await expandDirectoryOwnerToTree( + owner, 'kb-1', - new Set(), + chooseDirectoryPathPrefix(owner, new Set()), createSignal() ) @@ -180,22 +185,11 @@ describe('expandDirectoryOwnerToTree', () => { realFs.writeFileSync(path.join(rootDir, 'sheet.ods'), 'ods') copyFileIntoKnowledgeBaseAtMock.mockClear() - const { children } = await expandDirectoryOwnerToTree( - { - id: 'dir-owner-1', - baseId: 'kb-1', - groupId: null, - type: 'directory', - data: { - source: rootDir - }, - status: 'processing', - error: null, - createdAt: '2026-04-08T00:00:00.000Z', - updatedAt: '2026-04-08T00:00:00.000Z' - }, + const owner = createDirectoryOwner(rootDir) + const children = await expandDirectoryOwnerToTree( + owner, 'kb-1', - new Set(), + chooseDirectoryPathPrefix(owner, new Set()), createSignal() ) @@ -227,22 +221,11 @@ describe('expandDirectoryOwnerToTree', () => { realFs.writeFileSync(path.join(dirB, 'notes.md'), '# b') copyFileIntoKnowledgeBaseAtMock.mockClear() - const { children } = await expandDirectoryOwnerToTree( - { - id: 'dir-owner-1', - baseId: 'kb-1', - groupId: null, - type: 'directory', - data: { - source: rootDir - }, - status: 'processing', - error: null, - createdAt: '2026-04-08T00:00:00.000Z', - updatedAt: '2026-04-08T00:00:00.000Z' - }, + const owner = createDirectoryOwner(rootDir) + const children = await expandDirectoryOwnerToTree( + owner, 'kb-1', - new Set(), + chooseDirectoryPathPrefix(owner, new Set()), createSignal() ) @@ -254,31 +237,16 @@ describe('expandDirectoryOwnerToTree', () => { expect(new Set(copiedTargets).size).toBe(copiedTargets.length) }) - it('dedupes the top-level directory name with a `_N` suffix when it is already taken', async () => { + it('stores children under a deduped prefix passed in by the caller', async () => { tempRoot = createTempRoot() const rootDir = path.join(tempRoot, 'project') realFs.mkdirSync(rootDir, { recursive: true }) realFs.writeFileSync(path.join(rootDir, 'notes.md'), '# notes') - const { pathPrefix, children } = await expandDirectoryOwnerToTree( - { - id: 'dir-owner-1', - baseId: 'kb-1', - groupId: null, - type: 'directory', - data: { - source: rootDir - }, - status: 'processing', - error: null, - createdAt: '2026-04-08T00:00:00.000Z', - updatedAt: '2026-04-08T00:00:00.000Z' - }, - 'kb-1', - // A prior `project` directory already occupies that top-level name under raw/. - new Set(['project']), - createSignal() - ) + const owner = createDirectoryOwner(rootDir) + // A prior `project` directory already occupies that top-level name under raw/. + const pathPrefix = chooseDirectoryPathPrefix(owner, new Set(['project'])) + const children = await expandDirectoryOwnerToTree(owner, 'kb-1', pathPrefix, createSignal()) expect(pathPrefix).toBe('project_1') expect(children).toEqual([ @@ -292,87 +260,6 @@ describe('expandDirectoryOwnerToTree', () => { ]) }) - it('dedupes a dotted directory name after the whole basename, not before a fake extension', async () => { - tempRoot = createTempRoot() - // A folder literally named `report.v2`: the trailing `.v2` is part of the name, - // not a file extension, so the suffix must land after it (`report.v2_1`). - const rootDir = path.join(tempRoot, 'report.v2') - realFs.mkdirSync(rootDir, { recursive: true }) - realFs.writeFileSync(path.join(rootDir, 'notes.md'), '# notes') - - const { pathPrefix, children } = await expandDirectoryOwnerToTree( - { - id: 'dir-owner-1', - baseId: 'kb-1', - groupId: null, - type: 'directory', - data: { - source: rootDir - }, - status: 'processing', - error: null, - createdAt: '2026-04-08T00:00:00.000Z', - updatedAt: '2026-04-08T00:00:00.000Z' - }, - 'kb-1', - // A prior `report.v2` directory already occupies that top-level name under raw/. - new Set(['report.v2']), - createSignal() - ) - - expect(pathPrefix).toBe('report.v2_1') - expect(children).toEqual([ - { - type: 'file', - data: { - source: path.join(rootDir, 'notes.md'), - relativePath: 'report.v2_1/notes.md' - } - } - ]) - }) - - it('cleans up copied materials when a mid-expansion copy fails', async () => { - tempRoot = createTempRoot() - const rootDir = path.join(tempRoot, 'workspace') - realFs.mkdirSync(rootDir, { recursive: true }) - realFs.writeFileSync(path.join(rootDir, 'a.md'), '# a') - realFs.writeFileSync(path.join(rootDir, 'b.md'), '# b') - - copyFileIntoKnowledgeBaseAtMock.mockClear() - removeDirMock.mockClear() - const copyError = new Error('disk full') - copyFileIntoKnowledgeBaseAtMock - .mockImplementationOnce(async (_baseId, _externalPath, relativePath) => relativePath) - .mockImplementationOnce(async () => { - throw copyError - }) - - await expect( - expandDirectoryOwnerToTree( - { - id: 'dir-owner-1', - baseId: 'kb-1', - groupId: null, - type: 'directory', - data: { - source: rootDir - }, - status: 'processing', - error: null, - createdAt: '2026-04-08T00:00:00.000Z', - updatedAt: '2026-04-08T00:00:00.000Z' - }, - 'kb-1', - new Set(), - createSignal() - ) - ).rejects.toBe(copyError) - - expect(removeDirMock).toHaveBeenCalledTimes(1) - expect(removeDirMock).toHaveBeenCalledWith(getKnowledgeBaseFilePath('kb-1', 'workspace')) - }) - it('stops before reading when the runtime signal is already aborted', async () => { tempRoot = createTempRoot() const controller = new AbortController() @@ -380,24 +267,7 @@ describe('expandDirectoryOwnerToTree', () => { controller.abort(abortError) await expect( - expandDirectoryOwnerToTree( - { - id: 'dir-owner-1', - baseId: 'kb-1', - groupId: null, - type: 'directory', - data: { - source: tempRoot - }, - status: 'processing', - error: null, - createdAt: '2026-04-08T00:00:00.000Z', - updatedAt: '2026-04-08T00:00:00.000Z' - }, - 'kb-1', - new Set(), - controller.signal - ) + expandDirectoryOwnerToTree(createDirectoryOwner(tempRoot), 'kb-1', 'root', controller.signal) ).rejects.toBe(abortError) }) }) diff --git a/src/main/features/knowledge/pipeline/sources/directory.ts b/src/main/features/knowledge/pipeline/sources/directory.ts index 57179af673c..f764c749978 100644 --- a/src/main/features/knowledge/pipeline/sources/directory.ts +++ b/src/main/features/knowledge/pipeline/sources/directory.ts @@ -1,15 +1,12 @@ import fs from 'node:fs/promises' import path from 'node:path' -import { loggerService } from '@logger' -import { removeDir } from '@main/utils/file/fs' import { nextFreeKnowledgeRelativePath } from '@main/utils/knowledge' import type { DirectoryItemData, FileItemData, KnowledgeItem } from '@shared/data/types/knowledge' import { knowledgeSupportedFileExts } from '@shared/utils/file' -import { copyFileIntoKnowledgeBaseAt, getKnowledgeBaseFilePath } from '../../pathStorage' +import { copyFileIntoKnowledgeBaseAt } from '../../pathStorage' -const logger = loggerService.withContext('Knowledge:Directory') const KNOWLEDGE_SUPPORTED_FILE_EXT_SET = new Set(knowledgeSupportedFileExts) /** A scanned filesystem entry under a directory owner — only the fields this module reads. */ @@ -33,15 +30,6 @@ export type ExpandedDirectoryNode = data: Pick } -/** - * Result of expanding a directory owner: the deduped `raw/` prefix its files were - * stored under (to pin onto the owner's `relativePath`) plus the child tree. - */ -export interface ExpandedDirectoryTree { - pathPrefix: string - children: ExpandedDirectoryNode[] -} - async function readDirectoryTree( dirPath: string, signal: AbortSignal, @@ -135,51 +123,59 @@ async function expandDirectoryNode( } } -export async function expandDirectoryOwnerToTree( - owner: KnowledgeItem, - baseId: string, - reservedTopLevelNames: Set, - signal: AbortSignal -): Promise { +/** + * The deduped top-level `raw/` prefix a directory owner's files will be stored under — + * its own name (e.g. `raw/docs/...`) instead of the opaque owner UUID, so the on-disk + * layout mirrors what the user picked. When that name is already taken under raw/, + * dedupe it with a `_N` suffix (the same strategy file imports use, see + * reserveImportedFileRelativePath). Pure — no I/O — so the caller can pin it onto the + * container's `relativePath` BEFORE any byte is copied, making a mid-expansion crash + * recoverable (the retry reclaims `raw/` from the pinned row). + */ +export function chooseDirectoryPathPrefix(owner: KnowledgeItem, reservedTopLevelNames: Set): string { if (owner.type !== 'directory') { throw new Error(`Knowledge item '${owner.id}' must be type 'directory', received '${owner.type}'`) } // The original folder to scan lives in `source` (shared by every item type). `path` - // was retired in favour of a `relativePath` written back from `pathPrefix` below. + // was retired in favour of a `relativePath` written back from `pathPrefix`. const resolvedPath = path.resolve(owner.data.source) - // Store children under the directory's own name (e.g. `raw/docs/...`) instead of - // the opaque owner UUID, so the on-disk layout mirrors what the user picked. When - // that top-level name is already taken under raw/, dedupe it with a `_N` suffix — - // the same strategy file imports use (see reserveImportedFileRelativePath). - const pathPrefix = nextFreeKnowledgeRelativePath( + return nextFreeKnowledgeRelativePath( path.basename(resolvedPath), (candidate) => !reservedTopLevelNames.has(candidate), false // a directory basename is not a filename — keep any trailing ".ext" intact ) +} - // Expansion durably copies bytes into `raw//...` before any `knowledge_item` - // row references them (child rows are only created once the whole tree succeeds). A - // mid-expansion failure (disk full, permission error, an aborted signal) must not leave - // orphaned bytes under a name the DB doesn't know is occupied — the next attempt would - // pick the same `pathPrefix` again and collide with its own partial copy. This function - // owns the namespace it just reserved, so it cleans it up on any failure. - try { - const children = await readDirectoryTree(resolvedPath, signal) - const expandedChildren: ExpandedDirectoryNode[] = [] - - for (const child of children) { - const expandedChild = await expandDirectoryNode(baseId, pathPrefix, child, signal) - if (expandedChild) { - expandedChildren.push(expandedChild) - } - } +/** + * Scan a directory owner's on-disk tree and durably copy every supported file into + * `raw//...`. The prefix is chosen and pinned by the caller + * (`chooseDirectoryPathPrefix`) before this runs, so a mid-expansion crash leaves the + * container row already pointing at `pathPrefix`; the next attempt's + * `deletePreviousLeafExpansion` reclaims the whole `raw/` shell. This + * function therefore does not clean up on failure — the retry-level reclaimer does, + * and it also survives a hard kill this local cleanup could not. + */ +export async function expandDirectoryOwnerToTree( + owner: KnowledgeItem, + baseId: string, + pathPrefix: string, + signal: AbortSignal +): Promise { + if (owner.type !== 'directory') { + throw new Error(`Knowledge item '${owner.id}' must be type 'directory', received '${owner.type}'`) + } + + const resolvedPath = path.resolve(owner.data.source) + const children = await readDirectoryTree(resolvedPath, signal) + const expandedChildren: ExpandedDirectoryNode[] = [] - return { pathPrefix, children: expandedChildren } - } catch (error) { - await removeDir(getKnowledgeBaseFilePath(baseId, pathPrefix)).catch((cleanupError) => { - logger.warn('Failed to clean up partially expanded directory materials', { baseId, pathPrefix, cleanupError }) - }) - throw error + for (const child of children) { + const expandedChild = await expandDirectoryNode(baseId, pathPrefix, child, signal) + if (expandedChild) { + expandedChildren.push(expandedChild) + } } + + return expandedChildren } diff --git a/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts b/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts index 1e015950506..02d32db8810 100644 --- a/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts +++ b/src/main/features/knowledge/tasks/__tests__/jobHandlerTestUtils.ts @@ -1,4 +1,5 @@ import type { JobContext } from '@main/core/job/types' +import type * as FsUtils from '@main/utils/file/fs' import type { JobSnapshot } from '@shared/data/api/schemas/jobs' import type { KnowledgeBase, KnowledgeItemOf } from '@shared/data/types/knowledge' import { MockMainCacheServiceUtils } from '@test-mocks/main/CacheService' @@ -37,7 +38,8 @@ const mocks = vi.hoisted(() => ({ listExistingEmbeddingHashesMock: vi.fn(), embedKnowledgeTextsMock: vi.fn(), loggerWarnMock: vi.fn(), - scheduleItemMock: vi.fn() + scheduleItemMock: vi.fn(), + removeDirMock: vi.fn() })) export const { @@ -70,7 +72,8 @@ export const { listExistingEmbeddingHashesMock, embedKnowledgeTextsMock, loggerWarnMock, - scheduleItemMock + scheduleItemMock, + removeDirMock } = mocks /** @@ -168,6 +171,16 @@ vi.mock('../../pathStorage', async () => { } }) +// prepare-root reclaims the container's own `raw/` shell via removeDir on retry; +// spy on it while keeping every other fs helper real. Default no-op resolve in beforeEach. +vi.mock('@main/utils/file/fs', async () => { + const actual = await vi.importActual('@main/utils/file/fs') + return { + ...actual, + removeDir: removeDirMock + } +}) + vi.mock('../../pipeline/indexing/embed', () => ({ embedKnowledgeTexts: embedKnowledgeTextsMock })) @@ -392,6 +405,7 @@ beforeEach(() => { knowledgeItemUpdateIndexedRelativePathMock.mockReturnValue(createFileItem()) deleteItemsByIdsMock.mockReturnValue(undefined) deleteKnowledgeItemFilesBestEffortMock.mockResolvedValue(undefined) + removeDirMock.mockResolvedValue(undefined) probeKnowledgeFileMock.mockResolvedValue('readable') probeKnowledgeSourcePathMock.mockResolvedValue('readable') cancelMock.mockResolvedValue({ outcome: 'cancelled' }) diff --git a/src/main/features/knowledge/tasks/__tests__/prepareItem.test.ts b/src/main/features/knowledge/tasks/__tests__/prepareItem.test.ts index db062ed5860..a93ac25fdf3 100644 --- a/src/main/features/knowledge/tasks/__tests__/prepareItem.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/prepareItem.test.ts @@ -2,6 +2,7 @@ import type { KnowledgeItem } from '@shared/data/types/knowledge' import { beforeEach, describe, expect, it, vi } from 'vitest' const { + chooseDirectoryPathPrefixMock, expandDirectoryOwnerToTreeMock, knowledgeItemCreateActiveMock, knowledgeItemUpdateStatusMock, @@ -9,6 +10,7 @@ const { knowledgeItemGetItemsByBaseIdMock, loggerWarnMock } = vi.hoisted(() => ({ + chooseDirectoryPathPrefixMock: vi.fn(), expandDirectoryOwnerToTreeMock: vi.fn(), knowledgeItemCreateActiveMock: vi.fn(), knowledgeItemUpdateStatusMock: vi.fn(), @@ -35,6 +37,7 @@ vi.mock('@logger', () => ({ })) vi.mock('../../pipeline/sources/directory', () => ({ + chooseDirectoryPathPrefix: chooseDirectoryPathPrefixMock, expandDirectoryOwnerToTree: expandDirectoryOwnerToTreeMock })) @@ -102,7 +105,8 @@ describe('prepareKnowledgeItem', () => { beforeEach(() => { vi.clearAllMocks() - expandDirectoryOwnerToTreeMock.mockResolvedValue({ pathPrefix: 'docs', children: [] }) + chooseDirectoryPathPrefixMock.mockReturnValue('docs') + expandDirectoryOwnerToTreeMock.mockResolvedValue([]) knowledgeItemGetItemsByBaseIdMock.mockReturnValue([]) knowledgeItemCreateActiveMock.mockImplementation((_baseId: string, item: Partial) => ({ id: `${item.type}-created`, @@ -144,27 +148,33 @@ describe('prepareKnowledgeItem', () => { const childDir = createDirectoryItem('dir-child', root.id) const childFile = createFileItem('file-child', childDir.id) knowledgeItemCreateActiveMock.mockReturnValueOnce(childDir).mockReturnValueOnce(childFile) - expandDirectoryOwnerToTreeMock.mockResolvedValueOnce({ - pathPrefix: 'dir-root-prefix', - children: [ - { - type: 'directory', - data: childDir.data, - children: [ - { - type: 'file', - data: childFile.data - } - ] - } - ] - }) + chooseDirectoryPathPrefixMock.mockReturnValueOnce('dir-root-prefix') + expandDirectoryOwnerToTreeMock.mockResolvedValueOnce([ + { + type: 'directory', + data: childDir.data, + children: [ + { + type: 'file', + data: childFile.data + } + ] + } + ]) const options = createPrepareOptions(root) await expect(prepareKnowledgeItem(options)).resolves.toEqual([childFile]) - expect(expandDirectoryOwnerToTreeMock).toHaveBeenCalledWith(root, baseId, expect.any(Set), options.signal) + expect(expandDirectoryOwnerToTreeMock).toHaveBeenCalledWith(root, baseId, 'dir-root-prefix', options.signal) expect(knowledgeItemUpdateDirectoryRelativePathMock).toHaveBeenCalledWith(root.id, 'dir-root-prefix') + // The container prefix must be pinned BEFORE any byte is copied (expansion) or any child + // row is created, so a mid-expansion crash leaves the pinned row for the retry to reclaim. + expect(knowledgeItemUpdateDirectoryRelativePathMock.mock.invocationCallOrder[0]).toBeLessThan( + expandDirectoryOwnerToTreeMock.mock.invocationCallOrder[0] + ) + expect(knowledgeItemUpdateDirectoryRelativePathMock.mock.invocationCallOrder[0]).toBeLessThan( + knowledgeItemCreateActiveMock.mock.invocationCallOrder[0] + ) expect(knowledgeItemCreateActiveMock).toHaveBeenNthCalledWith(1, baseId, { groupId: root.id, type: 'directory', @@ -188,13 +198,12 @@ describe('prepareKnowledgeItem', () => { root.data = { source: '/abs/docs', relativePath: 'docs' } const sibling = createFileItem('file-sibling') // top-level relativePath `file-sibling.md` knowledgeItemGetItemsByBaseIdMock.mockReturnValueOnce([root, sibling]) - expandDirectoryOwnerToTreeMock.mockResolvedValueOnce({ - pathPrefix: 'docs', - children: [{ type: 'file', data: { source: '/abs/docs/a.md', relativePath: 'docs/a.md' } }] - }) + expandDirectoryOwnerToTreeMock.mockResolvedValueOnce([ + { type: 'file', data: { source: '/abs/docs/a.md', relativePath: 'docs/a.md' } } + ]) return prepareKnowledgeItem(createPrepareOptions(root)).then(() => { - const reserved = expandDirectoryOwnerToTreeMock.mock.calls[0][2] as Set + const reserved = chooseDirectoryPathPrefixMock.mock.calls[0][1] as Set expect(reserved.has('docs')).toBe(false) // Sibling names are still reserved — only the container under reindex is exempt. expect(reserved.has('file-sibling.md')).toBe(true) @@ -203,7 +212,8 @@ describe('prepareKnowledgeItem', () => { it('marks empty directory roots failed and returns no leaves', async () => { const root = createDirectoryItem('dir-root') - expandDirectoryOwnerToTreeMock.mockResolvedValueOnce({ pathPrefix: 'dir-root', children: [] }) + chooseDirectoryPathPrefixMock.mockReturnValueOnce('dir-root') + expandDirectoryOwnerToTreeMock.mockResolvedValueOnce([]) await expect(prepareKnowledgeItem(createPrepareOptions(root))).resolves.toEqual([]) @@ -215,7 +225,9 @@ describe('prepareKnowledgeItem', () => { expect(knowledgeItemUpdateStatusMock).toHaveBeenCalledWith(root.id, 'failed', { error: 'Directory contains no indexable files' }) - expect(knowledgeItemUpdateDirectoryRelativePathMock).not.toHaveBeenCalled() + // Pin now precedes the empty check (pin-first crash-safety), so an empty dir still + // reserves its top-level name — a failure-path change we accept as more correct. + expect(knowledgeItemUpdateDirectoryRelativePathMock).toHaveBeenCalledWith(root.id, 'dir-root') }) it('stops creating children when the runtime signal is aborted after expansion', async () => { @@ -224,18 +236,15 @@ describe('prepareKnowledgeItem', () => { const abortError = new Error('interrupted') expandDirectoryOwnerToTreeMock.mockImplementationOnce(async () => { controller.abort(abortError) - return { - pathPrefix: 'dir-root', - children: [ - { - type: 'file', - data: { - source: '/docs/file-child.md', - relativePath: 'file-child.md' - } + return [ + { + type: 'file', + data: { + source: '/docs/file-child.md', + relativePath: 'file-child.md' } - ] - } + } + ] }) await expect( diff --git a/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts b/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts index 89dd32e1f46..07d6109cb00 100644 --- a/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts +++ b/src/main/features/knowledge/tasks/__tests__/prepareRootJobHandler.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest' +import { getKnowledgeBaseFilePath } from '../../pathStorage' import { createCtx, createDirectoryItem, @@ -15,9 +16,16 @@ import { knowledgeItemUpdateStatusMock, knowledgeLockManager, prepareKnowledgeItemMock, + removeDirMock, scheduleItemMock } from './jobHandlerTestUtils' +/** A directory container whose `raw/` was pinned by a prior attempt (pin-before-copy). */ +function createPinnedDirectoryItem(relativePath: string) { + const container = createDirectoryItem() + return { ...container, data: { ...container.data, relativePath } } +} + describe('prepare-root job handler', () => { it('clears stale expansion and schedules recreated leaves', async () => { const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService) @@ -82,11 +90,54 @@ describe('prepare-root job handler', () => { expect(deleteItemsByIdsMock).not.toHaveBeenCalledWith('kb-1', expect.arrayContaining(['deleting-note'])) }) + it('reclaims the container pinned prefix shell even when no descendant rows exist to purge', async () => { + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService) + // A crash in the pin-before-copy window leaves orphan bytes under raw/docs but zero + // descendant rows. getSubtreeItems excludes the container, so the row-scoped purge sees + // nothing — only the explicit container removeDir can reclaim those orphan bytes. + knowledgeItemGetByIdMock.mockReturnValue(createPinnedDirectoryItem('docs')) + knowledgeItemGetSubtreeItemsMock.mockReturnValue([]) + + await handler.execute(createCtx({ baseId: 'kb-1', itemId: 'dir-1' }, 'prepare-job')) + + expect(deleteItemsByIdsMock).not.toHaveBeenCalled() + expect(removeDirMock).toHaveBeenCalledWith(getKnowledgeBaseFilePath('kb-1', 'docs')) + }) + + it('does not reclaim a container shell when no prefix was ever pinned', async () => { + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService) + // Default createDirectoryItem() carries data.source only, no relativePath — the guard + // must skip removeDir so a never-expanded directory does not blow away an unrelated path. + knowledgeItemGetByIdMock.mockReturnValue(createDirectoryItem()) + knowledgeItemGetSubtreeItemsMock.mockReturnValue([]) + + await handler.execute(createCtx({ baseId: 'kb-1', itemId: 'dir-1' }, 'prepare-job')) + + expect(removeDirMock).not.toHaveBeenCalled() + }) + + it('purges descendant rows before reclaiming the container shell', async () => { + const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService) + const activeChild = createNoteItem('active-note', 'dir-1') + knowledgeItemGetByIdMock.mockReturnValue(createPinnedDirectoryItem('docs')) + knowledgeItemGetSubtreeItemsMock.mockReturnValue([activeChild]) + + await handler.execute(createCtx({ baseId: 'kb-1', itemId: 'dir-1' }, 'prepare-job')) + + expect(deleteItemsByIdsMock).toHaveBeenCalledWith('kb-1', ['active-note']) + expect(removeDirMock).toHaveBeenCalledWith(getKnowledgeBaseFilePath('kb-1', 'docs')) + // Row-scoped purge runs first; the container removeDir then sweeps whatever the purge + // could not see (nested dir shells, bytes of rows that were never committed). + expect(deleteItemsByIdsMock.mock.invocationCallOrder[0]).toBeLessThan(removeDirMock.mock.invocationCallOrder[0]) + }) + it('skips expansion when the root becomes deleting inside the mutation lock', async () => { const handler = createPrepareRootJobHandler(knowledgeLockManager as never, ingestionService) + // Live on the pre-lock load, then deleting for every in-lock re-resolve (the shell-reclaim + // guard and the expansion guard both re-read the row); expansion must be skipped. knowledgeItemGetByIdMock .mockReturnValueOnce(createDirectoryItem()) - .mockReturnValueOnce(createDirectoryItem('dir-1', 'deleting')) + .mockReturnValue(createDirectoryItem('dir-1', 'deleting')) const ctx = createCtx({ baseId: 'kb-1', itemId: 'dir-1' }, 'prepare-job') await handler.execute(ctx) diff --git a/src/main/features/knowledge/tasks/prepareItem.ts b/src/main/features/knowledge/tasks/prepareItem.ts index a562091b62f..2ae2d1d6f62 100644 --- a/src/main/features/knowledge/tasks/prepareItem.ts +++ b/src/main/features/knowledge/tasks/prepareItem.ts @@ -9,7 +9,11 @@ import { import { type IndexableKnowledgeItem, isIndexableKnowledgeItem } from '../items' import { collectKnowledgeReservedRelativePaths } from '../pathStorage' -import { expandDirectoryOwnerToTree, type ExpandedDirectoryNode } from '../pipeline/sources/directory' +import { + chooseDirectoryPathPrefix, + expandDirectoryOwnerToTree, + type ExpandedDirectoryNode +} from '../pipeline/sources/directory' const logger = loggerService.withContext('KnowledgePrepare') const EMPTY_DIRECTORY_ERROR = 'Directory contains no indexable files' @@ -42,7 +46,16 @@ async function prepareDirectoryForRuntime( // Exclude this container itself: on reindex it already owns its `relativePath` // prefix, and counting it as reserved would self-collide it to `_1` every time. const reservedTopLevelNames = collectReservedTopLevelNames(baseId, item.id) - const { pathPrefix, children } = await expandDirectoryOwnerToTree(item, baseId, reservedTopLevelNames, signal) + const pathPrefix = chooseDirectoryPathPrefix(item, reservedTopLevelNames) + + // Pin the deduped `raw/` prefix onto the container BEFORE any byte is copied. Expansion + // durably writes files under `raw//...`; if a crash/kill/abort lands mid-copy, + // the pinned `relativePath` lets the next attempt's `deletePreviousLeafExpansion` reclaim + // the whole shell (orphan bytes are always a subset of `raw/`). The UI also + // shows the on-disk name (e.g. `docs_2`) and delete removes the shell by it. + knowledgeItemService.updateDirectoryRelativePath(item.id, pathPrefix) + + const children = await expandDirectoryOwnerToTree(item, baseId, pathPrefix, signal) signal.throwIfAborted() if (children.length === 0) { @@ -55,10 +68,6 @@ async function prepareDirectoryForRuntime( return [] } - // Pin the deduped `raw/` prefix the children were stored under onto the container, so the - // UI shows the on-disk name (e.g. `docs_2`) and delete can remove the whole shell by it. - knowledgeItemService.updateDirectoryRelativePath(item.id, pathPrefix) - return await createDirectoryChildren(baseId, item.id, children, signal) } diff --git a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts index 5a90fbd118d..6750bf5c58a 100644 --- a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts +++ b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts @@ -5,11 +5,13 @@ import { knowledgeItemService } from '@data/services/KnowledgeItemService' import { loggerService } from '@logger' import type { KeyedMutex } from '@main/core/concurrency/KeyedMutex' import type { JobContext, JobHandler } from '@main/core/job/types' +import { removeDir } from '@main/utils/file/fs' import type { KnowledgeItem } from '@shared/data/types/knowledge' import type { KnowledgeItemScheduler } from '../ingestion/KnowledgeIngestionService' import { markUnscheduledKnowledgeItemsFailed } from '../ingestion/statusCleanup' import { purgeKnowledgeSubtreeWithinLock } from '../ingestion/subtreePurge' +import { getKnowledgeBaseFilePath } from '../pathStorage' import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId, toKnowledgeItemId } from '../types' import type { KnowledgePrepareRootPayload } from './jobTypes' import { prepareKnowledgeItem } from './prepareItem' @@ -103,6 +105,19 @@ async function deletePreviousLeafExpansion( const descendants = knowledgeItemService.getSubtreeItems(baseId, [itemId]) const removableDescendants = descendants.filter((item) => item.status !== 'deleting') await purgeKnowledgeSubtreeWithinLock(base, removableDescendants, { baseId, itemId }) + + // `getSubtreeItems` excludes the container row, so the purge above never touches the + // container's own `raw/` shell. A prior attempt pins `relativePath` before + // copying any byte (see prepareDirectoryForRuntime), so if orphan bytes exist the row + // records their prefix — reclaim the whole shell here. removeDir is idempotent (ENOENT + // no-op) when the shell was never created. + const result = resolveLiveKnowledgeItem(itemId) + if ('item' in result && result.item.type === 'directory') { + const prefix = result.item.data.relativePath + if (prefix) { + await removeDir(getKnowledgeBaseFilePath(baseId, prefix)) + } + } }) } From e00a160373ca5a0b128d5b5d4c02164ec0fbf23d Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Sun, 5 Jul 2026 16:01:17 +0800 Subject: [PATCH 35/37] fix(knowledge): terminate scoped job cancellation when a full page misses the subtree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cancelActiveKnowledgeJobs drained the active-job queue by re-listing a full page until the raw page length dropped below KNOWLEDGE_ACTIVE_JOB_LIMIT. For a scoped (subtree) cancel, the subtree filter runs only in memory, so a full page whose every job misses the scope cancelled nothing, never shrank the active set, and re-queried the same page forever — hanging delete-subtree/replace-on-add and busy-spinning a core. Break when the page comes back short OR when a full page yielded nothing cancellable this round, so a scoped cancel with no in-scope jobs stops instead of looping. Base-wide drain behavior is preserved (a full page of matching jobs still loops until drained). Adds a regression test that returns the same all-miss full page on every list() call — it hangs on the old code and asserts a single list() with no cancels on the new. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: eeee0717 --- .../tasks/utils/__tests__/cancel.test.ts | 24 +++++++++++++++++++ .../features/knowledge/tasks/utils/cancel.ts | 8 ++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts b/src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts index 880274201ce..983610f41ba 100644 --- a/src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts +++ b/src/main/features/knowledge/tasks/utils/__tests__/cancel.test.ts @@ -196,6 +196,30 @@ describe('cancelActiveKnowledgeJobs', () => { expect(cancelMock).toHaveBeenCalledTimes(KNOWLEDGE_ACTIVE_JOB_LIMIT) }) + it('terminates a full page of scoped-out jobs instead of re-querying it forever', async () => { + knowledgeItemGetSubtreeItemsMock.mockReturnValue([createItem('target')]) + // A full page whose every job touches an item OUTSIDE the requested subtree: nothing + // is cancellable, so the active set never shrinks. `mockResolvedValue` (not `Once`) + // returns the same page on every call, so a loop that only breaks on a short page + // would spin forever here — the guard must stop on the first no-progress round. + const fullPageOfMisses = Array.from({ length: KNOWLEDGE_ACTIVE_JOB_LIMIT }, (_, index) => + createJobSnapshot({ + id: `index-job-${index}`, + type: 'knowledge.index-documents', + input: { baseId: 'kb-1', itemId: `other-${index}` } + }) + ) + listMock.mockResolvedValue(fullPageOfMisses) + + await cancelActiveKnowledgeJobs('kb-1', 'knowledge-delete-subtree', { + rootItemIds: ['target'], + onCancelTimeout: 'throw' + }) + + expect(listMock).toHaveBeenCalledTimes(1) + expect(cancelMock).not.toHaveBeenCalled() + }) + it('stops after a single empty page without cancelling anything', async () => { listMock.mockResolvedValueOnce([]) diff --git a/src/main/features/knowledge/tasks/utils/cancel.ts b/src/main/features/knowledge/tasks/utils/cancel.ts index 30fe6d5fbbb..90c161e4a54 100644 --- a/src/main/features/knowledge/tasks/utils/cancel.ts +++ b/src/main/features/knowledge/tasks/utils/cancel.ts @@ -78,7 +78,13 @@ export async function cancelActiveKnowledgeJobs( await Promise.all([...jobIds, ...fileProcessingJobIds].map(cancelOne)) - if (activeJobs.length < KNOWLEDGE_ACTIVE_JOB_LIMIT) break + // Stop when the page came back short (nothing more to drain) or when a full page + // yielded nothing cancellable this round. The short-page check alone is not enough: + // a scoped cancel whose entire full page misses the subtree cancels nothing, so the + // active set never shrinks and the same page would be re-queried forever. + if (activeJobs.length < KNOWLEDGE_ACTIVE_JOB_LIMIT || jobIds.length + fileProcessingJobIds.length === 0) { + break + } } } From 4ed524c1e10dafe0e082996a88a3264f1700688b Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Sun, 5 Jul 2026 16:01:42 +0800 Subject: [PATCH 36/37] docs(knowledge): sync drifted references after the knowledge structure refactor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This branch dropped the idle item status and the KnowledgeLockManager class (the per-base lock is now a core KeyedMutex instance created by the KnowledgeService facade) and moved several helpers/methods, but prose and comments still referenced the old shapes. - Remove idle from status lists (knowledge-service/workflow-architecture/ operation-guards) and code comments; the migrator maps no-uniqueId items to failed (never_indexed), so its README now says so instead of idle. - Replace KnowledgeLockManager references with the concrete per-base mutation lock / KeyedMutex.runExclusive wording, drop the deleted withBaseMutationLock method reference, and correct the base/-owns-the-lock factual error (the facade creates it and shares it). - Fix stale symbol references: toKnowledgeRelativePath -> toMaterialRelativePath, KnowledgeService.recoverInterruptedItems -> KnowledgeIngestionService, and move the material-path derivation note from pipeline/indexing/ to items.ts in the feature README directory map. - Correct the feature README query/ role: it is not purely a read side — the Concept ID tool surface (KnowledgeConceptService) also exposes kb_manage delete/refresh writes that delegate to ingestion/. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: eeee0717 --- .../experiment/knowledge-technical-design.md | 4 ++-- docs/references/knowledge/knowledge-service.md | 9 ++++----- docs/references/knowledge/operation-guards.md | 2 +- docs/references/knowledge/workflow-architecture.md | 8 ++++---- .../data/migration/v2/migrators/KnowledgeMigrator.ts | 2 +- .../v2/migrators/README-KnowledgeMigrator.md | 4 ++-- src/main/data/services/KnowledgeItemService.ts | 6 +++--- src/main/features/knowledge/README.md | 12 ++++++------ src/shared/data/types/knowledge.ts | 6 +++--- 9 files changed, 26 insertions(+), 27 deletions(-) diff --git a/docs/references/knowledge/experiment/knowledge-technical-design.md b/docs/references/knowledge/experiment/knowledge-technical-design.md index 675f7a4a3a7..ae31fb0c15b 100644 --- a/docs/references/knowledge/experiment/knowledge-technical-design.md +++ b/docs/references/knowledge/experiment/knowledge-technical-design.md @@ -149,7 +149,7 @@ A vector base's `knowledge_base.embeddingModelId` / `dimensions` must be valid t 1. Relational tables use generic SQLite DDL only; FTS5 is built into both engines; CJK handling lives in the application layer. 2. **Decision A1**: the canonical vector storage is a plain `BLOB` column holding little-endian float32 bytes (a generic SQLite BLOB, not any engine-proprietary vector type); it is the source of truth and both engines read the same bytes. 3. First-version vector retrieval is a brute-force scan over the canonical BLOBs (sqlite-vec's `vec_distance_cosine(col, ?)`, binding the query vector as raw little-endian float32 BLOB bytes — no `vector32()` wrapper), exposed through the `VectorIndex` adapter; **no** vec0 / ANN derived index (left as a purely additive change after performance evaluation). -4. A thin `SqliteDriver` port (execute / transaction / close) so the store is written once; better-sqlite3 keeps one synchronous, persistent connection — PRAGMAs (e.g. WAL) are set once and persist — and per-base writes are serialized by `KnowledgeLockManager.withBaseMutationLock(baseId)`, so the driver carries no write mutex or busy-retry of its own. +4. A thin `SqliteDriver` port (execute / transaction / close) so the store is written once; better-sqlite3 keeps one synchronous, persistent connection — PRAGMAs (e.g. WAL) are set once and persist — and per-base writes are serialized by the per-base mutation lock (`KeyedMutex.runExclusive`), so the driver carries no write mutex or busy-retry of its own. ## 6. Retrieval @@ -184,6 +184,6 @@ A vector base's `knowledge_base.embeddingModelId` / `dimensions` must be valid t - An explicit `maxParallelCalls` (plus token-aware batching) for `AiService.embedMany`, so one large document cannot fan out unbounded batches, exceed provider per-request token limits, and discard embeddings already paid for in a failed attempt. - Startup-recovery cross-cancellation: a crash-recovered delete-subtree job and the `recoverDeletingItems` re-enqueue get different idempotency keys and cancel each other via roots-intersection (`jobTouchesSubtree`); cancel only jobs whose roots are fully covered by the current job's roots. - Hybrid search runs its two lanes as independent read snapshots; a rebuild committing between them can transiently return both copies of a chunk — close with a shared read transaction or a second dedupe by material id + unit index. - - The per-base index driver's `close()` no longer needs to take a write mutex: better-sqlite3 uses one synchronous, persistent connection and per-base writes are serialized by `KnowledgeLockManager.withBaseMutationLock(baseId)`, so there is no driver-level async write mutex to race — shutdown safety rests on JobManager draining before the store service stops. + - The per-base index driver's `close()` no longer needs to take a write mutex: better-sqlite3 uses one synchronous, persistent connection and per-base writes are serialized by the per-base mutation lock (`KeyedMutex.runExclusive`), so there is no driver-level async write mutex to race — shutdown safety rests on JobManager draining before the store service stops. - Retrieval-surface follow-ups (PR C): the `searchMode` `default`→`vector` rename is externally visible through the gateway's pass-through base entity, and a permanent open failure (legacy layout) currently maps to a retryable 503. - PR A's full test matrix and risk notes live in this repo's test suites (`src/main/features/knowledge/**/__tests__`) and the PR #15973 description. diff --git a/docs/references/knowledge/knowledge-service.md b/docs/references/knowledge/knowledge-service.md index 9a2301c512c..64d8be02b6d 100644 --- a/docs/references/knowledge/knowledge-service.md +++ b/docs/references/knowledge/knowledge-service.md @@ -22,13 +22,13 @@ The current implementation is split into four responsibility areas: - Do not perform vector-store mutations. 3. `KnowledgeService` - Thin lifecycle facade: registers Knowledge JobManager handlers, runs boot recovery, and delegates every public method to `base/`, `ingestion/`, and `query/`. - - `base/` (`KnowledgeBaseAdminService`) creates/deletes/restores bases through data services and vector store services, and owns the per-base `KnowledgeLockManager`. + - `base/` (`KnowledgeBaseAdminService`) creates/deletes/restores bases through data services and vector store services. The per-base mutation lock (a core `KeyedMutex`) is created by the `KnowledgeService` facade and shared with `base/`, `ingestion/`, and the job handlers. - `ingestion/` (`KnowledgeIngestionService`) collapses delete/reindex item inputs to top-level roots, enforces runtime guards, and schedules the next workflow step. - `query/` (`KnowledgeQueryService` / `KnowledgeConceptService`) serves search and the Concept ID tool surface. 4. Knowledge job handlers - Execute durable workflow stages through JobManager. - Use `KnowledgeIngestionService` for next-step scheduling. - - Use `KnowledgeLockManager` for same-base mutations and vector cleanup. + - Use the per-base mutation lock (`KeyedMutex.runExclusive`) for same-base mutations and vector cleanup. ```text caller @@ -42,7 +42,7 @@ caller -> JobManager -> knowledge.prepare-root / knowledge.index-documents -> knowledge.delete-subtree / knowledge.reindex-subtree - -> KnowledgeLockManager + -> KeyedMutex.runExclusive -> KnowledgeBaseService / KnowledgeItemService -> KnowledgeVectorStoreService ``` @@ -131,11 +131,10 @@ Knowledge runtime work is persisted in JobManager. `KnowledgeService.onInit` reg - `knowledge.delete-subtree` - `knowledge.reindex-subtree` -Each base uses queue `base.${baseId}`. JobManager owns queue persistence, dispatch, retry, cancellation, timeout, and startup recovery. Knowledge code uses `KnowledgeLockManager` to serialize same-base vector and item mutations inside the current process. +Each base uses queue `base.${baseId}`. JobManager owns queue persistence, dispatch, retry, cancellation, timeout, and startup recovery. Knowledge code uses the per-base mutation lock (`KeyedMutex.runExclusive`) to serialize same-base vector and item mutations inside the current process. Current item statuses are: -- `idle` - `preparing` - `processing` - `reading` diff --git a/docs/references/knowledge/operation-guards.md b/docs/references/knowledge/operation-guards.md index 4ddd91bbfea..4e7c00b6006 100644 --- a/docs/references/knowledge/operation-guards.md +++ b/docs/references/knowledge/operation-guards.md @@ -59,7 +59,7 @@ Used only by `reindexItems`. - Runs after selected item ids have been collapsed to top-level roots. - Loads each selected root subtree with roots included. - Allows reindex only when every item in every selected subtree is terminal: `completed` or `failed`. -- Rejects active or deleting subtree state: `idle`, `preparing`, `processing`, `reading`, `embedding`, or `deleting`. +- Rejects active or deleting subtree state: `preparing`, `processing`, `reading`, `embedding`, or `deleting`. This is the backend authority for user-triggered reindex. UI may hide the reindex action for non-terminal rows, but the service guard must still reject stale or direct calls. diff --git a/docs/references/knowledge/workflow-architecture.md b/docs/references/knowledge/workflow-architecture.md index 206e17bf152..01a0320e872 100644 --- a/docs/references/knowledge/workflow-architecture.md +++ b/docs/references/knowledge/workflow-architecture.md @@ -11,14 +11,14 @@ API / user action -> KnowledgeIngestionService -> JobManager -> Knowledge job handlers - -> KnowledgeLockManager + -> KeyedMutex.runExclusive -> SQLite / index store / knowledge-owned files (raw/) ``` The design keeps three owners: - `KnowledgeIngestionService` decides the next workflow step. -- `KnowledgeLockManager` serializes same-base mutations and cleanup. +- The per-base mutation lock (`KeyedMutex.runExclusive`) serializes same-base mutations and cleanup. - Knowledge job handlers execute one durable stage and call the workflow service for the next step. Helpers may own source planning, lifecycle writes, knowledge-owned raw files, and FileProcessing adaptation. They should stay as modules until they need lifecycle-managed resources, IPC, timers, or long-lived state. @@ -78,10 +78,10 @@ Round 1 job types: ## Mutation And Crash Semantics -Same-base Knowledge mutations must go through `KnowledgeLockManager`. Main SQLite writes still use a synchronous write transaction (`DbService.withWriteTx`, or an equivalent `db.transaction()`); the lock manager is not a replacement for that write transaction. +Same-base Knowledge mutations must go through the per-base mutation lock (`KeyedMutex.runExclusive`). Main SQLite writes still use a synchronous write transaction (`DbService.withWriteTx`, or an equivalent `db.transaction()`); the mutation lock is not a replacement for that write transaction. Crash safety comes from durable jobs, durable item states, JobManager recovery, and idempotent cleanup. The in-memory mutation lock only serializes concurrent work in the current process. Delete and reindex span two stores: the main SQLite database and the per-base vector store. They cannot be one cross-store transaction. Consistency relies on durable re-entry and idempotent vector/artifact/row cleanup. -User-triggered reindex is not a cancellation primitive. The service admits reindex only when the entire selected subtree is already `completed` or `failed`. Active states (`idle`, `preparing`, `processing`, `reading`, `embedding`) and `deleting` are rejected; delete remains the operation that can be requested at any time. +User-triggered reindex is not a cancellation primitive. The service admits reindex only when the entire selected subtree is already `completed` or `failed`. Active states (`preparing`, `processing`, `reading`, `embedding`) and `deleting` are rejected; delete remains the operation that can be requested at any time. diff --git a/src/main/data/migration/v2/migrators/KnowledgeMigrator.ts b/src/main/data/migration/v2/migrators/KnowledgeMigrator.ts index 5d8051e1989..d45657180b4 100644 --- a/src/main/data/migration/v2/migrators/KnowledgeMigrator.ts +++ b/src/main/data/migration/v2/migrators/KnowledgeMigrator.ts @@ -609,7 +609,7 @@ export class KnowledgeMigrator extends BaseMigrator { // A resolved embedding model whose per-base legacy vector store is missing/empty/locked // yields dimensions===null. We must NOT drop the base (that loses the library with no // recoverable row): keep it as a `failed` row, like the dangling-model branch below, so - // the name/model/config/idle items survive and the UI offers a restore/re-index entry. + // the name/model/config/unindexed items survive and the UI offers a restore/re-index entry. // `vectorsWillMigrate` also gates directory expansion: a base whose vectors will not // migrate must not expand folders into `completed` children that would be empty shells. const vectorStoreUnresolved = embeddingResolution.kind === 'resolved' && resolvedDimensions.dimensions === null diff --git a/src/main/data/migration/v2/migrators/README-KnowledgeMigrator.md b/src/main/data/migration/v2/migrators/README-KnowledgeMigrator.md index f54e4b7b228..9f5dc7ad28b 100644 --- a/src/main/data/migration/v2/migrators/README-KnowledgeMigrator.md +++ b/src/main/data/migration/v2/migrators/README-KnowledgeMigrator.md @@ -52,7 +52,7 @@ - Legacy `processingStatus` is treated as runtime-only and not trusted for migration. - Item status is inferred from `uniqueId`: - `uniqueId` present and non-empty -> `completed` - - otherwise -> `idle` + - otherwise -> `failed` (`never_indexed`) 6. Vector dimension dependency - Completed bases require a resolved positive `knowledge_base.dimensions` value. @@ -91,7 +91,7 @@ | _no legacy grouping field_ | `groupId` | V1 exports are flat; migrated items are inserted without grouping metadata (`null`) | | `type` | `type` | Supported target types: file/url/note/directory. Legacy sitemap maps to url. | | `content` + Dexie lookups | `data` | Type-specific transform | -| `uniqueId` | `status` | `uniqueId` non-empty => `completed`, otherwise `idle` | +| `uniqueId` | `status` | `uniqueId` non-empty => `completed`, otherwise `failed` (`never_indexed`) | | `processingError` | `error` | Direct copy | | `created_at` | `createdAt` | Timestamp conversion | | `updated_at` | `updatedAt` | Timestamp conversion | diff --git a/src/main/data/services/KnowledgeItemService.ts b/src/main/data/services/KnowledgeItemService.ts index 2c080f9206e..cea463b2b58 100644 --- a/src/main/data/services/KnowledgeItemService.ts +++ b/src/main/data/services/KnowledgeItemService.ts @@ -29,8 +29,8 @@ const logger = loggerService.withContext('DataApi:KnowledgeItemService') const CONTAINER_CHILD_FAILURE_ERROR = 'One or more child items failed' /** * Item statuses that mean "indexing is in progress" — the item is being driven - * by a job and is neither finished (completed/failed), idle (never started), nor - * being deleted. Used to find items stranded by an interrupted job on boot. + * by a job and is neither finished (completed/failed) nor being deleted. Used to + * find items stranded by an interrupted job on boot. */ const KNOWLEDGE_ITEM_IN_FLIGHT_STATUSES = [ 'preparing', @@ -218,7 +218,7 @@ export class KnowledgeItemService { * A single UPDATE covers leaves and their ancestor containers together: a * container with an in-flight descendant is itself in-flight * (`reconcileContainers` keeps it `processing` while any child is active), so - * failing every in-flight row leaves no completed/idle container pointing at a + * failing every in-flight row leaves no completed container pointing at a * failed child — the rollup invariant holds without a separate reconcile pass. * * @returns the number of items marked failed. diff --git a/src/main/features/knowledge/README.md b/src/main/features/knowledge/README.md index 627b323dc7b..a9bf5fcbe86 100644 --- a/src/main/features/knowledge/README.md +++ b/src/main/features/knowledge/README.md @@ -28,17 +28,17 @@ orchestration, and it lives in `ingestion/` and `tasks/`. | Directory | Role | | --- | --- | -| `KnowledgeService.ts` | Lifecycle facade: registers job handlers, runs boot recovery, delegates every public method. No domain logic. | -| `base/` | Per-base domain: lifecycle admin (`KnowledgeBaseAdminService` — create with rollback, delete, restore, list), failed-base guard (`baseGuards.ts`), per-base mutation lock (`KnowledgeLockManager`). | +| `KnowledgeService.ts` | Lifecycle facade: registers job handlers, runs boot recovery, delegates every public method, and creates the shared per-base mutation lock (`KeyedMutex`). No domain logic. | +| `base/` | Per-base domain: lifecycle admin (`KnowledgeBaseAdminService` — create with rollback, delete, restore, list), failed-base guard (`baseGuards.ts`). | | `ingestion/` | Write-side orchestration: admission checks, item creation, add-conflict resolution, job enqueueing, subtree purge (`subtreePurge.ts`), boot recovery. | | `pipeline/sources/` | Input stage: directory expansion, url fetch (Jina reader), url/note snapshot capture, OKF frontmatter. | | `pipeline/readers/` | Preprocess stage: file → markdown/text `Document[]` readers (pdf/docx/epub/…). | -| `pipeline/indexing/` | Index stage: offset-preserving splitter + chunker, `AiService` embedding/rerank wrappers, material field derivation. | +| `pipeline/indexing/` | Index stage: offset-preserving splitter + chunker, `AiService` embedding/rerank wrappers. | | `pipeline/vectorstore/` | Persist stage: per-base `index.sqlite` lifecycle (`KnowledgeVectorStoreService`), the store itself (`indexStore/`, synchronous better-sqlite3 driver), vector deletion + index space reclamation (`vectorCleanup.ts`). | -| `query/` | Read side: hybrid search with visibility filtering (`KnowledgeQueryService`), Concept ID tool surface (`KnowledgeConceptService`). | +| `query/` | Read side + Concept ID tool surface: hybrid search with visibility filtering (`KnowledgeQueryService`); Concept ID read/grep/tree plus the `kb_manage` delete/refresh writes, which delegate to `ingestion/` (`KnowledgeConceptService`). | | `tasks/` | Job handlers — the pipeline executors (see below); `prepareItem.ts` is a prepare-root handler-private helper that expands a directory root into child items. | | `pathStorage.ts` | `raw/` path allocation: collision-free names, reservation, base file paths. | -| `items.ts` / `types.ts` | Shared item vocabulary (type aliases, predicates, source probing); branded ids, queue names, idempotency keys. | +| `items.ts` / `types.ts` | Shared item vocabulary (type aliases, predicates, source probing, material-path derivation); branded ids, queue names, idempotency keys. | ## Jobs @@ -62,7 +62,7 @@ job runs. ## Concurrency -`KnowledgeLockManager` is a per-base **application-level** mutex serializing multi-step business +The per-base mutation lock is a core `KeyedMutex` (acquired via `runExclusive`), an **application-level** mutex serializing multi-step business invariants that span the main DB, the index store, and the filesystem (e.g. add's read-conflicts-then-create-rows sequence). It is not about protecting SQLite itself — the per-base driver is synchronous, and single statements are atomic. Handlers acquire the lock only around the diff --git a/src/shared/data/types/knowledge.ts b/src/shared/data/types/knowledge.ts index 59326c96fe9..599c406f280 100644 --- a/src/shared/data/types/knowledge.ts +++ b/src/shared/data/types/knowledge.ts @@ -77,7 +77,7 @@ export const DEFAULT_KNOWLEDGE_BASE_STATUS: KnowledgeBaseStatus = 'completed' // user_model, so the base needs a new embedding model on restore. // `missing_vector_store`: the embedding model resolved, but the per-base legacy vector store // was missing/empty/locked so its dimensions could not be determined. The base (name, model, -// config, idle items) is kept as a restorable `failed` row instead of being dropped, so the +// config, unindexed items) is kept as a restorable `failed` row instead of being dropped, so the // user can re-index it — a transient lock is recoverable by re-running rather than a data loss. export const KNOWLEDGE_BASE_ERROR_CODES = ['missing_embedding_model', 'missing_vector_store'] as const export const KnowledgeBaseErrorCodeSchema = z.enum(KNOWLEDGE_BASE_ERROR_CODES) @@ -90,7 +90,7 @@ export const KNOWLEDGE_BASE_ERROR_MISSING_VECTOR_STORE: KnowledgeBaseErrorCode = * - `directory_not_migrated`: a v1-indexed `directory` whose container-level vectors could not * be re-attributed to per-file children (unreadable legacy sources, or no migratable vectors). * - `indexing_interrupted`: an indexing job was abandoned by an app quit / restart, so the item - * was parked at `failed` instead of silently resumed (see KnowledgeService.recoverInterruptedItems). + * was parked at `failed` instead of silently resumed (see KnowledgeIngestionService.recoverInterruptedItems). * - `never_indexed`: a migrated v1 item never started indexing (no `uniqueId`), so there is * nothing to resume from and it is parked at `failed` instead of a synthetic in-progress state. * Modeled as a zod enum (the same shape as the base error codes above) so the renderer's @@ -291,7 +291,7 @@ const KnowledgeItemSharedSchema = z.strictObject({ */ export const FileItemDataSchema = KnowledgeItemSharedSchema.extend({ // relativePath / indexedRelativePath are always produced by main-side helpers - // (copyFileIntoKnowledgeBaseAt, toKnowledgeRelativePath, ...), never raw caller + // (copyFileIntoKnowledgeBaseAt, toMaterialRelativePath, ...), never raw caller // input. The base-relative, POSIX-normalized, no-traversal invariant is // enforced imperatively by assertSafeKnowledgeRelativePath at the filesystem // boundary (getKnowledgeBaseFilePath). This schema only validates shape, so a From 976ec1523d51b7e76cdf01078c29edee926e8ff8 Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Sun, 5 Jul 2026 16:13:18 +0800 Subject: [PATCH 37/37] refactor(knowledge): hoist isDataApiNotFoundError to the shared DataApi error helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit query/visibility.ts imported isDataApiNotFoundError from tasks/utils/settled.ts, a query -> tasks reverse dependency that broke the read side's downward-only layering. The predicate is a fully generic DataApi check (isDataApiError(e) && e.code === NOT_FOUND) with no task-specific logic, so it belongs next to its siblings isDataApiError / isRetryableErrorCode in @shared/data/api/apiErrors.ts rather than in a knowledge task util. Move it there (verbatim, still a boolean predicate — no speculative type-guard narrowing) and re-point all five consumers (settled, liveItem, prepareRoot and indexDocuments handlers, query/visibility) at @shared/data/api. query/ now depends only downward on @shared, closing the leak. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: eeee0717 --- src/main/features/knowledge/query/visibility.ts | 2 +- .../features/knowledge/tasks/indexDocumentsJobHandler.ts | 3 ++- .../features/knowledge/tasks/prepareRootJobHandler.ts | 3 ++- src/main/features/knowledge/tasks/utils/liveItem.ts | 3 +-- src/main/features/knowledge/tasks/utils/settled.ts | 6 +----- src/shared/data/api/apiErrors.ts | 9 +++++++++ src/shared/data/api/index.ts | 1 + 7 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/main/features/knowledge/query/visibility.ts b/src/main/features/knowledge/query/visibility.ts index 8764236c31f..79a021e8073 100644 --- a/src/main/features/knowledge/query/visibility.ts +++ b/src/main/features/knowledge/query/visibility.ts @@ -1,9 +1,9 @@ import { knowledgeItemService } from '@data/services/KnowledgeItemService' import { loggerService } from '@logger' +import { isDataApiNotFoundError } from '@shared/data/api' import type { KnowledgeItem } from '@shared/data/types/knowledge' import { toMaterialRelativePath } from '../items' -import { isDataApiNotFoundError } from '../tasks/utils/settled' const logger = loggerService.withContext('Knowledge:Query') diff --git a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts index 9e2d39b7d24..d86855f4404 100644 --- a/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts +++ b/src/main/features/knowledge/tasks/indexDocumentsJobHandler.ts @@ -6,6 +6,7 @@ import { knowledgeItemService } from '@data/services/KnowledgeItemService' import { loggerService } from '@logger' import type { KeyedMutex } from '@main/core/concurrency/KeyedMutex' import type { JobContext, JobHandler } from '@main/core/job/types' +import { isDataApiNotFoundError } from '@shared/data/api' import type { KnowledgeBase } from '@shared/data/types/knowledge' import { isCompletedVectorKnowledgeBase } from '@shared/data/types/knowledge' @@ -23,7 +24,7 @@ import type { RebuildMaterialEmbeddingInput, RebuildMaterialInput } from '../pip import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId } from '../types' import type { KnowledgeIndexDocumentsPayload } from './jobTypes' import { resolveLiveKnowledgeItem } from './utils/liveItem' -import { isDataApiNotFoundError, markKnowledgeItemFailedOnSettled } from './utils/settled' +import { markKnowledgeItemFailedOnSettled } from './utils/settled' const logger = loggerService.withContext('Knowledge:IndexDocumentsJobHandler') diff --git a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts index 6750bf5c58a..b4685876e90 100644 --- a/src/main/features/knowledge/tasks/prepareRootJobHandler.ts +++ b/src/main/features/knowledge/tasks/prepareRootJobHandler.ts @@ -6,6 +6,7 @@ import { loggerService } from '@logger' import type { KeyedMutex } from '@main/core/concurrency/KeyedMutex' import type { JobContext, JobHandler } from '@main/core/job/types' import { removeDir } from '@main/utils/file/fs' +import { isDataApiNotFoundError } from '@shared/data/api' import type { KnowledgeItem } from '@shared/data/types/knowledge' import type { KnowledgeItemScheduler } from '../ingestion/KnowledgeIngestionService' @@ -16,7 +17,7 @@ import { knowledgeQueueName, reportKnowledgeProgress, toKnowledgeBaseId, toKnowl import type { KnowledgePrepareRootPayload } from './jobTypes' import { prepareKnowledgeItem } from './prepareItem' import { resolveLiveKnowledgeItem } from './utils/liveItem' -import { isDataApiNotFoundError, markKnowledgeItemFailedOnSettled } from './utils/settled' +import { markKnowledgeItemFailedOnSettled } from './utils/settled' const logger = loggerService.withContext('Knowledge:PrepareRootJobHandler') diff --git a/src/main/features/knowledge/tasks/utils/liveItem.ts b/src/main/features/knowledge/tasks/utils/liveItem.ts index 1e883cab8a4..fa56d9d4e5a 100644 --- a/src/main/features/knowledge/tasks/utils/liveItem.ts +++ b/src/main/features/knowledge/tasks/utils/liveItem.ts @@ -1,8 +1,7 @@ import { knowledgeItemService } from '@data/services/KnowledgeItemService' +import { isDataApiNotFoundError } from '@shared/data/api' import type { KnowledgeItem } from '@shared/data/types/knowledge' -import { isDataApiNotFoundError } from './settled' - export type LiveKnowledgeItemSkipReason = 'missing' | 'deleting' export type LiveKnowledgeItemResult = { item: KnowledgeItem } | { skip: LiveKnowledgeItemSkipReason } diff --git a/src/main/features/knowledge/tasks/utils/settled.ts b/src/main/features/knowledge/tasks/utils/settled.ts index cebf774fdcc..bee2b8b27e1 100644 --- a/src/main/features/knowledge/tasks/utils/settled.ts +++ b/src/main/features/knowledge/tasks/utils/settled.ts @@ -1,13 +1,9 @@ import { knowledgeItemService } from '@data/services/KnowledgeItemService' import type { JobSettledEvent } from '@main/core/job/types' import type { LoggerService } from '@main/core/logger/LoggerService' -import { ErrorCode, isDataApiError } from '@shared/data/api' +import { isDataApiNotFoundError } from '@shared/data/api' import { KNOWLEDGE_ITEM_ERROR_INDEXING_INTERRUPTED } from '@shared/data/types/knowledge' -export function isDataApiNotFoundError(error: unknown): boolean { - return isDataApiError(error) && error.code === ErrorCode.NOT_FOUND -} - export async function markKnowledgeItemFailedOnSettled( event: JobSettledEvent<{ itemId: string }>, logger: LoggerService, diff --git a/src/shared/data/api/apiErrors.ts b/src/shared/data/api/apiErrors.ts index 9d437102682..d3fe8d605e1 100644 --- a/src/shared/data/api/apiErrors.ts +++ b/src/shared/data/api/apiErrors.ts @@ -806,6 +806,15 @@ export function isDataApiError(error: unknown): error is DataApiError { return error instanceof DataApiError } +/** + * Check if an error is a DataApiError carrying the NOT_FOUND code. + * @param error - Any thrown value + * @returns true if the error is a NOT_FOUND DataApiError + */ +export function isDataApiNotFoundError(error: unknown): boolean { + return isDataApiError(error) && error.code === ErrorCode.NOT_FOUND +} + /** * Check if an object is a serialized DataApiError. * @param error - Any object diff --git a/src/shared/data/api/index.ts b/src/shared/data/api/index.ts index a90c4e2c518..42664ab3b7d 100644 --- a/src/shared/data/api/index.ts +++ b/src/shared/data/api/index.ts @@ -80,6 +80,7 @@ export { DataApiError, DataApiErrorFactory, isDataApiError, + isDataApiNotFoundError, isSerializedDataApiError, toDataApiError } from './apiErrors'