Skip to content

Commit 7fd146a

Browse files
jamby77jamby77
committed
feat(agent-memory): add MemoryStore.ensureIndex() to bootstrap the FT index
The package queried {name}:mem:idx for recall/forget/consolidate/capacity but never created it, leaving the index as an undocumented caller responsibility (flagged in review on #249/#250). Spec calls for the package to own index create/version-skew handling. - buildMemoryIndex: HNSW + memory schema (vector, scope/tag TAGs, numeric tunables, content TEXT) as reusable FT.CREATE args - MemoryStore.ensureIndex(): idempotent FT.INFO -> FT.CREATE, resolving the vector dimension from embedFn when not yet observed - AgentMemory.initialize() creates the index so the facade is usable without a hand-rolled FT.CREATE - Integration suite dogfoods ensureIndex() instead of hand-rolling the index
1 parent ce94197 commit 7fd146a

7 files changed

Lines changed: 281 additions & 47 deletions

File tree

‎packages/agent-memory/src/AgentMemory.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,10 @@ export class AgentMemory {
6363
}
6464

6565
async initialize(): Promise<void> {
66+
// Create the memory index before discovery so a freshly constructed facade
67+
// is immediately usable for remember/recall without the caller hand-rolling
68+
// the FT index. A create failure surfaces — the tier is unusable without it.
69+
await this.memory.ensureIndex();
6670
await Promise.all([
6771
this.cache.ensureDiscoveryReady().catch(() => undefined),
6872
this.memory.ensureDiscoveryReady(),

‎packages/agent-memory/src/MemoryStore.ts‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
import { randomUUID } from 'node:crypto';
22
import { SpanStatusCode, type Span } from '@opentelemetry/api';
3-
import { encodeFloat32, parseFtSearchResponse } from '@betterdb/valkey-search-kit';
3+
import {
4+
encodeFloat32,
5+
isIndexNotFoundError,
6+
parseFtSearchResponse,
7+
} from '@betterdb/valkey-search-kit';
48
import { buildMemoryRecord } from './buildMemoryRecord';
9+
import { buildMemoryIndexArgs, memoryIndexName } from './buildMemoryIndex';
510
import {
611
buildConsolidateFilter,
712
buildRecallQuery,
@@ -262,6 +267,26 @@ export class MemoryStore {
262267
}
263268
}
264269

270+
/**
271+
* Create the `{name}:mem:idx` vector index if it does not already exist.
272+
* Idempotent — an existing index is left untouched. Resolves the vector
273+
* dimension from `embedFn` when it has not been observed yet. Call once
274+
* before the first remember/recall; the AgentMemory facade does this in
275+
* initialize().
276+
*/
277+
async ensureIndex(): Promise<void> {
278+
try {
279+
await this.client.call('FT.INFO', memoryIndexName(this.name));
280+
return;
281+
} catch (err) {
282+
if (!isIndexNotFoundError(err)) {
283+
throw err;
284+
}
285+
}
286+
const dims = await this.resolveDims();
287+
await this.client.call('FT.CREATE', ...buildMemoryIndexArgs(this.name, dims));
288+
}
289+
265290
async recall(query: string, options: RecallOptions = {}): Promise<MemoryHit[]> {
266291
return this.traced('recall', async (span) => {
267292
const startedAt = Date.now();
@@ -668,6 +693,20 @@ export class MemoryStore {
668693
this.telemetry.metrics.items.labels(this.storeLabels).dec(removed);
669694
}
670695

696+
private async resolveDims(): Promise<number> {
697+
if (this.dims !== undefined) {
698+
return this.dims;
699+
}
700+
const probe = await this.embedFn('probe');
701+
if (probe.length === 0) {
702+
throw new Error(
703+
'Cannot resolve memory vector dimension: embedFn returned a zero-length embedding',
704+
);
705+
}
706+
this.dims = probe.length;
707+
return this.dims;
708+
}
709+
671710
private async embed(content: string): Promise<number[]> {
672711
this.telemetry.metrics.embeddingCalls.labels(this.storeLabels).inc();
673712
const vector = await this.embedFn(content);

‎packages/agent-memory/src/__tests__/AgentMemory.test.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,27 @@ describe('AgentMemory facade', () => {
119119
expect(memoryClose).toHaveBeenCalled();
120120
});
121121

122+
it('initialize() creates the memory index when it does not exist', async () => {
123+
const client = fakeValkey();
124+
client.call = vi.fn(async (command: string) => {
125+
if (command === 'FT.INFO') {
126+
throw new Error("Unknown index name 'betterdb_ac:mem:idx'");
127+
}
128+
return 'OK';
129+
});
130+
const mem = new AgentMemory(
131+
makeOptions({ client: client as unknown as AgentMemoryOptions['client'] }),
132+
);
133+
134+
await mem.initialize();
135+
136+
const create = client.call.mock.calls.find((c) => c[0] === 'FT.CREATE');
137+
expect(create).toBeDefined();
138+
expect(create?.[1]).toBe('betterdb_ac:mem:idx');
139+
140+
await mem.close();
141+
});
142+
122143
it('registers a memory discovery marker by default', async () => {
123144
const client = fakeValkey();
124145
const mem = new AgentMemory(

‎packages/agent-memory/src/__tests__/MemoryStore.ensureIndex.test.ts‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import { describe, it, expect, vi } from 'vitest';
2+
import { MemoryStore } from '../MemoryStore';
3+
import { fakeEmbed } from './helpers/fakeEmbed';
4+
import { mockClient } from './helpers/mockClient';
5+
import { buildMemoryIndexArgs } from '../buildMemoryIndex';
6+
7+
function indexNotFound(): Error {
8+
return new Error("Unknown index name 'mem:mem:idx'");
9+
}
10+
11+
describe('MemoryStore.ensureIndex', () => {
12+
it('creates the index with the memory schema when it does not exist', async () => {
13+
const client = mockClient((command) => {
14+
if (command === 'FT.INFO') {
15+
throw indexNotFound();
16+
}
17+
return 'OK';
18+
});
19+
const store = new MemoryStore({ client, name: 'mem', embedFn: fakeEmbed(16) });
20+
21+
await store.ensureIndex();
22+
23+
const create = client.call.mock.calls.find((c) => c[0] === 'FT.CREATE');
24+
expect(create).toEqual(['FT.CREATE', ...buildMemoryIndexArgs('mem', 16)]);
25+
});
26+
27+
it('is idempotent — does not re-create an existing index', async () => {
28+
const client = mockClient((command) => {
29+
if (command === 'FT.INFO') {
30+
return ['index_name', 'mem:mem:idx'];
31+
}
32+
return 'OK';
33+
});
34+
const store = new MemoryStore({ client, name: 'mem', embedFn: fakeEmbed(16) });
35+
36+
await store.ensureIndex();
37+
38+
expect(client.call.mock.calls.some((c) => c[0] === 'FT.CREATE')).toBe(false);
39+
});
40+
41+
it('resolves the vector dimension from embedFn when none has been observed', async () => {
42+
const client = mockClient((command) => {
43+
if (command === 'FT.INFO') {
44+
throw indexNotFound();
45+
}
46+
return 'OK';
47+
});
48+
const store = new MemoryStore({ client, name: 'mem', embedFn: fakeEmbed(32) });
49+
50+
await store.ensureIndex();
51+
52+
const create = client.call.mock.calls.find((c) => c[0] === 'FT.CREATE');
53+
const dimIdx = create?.indexOf('DIM') ?? -1;
54+
expect(create?.[dimIdx + 1]).toBe('32');
55+
});
56+
57+
it('reuses the dimension already observed from a write without re-probing', async () => {
58+
const embedFn = vi.fn(fakeEmbed(16));
59+
const client = mockClient((command) => {
60+
if (command === 'FT.INFO') {
61+
throw indexNotFound();
62+
}
63+
return 'OK';
64+
});
65+
const store = new MemoryStore({ client, name: 'mem', embedFn });
66+
await store.remember('seed');
67+
const callsAfterWrite = embedFn.mock.calls.length;
68+
69+
await store.ensureIndex();
70+
71+
expect(embedFn.mock.calls.length).toBe(callsAfterWrite);
72+
const create = client.call.mock.calls.find((c) => c[0] === 'FT.CREATE');
73+
const dimIdx = create?.indexOf('DIM') ?? -1;
74+
expect(create?.[dimIdx + 1]).toBe('16');
75+
});
76+
77+
it('rethrows FT.INFO errors that are not "index not found"', async () => {
78+
const client = mockClient((command) => {
79+
if (command === 'FT.INFO') {
80+
throw new Error('CONNECTION BROKEN');
81+
}
82+
return 'OK';
83+
});
84+
const store = new MemoryStore({ client, name: 'mem', embedFn: fakeEmbed(16) });
85+
86+
await expect(store.ensureIndex()).rejects.toThrow(/connection broken/i);
87+
expect(client.call.mock.calls.some((c) => c[0] === 'FT.CREATE')).toBe(false);
88+
});
89+
});

‎packages/agent-memory/src/__tests__/MemoryStore.integration.test.ts‎

Lines changed: 2 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -45,51 +45,6 @@ async function dropAndClean(): Promise<void> {
4545
}
4646
}
4747

48-
async function createIndex(): Promise<void> {
49-
await client.call(
50-
'FT.CREATE',
51-
INDEX,
52-
'ON',
53-
'HASH',
54-
'PREFIX',
55-
'1',
56-
`${NAME}:mem:`,
57-
'SCHEMA',
58-
'vector',
59-
'VECTOR',
60-
'FLAT',
61-
'6',
62-
'TYPE',
63-
'FLOAT32',
64-
'DIM',
65-
String(DIMS),
66-
'DISTANCE_METRIC',
67-
'COSINE',
68-
'threadId',
69-
'TAG',
70-
'agentId',
71-
'TAG',
72-
'namespace',
73-
'TAG',
74-
'tags',
75-
'TAG',
76-
'SEPARATOR',
77-
',',
78-
'source',
79-
'TAG',
80-
'importance',
81-
'NUMERIC',
82-
'created_at',
83-
'NUMERIC',
84-
'last_accessed_at',
85-
'NUMERIC',
86-
'access_count',
87-
'NUMERIC',
88-
'content',
89-
'TEXT',
90-
);
91-
}
92-
9348
beforeAll(async () => {
9449
client = new Valkey(VALKEY_URL, { lazyConnect: true, retryStrategy: () => null });
9550
try {
@@ -101,12 +56,13 @@ beforeAll(async () => {
10156
return;
10257
}
10358
await dropAndClean();
104-
await createIndex();
10559
store = new MemoryStore({
10660
client: client as unknown as MemoryStoreClient,
10761
name: NAME,
10862
embedFn: fakeEmbed(DIMS),
10963
});
64+
// Dogfood the package's own index bootstrap instead of hand-rolling FT.CREATE.
65+
await store.ensureIndex();
11066
});
11167

11268
afterAll(async () => {

‎packages/agent-memory/src/__tests__/buildMemoryIndex.test.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import { describe, it, expect } from 'vitest';
2+
import {
3+
buildMemoryIndexArgs,
4+
memoryIndexName,
5+
memoryKeyPrefix,
6+
MEMORY_INDEX_ALGORITHM,
7+
} from '../buildMemoryIndex';
8+
9+
describe('buildMemoryIndex', () => {
10+
it('names the index and key prefix off the store name', () => {
11+
expect(memoryIndexName('mem')).toBe('mem:mem:idx');
12+
expect(memoryKeyPrefix('mem')).toBe('mem:mem:');
13+
});
14+
15+
it('builds an FT.CREATE arg list scoped to the memory keyspace', () => {
16+
const args = buildMemoryIndexArgs('mem', 16);
17+
18+
expect(args.slice(0, 7)).toEqual(['mem:mem:idx', 'ON', 'HASH', 'PREFIX', '1', 'mem:mem:', 'SCHEMA']);
19+
});
20+
21+
it('declares the vector field with the configured dimension and cosine metric', () => {
22+
const args = buildMemoryIndexArgs('mem', 16);
23+
const vec = args.indexOf('vector');
24+
25+
expect(args.slice(vec, vec + 12)).toEqual([
26+
'vector',
27+
'VECTOR',
28+
MEMORY_INDEX_ALGORITHM,
29+
'6',
30+
'TYPE',
31+
'FLOAT32',
32+
'DIM',
33+
'16',
34+
'DISTANCE_METRIC',
35+
'COSINE',
36+
'threadId',
37+
'TAG',
38+
]);
39+
});
40+
41+
it('indexes the scope/tag fields as TAG and tags as comma-separated', () => {
42+
const args = buildMemoryIndexArgs('mem', 16);
43+
const joined = args.join(' ');
44+
45+
expect(joined).toContain('threadId TAG');
46+
expect(joined).toContain('agentId TAG');
47+
expect(joined).toContain('namespace TAG');
48+
expect(joined).toContain('tags TAG SEPARATOR ,');
49+
expect(joined).toContain('source TAG');
50+
});
51+
52+
it('indexes the numeric tunables and the content text field', () => {
53+
const args = buildMemoryIndexArgs('mem', 16);
54+
const joined = args.join(' ');
55+
56+
for (const field of ['importance', 'created_at', 'last_accessed_at', 'access_count']) {
57+
expect(joined).toContain(`${field} NUMERIC`);
58+
}
59+
expect(joined).toContain('content TEXT');
60+
});
61+
62+
it('rejects a non-positive or non-integer dimension', () => {
63+
expect(() => buildMemoryIndexArgs('mem', 0)).toThrow(/positive integer/i);
64+
expect(() => buildMemoryIndexArgs('mem', -4)).toThrow(/positive integer/i);
65+
expect(() => buildMemoryIndexArgs('mem', 1.5)).toThrow(/positive integer/i);
66+
});
67+
});

‎packages/agent-memory/src/buildMemoryIndex.ts‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import { VECTOR_FIELD } from './buildRecallQuery';
2+
3+
export const MEMORY_INDEX_ALGORITHM = 'HNSW';
4+
5+
export function memoryIndexName(name: string): string {
6+
return `${name}:mem:idx`;
7+
}
8+
9+
export function memoryKeyPrefix(name: string): string {
10+
return `${name}:mem:`;
11+
}
12+
13+
export function buildMemoryIndexArgs(name: string, dims: number): string[] {
14+
if (!Number.isInteger(dims) || dims <= 0) {
15+
throw new Error(`memory index dimension must be a positive integer, got: ${dims}`);
16+
}
17+
return [
18+
memoryIndexName(name),
19+
'ON',
20+
'HASH',
21+
'PREFIX',
22+
'1',
23+
memoryKeyPrefix(name),
24+
'SCHEMA',
25+
VECTOR_FIELD,
26+
'VECTOR',
27+
MEMORY_INDEX_ALGORITHM,
28+
'6',
29+
'TYPE',
30+
'FLOAT32',
31+
'DIM',
32+
String(dims),
33+
'DISTANCE_METRIC',
34+
'COSINE',
35+
'threadId',
36+
'TAG',
37+
'agentId',
38+
'TAG',
39+
'namespace',
40+
'TAG',
41+
'tags',
42+
'TAG',
43+
'SEPARATOR',
44+
',',
45+
'source',
46+
'TAG',
47+
'importance',
48+
'NUMERIC',
49+
'created_at',
50+
'NUMERIC',
51+
'last_accessed_at',
52+
'NUMERIC',
53+
'access_count',
54+
'NUMERIC',
55+
'content',
56+
'TEXT',
57+
];
58+
}

0 commit comments

Comments
 (0)