MemoryBearBenchmark/.env.example at main · SuanmoSuanyangTechnology/MemoryBearBenchmark · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
# Copy to .env (already gitignored) and fill in real values.
# Scripts look for .env in the current working directory first,
# then at this repo's root (next to this file).

# =========================================================================
# Quick reproduction (the only section external users need):
#   uv run src/lme/reproduce.py
#   uv run src/locomo/reproduce.py
# Both reproduce our published results from the released per-question
# memories under results/<benchmark>/. Everything below this section is
# for the internal full pipeline.
# =========================================================================
# Your LLM provider's API key (used for both reader and judge). Any
# OpenAI-compatible provider works.
# (OPENAI_API_KEY / OPENAI_BASE_URL are also honored, as fallbacks.)
LLM_API_KEY=your-llm-api-key
# LLM_BASE_URL=your-llm-url
# Default model for both reader and judge; the --reader-model /
# --judge-model flags override it. Unset = gpt-4o.
# LLM_MODEL=your-llm-model

# =========================================================================
# Internal full pipeline — LongMemEval (requires MemoryBear service access)
# =========================================================================

# --- Step 1: src/lme/fetch_mem_hyp.py ---
MEMORY_SERVICE_URL=your-url
MEMORY_API_KEY=your-memory-api-key
MEMORY_CONFIG_ID=your-memory-config-uuid
MEMORY_SEARCH_SWITCH=choose-a-switch
# Ingest manifest json (or a directory of them): drives which case_ids to fetch
# and each case's end_user_id. A single file = just that batch; a directory =
# all manifests merged (latest timestamp wins on duplicate case_id).
MEMORY_MANIFEST=manifests/lme/
# Reference json: question texts are extracted from here by question_id
LONGMEMEVAL_REF_FILE=data/lme/
# The single output / shared artifact: a JSON array of per-fetch trial records
# (full read response + autoeval_label). fetch_mem_hyp.py appends to it,
# evaluate_qa.py judges the new (un-labeled) ones.
TRIALS_FILE=results/lme/trials.json
# Optional overrides:
# MEMORY_READ_PATH=/v1/memory/read/sync
# Header used to send the API key. Default "Authorization" (sent as "Bearer <key>");
# set to e.g. X-API-Key if the service expects a custom header (sent as the raw key).
# MEMORY_API_KEY_HEADER=Authorization

# --- Step 2: src/lme/generate_hyp_interm.py (reader) ---
READER_MODEL=your-model
READER_API_KEY=your-reader-api-key
# OpenAI-compatible base url; default is DashScope's compatible-mode endpoint.
# READER_API_BASE=https://dashscope.aliyuncs.com/compatible-mode/v1

# --- Step 3: src/lme/evaluate_qa.py (LLM judge) ---
DASHSCOPE_API_KEY=your-dashscope-api-key
# DASHSCOPE_API_BASE=https://dashscope.aliyuncs.com/compatible-mode/v1

# =========================================================================
# Internal full pipeline — LoCoMo (requires MemoryBear service access)
# Used by src/locomo/{build_user_map,preflight,run_predictions,rerank,score}.py
# =========================================================================

# ---- Memory system (recall / search endpoint) ----
# Full POST URL of the search endpoint (NOT just the host).
MEM_SERVICE_URL=
MEM_API_KEY=
SEARCH_SWITCH=
MEM_CONFIG_ID=
TOP_K_LIMIT=
# skip_summary: true/false. Empty = omit the field (server default).
MEM_SKIP_SUMMARY=false
# How the API key is sent. Defaults below = header "Authorization: Bearer <key>".
MEM_AUTH_HEADER=Authorization
MEM_AUTH_PREFIX=Bearer

# ---- LLM used to turn retrieved memories into a short answer (and judge) ----
# OpenAI-compatible chat-completions endpoint (DashScope compatible-mode works).
# LLM_API_KEY is defined once at the top of this file and shared; the pipeline
# scripts read it from there.
LLM_URL=
LLM_MODEL=

# ---- Paths (all optional; defaults in src/locomo/config.py) ----
# Relative paths are resolved against src/locomo/.
# LOCOMO_JSON : dataset file (questions / speakers), default ../../data/locomo/locomo10.json
# MANIFEST_DIR: ingestion manifests read by build_user_map.py, default ../../manifests/locomo
# USER_MAP    : build_user_map.py output = run_predictions.py input
# RESULTS_DIR : predictions, caches and scores, default ../../results/locomo
# LOCOMO_JSON=../../data/locomo/locomo10.json
# MANIFEST_DIR=../../manifests/locomo
# USER_MAP=user_map.json
# RESULTS_DIR=../../results/locomo

# ---- Client-side rerank (src/locomo/rerank.py; only used when RERANK_ENABLED=1) ----
# 1 = fetch TOP_K_LIMIT candidates, rerank them client-side, keep RERANK_TOP_N.
# 0 = original path (context = data.answer + entity descriptions).
# The published run used RERANK_ENABLED=1, TOP_K_LIMIT=50, RERANK_TOP_N=10.
RERANK_ENABLED=0
RERANK_MODEL=
RERANK_URL=
# Empty = reuse LLM_API_KEY (same DashScope account).
RERANK_API_KEY=
RERANK_TOP_N=