diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..9715d72 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,34 @@ +--- +name: Bug Report +about: Report a bug to help us improve +title: "[Bug] " +labels: bug +assignees: '' +--- + +**Describe the bug** +A clear description of what the bug is. + +**To reproduce** +Steps to reproduce: +1. ... +2. ... +3. ... + +**Expected behavior** +What you expected to happen. + +**Actual behavior** +What actually happened. + +**Environment** +- OS: [e.g., macOS 15, Ubuntu 24.04] +- Node.js: [e.g., 20.11.0] +- Docker: [e.g., 27.0.0] +- Browser (if webapp): [e.g., Chrome 130] + +**Logs** +If applicable, paste relevant logs. + +**Additional context** +Any other context about the problem. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..fda9e28 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,19 @@ +--- +name: Feature Request +about: Suggest a new feature or improvement +title: "[Feature] " +labels: enhancement +assignees: '' +--- + +**Problem** +What problem does this feature solve? + +**Proposed solution** +Describe what you'd like to happen. + +**Alternatives considered** +Any alternative solutions you've considered. + +**Additional context** +Any other context, mockups, or examples. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..99d3404 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,20 @@ +## Summary + + + +## Changes + + +- + +## Checklist + +- [ ] `npm run typecheck` passes +- [ ] `npm run lint` passes +- [ ] Tests added/updated for new functionality +- [ ] Documentation updated (if applicable) +- [ ] No secrets or credentials in committed files + +## Related Issues + + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..cf7d6f0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,41 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + typecheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + - run: npm ci + - run: npm run typecheck + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + - run: npm ci + - run: npm run lint + + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + - run: npm ci + - run: npm run test diff --git a/.gitignore b/.gitignore index 3a05687..f9d8f3b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,20 @@ coverage .next dist .claude-flow -ngrok.config.yml \ No newline at end of file +ngrok.config.yml + +# Benchmark run data +tests/benchmarks/runs/ + +# Environment files +.env.*.local +.env.production +.env.benchmark + +# Keys and certificates +*.pem +*.key + +# Swarm data +.swarm/ +tests/benchmarks/src/.swarm/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1a6a786 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,28 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.1.0] - 2026-03-30 + +### Added + +- MCP server with full Model Context Protocol support (HTTP/SSE and stdio) +- REST API for facts, relations, knowledge cards, workspaces, and users +- Web dashboard (Next.js) for browsing and managing knowledge +- ArangoDB-backed knowledge graph with vector embeddings +- Hybrid search: vector similarity + graph traversal +- Auto-consolidation: background workers merge related facts into knowledge cards +- BGE cross-encoder reranker for improved search relevance +- File upload and fact extraction (PDF, DOCX, Excel, CSV) +- Google and GitHub OAuth authentication +- API key authentication +- Multi-workspace support with isolation +- Workspace-scoped security for all API endpoints +- Audit trails for all fact operations +- Embedding queue for real-time async processing +- Docker Compose configs for development and production +- Comprehensive benchmark suite (HotpotQA, RelationRecall, LongMemEval) +- Environment variable waterfall configuration (root -> service overrides) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..accec77 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,60 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a welcoming and respectful experience for everyone. + +We pledge to act and interact in ways that contribute to an open, inclusive, +and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment: + +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +- Focusing on what is best for the overall community + +Examples of unacceptable behavior: + +- Trolling, insulting or derogatory comments, and personal attacks +- Public or private harassment of any kind +- Publishing others' private information without explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Project maintainers are responsible for clarifying and enforcing our standards +of acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate or harmful. + +Project maintainers have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. + +## Enforcement + +Instances of unacceptable behavior may be reported to the project team at +**opensource@camplight.net**. All complaints will be reviewed and investigated +promptly and fairly. The project team is obligated to maintain confidentiality +with regard to the reporter of an incident. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d7e8224 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,121 @@ +# Contributing to KnowledgePlane + +Thank you for your interest in contributing to KnowledgePlane! We welcome contributions from everyone, whether it is a bug report, feature request, documentation improvement, or code change. + +## How to Contribute + +### Development Setup + +1. Fork and clone the repository: + + ```bash + git clone https://github.com/camplight/knowledgeplane.git + cd knowledgeplane + ``` + +2. Install dependencies and bootstrap the monorepo: + + ```bash + npm run bootstrap + ``` + +3. Start the development environment: + + ```bash + npm run dev + ``` + +For detailed setup instructions (ArangoDB, environment variables, Docker, etc.), see [DEVELOPMENT.md](DEVELOPMENT.md). + +### Branch Naming + +Use the following prefixes for your branches: + +- `feature/` -- New features (e.g., `feature/workspace-sharing`) +- `fix/` -- Bug fixes (e.g., `fix/search-timeout`) +- `docs/` -- Documentation changes (e.g., `docs/api-reference`) + +### Commit Messages + +We follow [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/). Each commit message should be structured as: + +``` +: +``` + +Supported types: + +- `feat:` -- A new feature +- `fix:` -- A bug fix +- `docs:` -- Documentation changes +- `chore:` -- Maintenance tasks (CI, tooling, dependencies) +- `refactor:` -- Code changes that neither fix a bug nor add a feature +- `test:` -- Adding or updating tests +- `perf:` -- Performance improvements + +Examples: + +``` +feat: add vector search to knowledge card queries +fix: handle empty embedding field in consolidation worker +docs: update REST API endpoint documentation +test: add integration tests for file upload extraction +``` + +### Pull Request Guidelines + +1. **Describe your changes** -- Provide a clear summary of what the PR does and why. +2. **Reference issues** -- Link related GitHub issues (e.g., "Closes #42"). +3. **Add tests** -- New features and bug fixes should include tests. +4. **Pass checks** -- Before submitting, run: + + ```bash + npm run typecheck + npm run lint + npm run test + ``` + +5. **Keep PRs focused** -- One logical change per PR. Split large changes into smaller, reviewable pieces. +6. **Update documentation** -- If your change affects public APIs or user-facing behavior, update the relevant docs. + +### Code Style + +- **TypeScript** -- All source code is written in TypeScript. Follow the existing patterns in the codebase. +- **No new dependencies without discussion** -- If your change requires a new dependency, open an issue first to discuss the rationale and alternatives. +- **Existing patterns** -- Match the conventions, naming, and structure already present in the code. + +### Testing + +Run the full test suite: + +```bash +npm run test +``` + +Add tests for any new features or bug fixes. We use the existing test infrastructure in the `tests/` directory. + +### Reporting Bugs + +Use the [GitHub issue tracker](https://github.com/camplight/knowledgeplane/issues) to report bugs. Please include: + +- A clear and descriptive title +- Steps to reproduce the issue +- Expected vs. actual behavior +- Environment details (OS, Node.js version, ArangoDB version) +- Any relevant logs or error messages + +### Feature Requests + +Use the [GitHub issue tracker](https://github.com/camplight/knowledgeplane/issues) to request features. Please include: + +- A clear description of the feature and the problem it solves +- Any alternatives you have considered +- Context on how you would use this feature + +### Code of Conduct + +This project adheres to a [Code of Conduct](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this standard. Please report unacceptable behavior to opensource@camplight.net. + +--- + +Thank you for helping make KnowledgePlane better! diff --git a/README.md b/README.md index 776cb3e..e652332 100644 --- a/README.md +++ b/README.md @@ -1,117 +1,132 @@ -# knowledgeplane +

+ Knowledge Plane +

-**Autonomous team knowledge for AI agents (MCP server).** +

Knowledge Plane

-- Team/project **namespaces**, RBAC scaffolding, audit logs -- **Hybrid retrieval** (pgvector + BM25 via `pgroonga` optional) -- **MCP tools** for read/write/search facts -- OpenAPI + tiny JS SDK -- Docker-first deploy +

+ Shared memory for AI agents and teams -- stops your tools from forgetting. +

-**📚 Documentation:** -- **[Getting Started](./GETTING_STARTED.md)** - Quick start guide -- **[Development Guide](./DEVELOPMENT.md)** - Localhost setup with ngrok -- **[Deployment Guide](./DEPLOYMENT.md)** - Cloud deployment (Digital Ocean, etc.) -- **[Environment Setup](./ENV_SETUP.md)** - Environment variables configuration -- **[API Specification](./docs/SPEC.md)** - Complete API documentation +

+ License + Node + Docker +

-### Quickstart -**Development Mode (with hot reload):** +## What is Knowledge Plane? -```bash -# 1) Clone & bootstrap -npm run bootstrap - -# 2) Set up environment variables -./scripts/setup-env.sh # Creates .env files from examples -# Edit .env files with your values (see DEVELOPMENT.md) +Knowledge Plane is an MCP server that gives AI agents and teams persistent, shared memory. It stores facts as a knowledge graph with vector embeddings, automatically consolidates related facts into knowledge cards, and provides hybrid search (vector + graph traversal). Native MCP protocol support means any MCP-compatible AI tool (Claude, Cursor, etc.) can read and write shared knowledge. -# 3) Start infrastructure + dev server (auto-reloads on code changes) -npm run dev - -# 4) Configure and start ngrok for MCP/OAuth callbacks -cp ngrok.config.example ngrok.config.yml -# Edit ngrok.config.yml and set your ngrok authtoken -ngrok start --config ./ngrok.config.yml mcp-server - -# The command will: -# - Start ArangoDB in Docker (port 8529) -# - Wait for database to be ready -# - Start MCP server in watch mode (port 8080) -# - Start webapp in dev mode (port 3000) -# - Start background workers -``` +## Features -**For detailed development setup including ngrok and OAuth configuration, see [DEVELOPMENT.md](./DEVELOPMENT.md)** +- **Persistent agent memory** -- Facts stored in ArangoDB knowledge graph with vector embeddings +- **MCP-native** -- Drop-in memory for Claude Desktop, Cursor, and any MCP client +- **Hybrid search** -- Vector similarity + graph traversal + optional BM25 +- **Auto-consolidation** -- Background workers merge related facts into knowledge cards +- **File upload and extraction** -- Upload documents, extract facts automatically +- **Web dashboard** -- Browse, search, and manage your knowledge graph +- **Multi-workspace** -- Isolated knowledge spaces per team or project +- **OAuth + API keys** -- Google/GitHub OAuth and API key authentication +- **REST API** -- Full CRUD API alongside the MCP server +- **Audit trails** -- Track who created and modified every fact -### ngrok Config (Reserved Domain) +## Quick Start -Use the provided ngrok config files to expose the local MCP server at: -`https://boa-driving-distinctly.ngrok-free.app` +```bash +# Clone and install +git clone https://github.com/camplight/knowledgeplane.git +cd knowledgeplane +npm run bootstrap -- `ngrok.config.example` is committed as the template -- `ngrok.config.yml` is for local use and is gitignored +# Set up environment +./scripts/setup-env.sh # Creates .env files from examples -```bash -cp ngrok.config.example ngrok.config.yml -# Set your authtoken in ngrok.config.yml -ngrok start --config ./ngrok.config.yml mcp-server +# Start everything (ArangoDB + all services) +npm run dev ``` -**Production Mode:** +Services start at: -```bash -# Start full stack (ArangoDB + all services in Docker) -docker compose -f infra/docker-compose.yml up --build -``` +| Service | URL | +|---------|-----| +| Web Dashboard | http://localhost:3000 | +| MCP Server | http://localhost:8080 | +| REST API | http://localhost:8081 | +| ArangoDB | http://localhost:8529 | -**For quick cloud deployment (Railway recommended), see [DEPLOYMENT.md](./DEPLOYMENT.md)** +For detailed setup including OAuth and ngrok, see [Getting Started](./GETTING_STARTED.md). -**Stop development servers:** -```bash -npm run dev:stop # Stops Docker containers -``` +## MCP Integration -### MCP integration (Claude Desktop) +### Claude Desktop -**Option 1: Connect via URL (HTTP/SSE)** +Add to your Claude Desktop MCP config: -Add to your `mcp.json`: +**Option 1: HTTP/SSE (recommended)** ```json { - "clients": { + "mcpServers": { "knowledgeplane": { "url": "http://localhost:8080/mcp", "headers": { - "Authorization": "Bearer DEV_API_KEY" + "Authorization": "Bearer YOUR_API_KEY" } } } } ``` -**Option 2: Connect via stdio adapter** - -Add to your `mcp.json`: +**Option 2: stdio adapter** ```json { - "clients": { + "mcpServers": { "knowledgeplane": { "command": "node", - "args": ["server/dist/mcp/adapter.js"], + "args": ["apps/mcp-server/dist/mcp/adapter.js"], "env": { "KNOWLEDGEPLANE_API_URL": "http://localhost:8080", - "KNOWLEDGEPLANE_API_KEY": "DEV_API_KEY" + "KNOWLEDGEPLANE_API_KEY": "YOUR_API_KEY" } } } } ``` -### License +## Architecture + +``` +knowledgeplane/ +├── apps/ +│ ├── mcp-server/ # MCP protocol server (Fastify) +│ ├── rest-api/ # REST API (Express) +│ ├── webapp/ # Web dashboard (Next.js) +│ └── background-workers/ # Consolidation & embeddings +├── packages/ +│ ├── db/ # ArangoDB models & queries +│ ├── aimodel/ # LLM abstraction layer +│ ├── api-core/ # Shared API utilities +│ └── file-processor/ # Document parsing & extraction +└── infra/ # Docker Compose configs +``` + +## Documentation + +| Guide | Description | +|-------|-------------| +| [Getting Started](./GETTING_STARTED.md) | Quick start and setup | +| [Development Guide](./DEVELOPMENT.md) | Local development with ngrok | +| [Deployment Guide](./DEPLOYMENT.md) | Cloud deployment (Railway, etc.) | +| [Environment Setup](./ENV_SETUP.md) | All environment variables | +| [API Specification](./docs/SPEC.md) | Complete API reference | + +## Contributing + +We welcome contributions! Please see [CONTRIBUTING.md](./CONTRIBUTING.md) for guidelines. -Apache-2.0 +## License +[Apache-2.0](./LICENSE) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..23d7ad1 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,48 @@ +# Security Policy + +## Reporting a Vulnerability + +If you discover a security vulnerability in Knowledge Plane, please report it responsibly. + +**Email:** security@camplight.net + +**Please do NOT open a public GitHub issue for security vulnerabilities.** + +### What to Include + +- Description of the vulnerability +- Steps to reproduce +- Potential impact assessment +- Suggested fix (if you have one) + +### Response Timeline + +- **Acknowledgment:** Within 48 hours +- **Initial assessment:** Within 5 business days +- **Fix timeline:** Depends on severity, typically within 30 days for critical issues + +### Disclosure + +We follow coordinated disclosure: + +1. Confirm the vulnerability and determine its impact +2. Develop and test a fix +3. Release the fix and publish an advisory +4. Credit the reporter (if desired) in the CHANGELOG and release notes + +We ask that you do not publicly disclose the vulnerability until a fix is available. + +## Scope + +- All code in this repository +- The official Docker images +- The deployment configurations in `infra/` + +## Out of Scope + +- Third-party dependencies (please report to their maintainers directly) +- Issues in services Knowledge Plane connects to (ArangoDB, ngrok, etc.) + +## Recognition + +We appreciate security researchers who help keep Knowledge Plane safe. With your permission, we will acknowledge your contribution in our CHANGELOG and release notes. diff --git a/apps/webapp/app/editor/components/RelationEditForm.tsx b/apps/webapp/app/editor/components/RelationEditForm.tsx index dfa59b4..7d85158 100644 --- a/apps/webapp/app/editor/components/RelationEditForm.tsx +++ b/apps/webapp/app/editor/components/RelationEditForm.tsx @@ -49,15 +49,6 @@ export function RelationEditForm({ const newType = selectedType; const newFactId = selectedFactId; - console.log("RelationEditForm submit:", { - newType, - newFactId, - currentType, - currentFactId, - relationId, - availableFactsCount: availableFacts.length - }); - if (!newType || newType.trim() === "") { console.error("RelationEditForm: Missing or empty type"); alert("Please select a relation type"); diff --git a/apps/webapp/app/editor/page.tsx b/apps/webapp/app/editor/page.tsx index cc1fbf9..96d8f84 100644 --- a/apps/webapp/app/editor/page.tsx +++ b/apps/webapp/app/editor/page.tsx @@ -332,16 +332,6 @@ export default function EditorPage() { const normalizedCurrentFactId = normalizeFactId(currentFactId); const normalizedNewFactId = normalizeFactId(newFactId); - console.log("handleUpdateRelation called:", { - relationId, - type, - newFactId, - normalizedNewFactId, - currentFactId, - normalizedCurrentFactId, - variant - }); - // Validate relation ID - should not be a fact ID if (!relationId || relationId.trim() === "") { console.error("Missing relation ID"); @@ -361,7 +351,6 @@ export default function EditorPage() { const factIdChanged = normalizedNewFactId !== normalizedCurrentFactId && newFactId !== currentFactId; if (factIdChanged) { - console.log("Fact ID changed, deleting and recreating relation"); // Delete old relation and create new one deleteRelationMutation.mutate( { id: relationId }, @@ -392,7 +381,6 @@ export default function EditorPage() { ); } else { // Just update the type - console.log("Updating relation type only", { relationId, type }); if (!type || type.trim() === "") { console.error("Cannot update relation: type is empty"); alert("Relation type cannot be empty"); diff --git a/docs/images/logo.png b/docs/images/logo.png new file mode 100644 index 0000000..9ac16f9 Binary files /dev/null and b/docs/images/logo.png differ diff --git a/packages/db/src/models/FactRelation.ts b/packages/db/src/models/FactRelation.ts index 639f35f..d6894fc 100644 --- a/packages/db/src/models/FactRelation.ts +++ b/packages/db/src/models/FactRelation.ts @@ -66,16 +66,6 @@ export class FactRelation { const normalizedFromFact = this._normalizeFactId(input.from_fact); const normalizedToFact = this._normalizeFactId(input.to_fact); - console.log("Fact validation:", { - inputFromFact: input.from_fact, - normalizedFromFact, - verifiedFromId, - inputToFact: input.to_fact, - normalizedToFact, - verifiedToId, - fromFactKey: fromFact._key, - toFactKey: toFact._key, - }); } catch (error: any) { // If it's already our validation error, rethrow it if (error.message?.includes("not found")) { @@ -569,11 +559,6 @@ export class FactRelation { const cursor = await collections.relations.database.query(aql, bindVars); const results = await cursor.all(); - // Debug: log raw results to see what we're getting - if (results.length > 0) { - console.log("getRelatedFacts raw results:", results.length, results[0]); - } - const validResults = results .filter((r: any) => { // Validate that we have both relation and fact @@ -682,15 +667,6 @@ export class FactRelation { const cursor = await collections.relations.database.query(aql, bindVars); const results = await cursor.all(); - // Debug: log raw results to see what we're getting - if (results.length > 0) { - console.log( - "getIncomingRelations raw results:", - results.length, - results[0], - ); - } - const validResults = results .filter((r: any) => { // Validate that we have both relation and fact