feat: add pagination envelope {data,page,count,items_per_page} to /query

Author: VaiTon

frontend/src/api.ts

@@ -8,6 +8,7 @@ import type {
   TickInfo,
   Flow,
   FlowsQuery,
+  FlowsResponse,
 } from "./types";
 
 export const tulipApi = createApi({
@@ -22,7 +23,7 @@ export const tulipApi = createApi({
     getFlow: builder.query<FullFlow, string>({
       query: (id) => `/flow/${id}`,
     }),
-    getFlows: builder.query<Flow[], FlowsQuery>({
+    getFlows: builder.query<FlowsResponse, FlowsQuery>({
       query: (query) => ({
         url: `/query`,
         method: "POST",

frontend/src/components/Corrie.tsx

@@ -99,7 +99,7 @@ export function Corrie() {
 
   // TODO: fix the below transformation - move it to server
   // Diederik gives you a beer once it has been fixed
-  const transformedFlowData = flowData?.map((flow) => ({
+  const transformedFlowData = flowData?.data.map((flow) => ({
     ...flow,
     service_tag:
       services?.find((s) => s.ip === flow.dst_ip && s.port === flow.dst_port)

frontend/src/components/FlowList.tsx

@@ -157,12 +157,12 @@ export function FlowList() {
         offset: allFlows.length,
       }).unwrap();
 
-      if (result.length < PAGE_SIZE) {
+      if (allFlows.length + result.data.length >= result.count) {
         setHasMore(false);
       }
 
       // Transform new flows with service tags
-      const transformedNewFlows = result.map((flow) => ({
+      const transformedNewFlows = result.data.map((flow) => ({
         ...flow,
         service_tag:
           services?.find(
@@ -188,15 +188,15 @@ export function FlowList() {
   // Reset flows when filters change
   useEffect(() => {
     if (flowData) {
-      const transformed = flowData.map((flow) => ({
+      const transformed = flowData.data.map((flow) => ({
         ...flow,
         service_tag:
           services?.find(
             (s) => s.ip === flow.dst_ip && s.port === flow.dst_port,
           )?.name ?? "unknown",
       }));
       setAllFlows(transformed);
-      setHasMore(flowData.length === PAGE_SIZE);
+      setHasMore(transformed.length < flowData.count);
     } else if (!isLoading) {
       // Clear flows if no data and not loading
       setAllFlows([]);

frontend/src/pages/Fingerprinter.tsx

@@ -36,7 +36,7 @@ export function Fingerprinter() {
     );
   }
 
-  if (!data || !Array.isArray(data) || data.length === 0) {
+  if (!data || !data.data || data.data.length === 0) {
     return (
       <div className="flex justify-center items-center h-[60vh]">
         <span className="text-2xl font-semibold text-gray-500">
@@ -50,7 +50,7 @@ export function Fingerprinter() {
     <div className="max-w-2xl mx-auto mt-10">
       <h2 className="text-3xl font-bold mb-6 text-center">Timeline for Fingerprint {idNumber}</h2>
       <ol className="relative border-l-2 border-blue-500">
-        {data.map((item, idx) => (
+        {data.data.map((item, idx) => (
           <li key={item._id || idx} className="mb-10 ml-6">
             <Link
               to={`/flow/${item._id}`}

frontend/src/pages/Fingerprints.tsx

@@ -57,7 +57,7 @@ function FingerprintSection({ fingerprint }: { fingerprint: number }) {
     );
   }
 
-  if (!flows || flows.length === 0) {
+  if (!flows || !flows.data || flows.data.length === 0) {
     return (
       <section
         key={fingerprint}
@@ -80,12 +80,12 @@ function FingerprintSection({ fingerprint }: { fingerprint: number }) {
           {fingerprint}
         </span>
         <span className="ml-2 bg-blue-100 dark:bg-blue-900 dark:text-blue-200 text-blue-800 text-xs font-semibold px-2.5 py-0.5 rounded">
-          {flows.length} flow{flows.length !== 1 ? "s" : ""}
+          {flows.data.length} flow{flows.data.length !== 1 ? "s" : ""}
         </span>
       </div>
       <div>
         <ul className="space-y-1">
-          {flows.slice(0, 5).map((flow, idx: number) => (
+          {flows.data.slice(0, 5).map((flow, idx: number) => (
             <li key={flow._id || idx}>
               Flow
               <Link

frontend/src/pages/FlowView.tsx

@@ -780,8 +780,8 @@ function FlowFingerprintTimeline({
         <div className="text-red-500 dark:text-red-400">
           Error fetching related flows
         </div>
-      ) : relatedFlows && relatedFlows.length > 0 ? (
-        renderTimeline(relatedFlows)
+      ) : relatedFlows?.data && relatedFlows.data.length > 0 ? (
+        renderTimeline(relatedFlows.data)
       ) : (
         <div className="text-gray-500 dark:text-gray-400">
           No related flows found.

frontend/src/types.ts

@@ -59,6 +59,13 @@ export type FlowsQuery = {
   fingerprints?: number[];
 };
 
+export type FlowsResponse = {
+  data: Flow[];
+  page: number;
+  count: number;
+  items_per_page: number;
+};
+
 export type Service = {
   ip: string;
   port: number;

plan-progressiveImprovements.prompt.md

@@ -0,0 +1,71 @@
+# Plan: Progressive Improvement of Tulip
+
+**TL;DR:** Address the highest-impact issues first (performance regressions), then improve code robustness (error handling, context propagation), then clean up UX hacks. Each phase is independently shippable.
+
+---
+
+## Phase 1 — Performance Quick Wins (~2–3 days, highest ROI)
+
+**1.1 Fix N+1 signature queries** — services/cmd/api/api.go ~L236–256
+Currently fetches each SuricataSig individually in a loop per flow. Add `GetSignaturesBatch(ctx, ids []string) []SuricataSig` to services/pkg/db/db.go backed by a `$in` query in services/pkg/db/mongo.go. 50 flows × 3 sigs = 150 round trips → 1.
+
+**1.2 Add compound index for enricher lookups** — services/pkg/db/mongo.go ~L201–206
+`AddSignatureToFlow` and `AddTagsToFlow` filter on `{src_ip, dst_ip, src_port, dst_port, time}` but no compound index covers this. Add index: `{src_ip: 1, dst_ip: 1, src_port: 1, dst_port: 1, time: 1}`.
+
+**1.3 Fix redundant tag aggregation** — services/pkg/db/mongo.go ~L60–111
+`GetTagList()` runs two full collection scans and merges with an O(n²) loop. Replace with a single authoritative source: keep only the `tags` collection lookup (fed by `InsertTag` calls which already exist) and drop the flows aggregation pass.
+
+**1.4 Add text index for flow content search** — services/pkg/db/mongo.go ~L465
+`flow.data` regex search does a full collection scan. Add a denormalized `search_data` root-level field populated at insert time by joining all `FlowItem.Data` values — enabling the existing text index to cover it.
+
+---
+
+## Phase 2 — Robustness & API Correctness (~3–4 days)
+
+**2.1 Propagate request context through DB calls** — services/pkg/db/mongo.go
+31 uses of `context.TODO()` mean in-flight DB queries can't be cancelled on client disconnect or timeout. Change all DB interface methods in services/pkg/db/db.go to accept `ctx context.Context` and pass the Echo request context from services/cmd/api/api.go.
+
+**2.2 Return errors from tag inserts** — services/pkg/db/mongo.go ~L415–424
+`InsertTags` / `InsertTag` silently swallow all errors. Change to return `error`; callers can ignore duplicate-key errors specifically (`mongo.IsDuplicateKeyError`) and log others.
+
+**2.3 Fix empty-tags edge case** — services/cmd/api/api.go + frontend/src/api.ts ~L33
+Empty `includeTags`/`excludeTags` arrays cause incorrect query behavior (known TODO with a beer bounty). Backend: guard — if array is empty, skip the `$all`/`$nin` clause. Remove the frontend workaround once backend handles it.
+
+**2.4 Add pagination metadata to `/query` response** — services/cmd/api/api.go ~L166
+Wrap the response: `{data: [...], meta: {total: N, hasMore: bool}}`. Run `CountFlows` in parallel with `GetFlows`. Frontend can use `hasMore` to disable infinite scroll rather than guessing.
+
+---
+
+## Phase 3 — Technical Debt & UX Hacks (~1 week, incremental)
+
+**3.1 Fix timezone display in chart** — frontend/src/components/Corrie.tsx ~L235, L324
+FIXME comment: datetime axis shows wrong timezone. Audit Chart.js time scale config; use UTC-aware formatting with `date-fns-tz` or standardize on UTC display throughout.
+
+**3.2 Replace O(n) flow index scan with Map** — frontend/src/pages/FlowList.tsx ~L251–280
+Keyboard navigation triggers a linear search over all loaded flows on each keypress. Replace with a `Map<flowId, index>` maintained alongside the flow list — O(1) lookup.
+
+**3.3 Fix scroll ↔ keyboard focus desync** — frontend/src/pages/FlowView.tsx ~L579
+TODO: manual scroll doesn't update `currentFlow`, causing keyboard navigation to jump. Add an `IntersectionObserver` or scroll event listener to sync the active flow with the visible item.
+
+**3.4 Decouple API layer from BSON** — services/cmd/api/api.go ~L73
+TODO comment: "API layer should not be aware of database structure." Move filter-building logic into a `db.FlowQuery` struct in services/pkg/db/db.go; api.go populates the struct, mongo.go translates it to BSON. Enables unit-testing API logic without a real DB.
+
+---
+
+## Verification per Phase
+- **Phase 1:** Benchmark `/query` + enrichment time before/after with `mongosh` + `hey`; run existing Go tests
+- **Phase 2:** Integration test empty-tag query; confirm context cancellation propagates; check `/query` response shape in frontend
+- **Phase 3:** Manual test keyboard nav + scroll; regression test chart rendering; run `npx eslint` on changed frontend files
+
+---
+
+## Priority order if time is limited
+1. Step 1.1 (N+1 fix) — biggest single perf gain, small code change
+2. Step 1.2 (compound index) — zero-risk, one line
+3. Step 2.1 (context propagation) — correctness, enables cancellation
+4. Step 2.3 (empty tags) — fixes a known bug with a beer bounty on it
+
+## Decisions
+- MongoDB stays — migration not worth it at CTF scale
+- Each phase is independently mergeable
+- Step 3.4 (BSON decoupling) is optional/long-term — only pursue if a DB swap becomes relevant later

services/cmd/api/api.go

@@ -16,6 +16,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
 	"tulip/pkg/db"
 
 	"github.com/labstack/echo/v4"
@@ -250,6 +251,16 @@ func (api *Router) query(c echo.Context) error {
 		return c.JSON(http.StatusInternalServerError, map[string]string{"error": err.Error()})
 	}
 
+	// Count total matching flows in parallel with signature fetching.
+	var totalCount int64
+	var countErr error
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		totalCount, countErr = api.DB.CountFlowsByOpts(c.Request().Context(), opts)
+	}()
+
 	// Collect all unique signature IDs across all flows for a single batch query.
 	allSigIDs := make([]string, 0)
 	sigIDSeen := make(map[string]bool)
@@ -305,7 +316,23 @@ func (api *Router) query(c echo.Context) error {
 		apiResults[i] = res
 	}
 
-	return c.JSON(http.StatusOK, apiResults)
+	wg.Wait()
+	if countErr != nil {
+		slog.Error("Failed to count flows", slog.Any("err", countErr))
+		return c.JSON(http.StatusInternalServerError, apiError{"Could not count flows. See server logs for details."})
+	}
+
+	page := 1
+	if opts.Limit > 0 {
+		page = opts.Offset/opts.Limit + 1
+	}
+
+	return c.JSON(http.StatusOK, map[string]any{
+		"data":           apiResults,
+		"page":           page,
+		"count":          totalCount,
+		"items_per_page": opts.Limit,
+	})
 }
 
 func (api *Router) getTags(c echo.Context) error {

services/cmd/api/query_test.go

@@ -35,6 +35,9 @@ func (m *mockDB) GetSignaturesBatch(ctx context.Context, ids []string) ([]db.Sur
 }
 func (m *mockDB) InsertFlows(context.Context, []db.FlowEntry) error            { return nil }
 func (m *mockDB) CountFlows(context.Context, bson.D) (int64, error)            { return 0, nil }
+func (m *mockDB) CountFlowsByOpts(context.Context, *db.FindFlowsOptions) (int64, error) {
+	return 0, nil
+}
 func (m *mockDB) SetStar(context.Context, string, bool) error                  { return nil }
 func (m *mockDB) GetFlowDetail(context.Context, string) (*db.FlowEntry, error) { return nil, nil }
 func (m *mockDB) GetTagList(context.Context) ([]string, error)                 { return nil, nil }
@@ -108,15 +111,20 @@ func TestQueryExcludeTagsForwarded(t *testing.T) {
 	assert.Equal(t, []string{"starred"}, mdb.lastOpts.ExcludeTags)
 }
 
-// TestQueryResponseIsJSON verifies that /query always returns a JSON array.
+// TestQueryResponseIsJSON verifies that /query returns a paginated JSON envelope.
 func TestQueryResponseIsJSON(t *testing.T) {
 	mdb := &mockDB{}
 	router := newTestRouter(mdb)
 
 	rec := doQuery(t, router, `{}`)
 
 	assert.Equal(t, http.StatusOK, rec.Code)
-	var result []json.RawMessage
+	var result struct {
+		Data         []json.RawMessage `json:"data"`
+		Page         int               `json:"page"`
+		Count        int64             `json:"count"`
+		ItemsPerPage int               `json:"items_per_page"`
+	}
 	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &result))
 }