Merge pull request #26 from MHaggis/MHaggis/fix-link-previews

Track nightly sync runs and fix dashboard sync status

Author: MHaggis

.github/workflows/nightly-sync.yml

@@ -96,5 +96,6 @@ jobs:
       - name: Seed Supabase from SQLite
         run: cd web && npx tsx scripts/seed-from-sqlite.ts
         env:
+          SYNC_SOURCE_TYPE: nightly_full
           NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
           SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_KEY }}

web/scripts/seed-from-sqlite.ts

@@ -34,26 +34,130 @@ if (!supabaseUrl || !supabaseKey) {
 
 const supabase = createClient(supabaseUrl, supabaseKey);
 const sqlite = new Database(SQLITE_PATH, { readonly: true });
+const SYNC_SOURCE_TYPE = process.env.SYNC_SOURCE_TYPE || 'nightly_full';
+const MAX_SYNC_ERROR_CHARS = 2000;
 
 async function main() {
-  console.log('=== Seeding Supabase from SQLite ===');
-  console.log(`SQLite: ${SQLITE_PATH}`);
-  console.log(`Supabase: ${supabaseUrl}`);
-  console.log('');
-
-  await seedDetections();
-  await seedDetectionTechniques();
-  await seedTechniqueTactics();
-  await seedAttackTechniques();
-  await seedAttackActors();
-  await seedAttackSoftware();
-  await seedActorTechniques();
-  await seedSoftwareTechniques();
-  await seedProcedureReference();
-  await seedStories();
-
-  console.log('\n=== Seed complete! ===');
-  sqlite.close();
+  let syncRunId: string | null = null;
+  let preRunDetections = 0;
+
+  try {
+    console.log('=== Seeding Supabase from SQLite ===');
+    console.log(`SQLite: ${SQLITE_PATH}`);
+    console.log(`Supabase: ${supabaseUrl}`);
+    console.log(`Sync source: ${SYNC_SOURCE_TYPE}`);
+    console.log('');
+
+    preRunDetections = await getDetectionCount();
+    syncRunId = await startSyncRun(preRunDetections);
+
+    await seedDetections();
+    await seedDetectionTechniques();
+    await seedTechniqueTactics();
+    await seedAttackTechniques();
+    await seedAttackActors();
+    await seedAttackSoftware();
+    await seedActorTechniques();
+    await seedSoftwareTechniques();
+    await seedProcedureReference();
+    await seedStories();
+
+    const postRunDetections = await getDetectionCount();
+    const detectionsAdded = Math.max(postRunDetections - preRunDetections, 0);
+
+    await completeSyncRun(syncRunId, {
+      detectionsTotal: postRunDetections,
+      detectionsAdded,
+      detectionsUpdated: 0,
+    });
+
+    console.log('\n=== Seed complete! ===');
+  } catch (err) {
+    if (syncRunId) {
+      await failSyncRun(syncRunId, formatSyncError(err));
+    }
+    throw err;
+  } finally {
+    sqlite.close();
+  }
+}
+
+async function getDetectionCount(): Promise<number> {
+  const { count, error } = await supabase
+    .from('detections')
+    .select('id', { count: 'exact', head: true });
+
+  if (error || count === null) {
+    throw new Error(`Failed to query detection count: ${error?.message || 'count unavailable'}`);
+  }
+
+  return count;
+}
+
+async function startSyncRun(preRunDetections: number): Promise<string> {
+  const { data, error } = await supabase
+    .from('sync_runs')
+    .insert({
+      source_type: SYNC_SOURCE_TYPE,
+      started_at: new Date().toISOString(),
+      detections_total: preRunDetections,
+      detections_added: 0,
+      detections_updated: 0,
+      status: 'running',
+      error: null,
+    })
+    .select('id')
+    .single();
+
+  if (error || !data?.id) {
+    throw new Error(`Failed to create sync run: ${error?.message || 'missing sync run id'}`);
+  }
+
+  console.log(`Started sync run: ${data.id}`);
+  return data.id as string;
+}
+
+async function completeSyncRun(
+  syncRunId: string,
+  metrics: { detectionsTotal: number; detectionsAdded: number; detectionsUpdated: number }
+): Promise<void> {
+  const { error } = await supabase
+    .from('sync_runs')
+    .update({
+      completed_at: new Date().toISOString(),
+      detections_total: metrics.detectionsTotal,
+      detections_added: metrics.detectionsAdded,
+      detections_updated: metrics.detectionsUpdated,
+      status: 'completed',
+      error: null,
+    })
+    .eq('id', syncRunId);
+
+  if (error) {
+    throw new Error(`Failed to finalize sync run ${syncRunId}: ${error.message}`);
+  }
+}
+
+async function failSyncRun(syncRunId: string, errorMessage: string): Promise<void> {
+  const { error } = await supabase
+    .from('sync_runs')
+    .update({
+      completed_at: new Date().toISOString(),
+      status: 'failed',
+      error: errorMessage,
+    })
+    .eq('id', syncRunId);
+
+  if (error) {
+    console.error(`Failed to mark sync run ${syncRunId} as failed: ${error.message}`);
+  }
+}
+
+function formatSyncError(err: unknown): string {
+  const message = err instanceof Error
+    ? `${err.message}${err.stack ? `\n${err.stack}` : ''}`
+    : String(err);
+  return message.slice(0, MAX_SYNC_ERROR_CHARS);
 }
 
 async function upsertBatch(table: string, rows: Record<string, unknown>[], batchSize = 500) {

web/supabase/migrations/015_sync_runs_backfill.sql

@@ -0,0 +1,22 @@
+-- Backfill one bootstrap sync run so dashboard has immediate status
+-- Idempotent: only inserts if sync_runs currently has zero rows
+INSERT INTO sync_runs (
+  source_type,
+  started_at,
+  completed_at,
+  detections_added,
+  detections_updated,
+  detections_total,
+  status,
+  error
+)
+SELECT
+  'bootstrap',
+  now(),
+  now(),
+  0,
+  0,
+  (SELECT COUNT(*) FROM detections),
+  'completed',
+  NULL
+WHERE NOT EXISTS (SELECT 1 FROM sync_runs);