Import report country code data (#170)

* Import report country code data

* Update backfill command to import country data

Author: jgraham

server/reportmanager/management/commands/backfill_missing_report_data.py

@@ -47,6 +47,7 @@
 
 from django.conf import settings
 from django.core.management import BaseCommand
+from django.db.models import Q
 from google.cloud import bigquery
 from google.oauth2 import service_account
 
@@ -62,6 +63,7 @@ class BackfillData:
     ml_valid_probability: float | None
     language_code: str | None
     translated_text: str | None
+    country: str | None
 
 
 class Command(BaseCommand):
@@ -80,17 +82,19 @@ def handle(self, *args, **options) -> None:
 
     def run_backfill(self) -> None:
         # Find reports needing ML updates (only those with non-empty comments)
-        reports_to_update = ReportEntry.objects.filter(
-            ml_valid_probability__isnull=True, comments__isnull=False
-        ).exclude(comments="")
+        reports_to_update = (
+            ReportEntry.objects.filter(comments__isnull=False)
+            .exclude(comments="")
+            .filter(Q(ml_valid_probability__isnull=True) | Q(country__isnull=True))
+        )
 
         total_reports = reports_to_update.count()
 
         if total_reports == 0:
-            LOG.info("No reports need ML backfill")
+            LOG.info("No reports need backfill")
             return
 
-        LOG.info("Found %d reports needing ML backfill", total_reports)
+        LOG.info("Found %d reports needing backfill", total_reports)
 
         all_reports = list(reports_to_update)
         batches = list(batched(all_reports, self.BQ_BATCH_SIZE))
@@ -114,12 +118,12 @@ def run_backfill(self) -> None:
                 len(report_batch),
             )
 
-            uuid_batch: list[str] = [str(report.uuid) for report in report_batch]
+            uuid_batch = {str(report.uuid): report for report in report_batch}
 
             query: str = f"""
                 SELECT r.uuid,
                        c.label as ml_label, c.probability as ml_probability,
-                       t.language_code, t.translated_text
+                       t.language_code, t.translated_text, r.country
                 FROM `{settings.BIGQUERY_TABLE}` as r
                 INNER JOIN `{settings.BIGQUERY_CLASSIFICATION_TABLE}` c
                     ON r.uuid = c.report_uuid
@@ -138,13 +142,18 @@ def run_backfill(self) -> None:
 
             bq_data: dict[str, BackfillData] = {}
             for row in result:
-                ml_valid_probability = transform_ml_label(
-                    row.ml_label, row.ml_probability
-                )
+                report = uuid_batch[row.uuid]
+                if report.ml_valid_probability is None:
+                    ml_valid_probability = transform_ml_label(
+                        row.ml_label, row.ml_probability
+                    )
+                else:
+                    ml_valid_probability = report.ml_valid_probability
                 bq_data[row.uuid] = BackfillData(
                     ml_valid_probability=ml_valid_probability,
                     language_code=row.language_code,
                     translated_text=row.translated_text,
+                    country=row.country,
                 )
 
             LOG.info("Fetched data for %d reports from BigQuery", len(bq_data))
@@ -160,13 +169,15 @@ def run_backfill(self) -> None:
                 if uuid in bq_data:
                     data = bq_data[uuid]
                     updated = False
+                    retriage = False
 
                     if (
                         report.ml_valid_probability is None
                         and data.ml_valid_probability is not None
                     ):
                         report.ml_valid_probability = data.ml_valid_probability
                         updated = True
+                        retriage = True
 
                     if (
                         report.comments_translated is None
@@ -178,12 +189,17 @@ def run_backfill(self) -> None:
                             data.translated_text
                         )
                         updated = True
+                        retriage = True
+
+                    if report.country is None and data.country is not None:
+                        report.country = data.country
+                        updated = True
 
                     if updated:
                         reports_to_update.append(report)
 
                         # Clear bucket assignment to re-triage these reports
-                        if report.cluster_id is None:
+                        if retriage and report.cluster_id is None:
                             report.bucket_id = None
 
             if reports_to_update:
@@ -195,6 +211,7 @@ def run_backfill(self) -> None:
                         "comments_original_language",
                         "comments_preprocessed",
                         "bucket_id",
+                        "country",
                     ],
                     batch_size=self.DB_BATCH_SIZE,
                 )

server/reportmanager/management/commands/import_reports_from_bigquery.py

@@ -41,7 +41,7 @@ def handle(self, *args, **options):
         result = client.query(
             f"""SELECT
                     r.*, t.language_code, t.translated_text,
-                    c.label as ml_label, c.probability as ml_probability
+                    c.label as ml_label, c.probability as ml_probability,
                 FROM `{settings.BIGQUERY_TABLE}` as r
                 LEFT JOIN `{settings.BIGQUERY_TRANSLATIONS_TABLE}` t
                     ON r.uuid = t.report_uuid
@@ -68,6 +68,7 @@ def handle(self, *args, **options):
                 comments=row.comments,
                 comments_translated=row.translated_text,
                 comments_original_language=row.language_code,
+                country=row.country,
                 details=row.details,
                 reported_at=row.reported_at.replace(tzinfo=UTC),
                 url=urlsplit(row.url),

server/reportmanager/migrations/0017_reportentry_country.py

@@ -0,0 +1,18 @@
+# Generated by Django 6.0.3 on 2026-04-23 19:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('reportmanager', '0016_joblock'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='reportentry',
+            name='country',
+            field=models.TextField(max_length=2, null=True),
+        ),
+    ]

server/reportmanager/models.py

@@ -655,6 +655,7 @@ def create_from_report(self, report, bucket_id=None, cluster_id=None):
             cluster_id=cluster_id,
             domain=domain,
             comments_preprocessed=preprocessed,
+            country=report.country,
         )
 
 
@@ -680,6 +681,7 @@ class ReportEntry(models.Model):
     )
     domain: models.CharField = models.CharField(max_length=255, null=True)
     comments_preprocessed: models.TextField = models.TextField(null=True)
+    country: models.TextField = models.TextField(max_length=2, null=True)
 
     objects = ReportEntryManager()
 

src/webcompat/models.py

@@ -55,6 +55,7 @@ class Report:
     breakage_category: str | None = None
     ml_valid_probability: float | None = None
     cluster_id: str | None = None
+    country: str | None = None
 
     @classmethod
     def load(cls, data: str) -> Report: