Merge pull request #89 from govtechmy/fix/phone-number-format

fix(gsheet): fetch gsheet as xlsx, not csv

Author: asyiqinasir

src/core/gsheet.py

@@ -56,4 +56,17 @@ def fetch_csv_data(gsheet_id: str, gid: str) -> tuple[bytes, str | None]:
     content_disp = response.headers.get("Content-Disposition", "")
     filename = _extract_filename(content_disp)
 
+    return response.content, filename
+
+def fetch_xlsx_data(gsheet_id: str, gid: str) -> tuple[bytes, str | None]:
+    """Fetch Google Sheet as XLSX to preserve actual cell values (not display format)."""
+    url = f"https://docs.google.com/spreadsheets/d/{gsheet_id}/export?format=xlsx&gid={gid}"
+    logger.info("Fetching XLSX data from URL: %s", url)
+
+    response = requests.get(url)
+    response.raise_for_status()
+
+    content_disp = response.headers.get("Content-Disposition", "")
+    filename = _extract_filename(content_disp)
+
     return response.content, filename

src/core/s3.py

@@ -28,6 +28,23 @@ def _upload_to_s3(csv_bytes: bytes, bucket: str, prefix: str, source_filename: s
 
     return s3_key
 
+def _upload_xlsx_to_s3(xlsx_bytes: bytes, bucket: str, prefix: str, source_filename: str | None = None,) -> str:
+    if not bucket:
+        bucket = get_s3_bucket_name()
+
+    timestamp = int(time.time())
+    version = _extract_file_version(source_filename) or "unknown"
+    s3_key = f"{prefix}/{version}/{timestamp}.xlsx"
+
+    s3.put_object(
+        Bucket=bucket,
+        Key=s3_key,
+        Body=xlsx_bytes,
+        ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+    )
+
+    return s3_key
+
 def _list_csv_files_in_s3(bucket: str, prefix: str) -> list[str]:
     response = s3.list_objects_v2(Bucket=bucket, Prefix=prefix)
 
@@ -46,6 +63,11 @@ def _read_csv_from_s3(bucket: str, s3_key: str) -> pd.DataFrame:
     response = s3.get_object(Bucket=bucket, Key=s3_key)
     return pd.read_csv(response["Body"], dtype=str).fillna("")
 
+def _read_xlsx_from_s3(bucket: str, s3_key: str) -> pd.DataFrame:
+    import io
+    response = s3.get_object(Bucket=bucket, Key=s3_key)
+    return pd.read_excel(io.BytesIO(response["Body"].read()), dtype=str).fillna("")
+
 def upload_json_to_s3(payload: dict | list, bucket: Optional[str], key: str) -> str:
     """
     Upload JSON to a fixed S3 key (no timestamp).

src/pipeline/ingestion.py

@@ -17,8 +17,8 @@
 
 from src.config import Settings, get_settings
 from src.models import Sekolah
-from src.core.gsheet import fetch_csv_data, _extract_file_version
-from src.core.s3 import (_upload_to_s3, _latest_csv_from_s3, _read_csv_from_s3)
+from src.core.gsheet import fetch_xlsx_data, _extract_file_version
+from src.core.s3 import (_upload_xlsx_to_s3, _read_xlsx_from_s3)
 from src.models.sekolah import SekolahStatus
 from src.pipeline.status_sync import sync_entiti_statuses, delete_inactive_entiti
 from src.core.time import _utc_now
@@ -99,13 +99,13 @@ def _read_google_sheet(sheet_id: str, gid: str) -> Iterable[Dict[str, Any]]:
 
 
 def _load_rows(settings: Settings) -> tuple[Iterable[Dict[str, Any]], str | None]:
-    csv_bytes, file_name = fetch_csv_data(settings.gsheet_id, settings.gsheet_gid)
-    logger.info("Uploading CSV data to S3 bucket %s", settings.s3_bucket_dataproc)
-    s3_key = _upload_to_s3(csv_bytes, settings.s3_bucket_dataproc, settings.s3_prefix_sekolah, file_name)
-    logger.info("CSV uploaded to S3 at key: %s", s3_key)
+    xlsx_bytes, file_name = fetch_xlsx_data(settings.gsheet_id, settings.gsheet_gid)
+    logger.info("Uploading XLSX data to S3 bucket %s", settings.s3_bucket_dataproc)
+    s3_key = _upload_xlsx_to_s3(xlsx_bytes, settings.s3_bucket_dataproc, settings.s3_prefix_sekolah, file_name)
+    logger.info("XLSX uploaded to S3 at key: %s", s3_key)
 
-    df = _read_csv_from_s3(settings.s3_bucket_dataproc, s3_key)
-    logger.info("CSV loaded from S3: %d rows, %d columns", df.shape[0], df.shape[1])
+    df = _read_xlsx_from_s3(settings.s3_bucket_dataproc, s3_key)
+    logger.info("XLSX loaded from S3: %d rows, %d columns", df.shape[0], df.shape[1])
     return df.to_dict(orient="records"), file_name