event_data_integration.py

# ---------------------------------------------------------------------------
#  Event and sports data for NYC taxi features
# ---------------------------------------------------------------------------
# Data sources:
#   - Ticketmaster Discovery API: concerts, sports, festivals, theater
#     → event_within_2km, num_events_within_5km, event_size_proxy.
#   - NYC Open Data (opendata.cityofnewyork.us): Special Events Permits,
#     Street Closures, Film/Parade Permits.
#   - Sports schedules (Yankees, Mets, Knicks, Rangers, Giants)
#     → is_game_day, game_within_3_hours, home_game, venue_proximity_score.
# Priorities: Implement sports first (biggest impact), then Ticketmaster,
# then NYC permits if you want permit-level granularity.
# ---------------------------------------------------------------------------

import os
from datetime import datetime, timedelta
from typing import Optional

import pandas as pd
import numpy as np
from dotenv import load_dotenv

# Optional: for distance calculations (install: pip install requests haversine)
try:
    import requests
except ImportError:
    requests = None

# Load environment variables so TICKETMASTER_API_KEY can be read from a .env file.
load_dotenv()

# ---------------------------------------------------------------------------
# 1. Ticketmaster Discovery API - Special Events 
# https://developer.ticketmaster.com/  — get API key, free tier available
# ---------------------------------------------------------------------------

TICKETMASTER_API_KEY = os.environ.get("TICKETMASTER_API_KEY")
TICKETMASTER_BASE = "https://app.ticketmaster.com/discovery/v2/"


def fetch_ticketmaster_events_nyc(
    start_date: str = "2022-05-01",
    end_date: str = "2022-05-31",
    city: str = "New York",
    state_code: str = "NY",
    country_code: str = "US",
    api_key: str = TICKETMASTER_API_KEY,
) -> pd.DataFrame:
    """
    Fetches events in NYC, NY, USA for a date range via Ticketmaster Discovery API.
    Uses city, stateCode, countryCode to filter; retrieves lat/lon, startDateTime, endDateTime per event.
    """
    if requests is None:
        raise ImportError("pip install requests")
    key = api_key or TICKETMASTER_API_KEY
    if not key:
        raise ValueError("Set TICKETMASTER_API_KEY or pass api_key=...")

    all_events = []
    page = 0
    total_pages = 1

    while page < total_pages:
        url = (
            f"{TICKETMASTER_BASE}events.json"
            f"?apikey={key}"
            f"&city={city}&stateCode={state_code}&countryCode={country_code}"
            f"&startDateTime={start_date}T00:00:00Z&endDateTime={end_date}T23:59:59Z"
            f"&size=200&page={page}"
        )
        r = requests.get(url)
        r.raise_for_status()
        data = r.json()
        if "_embedded" not in data or "events" not in data["_embedded"]:
            break
        events = data["_embedded"]["events"]
        total_pages = data.get("page", {}).get("totalPages", 1)

        for e in events:
            start_info = e.get("dates", {}).get("start", {})
            start_dt = start_info.get("dateTime") or start_info.get("localDate")
            start_datetime = pd.to_datetime(start_dt) if start_dt else None

            end_info = e.get("dates", {}).get("end", {})
            end_dt = end_info.get("dateTime") or end_info.get("localDate")
            end_datetime = pd.to_datetime(end_dt) if end_dt else None

            venues = e.get("_embedded", {}).get("venues", [{}])
            v = venues[0] if venues else {}
            loc = v.get("location", {})
            lat = loc.get("latitude")
            lon = loc.get("longitude")
            if lat is None and lon is None:
                ll = v.get("latlong")  # some APIs return "lat,lon" string
                if isinstance(ll, str):
                    parts = ll.split(",")
                    if len(parts) == 2:
                        try:
                            lat, lon = float(parts[0].strip()), float(parts[1].strip())
                        except ValueError:
                            lat, lon = None, None
            if lat is not None and lon is not None:
                lat, lon = float(lat), float(lon)

            all_events.append({
                "event_id": e.get("id"),
                "name": e.get("name"),
                "start_datetime": start_datetime,
                "end_datetime": end_datetime,
                "venue_lat": lat,
                "venue_lon": lon,
            })
        page += 1

    df = pd.DataFrame(all_events)
    # If there were no events, return an empty DataFrame with the expected columns
    if df.empty:
        return pd.DataFrame(
            columns=["event_id", "name", "start_datetime", "end_datetime", "venue_lat", "venue_lon"]
        )
    # Otherwise, drop rows that are missing coordinates
    if "venue_lat" in df.columns and "venue_lon" in df.columns:
        df = df.dropna(subset=["venue_lat", "venue_lon"])
    return df

# ---------------------------------------------------------------------------
# 2. NYC Open Data — Special Events associated with NYC Department of Parks & Recreation
# https://opendata.cityofnewyork.us/
# ---------------------------------------------------------------------------

def load_nyc_open_data_parks_events(
    start_date: str = "2022-05-01",
    end_date: str = "2022-05-31",
    csv_path: str = "data/nyc_parks_special_events.csv",
) -> pd.DataFrame:
    """
    Loads NYC Parks special events CSV and returns a DataFrame with the same columns
    as Ticketmaster events: event_id, name, start_datetime, end_datetime, venue_lat, venue_lon.
    Parks data has no lat/lon or end time; those are set to NaN/NaT.
    """
    if not csv_path or not os.path.exists(csv_path):
        return pd.DataFrame()

    df = pd.read_csv(csv_path)
    datetime_col = "Date and Time"
    name_col = "Event Name"
    if datetime_col not in df.columns or name_col not in df.columns:
        return pd.DataFrame()

    df["start_datetime"] = pd.to_datetime(df[datetime_col], errors="coerce")
    df = df.dropna(subset=["start_datetime"])
    start = pd.to_datetime(start_date).normalize()
    end = pd.to_datetime(end_date) + pd.Timedelta(days=1)
    df = df[(df["start_datetime"] >= start) & (df["start_datetime"] < end)]

    out = pd.DataFrame({
        "event_id": ["parks_" + str(i) for i in range(len(df))],
        "name": df[name_col].astype(str),
        "start_datetime": df["start_datetime"].values,
        "end_datetime": pd.NaT,
        "venue_lat": np.nan,
        "venue_lon": np.nan,
    })
    return out


def extract_ticketmaster_events_nyc(
    start_date: str = "2022-05-01",
    end_date: str = "2022-05-31",
    city: str = "New York",
    state_code: str = "NY",
    country_code: str = "US",
    api_key: Optional[str] = None,
) -> pd.DataFrame:
    """
    Convenience wrapper around ``fetch_ticketmaster_events_nyc`` that mirrors
    the pattern of ``load_weather_nyc`` in ``weather_integration.py``.

    It reads the Ticketmaster API key from the environment (via ``.env`` or
    OS variables), prints a short debug line about key presence, and then
    delegates to ``fetch_ticketmaster_events_nyc`` to actually call the API.

    Args:
        start_date: Start of the event window (YYYY-MM-DD).
        end_date: End of the event window (YYYY-MM-DD).
        city: City filter (default "New York").
        state_code: State filter (default "NY").
        country_code: Country filter (default "US").
        api_key: Optional explicit key; if omitted, falls back to
            ``TICKETMASTER_API_KEY``.

    Returns:
        pd.DataFrame: Raw Ticketmaster events for the given window.
    """
    key = (api_key or os.environ.get("TICKETMASTER_API_KEY", "")).strip()
    print(f"[Events/Ticketmaster] API key present: {bool(key)} (length={len(key) if key else 0})")
    if not key:
        raise ValueError(
            "Ticketmaster API key is required. "
            "Set TICKETMASTER_API_KEY in your environment or pass api_key=..."
        )

    return fetch_ticketmaster_events_nyc(
        start_date=start_date,
        end_date=end_date,
        city=city,
        state_code=state_code,
        country_code=country_code,
        api_key=key,
    )


def extract_parks_events_nyc(
    start_date: str = "2022-05-01",
    end_date: str = "2022-05-31",
    csv_path: str = "data/nyc_parks_special_events.csv",
) -> pd.DataFrame:
    """
    High-level extractor for NYC Parks special events.

    This simply wraps ``load_nyc_open_data_parks_events`` so that the notebook
    can follow the same pattern used for weather data, for example:

    .. code-block:: python

        parks_raw = extract_parks_events_nyc(...)
        parks_std = standardize_events(parks_raw, ...)

    Args:
        start_date: Start of the event window (YYYY-MM-DD).
        end_date: End of the event window (YYYY-MM-DD).
        csv_path: Path to the NYC Parks special events CSV file.

    Returns:
        pd.DataFrame: Raw Parks events for the given window.
    """
    print(f"[Events/Parks] Loading Parks events from {csv_path}")
    return load_nyc_open_data_parks_events(
        start_date=start_date,
        end_date=end_date,
        csv_path=csv_path,
    )


# ---------------------------------------------------------------------------
# 4. Sports schedules — Yankees, Mets, Knicks, Rangers, Giants
# Scrape or API; here we assume a pre-built CSV or manual schedule table
# ---------------------------------------------------------------------------

# Approximate venue coordinates (lat, lon) for NYC-area major venues
VENUE_COORDS = {
    "yankee_stadium": (40.8296, -73.9262),   # Bronx
    "citi_field": (40.7571, -73.8458),       # Queens (Mets)
    "madison_square_garden": (40.7505, -73.9934),
    "barclays_center": (40.6826, -73.9754),
    "metlife_stadium": (40.8128, -74.0742),   # East Rutherford, NJ (Giants)
}

def load_sports_schedule(
    csv_path: str,
    date_col: str = "date",
    time_col: Optional[str] = "start_time",
    venue_col: str = "venue",
    home_team_col: str = "home_team",
) -> pd.DataFrame:
    """
    Loads sports schedule from CSV and adds game_datetime, venue_lat, venue_lon.

    Venue values should match keys in VENUE_COORDS (e.g. yankee_stadium, citi_field,
    madison_square_garden, metlife_stadium) for lat/lon to be filled.

    Args:
        csv_path: Path to CSV with at least date and venue columns.
        date_col: Column name for game date.
        time_col: Optional column for start time (for game_datetime).
        venue_col: Column name for venue (matched against VENUE_COORDS).
        home_team_col: Column name for home team (unused in default logic).

    Returns:
        pd.DataFrame: Schedule with date_col, game_datetime, venue_lat, venue_lon.
    """
    df = pd.read_csv(csv_path)
    df[date_col] = pd.to_datetime(df[date_col]).dt.normalize()
    if time_col and time_col in df.columns:
        df["game_datetime"] = pd.to_datetime(df[date_col].astype(str) + " " + df[time_col].astype(str), errors="coerce")
    else:
        df["game_datetime"] = df[date_col]
    df["venue_lat"] = df[venue_col].map(lambda v: VENUE_COORDS.get(str(v).lower().replace(" ", "_"), (None, None))[0])
    df["venue_lon"] = df[venue_col].map(lambda v: VENUE_COORDS.get(str(v).lower().replace(" ", "_"), (None, None))[1])
    return df


# ---------------------------------------------------------------------------
# 4. Distance helpers (haversine) — taxi pickup vs event/venue
# ---------------------------------------------------------------------------

def haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
    """
    Great-circle distance in km between two (lat, lon) points.

    Args:
        lat1, lon1: Latitude and longitude of first point (degrees).
        lat2, lon2: Latitude and longitude of second point (degrees).

    Returns:
        float: Distance in kilometers.
    """
    R = 6371  # Earth radius km
    lat1, lon1, lat2, lon2 = map(np.radians, [lat1, lon1, lat2, lon2])
    dlat = lat2 - lat1
    dlon = lon2 - lon1
    a = np.sin(dlat / 2) ** 2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon / 2) ** 2
    c = 2 * np.arcsin(np.sqrt(np.minimum(a, 1.0)))
    return R * c


def add_event_features_to_trips(
    df: pd.DataFrame,
    events: pd.DataFrame,
    pickup_lat_col: str = "pickup_lat",
    pickup_lon_col: str = "pickup_lon",
    pickup_datetime_col: str = "pickup_datetime",
    event_datetime_col: str = "event_datetime",
    event_lat: str = "venue_lat",
    event_lon: str = "venue_lon",
    within_km: float = 2.0,
    within_km_count: float = 5.0,
    window_hours: float = 3.0,
    capacity_col: Optional[str] = "capacity",
) -> pd.DataFrame:
    """
    Adds event-based features to trip rows: event_within_2km, num_events_within_5km, event_size_proxy.

    For each trip, checks events within a time window and distance radii. Requires df to have
    pickup lat/lon; if only PULocationID exists, merge with zone centroids first.

    Args:
        df: Trip DataFrame with pickup lat/lon and datetime columns.
        events: Event DataFrame with event_datetime, venue_lat, venue_lon; optional capacity_col.
        pickup_lat_col, pickup_lon_col, pickup_datetime_col: Column names in df.
        event_datetime_col, event_lat, event_lon: Column names in events.
        within_km: Radius (km) for binary "event nearby" (e.g. 2 km).
        within_km_count: Radius (km) for counting events (e.g. 5 km).
        window_hours: Time window ± pickup time (hours) to consider events.
        capacity_col: Optional column in events for event_size_proxy.

    Returns:
        pd.DataFrame: df with added columns event_within_2km, num_events_within_5km, event_size_proxy.
    """
    out = df.copy()
    if pickup_lat_col not in out.columns or pickup_lon_col not in out.columns:
        out["event_within_2km"] = 0
        out["num_events_within_5km"] = 0
        out["event_size_proxy"] = 0
        return out

    events = events.dropna(subset=[event_lat, event_lon, event_datetime_col])
    event_dt = pd.to_datetime(events[event_datetime_col])

    e2km = []
    e5km_count = []
    e_size = []
    for _, row in out.iterrows():
        t0 = row[pickup_datetime_col]
        if pd.isna(t0):
            e2km.append(0)
            e5km_count.append(0)
            e_size.append(0)
            continue
        t0 = pd.to_datetime(t0)
        window_start = t0 - timedelta(hours=window_hours)
        window_end = t0 + timedelta(hours=window_hours)
        mask = (event_dt >= window_start) & (event_dt <= window_end)
        ev = events.loc[mask]
        lat1, lon1 = row[pickup_lat_col], row[pickup_lon_col]
        within_2 = 0
        count_5 = 0
        max_cap = 0
        for _, e in ev.iterrows():
            d = haversine_km(lat1, lon1, e[event_lat], e[event_lon])
            if d <= within_km:
                within_2 = 1
            if d <= within_km_count:
                count_5 += 1
                cap = e.get(capacity_col) if capacity_col else None
                if cap is not None and not np.isnan(cap):
                    max_cap = max(max_cap, float(cap))
        if count_5 > 0 and max_cap == 0:
            max_cap = 1  # proxy when capacity missing
        e2km.append(within_2)
        e5km_count.append(count_5)
        e_size.append(max_cap)
    out["event_within_2km"] = e2km
    out["num_events_within_5km"] = e5km_count
    out["event_size_proxy"] = e_size
    return out


def add_sports_features_to_trips(
    df: pd.DataFrame,
    games: pd.DataFrame,
    pickup_date_col: str = "pickup_date",
    pickup_datetime_col: str = "pickup_datetime",
    pickup_lat_col: str = "pickup_lat",
    pickup_lon_col: str = "pickup_lon",
    game_datetime_col: str = "game_datetime",
    game_date_col: str = "date",
    venue_lat: str = "venue_lat",
    venue_lon: str = "venue_lon",
    within_hours: float = 3.0,
) -> pd.DataFrame:
    """
    Adds sports-related features: is_game_day, game_within_3_hours, home_game, venue_proximity_score.

    For each trip, checks whether the pickup date has games and whether any game start is within
    ±within_hours of pickup time; computes proximity score from min distance to same-day venues.

    Args:
        df: Trip DataFrame with pickup date/datetime and optional pickup lat/lon.
        games: Schedule DataFrame with game date, game_datetime, venue_lat, venue_lon.
        pickup_date_col, pickup_datetime_col, pickup_lat_col, pickup_lon_col: Column names in df.
        game_datetime_col, game_date_col, venue_lat, venue_lon: Column names in games.
        within_hours: Hours before/after game start to set game_within_3_hours=1.

    Returns:
        pd.DataFrame: df with added columns is_game_day, game_within_3_hours, home_game, venue_proximity_score.
    """
    out = df.copy()
    if pickup_date_col not in out.columns and pickup_datetime_col in out.columns:
        out[pickup_date_col] = pd.to_datetime(out[pickup_datetime_col]).dt.normalize()

    game_dates = pd.to_datetime(games[game_date_col]).dt.normalize()
    game_dt = pd.to_datetime(games[game_datetime_col])

    is_game_day = []
    game_within_3h = []
    home_game = []
    venue_proximity = []
    for _, row in out.iterrows():
        d = row[pickup_date_col]
        t0 = row.get(pickup_datetime_col)
        if pd.isna(d):
            is_game_day.append(0)
            game_within_3h.append(0)
            home_game.append(1 if 0 else 0)
            venue_proximity.append(0.0)
            continue
        d = pd.Timestamp(d).normalize()
        same_day = (game_dates == d)
        day_games = games.loc[same_day]
        is_day = 1 if len(day_games) > 0 else 0
        is_game_day.append(is_day)
        if t0 is None or pd.isna(t0):
            t0 = d
        t0 = pd.to_datetime(t0)
        within = 0
        min_km = np.inf
        for _, g in day_games.iterrows():
            gt = g.get(game_datetime_col)
            if pd.notna(gt):
                gt = pd.to_datetime(gt)
                if abs((gt - t0).total_seconds()) <= within_hours * 3600:
                    within = 1
            if pickup_lat_col in row and pickup_lon_col in row and pd.notna(row[pickup_lat_col]):
                km = haversine_km(row[pickup_lat_col], row[pickup_lon_col], g[venue_lat], g[venue_lon])
                min_km = min(min_km, km)
        game_within_3h.append(within)
        home_game.append(is_day)  # refine with actual home/away if you have it
        venue_proximity.append(1.0 / (1.0 + min_km) if min_km != np.inf else 0.0)
    out["is_game_day"] = is_game_day
    out["game_within_3_hours"] = game_within_3h
    out["home_game"] = home_game
    out["venue_proximity_score"] = venue_proximity
    return out


# ---------------------------------------------------------------------------
# 5. Zone centroids — map PULocationID to lat/lon for distance features
# ---------------------------------------------------------------------------

def load_zone_centroids(csv_path: str = "data/taxi_zone_centroids.csv") -> pd.DataFrame:
    """
    Loads zone centroid table mapping zone ID to latitude and longitude.

    Used to add pickup_lat, pickup_lon from PULocationID. NYC TLC publishes zone lookups;
    centroids can be derived from shapefiles or a pre-made CSV.

    Args:
        csv_path: Path to CSV with zone id and lat/lon columns.

    Returns:
        pd.DataFrame: Zone centroids, or empty DataFrame if file not found.
    """
    if not os.path.exists(csv_path):
        return pd.DataFrame()
    z = pd.read_csv(csv_path)
    return z


def merge_pickup_lat_lon(
    df: pd.DataFrame,
    zone_id_col: str = "PULocationID",
    centroids: Optional[pd.DataFrame] = None,
    centroid_id_col: str = "LocationID",
    lat_col: str = "lat",
    lon_col: str = "lon",
) -> pd.DataFrame:
    """
    Adds pickup_lat and pickup_lon to df by merging with zone centroids on zone ID.

    Left-merge on zone_id_col; centroid columns are renamed to pickup_lat, pickup_lon.

    Args:
        df: Trip DataFrame with zone ID column (e.g. PULocationID).
        zone_id_col: Column in df containing zone identifier.
        centroids: DataFrame with zone id and lat/lon columns.
        centroid_id_col, lat_col, lon_col: Column names in centroids.

    Returns:
        pd.DataFrame: df with added pickup_lat, pickup_lon (NaN where no match).
    """
    out = df.copy()
    if centroids is None or len(centroids) == 0:
        return out
    c = centroids[[centroid_id_col, lat_col, lon_col]].copy()
    c = c.rename(columns={lat_col: "pickup_lat", lon_col: "pickup_lon"})
    out = out.merge(c, left_on=zone_id_col, right_on=centroid_id_col, how="left")
    if centroid_id_col != zone_id_col and centroid_id_col in out.columns:
        out = out.drop(columns=[centroid_id_col])
    return out


#