From 3e3545dd0ad955afd1babf1a5b2a69b5ef87adef Mon Sep 17 00:00:00 2001 From: raj Date: Fri, 1 May 2026 21:52:22 +0530 Subject: [PATCH 1/8] Add resource activity report script for bulk CSV export Python script to export resource create/delete/update activity from Guardrails workspaces via GraphQL API. Supports multiple workspaces, configurable resource types, and auto-detection of the Turbot automation identity. Designed for cases where the console Resource Activities report times out on large notification datasets. --- .../get-resource-activity-report/README.md | 148 +++++++ .../requirements.txt | 2 + .../resource_activity_report.py | 407 ++++++++++++++++++ 3 files changed, 557 insertions(+) create mode 100644 guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md create mode 100644 guardrails_utilities/python_utils/notifications/get-resource-activity-report/requirements.txt create mode 100644 guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md new file mode 100644 index 000000000..647ba0b6e --- /dev/null +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md @@ -0,0 +1,148 @@ +# Get Resource Activity Report + +Export resource create/delete/update activity from one or more Guardrails workspaces to CSV. Filters by actor identity (defaults to the Turbot automation identity) and resource type. + +This script is useful when the console Resource Activities report times out on workspaces with large notification volumes. It fetches only resource-level CRUD notifications (not control/policy processing activity), keeping the dataset manageable. + +## Features + +- **Multi-workspace support** — run against multiple workspaces in a single invocation +- **Auto-detects Turbot Identity** — finds the automation actor ID automatically +- **Configurable resource type** — defaults to EC2 Snapshots, works with any resource type +- **Configurable time range** — `--days` parameter for lookback window +- **CSV output** — matches the console Resource Activities report format +- **Handles large datasets** — paginated queries with timeout retry logic + +## Prerequisites + +- [Python 3.8+](https://www.python.org/downloads/) +- [Turbot CLI credentials](https://turbot.com/guardrails/docs/reference/cli/installation#set-up-your-turbot-guardrails-credentials) configured at `~/.config/turbot/credentials.yml` + +## Setup + +```bash +cd get-resource-activity-report +pip install -r requirements.txt +``` + +### Credentials + +The script uses the same credentials file as the Turbot CLI (`~/.config/turbot/credentials.yml`): + +```yaml +my-workspace: + workspace: "https://my-workspace.turbot.com" + accessKey: "your-access-key" + secretKey: "your-secret-key" + +another-workspace: + workspace: "https://another-workspace.turbot.com" + accessKey: "your-access-key" + secretKey: "your-secret-key" +``` + +## Usage + +### Basic — EC2 Snapshots, last 30 days + +```bash +python resource_activity_report.py --profile my-workspace +``` + +### Custom time range + +```bash +# Last 90 days +python resource_activity_report.py --profile my-workspace --days 90 + +# Last 7 days +python resource_activity_report.py --profile my-workspace --days 7 +``` + +### Multiple workspaces + +```bash +python resource_activity_report.py \ + --profile workspace-a \ + --profile workspace-b \ + --profile workspace-c \ + --days 30 +``` + +### Different resource type + +```bash +# S3 Buckets +python resource_activity_report.py --profile my-workspace \ + --resource-type "tmod:@turbot/aws-s3#/resource/types/bucket" + +# EC2 Instances +python resource_activity_report.py --profile my-workspace \ + --resource-type "tmod:@turbot/aws-ec2#/resource/types/instance" +``` + +### Specific actor identity + +```bash +python resource_activity_report.py --profile my-workspace \ + --days 90 --actor-id 123456789012345 +``` + +### Custom output directory + +```bash +python resource_activity_report.py --profile my-workspace \ + --days 30 --output-dir ./reports +``` + +## Output + +One CSV file per workspace: `{profile}-resource-activity-{days}d-{date}.csv` + +### Columns + +| Column | Description | +|--------|-------------| +| Timestamp | Activity timestamp (DD-Mon-YYYY HH:MM:SS) | +| NotificationType | RESOURCE CREATED, RESOURCE DELETED, or RESOURCE UPDATED | +| Type / Message | Resource type category and name | +| Resource | Resource title (e.g., snapshot ID, bucket name) | +| Actor | Actor identity name | +| ResourceId | Guardrails resource ID | +| TrunkPath | Full resource hierarchy path | +| Detail URL | Direct link to the notification in the console | + +### Example output + +``` +Timestamp,NotificationType,Type / Message,Resource,Actor,ResourceId,TrunkPath,Detail URL +01-May-2026 04:47:35,RESOURCE DELETED,Object > Snapshot,snap-08170c6ca...,Turbot > Turbot Identity,384593893452312,(deleted),https://... +``` + +## How it works + +1. Connects to each workspace using the credentials.yml profile +2. Auto-detects the Turbot Identity actor ID (resource type `turbotIdentity`) +3. Queries all resource CRUD notifications for the given resource type and actor via paginated GraphQL +4. Filters to the requested date range client-side +5. Writes one CSV per workspace + +### Performance note + +The script intentionally avoids combining `actorIdentityId` with `timestamp` filters in the GraphQL query. On workspaces with millions of notifications, this filter combination causes backend query timeouts. Instead, it fetches all resource CRUD notifications (typically hundreds to low thousands) and applies the date filter in Python. This approach completes reliably in under a minute. + +## Command-line reference + +``` +usage: resource_activity_report.py [-h] --profile PROFILE [--days DAYS] + [--resource-type RESOURCE_TYPE] + [--actor-id ACTOR_ID] + [--output-dir OUTPUT_DIR] + +options: + --profile PROFILE Turbot CLI profile name (repeatable) + --days DAYS Days to look back (default: 30) + --resource-type TYPE Resource type URI (default: EC2 Snapshot) + --actor-id ACTOR_ID Actor identity ID (default: auto-detect) + --output-dir OUTPUT_DIR Output directory (default: current) +``` diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/requirements.txt b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/requirements.txt new file mode 100644 index 000000000..7daf2ab7f --- /dev/null +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/requirements.txt @@ -0,0 +1,2 @@ +requests>=2.28.0 +PyYAML>=6.0 diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py new file mode 100644 index 000000000..9fa009837 --- /dev/null +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py @@ -0,0 +1,407 @@ +#!/usr/bin/env python3 +""" +Guardrails Resource Activity Report + +Pulls resource create/delete/update activity by a specific actor (e.g. Turbot +automation identity) from one or more Guardrails workspaces. Outputs CSV +matching the console Resource Activities report format. + +Designed for cases where the console Resource Activities report times out on +large notification datasets. The script fetches only resource-level CRUD +notifications (not control/policy processing), making it fast even on +workspaces with millions of notifications. + +Authentication uses ~/.config/turbot/credentials.yml (same as Turbot CLI). +""" + +import argparse +import csv +import os +import sys +import time +from base64 import b64encode +from datetime import datetime, timedelta, timezone + +import requests +import yaml + +CREDENTIALS_PATH = os.path.expanduser("~/.config/turbot/credentials.yml") +GRAPHQL_PATH = "api/v5/graphql" +PAGE_SIZE = 500 + +NOTIFICATIONS_QUERY = """ +query ResourceActivity($filter: [String!], $paging: String) { + notifications(filter: $filter, paging: $paging, dataSource: DB) { + items { + turbot { + id + createTimestamp + } + notificationType + resource { + type { + title + category { + title + } + } + trunk { + title + } + turbot { + id + title + } + } + actor { + identity { + trunk { + title + } + turbot { + id + title + } + } + } + } + paging { + next + } + } +} +""" + +TURBOT_IDENTITY_QUERY = """ +query FindTurbotIdentity($filter: [String!]) { + resources(filter: $filter) { + items { + turbot { + id + title + } + } + } +} +""" + + +def load_profile(profile_name): + """Load workspace credentials from ~/.config/turbot/credentials.yml.""" + if not os.path.exists(CREDENTIALS_PATH): + print(f"Error: Credentials file not found at {CREDENTIALS_PATH}") + print( + "Create it with your workspace profiles. See: " + "https://turbot.com/guardrails/docs/reference/cli/installation" + "#set-up-your-turbot-guardrails-credentials" + ) + sys.exit(1) + + with open(CREDENTIALS_PATH, "r") as f: + creds = yaml.safe_load(f) + + if profile_name not in creds: + available = ", ".join(creds.keys()) + print(f"Error: Profile '{profile_name}' not found. Available: {available}") + sys.exit(1) + + profile = creds[profile_name] + for key in ("workspace", "accessKey", "secretKey"): + if key not in profile: + print(f"Error: Profile '{profile_name}' missing '{key}'") + sys.exit(1) + + workspace = profile["workspace"].rstrip("/") + auth_bytes = f"{profile['accessKey']}:{profile['secretKey']}".encode("utf-8") + auth_token = b64encode(auth_bytes).decode() + + return workspace, { + "endpoint": f"{workspace}/{GRAPHQL_PATH}", + "headers": { + "Authorization": f"Basic {auth_token}", + "Content-Type": "application/json", + }, + } + + +def graphql_request(config, query, variables=None): + """Execute a GraphQL query against the workspace.""" + payload = {"query": query} + if variables: + payload["variables"] = variables + + response = requests.post( + config["endpoint"], + json=payload, + headers=config["headers"], + timeout=180, + ) + response.raise_for_status() + result = response.json() + + if "errors" in result: + for err in result["errors"]: + msg = str(err.get("message", "unknown error")) + print(f" GraphQL error: {msg}") + + return result + + +def get_turbot_identity_id(config): + """Auto-detect the Turbot Identity actor ID in the workspace.""" + result = graphql_request( + config, + TURBOT_IDENTITY_QUERY, + { + "filter": [ + "resourceTypeId:'tmod:@turbot/turbot-iam#/resource/types/turbotIdentity'", + "limit:1", + ] + }, + ) + items = (result.get("data") or {}).get("resources", {}).get("items", []) + if items: + return items[0]["turbot"]["id"] + return None + + +def fetch_all_pages(config, filter_str): + """Fetch all pages for a given filter string.""" + all_items = [] + next_page = None + page_num = 0 + + while True: + page_num += 1 + variables = {"filter": filter_str, "paging": next_page} + + try: + result = graphql_request(config, NOTIFICATIONS_QUERY, variables) + except requests.exceptions.Timeout: + print(f" Page {page_num}: timeout — retrying in 10s...") + time.sleep(10) + try: + result = graphql_request(config, NOTIFICATIONS_QUERY, variables) + except requests.exceptions.Timeout: + print(f" Page {page_num}: timeout again — stopping") + break + + data = result.get("data") or {} + notifications = data.get("notifications") or {} + items = notifications.get("items") or [] + all_items.extend(items) + + paging = notifications.get("paging") or {} + print(f" Page {page_num}: {len(items)} items (total so far: {len(all_items)})") + + if paging and paging.get("next"): + next_page = paging["next"] + else: + break + + return all_items + + +def fetch_resource_activity(config, resource_type_id, days, actor_id=None): + """ + Fetch resource CRUD activity for a given resource type and actor. + + Fetches all resource_created/deleted/updated notifications without date + filters (to avoid backend query plan timeouts on large datasets), then + filters to the requested date range client-side. + """ + if not actor_id: + print(" Detecting Turbot Identity ID...", end=" ") + detected_id = get_turbot_identity_id(config) + if detected_id: + actor_id = str(detected_id) + print(actor_id) + else: + print("FAILED — specify --actor-id manually") + return [] + + safe_actor_id = str(actor_id) + filter_str = ( + f"resourceType:'{resource_type_id}'" + f" actorIdentityId:'{safe_actor_id}'" + f" notificationType:resource_created,resource_deleted,resource_updated" + f" sort:-createTimestamp" + f" limit:{PAGE_SIZE}" + ) + + print(f" Fetching resource activity for actor {safe_actor_id}...") + all_items = fetch_all_pages(config, filter_str) + + if days and all_items: + cutoff = datetime.now(timezone.utc) - timedelta(days=days) + before_count = len(all_items) + all_items = [ + i + for i in all_items + if datetime.fromisoformat( + i["turbot"]["createTimestamp"].replace("Z", "+00:00") + ) + >= cutoff + ] + print( + f" Date filter (last {days}d): {before_count} → {len(all_items)} items" + ) + + return all_items + + +def format_row(item, workspace_url): + """Format a notification item as a CSV row.""" + ts_raw = item["turbot"]["createTimestamp"] + ts = datetime.fromisoformat(ts_raw.replace("Z", "+00:00")) + ts_fmt = ts.strftime("%d-%b-%Y %H:%M:%S") + ntype = item["notificationType"].upper().replace("_", " ") + + res_type = item["resource"].get("type") or {} + cat_title = (res_type.get("category") or {}).get("title", "") + type_title = res_type.get("title", "") + type_msg = f"{cat_title} > {type_title}" if cat_title else type_title + + resource_title = item["resource"]["turbot"]["title"] + resource_id = item["resource"]["turbot"]["id"] + trunk = (item["resource"].get("trunk") or {}).get("title", "(deleted)") + + actor = item.get("actor") or {} + identity = actor.get("identity") or {} + actor_name = (identity.get("trunk") or {}).get("title", "") + if not actor_name: + actor_name = (identity.get("turbot") or {}).get("title", "") + + detail_url = f"{workspace_url}/apollo/notifications/{item['turbot']['id']}" + + return { + "Timestamp": ts_fmt, + "NotificationType": ntype, + "Type / Message": type_msg, + "Resource": resource_title, + "Actor": actor_name, + "ResourceId": resource_id, + "TrunkPath": trunk, + "Detail URL": detail_url, + } + + +CSV_FIELDNAMES = [ + "Timestamp", + "NotificationType", + "Type / Message", + "Resource", + "Actor", + "ResourceId", + "TrunkPath", + "Detail URL", +] + + +def write_csv(items, workspace_url, output_path): + """Write items to CSV file.""" + with open(output_path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=CSV_FIELDNAMES) + writer.writeheader() + for item in items: + writer.writerow(format_row(item, workspace_url)) + return len(items) + + +def main(): + parser = argparse.ArgumentParser( + description="Pull resource activity from Guardrails workspaces", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # EC2 snapshots deleted by Turbot, last 90 days + %(prog)s --profile myworkspace --days 90 + + # Multiple workspaces + %(prog)s --profile ws1 --profile ws2 --days 30 + + # S3 buckets instead of snapshots + %(prog)s --profile myworkspace --days 30 \\ + --resource-type "tmod:@turbot/aws-s3#/resource/types/bucket" + + # Specific actor identity + %(prog)s --profile myworkspace --days 90 --actor-id 123456789012345 + """, + ) + parser.add_argument( + "--profile", + action="append", + required=True, + help="Turbot CLI profile name (repeatable for multiple workspaces)", + ) + parser.add_argument( + "--days", + type=int, + default=30, + help="Number of days to look back (default: 30)", + ) + parser.add_argument( + "--resource-type", + default="tmod:@turbot/aws-ec2#/resource/types/snapshot", + help="Resource type URI to filter on (default: EC2 Snapshot)", + ) + parser.add_argument( + "--actor-id", + help="Actor identity ID to filter on (default: auto-detect Turbot Identity)", + ) + parser.add_argument( + "--output-dir", + default=".", + help="Output directory for CSV files (default: current directory)", + ) + + args = parser.parse_args() + os.makedirs(args.output_dir, exist_ok=True) + + print(f"Resource Activity Report — last {args.days} days") + print(f"Profiles: {', '.join(args.profile)}") + print(f"Resource type: {args.resource_type}") + print(f"Output: {os.path.abspath(args.output_dir)}") + print() + + for profile_name in args.profile: + print(f"[{profile_name}]") + workspace, config = load_profile(profile_name) + print(f" Workspace: {workspace}") + + items = fetch_resource_activity( + config, args.resource_type, args.days, args.actor_id + ) + + if not items: + print(" No resource activity found.") + print() + continue + + created = sum( + 1 for i in items if i["notificationType"] == "resource_created" + ) + deleted = sum( + 1 for i in items if i["notificationType"] == "resource_deleted" + ) + updated = sum( + 1 for i in items if i["notificationType"] == "resource_updated" + ) + + date_str = datetime.now(timezone.utc).strftime("%Y%m%d") + filename = f"{profile_name}-resource-activity-{args.days}d-{date_str}.csv" + output_path = os.path.join(args.output_dir, filename) + + count = write_csv(items, workspace, output_path) + print( + f" Results: {count} total" + f" ({created} created, {deleted} deleted, {updated} updated)" + ) + print(f" CSV: {output_path}") + print() + + print("Done.") + + +if __name__ == "__main__": + main() From a09fa4c5f978b687e3f820376ca03b5a6bf5b3b0 Mon Sep 17 00:00:00 2001 From: raj Date: Fri, 1 May 2026 22:18:11 +0530 Subject: [PATCH 2/8] Increase API timeout to 300s and add retry with backoff Large workspaces with millions of notifications can take over 180s on the first query. Bumps timeout to 300s and retries up to 3 times with progressive backoff (15s, 30s, 45s). --- .../resource_activity_report.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py index 9fa009837..6bfee71c7 100644 --- a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py @@ -134,7 +134,7 @@ def graphql_request(config, query, variables=None): config["endpoint"], json=payload, headers=config["headers"], - timeout=180, + timeout=300, ) response.raise_for_status() result = response.json() @@ -175,16 +175,19 @@ def fetch_all_pages(config, filter_str): page_num += 1 variables = {"filter": filter_str, "paging": next_page} - try: - result = graphql_request(config, NOTIFICATIONS_QUERY, variables) - except requests.exceptions.Timeout: - print(f" Page {page_num}: timeout — retrying in 10s...") - time.sleep(10) + max_retries = 3 + for attempt in range(1, max_retries + 1): try: result = graphql_request(config, NOTIFICATIONS_QUERY, variables) - except requests.exceptions.Timeout: - print(f" Page {page_num}: timeout again — stopping") break + except requests.exceptions.Timeout: + if attempt < max_retries: + wait = attempt * 15 + print(f" Page {page_num}: timeout (attempt {attempt}/{max_retries}) — retrying in {wait}s...") + time.sleep(wait) + else: + print(f" Page {page_num}: timeout after {max_retries} attempts — stopping") + return all_items data = result.get("data") or {} notifications = data.get("notifications") or {} From df4aeba5605034b754dc5da0fe0dc10e145299b3 Mon Sep 17 00:00:00 2001 From: raj Date: Fri, 8 May 2026 14:07:52 +0530 Subject: [PATCH 3/8] Add turbot CLI-based resource deletion report with calendar-day boundaries Add fetch_resource_deletions.py that uses `turbot graphql` CLI for reliable paginated fetches of resource deletion notifications. Defaults to all resource types when a time boundary is set, supports --date for midnight-to-midnight UTC boundaries (no overlap between days), resource type aliases, and a safety guard against unbounded queries. Includes reference GraphQL queries captured from the Guardrails console and updated README with comprehensive examples. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../get-resource-activity-report/README.md | 281 ++++++++--- .../fetch_resource_deletions.py | 272 +++++++++++ ...e-resource-deleted-by-turbot-query.graphql | 449 +++++++++++++++++ ...rt-resource-deleted-by-tubot-query.graphql | 457 ++++++++++++++++++ .../resource_activity_report.py | 447 ++++++++++++----- .../resource_deleted_by_turbot.graphql | 19 + 6 files changed, 1717 insertions(+), 208 deletions(-) create mode 100644 guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py create mode 100644 guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/console-resource-deleted-by-turbot-query.graphql create mode 100644 guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/export-resource-deleted-by-tubot-query.graphql create mode 100644 guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_deleted_by_turbot.graphql diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md index 647ba0b6e..cdfc39ffe 100644 --- a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md @@ -1,148 +1,269 @@ # Get Resource Activity Report -Export resource create/delete/update activity from one or more Guardrails workspaces to CSV. Filters by actor identity (defaults to the Turbot automation identity) and resource type. +Export resource deletion activity from Guardrails workspaces to CSV. Two scripts are provided: -This script is useful when the console Resource Activities report times out on workspaces with large notification volumes. It fetches only resource-level CRUD notifications (not control/policy processing activity), keeping the dataset manageable. +| Script | Backend | Best for | +|--------|---------|----------| +| `fetch_resource_deletions.py` | **Turbot CLI** (`turbot graphql`) | Reliable paginated fetches with calendar-day boundaries, bypasses the console 5K export limit | +| `resource_activity_report.py` | Python `requests` (direct HTTP) | Multi-workspace batch runs, auto-detects Turbot Identity ID | -## Features - -- **Multi-workspace support** — run against multiple workspaces in a single invocation -- **Auto-detects Turbot Identity** — finds the automation actor ID automatically -- **Configurable resource type** — defaults to EC2 Snapshots, works with any resource type -- **Configurable time range** — `--days` parameter for lookback window -- **CSV output** — matches the console Resource Activities report format -- **Handles large datasets** — paginated queries with timeout retry logic +Both produce CSV output matching the console Resource Activities export format. ## Prerequisites - [Python 3.8+](https://www.python.org/downloads/) -- [Turbot CLI credentials](https://turbot.com/guardrails/docs/reference/cli/installation#set-up-your-turbot-guardrails-credentials) configured at `~/.config/turbot/credentials.yml` +- [Turbot CLI](https://turbot.com/guardrails/docs/reference/cli/installation) installed and configured +- Turbot CLI credentials at `~/.config/turbot/credentials.yml` ## Setup ```bash -cd get-resource-activity-report +cd guardrails_utilities/python_utils/notifications/get-resource-activity-report pip install -r requirements.txt ``` -### Credentials +Verify turbot CLI connectivity: -The script uses the same credentials file as the Turbot CLI (`~/.config/turbot/credentials.yml`): +```bash +turbot graphql --profile kochgp --query='{ resource(id:"tmod:@turbot/turbot#/") { turbot { title } } }' +``` -```yaml -my-workspace: - workspace: "https://my-workspace.turbot.com" - accessKey: "your-access-key" - secretKey: "your-secret-key" +--- -another-workspace: - workspace: "https://another-workspace.turbot.com" - accessKey: "your-access-key" - secretKey: "your-secret-key" +## fetch_resource_deletions.py (Recommended) + +Uses the turbot CLI for authentication and GraphQL transport. Paginates automatically and writes CSV incrementally (partial data is preserved if interrupted). + +### Key features + +- **Calendar-day boundaries** — `--date 2026-05-07` fetches midnight-to-midnight UTC, no overlap between consecutive days +- **All resource types by default** — captures snapshots, instances, volumes, Lambda functions, etc. in a single run +- **Bypasses the 5K console export limit** — paginated GraphQL fetches with no cap +- **Resource type aliases** — use `--resource-type snapshot` instead of the full `tmod:` URI +- **Safety guard** — blocks unbounded queries (all types + no time filter) to prevent fetching millions of rows + +### Examples + +#### All resource types deleted by Turbot on a single day + +```bash +python fetch_resource_deletions.py --profile kochgp --date 2026-05-07 +# Output: kochgp-resource-deleted-all-types-2026-05-07.csv +``` + +#### Only EC2 snapshots on a single day + +```bash +python fetch_resource_deletions.py --profile kochgp --date 2026-05-07 --resource-type snapshot +# Output: kochgp-resource-deleted-snapshot-2026-05-07.csv ``` -## Usage +#### Only EC2 instances on a single day + +```bash +python fetch_resource_deletions.py --profile kochgp --date 2026-05-07 --resource-type instance +# Output: kochgp-resource-deleted-instance-2026-05-07.csv +``` -### Basic — EC2 Snapshots, last 30 days +#### Date range — all types from May 1 to May 8 ```bash -python resource_activity_report.py --profile my-workspace +python fetch_resource_deletions.py --profile kochgp --since 2026-05-01 --until 2026-05-08 +# Output: kochgp-resource-deleted-all-types-2026-05-01.csv ``` -### Custom time range +#### Date range — snapshots only from May 1 to May 8 ```bash -# Last 90 days -python resource_activity_report.py --profile my-workspace --days 90 +python fetch_resource_deletions.py --profile kochgp --since 2026-05-01 --until 2026-05-08 --resource-type snapshot +# Output: kochgp-resource-deleted-snapshot-2026-05-01.csv +``` + +#### Rolling window — last 3 days, instances only -# Last 7 days -python resource_activity_report.py --profile my-workspace --days 7 +```bash +python fetch_resource_deletions.py --profile kochgp --days 3 --resource-type instance +# Output: kochgp-resource-deleted-instance-20260508.csv ``` -### Multiple workspaces +#### Rolling window — last 7 days, all types ```bash -python resource_activity_report.py \ - --profile workspace-a \ - --profile workspace-b \ - --profile workspace-c \ - --days 30 +python fetch_resource_deletions.py --profile kochgp --days 7 +# Output: kochgp-resource-deleted-all-types-20260508.csv ``` -### Different resource type +#### Custom output file ```bash -# S3 Buckets -python resource_activity_report.py --profile my-workspace \ - --resource-type "tmod:@turbot/aws-s3#/resource/types/bucket" +python fetch_resource_deletions.py --profile kochgp --date 2026-05-07 --output may7-report.csv +``` -# EC2 Instances -python resource_activity_report.py --profile my-workspace \ - --resource-type "tmod:@turbot/aws-ec2#/resource/types/instance" +#### Different workspace (kochkbs) + +```bash +python fetch_resource_deletions.py --profile kochkbs --date 2026-05-07 \ + --actor-id 123456789012345 \ + --workspace-url "https://kbs.turbotprod.kochind.cloud" +``` + +#### Specific resource type by full URI + +```bash +python fetch_resource_deletions.py --profile kochgp --date 2026-05-07 \ + --resource-type "tmod:@turbot/aws-lambda#/resource/types/functionVersion" ``` -### Specific actor identity +#### Open-ended since (no end date) ```bash -python resource_activity_report.py --profile my-workspace \ - --days 90 --actor-id 123456789012345 +python fetch_resource_deletions.py --profile kochgp --since 2026-05-01 --resource-type volume +# Fetches from May 1 to now +# Output: kochgp-resource-deleted-volume-2026-05-01.csv ``` -### Custom output directory +#### Day-by-day tracking for a week ```bash -python resource_activity_report.py --profile my-workspace \ - --days 30 --output-dir ./reports +for d in 01 02 03 04 05 06 07; do + python fetch_resource_deletions.py --profile kochgp --date 2026-05-$d +done +# Produces: kochgp-resource-deleted-all-types-2026-05-01.csv through -07.csv +# No overlap between files — each covers midnight-to-midnight UTC ``` -## Output +#### Compare two workspaces for the same day -One CSV file per workspace: `{profile}-resource-activity-{days}d-{date}.csv` +```bash +python fetch_resource_deletions.py --profile kochgp --date 2026-05-07 +python fetch_resource_deletions.py --profile kochkbs --date 2026-05-07 \ + --actor-id 123456789012345 \ + --workspace-url "https://kbs.turbotprod.kochind.cloud" +``` + +### Resource type aliases + +Short names you can use with `--resource-type` instead of full URIs: + +| Alias | Resource type URI | +|-------|-------------------| +| `snapshot` | `tmod:@turbot/aws-ec2#/resource/types/snapshot` | +| `instance` | `tmod:@turbot/aws-ec2#/resource/types/instance` | +| `volume` | `tmod:@turbot/aws-ec2#/resource/types/volume` | +| `ami` | `tmod:@turbot/aws-ec2#/resource/types/image` | +| `launch-template` | `tmod:@turbot/aws-ec2#/resource/types/launchTemplate` | +| `bucket` | `tmod:@turbot/aws-s3#/resource/types/bucket` | +| `lambda` | `tmod:@turbot/aws-lambda#/resource/types/function` | +| `function-version` | `tmod:@turbot/aws-lambda#/resource/types/functionVersion` | +| `role` | `tmod:@turbot/aws-iam#/resource/types/role` | +| `vpc` | `tmod:@turbot/aws-vpc-core#/resource/types/vpc` | +| `security-group` | `tmod:@turbot/aws-vpc-security#/resource/types/securityGroup` | +| `subscription` | `tmod:@turbot/aws-sns#/resource/types/subscription` | +| `ecs-service` | `tmod:@turbot/aws-ecs#/resource/types/service` | + +You can also pass any `tmod:@turbot/...` URI directly. + +### Command-line reference + +``` +usage: fetch_resource_deletions.py [-h] --profile PROFILE + [--date DATE] [--since SINCE] [--until UNTIL] [--days DAYS] + [--actor-id ACTOR_ID] [--resource-type RESOURCE_TYPE] + [--output OUTPUT] [--workspace-url WORKSPACE_URL] + +options: + --profile PROFILE Turbot CLI profile name (required) + +time range (pick one): + --date DATE Single calendar day, midnight-to-midnight UTC (YYYY-MM-DD) + --since SINCE Start date inclusive (YYYY-MM-DD) + --until UNTIL End date exclusive (YYYY-MM-DD), use with --since + --days DAYS Rolling window in days (default: 1) + + --actor-id ACTOR_ID Turbot actor identity ID (auto-detected for known workspaces) + --resource-type TYPE Resource type alias or full tmod URI (default: all types) + --output OUTPUT Output CSV file path (default: auto-generated) + --workspace-url URL Workspace base URL (auto-detected for known workspaces) +``` + +### Time range behavior + +| Option | Boundary | Example | +|--------|----------|---------| +| `--date 2026-05-07` | `timestamp:>2026-05-07 timestamp:<2026-05-08` | Midnight-to-midnight UTC, no overlap | +| `--since 2026-05-01 --until 2026-05-08` | `timestamp:>2026-05-01 timestamp:<2026-05-08` | Arbitrary range | +| `--since 2026-05-01` | `timestamp:>2026-05-01` | From May 1 to now | +| `--days 3` | `timestamp:>2026-05-05` | Rolling 3 days from today | + +### Safety guard + +To prevent accidental fetches of millions of rows, the script blocks queries that combine **all resource types** (no `--resource-type`) with **no absolute time boundary** (no `--date` or `--since`). Either add a time boundary or specify a resource type. + +--- + +## Output format ### Columns | Column | Description | |--------|-------------| -| Timestamp | Activity timestamp (DD-Mon-YYYY HH:MM:SS) | -| NotificationType | RESOURCE CREATED, RESOURCE DELETED, or RESOURCE UPDATED | -| Type / Message | Resource type category and name | -| Resource | Resource title (e.g., snapshot ID, bucket name) | -| Actor | Actor identity name | +| Timestamp | Activity timestamp (DD-Mon-YYYY HH:MM:SS UTC) | +| NotificationType | RESOURCE DELETED | +| Type / Message | Resource type category (e.g., Object > Snapshot) | +| Resource | Resource title (e.g., snap-08789b5a4ab739c33) | +| Actor | Actor identity name (e.g., Turbot Identity) | | ResourceId | Guardrails resource ID | -| TrunkPath | Full resource hierarchy path | +| TrunkPath | Resource hierarchy path, or (deleted) | | Detail URL | Direct link to the notification in the console | ### Example output -``` +```csv Timestamp,NotificationType,Type / Message,Resource,Actor,ResourceId,TrunkPath,Detail URL -01-May-2026 04:47:35,RESOURCE DELETED,Object > Snapshot,snap-08170c6ca...,Turbot > Turbot Identity,384593893452312,(deleted),https://... +07-May-2026 13:42:32,RESOURCE DELETED,Object > Snapshot,snap-03ae5c144a2bab0e8,Turbot Identity,385157353768573,(deleted),https://gp.turbotprod.kochind.cloud/apollo/notifications/385157685600695 +07-May-2026 11:32:25,RESOURCE DELETED,Object > Instance,i-0a1b2c3d4e5f67890,Turbot Identity,385149350192559,(deleted),https://gp.turbotprod.kochind.cloud/apollo/notifications/385149692057073 ``` -## How it works +--- -1. Connects to each workspace using the credentials.yml profile -2. Auto-detects the Turbot Identity actor ID (resource type `turbotIdentity`) -3. Queries all resource CRUD notifications for the given resource type and actor via paginated GraphQL -4. Filters to the requested date range client-side -5. Writes one CSV per workspace +## Credentials -### Performance note +Both scripts use `~/.config/turbot/credentials.yml` (same as Turbot CLI): -The script intentionally avoids combining `actorIdentityId` with `timestamp` filters in the GraphQL query. On workspaces with millions of notifications, this filter combination causes backend query timeouts. Instead, it fetches all resource CRUD notifications (typically hundreds to low thousands) and applies the date filter in Python. This approach completes reliably in under a minute. - -## Command-line reference +```yaml +kochgp: + workspace: "https://gp.turbotprod.kochind.cloud" + accessKey: "your-access-key" + secretKey: "your-secret-key" +kochkbs: + workspace: "https://kbs.turbotprod.kochind.cloud" + accessKey: "your-access-key" + secretKey: "your-secret-key" ``` -usage: resource_activity_report.py [-h] --profile PROFILE [--days DAYS] - [--resource-type RESOURCE_TYPE] - [--actor-id ACTOR_ID] - [--output-dir OUTPUT_DIR] -options: - --profile PROFILE Turbot CLI profile name (repeatable) - --days DAYS Days to look back (default: 30) - --resource-type TYPE Resource type URI (default: EC2 Snapshot) - --actor-id ACTOR_ID Actor identity ID (default: auto-detect) - --output-dir OUTPUT_DIR Output directory (default: current) +List configured profiles: + +```bash +turbot workspace list ``` + +--- + +## GraphQL reference files + +The `graphql-diff/` directory contains the full GraphQL queries captured from the Guardrails console for reference: + +| File | Description | +|------|-------------| +| `export-resource-deleted-by-tubot-query.graphql` | Console Export CSV query with all fragments and example variables | +| `console-resource-deleted-by-turbot-query.graphql` | Console display query (used for in-browser rendering) | +| `resource_deleted_by_turbot.graphql` | Minimal query used by `fetch_resource_deletions.py` | + +--- + +## Notes + +- **metadata.stats.total is approximate** — the total count in the GraphQL response does not apply all filter conditions (particularly timestamp boundaries). The actual item count from pagination is the accurate number. +- **Turbot Identity ID** — the actor identity ID for the Turbot automation identity varies per workspace. For `kochgp` it is `218162262814364`. For other workspaces, pass `--actor-id` or omit it to fetch deletions by all actors. +- **Timestamps are UTC** — all `--date`, `--since`, and `--until` values are interpreted as UTC midnight boundaries. diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py new file mode 100644 index 000000000..b7e5f324b --- /dev/null +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +""" +Fetch 'Resources Deleted by Turbot' from a Guardrails workspace using the turbot +CLI, with pagination and CSV output matching the console export format. + +Supports calendar-day boundaries (midnight-to-midnight UTC) for consistent +day-over-day tracking. Fetches all resource types by default when a time +boundary is provided; optionally filter to a specific resource type. + +Usage: + # All resource types deleted by Turbot on a single day + python fetch_resource_deletions.py --profile kochgp --date 2026-05-07 + + # Only snapshots + python fetch_resource_deletions.py --profile kochgp --date 2026-05-07 \ + --resource-type snapshot + + # Date range, all types + python fetch_resource_deletions.py --profile kochgp --since 2026-05-01 --until 2026-05-08 + + # Rolling window + python fetch_resource_deletions.py --profile kochgp --days 3 +""" + +import argparse +import csv +import json +import os +import subprocess +import sys +from datetime import datetime, timedelta + +QUERY_FILE = os.path.join(os.path.dirname(__file__), "resource_deleted_by_turbot.graphql") +PAGE_SIZE = 200 + +WORKSPACE_URLS = { + "kochgp": "https://gp.turbotprod.kochind.cloud", + "kochkbs": "https://kbs.turbotprod.kochind.cloud", + "kochkbxt": "https://kbxt.turbotprod.kochind.cloud", + "kochgdn": "https://gdn.turbotprod.kochind.cloud", + "kochinv": "https://inv.turbotprod.kochind.cloud", + "kochind": "https://kochind.turbotprod.kochind.cloud", + "kochkaes": "https://kaes.turbotprod.kochind.cloud", + "kochfhr": "https://fhr.turbotprod.kochind.cloud", + "kochkes": "https://kes.turbotprod.kochind.cloud", + "kochkmt": "https://kmt.turbotprod.kochind.cloud", + "kochmlx": "https://mlx.turbotprod.kochind.cloud", +} + +TURBOT_IDENTITY_IDS = { + "kochgp": "218162262814364", +} + +RESOURCE_TYPE_ALIASES = { + "snapshot": "tmod:@turbot/aws-ec2#/resource/types/snapshot", + "ec2-snapshot": "tmod:@turbot/aws-ec2#/resource/types/snapshot", + "instance": "tmod:@turbot/aws-ec2#/resource/types/instance", + "ec2-instance": "tmod:@turbot/aws-ec2#/resource/types/instance", + "volume": "tmod:@turbot/aws-ec2#/resource/types/volume", + "ec2-volume": "tmod:@turbot/aws-ec2#/resource/types/volume", + "ami": "tmod:@turbot/aws-ec2#/resource/types/image", + "launch-template": "tmod:@turbot/aws-ec2#/resource/types/launchTemplate", + "bucket": "tmod:@turbot/aws-s3#/resource/types/bucket", + "s3-bucket": "tmod:@turbot/aws-s3#/resource/types/bucket", + "function": "tmod:@turbot/aws-lambda#/resource/types/function", + "lambda": "tmod:@turbot/aws-lambda#/resource/types/function", + "function-version": "tmod:@turbot/aws-lambda#/resource/types/functionVersion", + "role": "tmod:@turbot/aws-iam#/resource/types/role", + "vpc": "tmod:@turbot/aws-vpc-core#/resource/types/vpc", + "security-group": "tmod:@turbot/aws-vpc-security#/resource/types/securityGroup", + "subscription": "tmod:@turbot/aws-sns#/resource/types/subscription", + "ecs-service": "tmod:@turbot/aws-ecs#/resource/types/service", +} + +CSV_HEADERS = [ + "Timestamp", "NotificationType", "Type / Message", + "Resource", "Actor", "ResourceId", "TrunkPath", "Detail URL", +] + + +def resolve_resource_type(value): + if value is None: + return None + if value.startswith("tmod:"): + return value + alias = RESOURCE_TYPE_ALIASES.get(value.lower()) + if alias: + return alias + print(f"Error: unknown resource type alias '{value}'.", file=sys.stderr) + print(f"Known aliases: {', '.join(sorted(RESOURCE_TYPE_ALIASES.keys()))}", file=sys.stderr) + print(f"Or pass a full tmod:@turbot/... URI.", file=sys.stderr) + sys.exit(1) + + +def run_query(profile, variables): + cmd = [ + "turbot", "graphql", + "--profile", profile, + "--format", "json", + "--query", QUERY_FILE, + "--variables", json.dumps(variables), + ] + result = subprocess.run(cmd, capture_output=True, text=True, timeout=600) + if result.returncode != 0: + print(f"turbot CLI error: {result.stderr.strip()}", file=sys.stderr) + sys.exit(1) + return json.loads(result.stdout) + + +def build_filter(actor_id, resource_type_uri, date, since, until, days): + parts = ["notificationType:resource_deleted"] + if actor_id: + parts.append(f"actorIdentityId:{actor_id}") + if resource_type_uri: + parts.append(f"resourceTypeId:'{resource_type_uri}'") + + if date: + next_day = (datetime.strptime(date, "%Y-%m-%d") + timedelta(days=1)).strftime("%Y-%m-%d") + parts.append(f"timestamp:>{date}") + parts.append(f"timestamp:<{next_day}") + elif since and until: + parts.append(f"timestamp:>{since}") + parts.append(f"timestamp:<{until}") + elif since: + parts.append(f"timestamp:>{since}") + else: + start = (datetime.utcnow() - timedelta(days=days)).strftime("%Y-%m-%d") + parts.append(f"timestamp:>{start}") + + filter_str = " ".join(parts) + filters = [filter_str, f"limit:{PAGE_SIZE}"] + + return filters + + +def format_timestamp(iso_ts): + dt = datetime.fromisoformat(iso_ts.replace("Z", "+00:00")) + return dt.strftime("%d-%b-%Y %H:%M:%S") + + +def format_actor(actor): + if not actor: + return "" + identity = actor.get("identity") or {} + persona = actor.get("persona") or {} + id_title = identity.get("title", "") + pe_title = persona.get("title", "") + if pe_title and pe_title != id_title: + return f"{id_title} > {pe_title}" + return id_title + + +def to_csv_row(item, workspace_url): + ts = format_timestamp(item["turbot"]["createTimestamp"]) + nt = item["notificationType"].replace("_", " ").upper() + resource = item.get("resource") or {} + res_turbot = resource.get("turbot") or {} + res_type_title = (resource.get("type") or {}).get("turbot", {}).get("title", "") + trunk_title = (resource.get("trunk") or {}).get("title") or "(deleted)" + type_msg = f"Object > {res_type_title}" if res_type_title else "" + actor_str = format_actor(item.get("actor")) + notif_id = item["turbot"]["id"] + detail_url = f"{workspace_url}/apollo/notifications/{notif_id}" + + return [ + ts, nt, type_msg, + res_turbot.get("title", ""), + actor_str, + res_turbot.get("id", ""), + trunk_title, + detail_url, + ] + + +def main(): + parser = argparse.ArgumentParser( + description="Fetch resource deletions by Turbot from a Guardrails workspace via turbot CLI", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +resource type aliases: + snapshot, instance, volume, ami, launch-template, bucket, + lambda, function-version, role, vpc, security-group, + subscription, ecs-service + + Or pass a full URI: tmod:@turbot/aws-ec2#/resource/types/snapshot + +examples: + %(prog)s --profile kochgp --date 2026-05-07 + %(prog)s --profile kochgp --date 2026-05-07 --resource-type snapshot + %(prog)s --profile kochgp --since 2026-05-01 --until 2026-05-08 + %(prog)s --profile kochgp --days 7 --resource-type instance +""", + ) + parser.add_argument("--profile", required=True, help="Turbot CLI profile name (e.g. kochgp)") + + time_group = parser.add_argument_group("time range (pick one)") + time_group.add_argument("--date", help="Single calendar day, midnight-to-midnight UTC (YYYY-MM-DD)") + time_group.add_argument("--since", help="Start date inclusive (YYYY-MM-DD)") + time_group.add_argument("--until", help="End date exclusive (YYYY-MM-DD), use with --since") + time_group.add_argument("--days", type=int, default=1, help="Rolling window in days (default: 1)") + + parser.add_argument("--actor-id", help="Turbot actor identity ID (auto-detected for known workspaces)") + parser.add_argument("--resource-type", + help="Resource type alias or tmod URI (default: all types)") + parser.add_argument("--output", help="Output CSV file path (default: auto-generated)") + parser.add_argument("--workspace-url", help="Workspace base URL (auto-detected for known workspaces)") + args = parser.parse_args() + + actor_id = args.actor_id or TURBOT_IDENTITY_IDS.get(args.profile) + workspace_url = args.workspace_url or WORKSPACE_URLS.get(args.profile, "") + resource_type_uri = resolve_resource_type(args.resource_type) + + if not actor_id: + print(f"Warning: no actor-id for profile '{args.profile}'. Fetching deletions by ALL actors.", + file=sys.stderr) + if not workspace_url: + print(f"Warning: no workspace URL for profile '{args.profile}'. Detail URLs will be incomplete.", + file=sys.stderr) + + if not resource_type_uri and not args.date and not args.since: + print("Error: fetching all resource types without a time boundary is unsafe (millions of rows).", + file=sys.stderr) + print("Add --date, --since, or --resource-type to bound the query.", file=sys.stderr) + sys.exit(1) + + filters = build_filter(actor_id, resource_type_uri, args.date, args.since, args.until, args.days) + + type_tag = args.resource_type or "all-types" + date_tag = args.date or args.since or datetime.now().strftime("%Y%m%d") + output_file = args.output or f"{args.profile}-resource-deleted-{type_tag}-{date_tag}.csv" + + scope = resource_type_uri or "all resource types" + print(f"Profile: {args.profile}") + print(f"Resource type: {scope}") + print(f"Filters: {filters}") + print(f"Output: {output_file}") + print() + + paging = None + page = 0 + total_written = 0 + + with open(output_file, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(CSV_HEADERS) + + while True: + page += 1 + variables = {"filter": filters} + if paging: + variables["paging"] = paging + + data = run_query(args.profile, variables) + notifications = data.get("notifications", {}) + items = notifications.get("items", []) + + for item in items: + writer.writerow(to_csv_row(item, workspace_url)) + f.flush() + total_written += len(items) + + print(f" Page {page}: {len(items)} items (cumulative: {total_written})") + + paging = notifications.get("paging", {}).get("next") + if not paging or not items: + break + + print(f"\nDone. {total_written} rows written to {output_file}") + + +if __name__ == "__main__": + main() diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/console-resource-deleted-by-turbot-query.graphql b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/console-resource-deleted-by-turbot-query.graphql new file mode 100644 index 000000000..5d76951de --- /dev/null +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/console-resource-deleted-by-turbot-query.graphql @@ -0,0 +1,449 @@ + +## Used GraphQL query to fetch notifications for snapshot deletions by a specific actor. The query includes various fragments to retrieve detailed information about the notifications, including control, grant, policy settings, and resource details. The filters applied in the query ensure that only relevant notifications are fetched based on the specified criteria. + +query Notifications($filter: [String!], $paging: String, $dataSource: NotificationDataSource) { + notifications(filter: $filter, paging: $paging, dataSource: $dataSource) { + items { + ...notification + data + __typename + } + paging { + next + __typename + } + metadata { + stats { + total + __typename + } + __typename + } + __typename + } +} + +fragment notification on Notification { + ...notificationFields + control { + ...controlNotification + __typename + } + oldControl { + ...controlNotification + __typename + } + grant { + ...grantNotification + __typename + } + oldGrant { + ...grantNotification + __typename + } + activeGrant { + ...activeGrantNotification + __typename + } + oldActiveGrant { + ...activeGrantNotification + __typename + } + policySetting { + ...policySettingNotification + __typename + } + oldPolicySetting { + ...policySettingNotification + __typename + } + policyValue { + ...policyValueNotification + __typename + } + oldPolicyValue { + ...policyValueNotification + __typename + } + resource { + ...resourceNotification + __typename + } + oldResource { + ...resourceNotification + __typename + } + actor { + ...actorNotification + __typename + } + turbot { + ...notificationMetadataFields + __typename + } + __typename +} + +fragment activeGrantNotification on ActiveGrant { + grant { + ...grantNotification + __typename + } + turbot { + ...activeGrantMetadataFields + __typename + } + __typename +} + +fragment activeGrantMetadataFields on TurbotActiveGrantMetadata { + id + versionId + createTimestamp + __typename +} + +fragment grantMetadataFields on TurbotGrantMetadata { + id + versionId + createTimestamp + __typename +} + +fragment permissionMetadataFields on TurbotPermissionMetadata { + id + path + title + __typename +} + +fragment trunkFields on TrunkItems { + title + items { + ...trunkItemFields + __typename + } + __typename +} + +fragment trunkItemFields on TrunkItem { + turbot { + id + path + title + __typename + } + __typename +} + +fragment actorNotification on Actor { + identity { + ...identityFields + __typename + } + persona { + ...identityFields + __typename + } + role { + ...identityFields + __typename + } + alternatePersona + __typename +} + +fragment identityFields on Identity { + picture + title + type { + uri + turbot { + ...resourceTypeMetadataFields + __typename + } + __typename + } + turbot { + ...resourceMetadataFields + __typename + } + __typename +} + +fragment resourceMetadataFields on TurbotResourceMetadata { + akas + id + parentId + path + title + versionId + __typename +} + +fragment controlNotification on Control { + state + type { + icon + trunk { + ...trunkFields + __typename + } + turbot { + ...controlTypeMetadataFields + __typename + } + __typename + } + turbot { + ...controlMetadataFields + __typename + } + __typename +} + +fragment controlMetadataFields on TurbotControlMetadata { + id + stateChangeTimestamp + versionId + __typename +} + +fragment controlTypeMetadataFields on TurbotControlTypeMetadata { + id + title + __typename +} + +fragment grantNotification on Grant { + roleName + groupName + type { + trunk { + ...trunkFields + __typename + } + turbot { + ...permissionMetadataFields + __typename + } + __typename + } + level { + turbot { + ...permissionMetadataFields + __typename + } + __typename + } + identity { + turbot { + ...resourceMetadataFields + __typename + } + __typename + } + turbot { + ...grantMetadataFields + __typename + } + __typename +} + +fragment notificationFields on Notification { + notificationType + icon + message + __typename +} + +fragment notificationMetadataFields on TurbotNotificationMetadata { + updateTimestamp + createTimestamp + id + processId + __typename +} + +fragment policySettingNotification on PolicySetting { + default + isCalculated + orphan + exception + precedence + template + templateInput + value + valueSource + note + validToTimestamp + resource { + turbot { + ...resourceMetadataFields + __typename + } + __typename + } + type { + icon + trunk { + ...trunkFields + __typename + } + turbot { + ...policyTypeMetadataFields + __typename + } + __typename + } + turbot { + ...policySettingMetadataFields + __typename + } + __typename +} + +fragment policySettingMetadataFields on TurbotPolicySettingMetadata { + id + timestamp + createTimestamp + updateTimestamp + versionId + __typename +} + +fragment policyTypeMetadataFields on TurbotPolicyTypeMetadata { + id + title + parentId + __typename +} + +fragment policyValueNotification on PolicyValue { + default + value + isCalculated + state + reason + precedence + lastProcess { + turbot { + id + __typename + } + __typename + } + setting { + resource { + turbot { + ...resourceMetadataFields + __typename + } + __typename + } + turbot { + ...policySettingMetadataFields + __typename + } + __typename + } + type { + ...policyTypeFields + trunk { + ...trunkFields + __typename + } + turbot { + ...policyTypeMetadataFields + __typename + } + __typename + } + turbot { + ...policyValueMetadataFields + __typename + } + __typename +} + +fragment policyTypeFields on PolicyType { + class + defaultTemplate + defaultTemplateInput + description + icon + readOnly + resolvedSchema + secret + secretLevel + targets + uri + __typename +} + +fragment policyValueMetadataFields on TurbotPolicyValueMetadata { + id + settingId + nextTickTimestamp + stateChangeTimestamp + createTimestamp + versionId + __typename +} + +fragment resourceNotification on Resource { + trunk { + ...trunkFields + __typename + } + type { + ...resourceTypeFields + trunk { + ...trunkFields + __typename + } + turbot { + ...resourceTypeMetadataFields + __typename + } + __typename + } + turbot { + ...resourceMetadataFields + __typename + } + object + __typename +} + +fragment resourceTypeFields on ResourceType { + description + icon + uri + __typename +} + +fragment resourceTypeMetadataFields on TurbotResourceTypeMetadata { + id + path + title + __typename +} + + +## required variables for the above query to fetch notifications for snapshot deletions by a specific actor: + +{ + "filter": "notificationType:resource_deleted actorIdentityId:218162262814364 resourceTypeId:'tmod:@turbot/aws-ec2#/resource/types/snapshot'", + "dataSource": "DB" +} + +## Or can be withiut DB filter to use default data source which may include cached data: + + +{ + "filter": "notificationType:resource_deleted actorIdentityId:218162262814364 resourceTypeId:'tmod:@turbot/aws-ec2#/resource/types/snapshot'" +} + +## We can use the above filters to get all the notifications for snapshot deletions by a specific actor. The first filter will query the database directly, while the second filter will use the default data source which may include cached data. + diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/export-resource-deleted-by-tubot-query.graphql b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/export-resource-deleted-by-tubot-query.graphql new file mode 100644 index 000000000..929ce0822 --- /dev/null +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/export-resource-deleted-by-tubot-query.graphql @@ -0,0 +1,457 @@ +## GraphQL query to retrieve notifications of type 'resource_deleted' for AWS EC2 snapshot resources, with pagination support. + +query Notifications($filter: [String!], $paging: String, $dataSource: NotificationDataSource) { + notifications(filter: $filter, paging: $paging, dataSource: $dataSource) { + items { + ...notification + data + __typename + } + paging { + next + __typename + } + metadata { + stats { + total + __typename + } + __typename + } + __typename + } +} + +fragment notification on Notification { + ...notificationFields + control { + ...controlNotification + __typename + } + oldControl { + ...controlNotification + __typename + } + grant { + ...grantNotification + __typename + } + oldGrant { + ...grantNotification + __typename + } + activeGrant { + ...activeGrantNotification + __typename + } + oldActiveGrant { + ...activeGrantNotification + __typename + } + policySetting { + ...policySettingNotification + __typename + } + oldPolicySetting { + ...policySettingNotification + __typename + } + policyValue { + ...policyValueNotification + __typename + } + oldPolicyValue { + ...policyValueNotification + __typename + } + resource { + ...resourceNotification + __typename + } + oldResource { + ...resourceNotification + __typename + } + actor { + ...actorNotification + __typename + } + turbot { + ...notificationMetadataFields + __typename + } + __typename +} + +fragment activeGrantNotification on ActiveGrant { + grant { + ...grantNotification + __typename + } + turbot { + ...activeGrantMetadataFields + __typename + } + __typename +} + +fragment activeGrantMetadataFields on TurbotActiveGrantMetadata { + id + versionId + createTimestamp + __typename +} + +fragment grantMetadataFields on TurbotGrantMetadata { + id + versionId + createTimestamp + __typename +} + +fragment permissionMetadataFields on TurbotPermissionMetadata { + id + path + title + __typename +} + +fragment trunkFields on TrunkItems { + title + items { + ...trunkItemFields + __typename + } + __typename +} + +fragment trunkItemFields on TrunkItem { + turbot { + id + path + title + __typename + } + __typename +} + +fragment actorNotification on Actor { + identity { + ...identityFields + __typename + } + persona { + ...identityFields + __typename + } + role { + ...identityFields + __typename + } + alternatePersona + __typename +} + +fragment identityFields on Identity { + picture + title + type { + uri + turbot { + ...resourceTypeMetadataFields + __typename + } + __typename + } + turbot { + ...resourceMetadataFields + __typename + } + __typename +} + +fragment resourceMetadataFields on TurbotResourceMetadata { + akas + id + parentId + path + title + versionId + __typename +} + +fragment controlNotification on Control { + state + type { + icon + trunk { + ...trunkFields + __typename + } + turbot { + ...controlTypeMetadataFields + __typename + } + __typename + } + turbot { + ...controlMetadataFields + __typename + } + __typename +} + +fragment controlMetadataFields on TurbotControlMetadata { + id + stateChangeTimestamp + versionId + __typename +} + +fragment controlTypeMetadataFields on TurbotControlTypeMetadata { + id + title + __typename +} + +fragment grantNotification on Grant { + roleName + groupName + type { + trunk { + ...trunkFields + __typename + } + turbot { + ...permissionMetadataFields + __typename + } + __typename + } + level { + turbot { + ...permissionMetadataFields + __typename + } + __typename + } + identity { + turbot { + ...resourceMetadataFields + __typename + } + __typename + } + turbot { + ...grantMetadataFields + __typename + } + __typename +} + +fragment notificationFields on Notification { + notificationType + icon + message + __typename +} + +fragment notificationMetadataFields on TurbotNotificationMetadata { + updateTimestamp + createTimestamp + id + processId + __typename +} + +fragment policySettingNotification on PolicySetting { + default + isCalculated + orphan + exception + precedence + template + templateInput + value + valueSource + note + validToTimestamp + resource { + turbot { + ...resourceMetadataFields + __typename + } + __typename + } + type { + icon + trunk { + ...trunkFields + __typename + } + turbot { + ...policyTypeMetadataFields + __typename + } + __typename + } + turbot { + ...policySettingMetadataFields + __typename + } + __typename +} + +fragment policySettingMetadataFields on TurbotPolicySettingMetadata { + id + timestamp + createTimestamp + updateTimestamp + versionId + __typename +} + +fragment policyTypeMetadataFields on TurbotPolicyTypeMetadata { + id + title + parentId + __typename +} + +fragment policyValueNotification on PolicyValue { + default + value + isCalculated + state + reason + precedence + lastProcess { + turbot { + id + __typename + } + __typename + } + setting { + resource { + turbot { + ...resourceMetadataFields + __typename + } + __typename + } + turbot { + ...policySettingMetadataFields + __typename + } + __typename + } + type { + ...policyTypeFields + trunk { + ...trunkFields + __typename + } + turbot { + ...policyTypeMetadataFields + __typename + } + __typename + } + turbot { + ...policyValueMetadataFields + __typename + } + __typename +} + +fragment policyTypeFields on PolicyType { + class + defaultTemplate + defaultTemplateInput + description + icon + readOnly + resolvedSchema + secret + secretLevel + targets + uri + __typename +} + +fragment policyValueMetadataFields on TurbotPolicyValueMetadata { + id + settingId + nextTickTimestamp + stateChangeTimestamp + createTimestamp + versionId + __typename +} + +fragment resourceNotification on Resource { + trunk { + ...trunkFields + __typename + } + type { + ...resourceTypeFields + trunk { + ...trunkFields + __typename + } + turbot { + ...resourceTypeMetadataFields + __typename + } + __typename + } + turbot { + ...resourceMetadataFields + __typename + } + object + __typename +} + +fragment resourceTypeFields on ResourceType { + description + icon + uri + __typename +} + +fragment resourceTypeMetadataFields on TurbotResourceTypeMetadata { + id + path + title + __typename +} + + +## Variables for the GraphQL query, specifying the filter for 'resource_deleted' notifications related to AWS EC2 snapshot resources, and pagination settings. + +{ + "filter": [ + "notificationType:resource_deleted resourceTypeId:'tmod:@turbot/aws-ec2#/resource/types/snapshot'", + "limit:300" + ], + "paging": "eyJzb3J0IjpbeyJ0ZXh0IjoiaWQiLCJvcGVyYXRvciI6Ii0ifSx7InRleHQiOiJ0aW1lc3RhbXAiLCJvcGVyYXRvciI6Ii0ifV0sIndoZXJlIjpbeyJwaXZvdCI6ImlkIiwib3BlcmF0b3IiOiI8IiwidmFsdWUiOiIzODUxMzkzOTExMjU4MjIifSx7InBpdm90IjoidGltZXN0YW1wIiwib3BlcmF0b3IiOiI8IiwidmFsdWUiOiIyMDI2LTA1LTA3VDA4OjQ0OjQ2LjI0NVoifV0sIm1vZGUiOiJuZXh0In0=" +} + +## Variables for the GraphQL query, specifying the filter for 'resource_deleted' notifications related to AWS EC2 snapshot resources by a specific actor, and pagination settings. + + + "filter": [ + "notificationType:resource_deleted actorIdentityId:218162262814364 resourceTypeId:'tmod:@turbot/aws-ec2#/resource/types/snapshot'", + "limit:300" + ], + "paging": "eyJzb3J0IjpbeyJ0ZXh0IjoiaWQiLCJvcGVyYXRvciI6Ii0ifSx7InRleHQiOiJ0aW1lc3RhbXAiLCJvcGVyYXRvciI6Ii0ifV0sIndoZXJlIjpbeyJwaXZvdCI6ImlkIiwib3BlcmF0b3IiOiI8IiwidmFsdWUiOiIzODUxMzkzOTExMjU4MjIifSx7InBpdm90IjoidGltZXN0YW1wIiwib3BlcmF0b3IiOiI8IiwidmFsdWUiOiIyMDI2LTA1LTA3VDA4OjQ0OjQ2LjI0NVoifV0sIm1vZGUiOiJuZXh0In0=" +} + +## Variables for the GraphQL query to fetch notifications for snapshot deletions by a specific actor, without pagination settings. + +{ + "filter":"notificationType:resource_deleted actorIdentityId:218162262814364 resourceTypeId:'tmod:@turbot/aws-ec2#/resource/types/snapshot'" +} \ No newline at end of file diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py index 6bfee71c7..c02b54287 100644 --- a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py @@ -27,11 +27,53 @@ CREDENTIALS_PATH = os.path.expanduser("~/.config/turbot/credentials.yml") GRAPHQL_PATH = "api/v5/graphql" -PAGE_SIZE = 500 +# 100 is the largest filter `limit:` value that still returns a working +# paging cursor on Guardrails. Values ≥500 hit a server-side cap (~255 items) +# AND null the cursor, killing pagination. Don't raise without re-validating. +PAGE_SIZE = 100 +DEFAULT_TIMEOUT = 300 +PROBE_TIMEOUT = 30 + +# Short aliases for common AWS resource types. Users can also pass full +# `tmod:@turbot/...` URIs directly. +RESOURCE_TYPE_ALIASES = { + "snapshot": "tmod:@turbot/aws-ec2#/resource/types/snapshot", + "ec2-snapshot": "tmod:@turbot/aws-ec2#/resource/types/snapshot", + "instance": "tmod:@turbot/aws-ec2#/resource/types/instance", + "ec2-instance": "tmod:@turbot/aws-ec2#/resource/types/instance", + "volume": "tmod:@turbot/aws-ec2#/resource/types/volume", + "ami": "tmod:@turbot/aws-ec2#/resource/types/image", + "vpc": "tmod:@turbot/aws-vpc-core#/resource/types/vpc", + "security-group": "tmod:@turbot/aws-vpc-security#/resource/types/securityGroup", + "bucket": "tmod:@turbot/aws-s3#/resource/types/bucket", + "s3-bucket": "tmod:@turbot/aws-s3#/resource/types/bucket", + "role": "tmod:@turbot/aws-iam#/resource/types/role", + "iam-role": "tmod:@turbot/aws-iam#/resource/types/role", + "user": "tmod:@turbot/aws-iam#/resource/types/user", + "iam-user": "tmod:@turbot/aws-iam#/resource/types/user", + "iam-policy": "tmod:@turbot/aws-iam#/resource/types/policy", + "lambda": "tmod:@turbot/aws-lambda#/resource/types/function", + "rds-instance": "tmod:@turbot/aws-rds#/resource/types/dbInstance", + "rds-cluster": "tmod:@turbot/aws-rds#/resource/types/dbCluster", + "rds-snapshot": "tmod:@turbot/aws-rds#/resource/types/dbSnapshot", + "kms-key": "tmod:@turbot/aws-kms#/resource/types/key", +} + + +def resolve_resource_types(values): + """Expand --resource-type values: alias or full URI; commas split a value.""" + resolved = [] + for v in values: + for token in v.split(","): + token = token.strip() + if not token: + continue + resolved.append(RESOURCE_TYPE_ALIASES.get(token, token)) + return resolved NOTIFICATIONS_QUERY = """ -query ResourceActivity($filter: [String!], $paging: String) { - notifications(filter: $filter, paging: $paging, dataSource: DB) { +query ResourceActivity($filter: [String!], $paging: String, $dataSource: NotificationDataSource) { + notifications(filter: $filter, paging: $paging, dataSource: $dataSource) { items { turbot { id @@ -85,6 +127,14 @@ } """ +COUNT_QUERY = """ +query CountActivity($filter: [String!]) { + notifications(filter: $filter) { + metadata { stats { total } } + } +} +""" + def load_profile(profile_name): """Load workspace credentials from ~/.config/turbot/credentials.yml.""" @@ -124,7 +174,7 @@ def load_profile(profile_name): } -def graphql_request(config, query, variables=None): +def graphql_request(config, query, variables=None, timeout=DEFAULT_TIMEOUT): """Execute a GraphQL query against the workspace.""" payload = {"query": query} if variables: @@ -134,7 +184,7 @@ def graphql_request(config, query, variables=None): config["endpoint"], json=payload, headers=config["headers"], - timeout=300, + timeout=timeout, ) response.raise_for_status() result = response.json() @@ -165,15 +215,58 @@ def get_turbot_identity_id(config): return None -def fetch_all_pages(config, filter_str): - """Fetch all pages for a given filter string.""" +DEFAULT_NOTIFICATION_TYPES = ("resource_created", "resource_deleted", "resource_updated") + + +def build_base_filter(resource_type_id, actor_id, since_date=None, until_date=None, notification_types=None): + """Build the filter string matching the console's report shape. + + - `resourceTypeId:` (not `resourceType:`) is the indexed field name. + - Numeric `actorIdentityId:` is unquoted (URI-style values still use quotes). + - Timestamps must be `YYYY-MM-DD` (date-only); the parser rejects full ISO8601. + - No `sort:` clause — server's default compound sort `(-id, -timestamp)` is + what makes cursor pagination stable across mass-delete bursts. + """ + ntypes = notification_types or DEFAULT_NOTIFICATION_TYPES + parts = [ + f"resourceTypeId:'{resource_type_id}'", + f"actorIdentityId:{actor_id}", + f"notificationType:{','.join(ntypes)}", + ] + if since_date: + parts.append(f"timestamp:>{since_date}") + if until_date: + parts.append(f"timestamp:<{until_date}") + return " ".join(parts) + + +def count_window(config, base_filter, timeout=PROBE_TIMEOUT): + """Return the stats total for `base_filter`, or None on failure.""" + try: + result = graphql_request(config, COUNT_QUERY, {"filter": base_filter}, timeout=timeout) + meta = ((result.get("data") or {}).get("notifications") or {}).get("metadata") or {} + return (meta.get("stats") or {}).get("total") + except Exception as e: + print(f" Count query failed: {e}") + return None + + +def fetch_window(config, base_filter, page_size=PAGE_SIZE, data_source=None): + """Paginate one window's results. + + `base_filter` is the per-window filter string; `limit:` is appended as a + separate filter array element to match the console's variable shape. + """ + filter_array = [base_filter, f"limit:{page_size}"] all_items = [] next_page = None page_num = 0 while True: page_num += 1 - variables = {"filter": filter_str, "paging": next_page} + variables = {"filter": filter_array, "paging": next_page} + if data_source: + variables["dataSource"] = data_source max_retries = 3 for attempt in range(1, max_retries + 1): @@ -183,11 +276,11 @@ def fetch_all_pages(config, filter_str): except requests.exceptions.Timeout: if attempt < max_retries: wait = attempt * 15 - print(f" Page {page_num}: timeout (attempt {attempt}/{max_retries}) — retrying in {wait}s...") + print(f" Page {page_num}: timeout (attempt {attempt}/{max_retries}) — retrying in {wait}s...") time.sleep(wait) else: - print(f" Page {page_num}: timeout after {max_retries} attempts — stopping") - return all_items + print(f" Page {page_num}: timeout after {max_retries} attempts — stopping window") + return all_items, False data = result.get("data") or {} notifications = data.get("notifications") or {} @@ -195,62 +288,60 @@ def fetch_all_pages(config, filter_str): all_items.extend(items) paging = notifications.get("paging") or {} - print(f" Page {page_num}: {len(items)} items (total so far: {len(all_items)})") + if page_num == 1 or page_num % 10 == 0 or not paging.get("next"): + print(f" Page {page_num}: {len(items)} items (total so far: {len(all_items)})") if paging and paging.get("next"): next_page = paging["next"] else: break - return all_items + return all_items, True -def fetch_resource_activity(config, resource_type_id, days, actor_id=None): - """ - Fetch resource CRUD activity for a given resource type and actor. +def build_windows(days, from_date=None, to_date=None): + """Return [(since_date, until_date, label), ...] one entry per UTC day. - Fetches all resource_created/deleted/updated notifications without date - filters (to avoid backend query plan timeouts on large datasets), then - filters to the requested date range client-side. - """ - if not actor_id: - print(" Detecting Turbot Identity ID...", end=" ") - detected_id = get_turbot_identity_id(config) - if detected_id: - actor_id = str(detected_id) - print(actor_id) - else: - print("FAILED — specify --actor-id manually") - return [] - - safe_actor_id = str(actor_id) - filter_str = ( - f"resourceType:'{resource_type_id}'" - f" actorIdentityId:'{safe_actor_id}'" - f" notificationType:resource_created,resource_deleted,resource_updated" - f" sort:-createTimestamp" - f" limit:{PAGE_SIZE}" - ) + Each window covers a full UTC day, expressed as YYYY-MM-DD strings + suitable for the `timestamp:>since timestamp:= cutoff - ] - print( - f" Date filter (last {days}d): {before_count} → {len(all_items)} items" + Without --from/--to, the convention is "the last N completed UTC days": + end = today's UTC midnight (start of today), start = end - N days. This + matches a daily-audit-ingest pattern where today's partial data is + deferred to tomorrow's run. + """ + if from_date and to_date: + start = _parse_date(from_date) + end = _parse_date(to_date) + else: + today_midnight = datetime.now(timezone.utc).replace( + hour=0, minute=0, second=0, microsecond=0 ) + end = today_midnight + start = end - timedelta(days=days) + + if start >= end: + return [] + + windows = [] + cursor = start + while cursor < end: + win_end = cursor + timedelta(days=1) + if win_end > end: + win_end = end + label = cursor.strftime("%Y-%m-%d") + windows.append((_format_date(cursor), _format_date(win_end), label)) + cursor = win_end + return windows - return all_items + +def _parse_date(s): + """Accept YYYY-MM-DD, return tz-aware UTC midnight datetime.""" + return datetime.strptime(s, "%Y-%m-%d").replace(tzinfo=timezone.utc) + + +def _format_date(dt): + return dt.strftime("%Y-%m-%d") def format_row(item, workspace_url): @@ -311,96 +402,196 @@ def write_csv(items, workspace_url, output_path): return len(items) +def run_workspace(profile_name, args, resource_types, notification_types): + """Fetch all windows for one workspace and write CSV(s).""" + workspace, config = load_profile(profile_name) + print(f" Workspace: {workspace}") + + actor_id = args.actor_id + if not actor_id: + print(" Detecting Turbot Identity ID...", end=" ") + detected = get_turbot_identity_id(config) + if not detected: + print("FAILED — specify --actor-id manually") + return + actor_id = str(detected) + print(actor_id) + + windows = build_windows(args.days, args.from_, args.to) + print(f" Date windows: {len(windows)} ({windows[0][0]} → {windows[-1][1]})") + + all_items = [] + summary_rows = [] + + for rt in resource_types: + print(f" Resource type: {rt}") + for since_date, until_date, label in windows: + base_filter = build_base_filter( + rt, actor_id, + since_date=since_date, until_date=until_date, + notification_types=notification_types, + ) + + expected = None + if not args.skip_preflight: + expected = count_window(config, base_filter, timeout=args.probe_timeout) + exp_str = expected if expected is not None else "?" + print(f" [{label}] expected={exp_str}") + + if args.preflight_only: + summary_rows.append((rt, label, expected, None, None)) + continue + + items, ok = fetch_window( + config, base_filter, + page_size=args.page_size, + data_source=args.data_source, + ) + print(f" [{label}] fetched={len(items)} ({'ok' if ok else 'partial'})") + + if expected is not None and ok and len(items) < expected * 0.95: + print(f" WARN: fetched < 95% of expected ({len(items)}/{expected})") + + summary_rows.append((rt, label, expected, len(items), ok)) + + if items and args.per_window_csv: + fname = f"{profile_name}-{_short_type(rt)}-{label}.csv" + path = os.path.join(args.output_dir, fname) + write_csv(items, workspace, path) + print(f" Per-window CSV: {path}") + + all_items.extend(items) + + if args.preflight_only: + return + + if not all_items: + print(" No resource activity found.") + return + + created = sum(1 for i in all_items if i["notificationType"] == "resource_created") + deleted = sum(1 for i in all_items if i["notificationType"] == "resource_deleted") + updated = sum(1 for i in all_items if i["notificationType"] == "resource_updated") + + date_str = datetime.now(timezone.utc).strftime("%Y%m%d") + span = ( + f"{args.from_}_to_{args.to}" + if args.from_ and args.to + else f"{args.days}d-{date_str}" + ) + filename = f"{profile_name}-resource-activity-{span}.csv" + output_path = os.path.join(args.output_dir, filename) + count = write_csv(all_items, workspace, output_path) + + print() + print(f" Consolidated: {count} total" + f" ({created} created, {deleted} deleted, {updated} updated)") + print(f" CSV: {output_path}") + + short_total = sum(1 for r in summary_rows if r[2] is not None and r[3] is not None and r[3] < r[2] * 0.95) + if short_total: + print(f" WARN: {short_total} window(s) returned < 95% of expected — re-run individual days with --from/--to") + + +def _short_type(uri): + """Extract a short label from a resource type URI for filenames.""" + if "/" in uri: + return uri.rsplit("/", 1)[-1] + return uri.replace(":", "_") + + def main(): parser = argparse.ArgumentParser( - description="Pull resource activity from Guardrails workspaces", + description="Pull resource activity from Guardrails workspaces (per-day windowed)", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: - # EC2 snapshots deleted by Turbot, last 90 days - %(prog)s --profile myworkspace --days 90 + # Last 7 days, all default activity types, default snapshot type + %(prog)s --profile kochgp --days 7 - # Multiple workspaces - %(prog)s --profile ws1 --profile ws2 --days 30 + # 7 days of snapshot deletions only, with per-window CSVs (resumable) + %(prog)s --profile kochgp --days 7 \\ + --resource-type snapshot --notification-type resource_deleted \\ + --per-window-csv - # S3 buckets instead of snapshots - %(prog)s --profile myworkspace --days 30 \\ - --resource-type "tmod:@turbot/aws-s3#/resource/types/bucket" - - # Specific actor identity - %(prog)s --profile myworkspace --days 90 --actor-id 123456789012345 + # Backfill a single window + %(prog)s --profile kochgp --resource-type snapshot \\ + --from 2026-05-01 --to 2026-05-02 """, ) - parser.add_argument( - "--profile", - action="append", - required=True, - help="Turbot CLI profile name (repeatable for multiple workspaces)", - ) - parser.add_argument( - "--days", - type=int, - default=30, - help="Number of days to look back (default: 30)", - ) - parser.add_argument( - "--resource-type", - default="tmod:@turbot/aws-ec2#/resource/types/snapshot", - help="Resource type URI to filter on (default: EC2 Snapshot)", - ) - parser.add_argument( - "--actor-id", - help="Actor identity ID to filter on (default: auto-detect Turbot Identity)", - ) - parser.add_argument( - "--output-dir", - default=".", - help="Output directory for CSV files (default: current directory)", - ) + parser.add_argument("--profile", action="append", + help="Turbot CLI profile name (repeatable).") + parser.add_argument("--days", type=int, default=7, + help="Lookback in days (default: 7). Ignored if --from/--to given.") + parser.add_argument("--from", dest="from_", help="Window start (YYYY-MM-DD or ISO8601).") + parser.add_argument("--to", dest="to", help="Window end (YYYY-MM-DD or ISO8601).") + parser.add_argument("--resource-type", action="append", + help="Resource type alias or full tmod URI (repeatable, comma-OK). " + "Default: snapshot. Run --list-types to see aliases.") + parser.add_argument("--notification-type", action="append", + choices=list(DEFAULT_NOTIFICATION_TYPES), + help="Limit to specific notification types (repeatable). " + "Default: all three.") + parser.add_argument("--actor-id", + help="Actor identity ID (default: auto-detect Turbot Identity).") + parser.add_argument("--data-source", choices=["ALL", "DB"], + help="Notification data source (default: server default = ALL).") + parser.add_argument("--page-size", type=int, default=PAGE_SIZE, + help=f"Page size for paginated fetch (default: {PAGE_SIZE}, matches console).") + parser.add_argument("--output-dir", default=".", + help="Output directory for CSV files.") + parser.add_argument("--per-window-csv", action="store_true", + help="Also write one CSV per day window (in addition to consolidated).") + parser.add_argument("--preflight-only", action="store_true", + help="Run per-window count probes and exit without fetching items.") + parser.add_argument("--skip-preflight", action="store_true", + help="Skip per-window count probes (faster but no expected/actual check).") + parser.add_argument("--probe-timeout", type=int, default=PROBE_TIMEOUT, + help=f"Per-window count timeout in seconds (default: {PROBE_TIMEOUT}).") + parser.add_argument("--list-types", action="store_true", + help="Print built-in resource-type aliases and exit.") args = parser.parse_args() + + if args.list_types: + print("Resource-type aliases:") + width = max(len(k) for k in RESOURCE_TYPE_ALIASES) + for alias, uri in sorted(RESOURCE_TYPE_ALIASES.items()): + print(f" {alias:<{width}} {uri}") + return + + if not args.profile: + parser.error("--profile is required (unless --list-types is used)") + if bool(args.from_) ^ bool(args.to): + parser.error("--from and --to must be given together") + os.makedirs(args.output_dir, exist_ok=True) - print(f"Resource Activity Report — last {args.days} days") - print(f"Profiles: {', '.join(args.profile)}") - print(f"Resource type: {args.resource_type}") - print(f"Output: {os.path.abspath(args.output_dir)}") + resource_types = ( + resolve_resource_types(args.resource_type) + if args.resource_type + else ["tmod:@turbot/aws-ec2#/resource/types/snapshot"] + ) + notification_types = args.notification_type # None means default trio in build_base_filter + + print("Resource Activity Report") + print(f"Profiles: {', '.join(args.profile)}") + print(f"Resource types: {', '.join(resource_types)}") + print(f"Notif types: {', '.join(notification_types or DEFAULT_NOTIFICATION_TYPES)}") + if args.from_: + print(f"Window: {args.from_} → {args.to}") + else: + print(f"Window: last {args.days} day(s)") + print(f"Page size: {args.page_size}") + print(f"Data source: {args.data_source or 'server default (ALL)'}") + if args.preflight_only: + print("Mode: pre-flight only (counts, no fetch)") + print(f"Output dir: {os.path.abspath(args.output_dir)}") print() for profile_name in args.profile: print(f"[{profile_name}]") - workspace, config = load_profile(profile_name) - print(f" Workspace: {workspace}") - - items = fetch_resource_activity( - config, args.resource_type, args.days, args.actor_id - ) - - if not items: - print(" No resource activity found.") - print() - continue - - created = sum( - 1 for i in items if i["notificationType"] == "resource_created" - ) - deleted = sum( - 1 for i in items if i["notificationType"] == "resource_deleted" - ) - updated = sum( - 1 for i in items if i["notificationType"] == "resource_updated" - ) - - date_str = datetime.now(timezone.utc).strftime("%Y%m%d") - filename = f"{profile_name}-resource-activity-{args.days}d-{date_str}.csv" - output_path = os.path.join(args.output_dir, filename) - - count = write_csv(items, workspace, output_path) - print( - f" Results: {count} total" - f" ({created} created, {deleted} deleted, {updated} updated)" - ) - print(f" CSV: {output_path}") + run_workspace(profile_name, args, resource_types, notification_types) print() print("Done.") diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_deleted_by_turbot.graphql b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_deleted_by_turbot.graphql new file mode 100644 index 000000000..7d8f499ef --- /dev/null +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_deleted_by_turbot.graphql @@ -0,0 +1,19 @@ +query Notifications($filter: [String!], $paging: String) { + notifications(filter: $filter, paging: $paging) { + items { + notificationType + resource { + turbot { id title } + type { turbot { title } } + trunk { title } + } + actor { + identity { title } + persona { title } + } + turbot { id createTimestamp } + } + paging { next } + metadata { stats { total } } + } +} From 7ea9f46c56a57652e78c3d49c15600839be0c122 Mon Sep 17 00:00:00 2001 From: raj Date: Fri, 8 May 2026 15:02:24 +0530 Subject: [PATCH 4/8] Remove customer-specific info from examples and hardcoded values Replace Koch workspace names, URLs, actor identity IDs, real resource IDs, and paging cursors with generic placeholders throughout the script, README, and GraphQL reference files. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../get-resource-activity-report/README.md | 70 +++++++++---------- .../fetch_resource_deletions.py | 34 ++++----- ...e-resource-deleted-by-turbot-query.graphql | 4 +- ...rt-resource-deleted-by-tubot-query.graphql | 8 +-- .../resource_activity_report.py | 6 +- 5 files changed, 57 insertions(+), 65 deletions(-) diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md index cdfc39ffe..bc9ddfd78 100644 --- a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md @@ -25,7 +25,7 @@ pip install -r requirements.txt Verify turbot CLI connectivity: ```bash -turbot graphql --profile kochgp --query='{ resource(id:"tmod:@turbot/turbot#/") { turbot { title } } }' +turbot graphql --profile my-workspace --query='{ resource(id:"tmod:@turbot/turbot#/") { turbot { title } } }' ``` --- @@ -47,98 +47,98 @@ Uses the turbot CLI for authentication and GraphQL transport. Paginates automati #### All resource types deleted by Turbot on a single day ```bash -python fetch_resource_deletions.py --profile kochgp --date 2026-05-07 -# Output: kochgp-resource-deleted-all-types-2026-05-07.csv +python fetch_resource_deletions.py --profile my-workspace --date 2026-05-07 +# Output: my-workspace-resource-deleted-all-types-2026-05-07.csv ``` #### Only EC2 snapshots on a single day ```bash -python fetch_resource_deletions.py --profile kochgp --date 2026-05-07 --resource-type snapshot -# Output: kochgp-resource-deleted-snapshot-2026-05-07.csv +python fetch_resource_deletions.py --profile my-workspace --date 2026-05-07 --resource-type snapshot +# Output: my-workspace-resource-deleted-snapshot-2026-05-07.csv ``` #### Only EC2 instances on a single day ```bash -python fetch_resource_deletions.py --profile kochgp --date 2026-05-07 --resource-type instance -# Output: kochgp-resource-deleted-instance-2026-05-07.csv +python fetch_resource_deletions.py --profile my-workspace --date 2026-05-07 --resource-type instance +# Output: my-workspace-resource-deleted-instance-2026-05-07.csv ``` #### Date range — all types from May 1 to May 8 ```bash -python fetch_resource_deletions.py --profile kochgp --since 2026-05-01 --until 2026-05-08 -# Output: kochgp-resource-deleted-all-types-2026-05-01.csv +python fetch_resource_deletions.py --profile my-workspace --since 2026-05-01 --until 2026-05-08 +# Output: my-workspace-resource-deleted-all-types-2026-05-01.csv ``` #### Date range — snapshots only from May 1 to May 8 ```bash -python fetch_resource_deletions.py --profile kochgp --since 2026-05-01 --until 2026-05-08 --resource-type snapshot -# Output: kochgp-resource-deleted-snapshot-2026-05-01.csv +python fetch_resource_deletions.py --profile my-workspace --since 2026-05-01 --until 2026-05-08 --resource-type snapshot +# Output: my-workspace-resource-deleted-snapshot-2026-05-01.csv ``` #### Rolling window — last 3 days, instances only ```bash -python fetch_resource_deletions.py --profile kochgp --days 3 --resource-type instance -# Output: kochgp-resource-deleted-instance-20260508.csv +python fetch_resource_deletions.py --profile my-workspace --days 3 --resource-type instance +# Output: my-workspace-resource-deleted-instance-20260508.csv ``` #### Rolling window — last 7 days, all types ```bash -python fetch_resource_deletions.py --profile kochgp --days 7 -# Output: kochgp-resource-deleted-all-types-20260508.csv +python fetch_resource_deletions.py --profile my-workspace --days 7 +# Output: my-workspace-resource-deleted-all-types-20260508.csv ``` #### Custom output file ```bash -python fetch_resource_deletions.py --profile kochgp --date 2026-05-07 --output may7-report.csv +python fetch_resource_deletions.py --profile my-workspace --date 2026-05-07 --output may7-report.csv ``` -#### Different workspace (kochkbs) +#### Different workspace with explicit actor ID ```bash -python fetch_resource_deletions.py --profile kochkbs --date 2026-05-07 \ +python fetch_resource_deletions.py --profile another-workspace --date 2026-05-07 \ --actor-id 123456789012345 \ - --workspace-url "https://kbs.turbotprod.kochind.cloud" + --workspace-url "https://another-workspace.cloud.turbot.com" ``` #### Specific resource type by full URI ```bash -python fetch_resource_deletions.py --profile kochgp --date 2026-05-07 \ +python fetch_resource_deletions.py --profile my-workspace --date 2026-05-07 \ --resource-type "tmod:@turbot/aws-lambda#/resource/types/functionVersion" ``` #### Open-ended since (no end date) ```bash -python fetch_resource_deletions.py --profile kochgp --since 2026-05-01 --resource-type volume +python fetch_resource_deletions.py --profile my-workspace --since 2026-05-01 --resource-type volume # Fetches from May 1 to now -# Output: kochgp-resource-deleted-volume-2026-05-01.csv +# Output: my-workspace-resource-deleted-volume-2026-05-01.csv ``` #### Day-by-day tracking for a week ```bash for d in 01 02 03 04 05 06 07; do - python fetch_resource_deletions.py --profile kochgp --date 2026-05-$d + python fetch_resource_deletions.py --profile my-workspace --date 2026-05-$d done -# Produces: kochgp-resource-deleted-all-types-2026-05-01.csv through -07.csv +# Produces: my-workspace-resource-deleted-all-types-2026-05-01.csv through -07.csv # No overlap between files — each covers midnight-to-midnight UTC ``` #### Compare two workspaces for the same day ```bash -python fetch_resource_deletions.py --profile kochgp --date 2026-05-07 -python fetch_resource_deletions.py --profile kochkbs --date 2026-05-07 \ +python fetch_resource_deletions.py --profile workspace-a --date 2026-05-07 +python fetch_resource_deletions.py --profile workspace-b --date 2026-05-07 \ --actor-id 123456789012345 \ - --workspace-url "https://kbs.turbotprod.kochind.cloud" + --workspace-url "https://workspace-b.cloud.turbot.com" ``` ### Resource type aliases @@ -210,7 +210,7 @@ To prevent accidental fetches of millions of rows, the script blocks queries tha | Timestamp | Activity timestamp (DD-Mon-YYYY HH:MM:SS UTC) | | NotificationType | RESOURCE DELETED | | Type / Message | Resource type category (e.g., Object > Snapshot) | -| Resource | Resource title (e.g., snap-08789b5a4ab739c33) | +| Resource | Resource title (e.g., snap-0abcdef1234567890) | | Actor | Actor identity name (e.g., Turbot Identity) | | ResourceId | Guardrails resource ID | | TrunkPath | Resource hierarchy path, or (deleted) | @@ -220,8 +220,8 @@ To prevent accidental fetches of millions of rows, the script blocks queries tha ```csv Timestamp,NotificationType,Type / Message,Resource,Actor,ResourceId,TrunkPath,Detail URL -07-May-2026 13:42:32,RESOURCE DELETED,Object > Snapshot,snap-03ae5c144a2bab0e8,Turbot Identity,385157353768573,(deleted),https://gp.turbotprod.kochind.cloud/apollo/notifications/385157685600695 -07-May-2026 11:32:25,RESOURCE DELETED,Object > Instance,i-0a1b2c3d4e5f67890,Turbot Identity,385149350192559,(deleted),https://gp.turbotprod.kochind.cloud/apollo/notifications/385149692057073 +07-May-2026 13:42:32,RESOURCE DELETED,Object > Snapshot,snap-0abcdef1234567890,Turbot Identity,123456789012345,(deleted),https://my-workspace.cloud.turbot.com/apollo/notifications/987654321098765 +07-May-2026 11:32:25,RESOURCE DELETED,Object > Instance,i-0abcdef1234567890,Turbot Identity,123456789012346,(deleted),https://my-workspace.cloud.turbot.com/apollo/notifications/987654321098766 ``` --- @@ -231,13 +231,13 @@ Timestamp,NotificationType,Type / Message,Resource,Actor,ResourceId,TrunkPath,De Both scripts use `~/.config/turbot/credentials.yml` (same as Turbot CLI): ```yaml -kochgp: - workspace: "https://gp.turbotprod.kochind.cloud" +my-workspace: + workspace: "https://my-workspace.cloud.turbot.com" accessKey: "your-access-key" secretKey: "your-secret-key" -kochkbs: - workspace: "https://kbs.turbotprod.kochind.cloud" +another-workspace: + workspace: "https://another-workspace.cloud.turbot.com" accessKey: "your-access-key" secretKey: "your-secret-key" ``` @@ -265,5 +265,5 @@ The `graphql-diff/` directory contains the full GraphQL queries captured from th ## Notes - **metadata.stats.total is approximate** — the total count in the GraphQL response does not apply all filter conditions (particularly timestamp boundaries). The actual item count from pagination is the accurate number. -- **Turbot Identity ID** — the actor identity ID for the Turbot automation identity varies per workspace. For `kochgp` it is `218162262814364`. For other workspaces, pass `--actor-id` or omit it to fetch deletions by all actors. +- **Turbot Identity ID** — the actor identity ID for the Turbot automation identity varies per workspace. Find it via the console under Permissions > Turbot Identity, or pass `--actor-id` explicitly. Omit it to fetch deletions by all actors. - **Timestamps are UTC** — all `--date`, `--since`, and `--until` values are interpreted as UTC midnight boundaries. diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py index b7e5f324b..3ad95f291 100644 --- a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py @@ -9,17 +9,17 @@ Usage: # All resource types deleted by Turbot on a single day - python fetch_resource_deletions.py --profile kochgp --date 2026-05-07 + python fetch_resource_deletions.py --profile my-workspace --date 2026-05-07 # Only snapshots - python fetch_resource_deletions.py --profile kochgp --date 2026-05-07 \ + python fetch_resource_deletions.py --profile my-workspace --date 2026-05-07 \ --resource-type snapshot # Date range, all types - python fetch_resource_deletions.py --profile kochgp --since 2026-05-01 --until 2026-05-08 + python fetch_resource_deletions.py --profile my-workspace --since 2026-05-01 --until 2026-05-08 # Rolling window - python fetch_resource_deletions.py --profile kochgp --days 3 + python fetch_resource_deletions.py --profile my-workspace --days 3 """ import argparse @@ -34,21 +34,13 @@ PAGE_SIZE = 200 WORKSPACE_URLS = { - "kochgp": "https://gp.turbotprod.kochind.cloud", - "kochkbs": "https://kbs.turbotprod.kochind.cloud", - "kochkbxt": "https://kbxt.turbotprod.kochind.cloud", - "kochgdn": "https://gdn.turbotprod.kochind.cloud", - "kochinv": "https://inv.turbotprod.kochind.cloud", - "kochind": "https://kochind.turbotprod.kochind.cloud", - "kochkaes": "https://kaes.turbotprod.kochind.cloud", - "kochfhr": "https://fhr.turbotprod.kochind.cloud", - "kochkes": "https://kes.turbotprod.kochind.cloud", - "kochkmt": "https://kmt.turbotprod.kochind.cloud", - "kochmlx": "https://mlx.turbotprod.kochind.cloud", + # Add your workspace profiles here: + # "my-workspace": "https://my-workspace.cloud.turbot.com", } TURBOT_IDENTITY_IDS = { - "kochgp": "218162262814364", + # Add Turbot Identity IDs per workspace (find via console > Permissions > Turbot Identity): + # "my-workspace": "123456789012345", } RESOURCE_TYPE_ALIASES = { @@ -185,13 +177,13 @@ def main(): Or pass a full URI: tmod:@turbot/aws-ec2#/resource/types/snapshot examples: - %(prog)s --profile kochgp --date 2026-05-07 - %(prog)s --profile kochgp --date 2026-05-07 --resource-type snapshot - %(prog)s --profile kochgp --since 2026-05-01 --until 2026-05-08 - %(prog)s --profile kochgp --days 7 --resource-type instance + %(prog)s --profile my-workspace --date 2026-05-07 + %(prog)s --profile my-workspace --date 2026-05-07 --resource-type snapshot + %(prog)s --profile my-workspace --since 2026-05-01 --until 2026-05-08 + %(prog)s --profile my-workspace --days 7 --resource-type instance """, ) - parser.add_argument("--profile", required=True, help="Turbot CLI profile name (e.g. kochgp)") + parser.add_argument("--profile", required=True, help="Turbot CLI profile name (e.g. my-workspace)") time_group = parser.add_argument_group("time range (pick one)") time_group.add_argument("--date", help="Single calendar day, midnight-to-midnight UTC (YYYY-MM-DD)") diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/console-resource-deleted-by-turbot-query.graphql b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/console-resource-deleted-by-turbot-query.graphql index 5d76951de..fe1bd3b28 100644 --- a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/console-resource-deleted-by-turbot-query.graphql +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/console-resource-deleted-by-turbot-query.graphql @@ -434,7 +434,7 @@ fragment resourceTypeMetadataFields on TurbotResourceTypeMetadata { ## required variables for the above query to fetch notifications for snapshot deletions by a specific actor: { - "filter": "notificationType:resource_deleted actorIdentityId:218162262814364 resourceTypeId:'tmod:@turbot/aws-ec2#/resource/types/snapshot'", + "filter": "notificationType:resource_deleted actorIdentityId:YOUR_ACTOR_IDENTITY_ID resourceTypeId:'tmod:@turbot/aws-ec2#/resource/types/snapshot'", "dataSource": "DB" } @@ -442,7 +442,7 @@ fragment resourceTypeMetadataFields on TurbotResourceTypeMetadata { { - "filter": "notificationType:resource_deleted actorIdentityId:218162262814364 resourceTypeId:'tmod:@turbot/aws-ec2#/resource/types/snapshot'" + "filter": "notificationType:resource_deleted actorIdentityId:YOUR_ACTOR_IDENTITY_ID resourceTypeId:'tmod:@turbot/aws-ec2#/resource/types/snapshot'" } ## We can use the above filters to get all the notifications for snapshot deletions by a specific actor. The first filter will query the database directly, while the second filter will use the default data source which may include cached data. diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/export-resource-deleted-by-tubot-query.graphql b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/export-resource-deleted-by-tubot-query.graphql index 929ce0822..d6c53605f 100644 --- a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/export-resource-deleted-by-tubot-query.graphql +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/export-resource-deleted-by-tubot-query.graphql @@ -437,21 +437,21 @@ fragment resourceTypeMetadataFields on TurbotResourceTypeMetadata { "notificationType:resource_deleted resourceTypeId:'tmod:@turbot/aws-ec2#/resource/types/snapshot'", "limit:300" ], - "paging": "eyJzb3J0IjpbeyJ0ZXh0IjoiaWQiLCJvcGVyYXRvciI6Ii0ifSx7InRleHQiOiJ0aW1lc3RhbXAiLCJvcGVyYXRvciI6Ii0ifV0sIndoZXJlIjpbeyJwaXZvdCI6ImlkIiwib3BlcmF0b3IiOiI8IiwidmFsdWUiOiIzODUxMzkzOTExMjU4MjIifSx7InBpdm90IjoidGltZXN0YW1wIiwib3BlcmF0b3IiOiI8IiwidmFsdWUiOiIyMDI2LTA1LTA3VDA4OjQ0OjQ2LjI0NVoifV0sIm1vZGUiOiJuZXh0In0=" + "paging": "YOUR_PAGING_CURSOR_HERE" } ## Variables for the GraphQL query, specifying the filter for 'resource_deleted' notifications related to AWS EC2 snapshot resources by a specific actor, and pagination settings. "filter": [ - "notificationType:resource_deleted actorIdentityId:218162262814364 resourceTypeId:'tmod:@turbot/aws-ec2#/resource/types/snapshot'", + "notificationType:resource_deleted actorIdentityId:YOUR_ACTOR_IDENTITY_ID resourceTypeId:'tmod:@turbot/aws-ec2#/resource/types/snapshot'", "limit:300" ], - "paging": "eyJzb3J0IjpbeyJ0ZXh0IjoiaWQiLCJvcGVyYXRvciI6Ii0ifSx7InRleHQiOiJ0aW1lc3RhbXAiLCJvcGVyYXRvciI6Ii0ifV0sIndoZXJlIjpbeyJwaXZvdCI6ImlkIiwib3BlcmF0b3IiOiI8IiwidmFsdWUiOiIzODUxMzkzOTExMjU4MjIifSx7InBpdm90IjoidGltZXN0YW1wIiwib3BlcmF0b3IiOiI8IiwidmFsdWUiOiIyMDI2LTA1LTA3VDA4OjQ0OjQ2LjI0NVoifV0sIm1vZGUiOiJuZXh0In0=" + "paging": "YOUR_PAGING_CURSOR_HERE" } ## Variables for the GraphQL query to fetch notifications for snapshot deletions by a specific actor, without pagination settings. { - "filter":"notificationType:resource_deleted actorIdentityId:218162262814364 resourceTypeId:'tmod:@turbot/aws-ec2#/resource/types/snapshot'" + "filter":"notificationType:resource_deleted actorIdentityId:YOUR_ACTOR_IDENTITY_ID resourceTypeId:'tmod:@turbot/aws-ec2#/resource/types/snapshot'" } \ No newline at end of file diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py index c02b54287..da46dd07e 100644 --- a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py @@ -507,15 +507,15 @@ def main(): epilog=""" Examples: # Last 7 days, all default activity types, default snapshot type - %(prog)s --profile kochgp --days 7 + %(prog)s --profile my-workspace --days 7 # 7 days of snapshot deletions only, with per-window CSVs (resumable) - %(prog)s --profile kochgp --days 7 \\ + %(prog)s --profile my-workspace --days 7 \\ --resource-type snapshot --notification-type resource_deleted \\ --per-window-csv # Backfill a single window - %(prog)s --profile kochgp --resource-type snapshot \\ + %(prog)s --profile my-workspace --resource-type snapshot \\ --from 2026-05-01 --to 2026-05-02 """, ) From 399d42b669aa5d0223e0c3055fe42ee2507db0f1 Mon Sep 17 00:00:00 2001 From: raj Date: Fri, 8 May 2026 15:20:53 +0530 Subject: [PATCH 5/8] Remove graphql-diff reference files and fix Detail URL to resource activity page Remove graphql-diff/ directory containing console query samples with workspace-specific variables. Fix Detail URL in CSV output to link to the resource activity page (/apollo/resources/{id}/activity) instead of the notification page. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../get-resource-activity-report/README.md | 18 +- .../fetch_resource_deletions.py | 4 +- ...e-resource-deleted-by-turbot-query.graphql | 449 ----------------- ...rt-resource-deleted-by-tubot-query.graphql | 457 ------------------ 4 files changed, 5 insertions(+), 923 deletions(-) delete mode 100644 guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/console-resource-deleted-by-turbot-query.graphql delete mode 100644 guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/export-resource-deleted-by-tubot-query.graphql diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md index bc9ddfd78..c39bc5d35 100644 --- a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md @@ -214,14 +214,14 @@ To prevent accidental fetches of millions of rows, the script blocks queries tha | Actor | Actor identity name (e.g., Turbot Identity) | | ResourceId | Guardrails resource ID | | TrunkPath | Resource hierarchy path, or (deleted) | -| Detail URL | Direct link to the notification in the console | +| Detail URL | Link to the resource activity page in the console | ### Example output ```csv Timestamp,NotificationType,Type / Message,Resource,Actor,ResourceId,TrunkPath,Detail URL -07-May-2026 13:42:32,RESOURCE DELETED,Object > Snapshot,snap-0abcdef1234567890,Turbot Identity,123456789012345,(deleted),https://my-workspace.cloud.turbot.com/apollo/notifications/987654321098765 -07-May-2026 11:32:25,RESOURCE DELETED,Object > Instance,i-0abcdef1234567890,Turbot Identity,123456789012346,(deleted),https://my-workspace.cloud.turbot.com/apollo/notifications/987654321098766 +07-May-2026 13:42:32,RESOURCE DELETED,Object > Snapshot,snap-0abcdef1234567890,Turbot Identity,123456789012345,(deleted),https://my-workspace.cloud.turbot.com/apollo/resources/123456789012345/activity +07-May-2026 11:32:25,RESOURCE DELETED,Object > Instance,i-0abcdef1234567890,Turbot Identity,123456789012346,(deleted),https://my-workspace.cloud.turbot.com/apollo/resources/123456789012346/activity ``` --- @@ -250,18 +250,6 @@ turbot workspace list --- -## GraphQL reference files - -The `graphql-diff/` directory contains the full GraphQL queries captured from the Guardrails console for reference: - -| File | Description | -|------|-------------| -| `export-resource-deleted-by-tubot-query.graphql` | Console Export CSV query with all fragments and example variables | -| `console-resource-deleted-by-turbot-query.graphql` | Console display query (used for in-browser rendering) | -| `resource_deleted_by_turbot.graphql` | Minimal query used by `fetch_resource_deletions.py` | - ---- - ## Notes - **metadata.stats.total is approximate** — the total count in the GraphQL response does not apply all filter conditions (particularly timestamp boundaries). The actual item count from pagination is the accurate number. diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py index 3ad95f291..d5b21961b 100644 --- a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py @@ -151,8 +151,8 @@ def to_csv_row(item, workspace_url): trunk_title = (resource.get("trunk") or {}).get("title") or "(deleted)" type_msg = f"Object > {res_type_title}" if res_type_title else "" actor_str = format_actor(item.get("actor")) - notif_id = item["turbot"]["id"] - detail_url = f"{workspace_url}/apollo/notifications/{notif_id}" + res_id = res_turbot.get("id", "") + detail_url = f"{workspace_url}/apollo/resources/{res_id}/activity" if res_id else "" return [ ts, nt, type_msg, diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/console-resource-deleted-by-turbot-query.graphql b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/console-resource-deleted-by-turbot-query.graphql deleted file mode 100644 index fe1bd3b28..000000000 --- a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/console-resource-deleted-by-turbot-query.graphql +++ /dev/null @@ -1,449 +0,0 @@ - -## Used GraphQL query to fetch notifications for snapshot deletions by a specific actor. The query includes various fragments to retrieve detailed information about the notifications, including control, grant, policy settings, and resource details. The filters applied in the query ensure that only relevant notifications are fetched based on the specified criteria. - -query Notifications($filter: [String!], $paging: String, $dataSource: NotificationDataSource) { - notifications(filter: $filter, paging: $paging, dataSource: $dataSource) { - items { - ...notification - data - __typename - } - paging { - next - __typename - } - metadata { - stats { - total - __typename - } - __typename - } - __typename - } -} - -fragment notification on Notification { - ...notificationFields - control { - ...controlNotification - __typename - } - oldControl { - ...controlNotification - __typename - } - grant { - ...grantNotification - __typename - } - oldGrant { - ...grantNotification - __typename - } - activeGrant { - ...activeGrantNotification - __typename - } - oldActiveGrant { - ...activeGrantNotification - __typename - } - policySetting { - ...policySettingNotification - __typename - } - oldPolicySetting { - ...policySettingNotification - __typename - } - policyValue { - ...policyValueNotification - __typename - } - oldPolicyValue { - ...policyValueNotification - __typename - } - resource { - ...resourceNotification - __typename - } - oldResource { - ...resourceNotification - __typename - } - actor { - ...actorNotification - __typename - } - turbot { - ...notificationMetadataFields - __typename - } - __typename -} - -fragment activeGrantNotification on ActiveGrant { - grant { - ...grantNotification - __typename - } - turbot { - ...activeGrantMetadataFields - __typename - } - __typename -} - -fragment activeGrantMetadataFields on TurbotActiveGrantMetadata { - id - versionId - createTimestamp - __typename -} - -fragment grantMetadataFields on TurbotGrantMetadata { - id - versionId - createTimestamp - __typename -} - -fragment permissionMetadataFields on TurbotPermissionMetadata { - id - path - title - __typename -} - -fragment trunkFields on TrunkItems { - title - items { - ...trunkItemFields - __typename - } - __typename -} - -fragment trunkItemFields on TrunkItem { - turbot { - id - path - title - __typename - } - __typename -} - -fragment actorNotification on Actor { - identity { - ...identityFields - __typename - } - persona { - ...identityFields - __typename - } - role { - ...identityFields - __typename - } - alternatePersona - __typename -} - -fragment identityFields on Identity { - picture - title - type { - uri - turbot { - ...resourceTypeMetadataFields - __typename - } - __typename - } - turbot { - ...resourceMetadataFields - __typename - } - __typename -} - -fragment resourceMetadataFields on TurbotResourceMetadata { - akas - id - parentId - path - title - versionId - __typename -} - -fragment controlNotification on Control { - state - type { - icon - trunk { - ...trunkFields - __typename - } - turbot { - ...controlTypeMetadataFields - __typename - } - __typename - } - turbot { - ...controlMetadataFields - __typename - } - __typename -} - -fragment controlMetadataFields on TurbotControlMetadata { - id - stateChangeTimestamp - versionId - __typename -} - -fragment controlTypeMetadataFields on TurbotControlTypeMetadata { - id - title - __typename -} - -fragment grantNotification on Grant { - roleName - groupName - type { - trunk { - ...trunkFields - __typename - } - turbot { - ...permissionMetadataFields - __typename - } - __typename - } - level { - turbot { - ...permissionMetadataFields - __typename - } - __typename - } - identity { - turbot { - ...resourceMetadataFields - __typename - } - __typename - } - turbot { - ...grantMetadataFields - __typename - } - __typename -} - -fragment notificationFields on Notification { - notificationType - icon - message - __typename -} - -fragment notificationMetadataFields on TurbotNotificationMetadata { - updateTimestamp - createTimestamp - id - processId - __typename -} - -fragment policySettingNotification on PolicySetting { - default - isCalculated - orphan - exception - precedence - template - templateInput - value - valueSource - note - validToTimestamp - resource { - turbot { - ...resourceMetadataFields - __typename - } - __typename - } - type { - icon - trunk { - ...trunkFields - __typename - } - turbot { - ...policyTypeMetadataFields - __typename - } - __typename - } - turbot { - ...policySettingMetadataFields - __typename - } - __typename -} - -fragment policySettingMetadataFields on TurbotPolicySettingMetadata { - id - timestamp - createTimestamp - updateTimestamp - versionId - __typename -} - -fragment policyTypeMetadataFields on TurbotPolicyTypeMetadata { - id - title - parentId - __typename -} - -fragment policyValueNotification on PolicyValue { - default - value - isCalculated - state - reason - precedence - lastProcess { - turbot { - id - __typename - } - __typename - } - setting { - resource { - turbot { - ...resourceMetadataFields - __typename - } - __typename - } - turbot { - ...policySettingMetadataFields - __typename - } - __typename - } - type { - ...policyTypeFields - trunk { - ...trunkFields - __typename - } - turbot { - ...policyTypeMetadataFields - __typename - } - __typename - } - turbot { - ...policyValueMetadataFields - __typename - } - __typename -} - -fragment policyTypeFields on PolicyType { - class - defaultTemplate - defaultTemplateInput - description - icon - readOnly - resolvedSchema - secret - secretLevel - targets - uri - __typename -} - -fragment policyValueMetadataFields on TurbotPolicyValueMetadata { - id - settingId - nextTickTimestamp - stateChangeTimestamp - createTimestamp - versionId - __typename -} - -fragment resourceNotification on Resource { - trunk { - ...trunkFields - __typename - } - type { - ...resourceTypeFields - trunk { - ...trunkFields - __typename - } - turbot { - ...resourceTypeMetadataFields - __typename - } - __typename - } - turbot { - ...resourceMetadataFields - __typename - } - object - __typename -} - -fragment resourceTypeFields on ResourceType { - description - icon - uri - __typename -} - -fragment resourceTypeMetadataFields on TurbotResourceTypeMetadata { - id - path - title - __typename -} - - -## required variables for the above query to fetch notifications for snapshot deletions by a specific actor: - -{ - "filter": "notificationType:resource_deleted actorIdentityId:YOUR_ACTOR_IDENTITY_ID resourceTypeId:'tmod:@turbot/aws-ec2#/resource/types/snapshot'", - "dataSource": "DB" -} - -## Or can be withiut DB filter to use default data source which may include cached data: - - -{ - "filter": "notificationType:resource_deleted actorIdentityId:YOUR_ACTOR_IDENTITY_ID resourceTypeId:'tmod:@turbot/aws-ec2#/resource/types/snapshot'" -} - -## We can use the above filters to get all the notifications for snapshot deletions by a specific actor. The first filter will query the database directly, while the second filter will use the default data source which may include cached data. - diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/export-resource-deleted-by-tubot-query.graphql b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/export-resource-deleted-by-tubot-query.graphql deleted file mode 100644 index d6c53605f..000000000 --- a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/graphql-diff/export-resource-deleted-by-tubot-query.graphql +++ /dev/null @@ -1,457 +0,0 @@ -## GraphQL query to retrieve notifications of type 'resource_deleted' for AWS EC2 snapshot resources, with pagination support. - -query Notifications($filter: [String!], $paging: String, $dataSource: NotificationDataSource) { - notifications(filter: $filter, paging: $paging, dataSource: $dataSource) { - items { - ...notification - data - __typename - } - paging { - next - __typename - } - metadata { - stats { - total - __typename - } - __typename - } - __typename - } -} - -fragment notification on Notification { - ...notificationFields - control { - ...controlNotification - __typename - } - oldControl { - ...controlNotification - __typename - } - grant { - ...grantNotification - __typename - } - oldGrant { - ...grantNotification - __typename - } - activeGrant { - ...activeGrantNotification - __typename - } - oldActiveGrant { - ...activeGrantNotification - __typename - } - policySetting { - ...policySettingNotification - __typename - } - oldPolicySetting { - ...policySettingNotification - __typename - } - policyValue { - ...policyValueNotification - __typename - } - oldPolicyValue { - ...policyValueNotification - __typename - } - resource { - ...resourceNotification - __typename - } - oldResource { - ...resourceNotification - __typename - } - actor { - ...actorNotification - __typename - } - turbot { - ...notificationMetadataFields - __typename - } - __typename -} - -fragment activeGrantNotification on ActiveGrant { - grant { - ...grantNotification - __typename - } - turbot { - ...activeGrantMetadataFields - __typename - } - __typename -} - -fragment activeGrantMetadataFields on TurbotActiveGrantMetadata { - id - versionId - createTimestamp - __typename -} - -fragment grantMetadataFields on TurbotGrantMetadata { - id - versionId - createTimestamp - __typename -} - -fragment permissionMetadataFields on TurbotPermissionMetadata { - id - path - title - __typename -} - -fragment trunkFields on TrunkItems { - title - items { - ...trunkItemFields - __typename - } - __typename -} - -fragment trunkItemFields on TrunkItem { - turbot { - id - path - title - __typename - } - __typename -} - -fragment actorNotification on Actor { - identity { - ...identityFields - __typename - } - persona { - ...identityFields - __typename - } - role { - ...identityFields - __typename - } - alternatePersona - __typename -} - -fragment identityFields on Identity { - picture - title - type { - uri - turbot { - ...resourceTypeMetadataFields - __typename - } - __typename - } - turbot { - ...resourceMetadataFields - __typename - } - __typename -} - -fragment resourceMetadataFields on TurbotResourceMetadata { - akas - id - parentId - path - title - versionId - __typename -} - -fragment controlNotification on Control { - state - type { - icon - trunk { - ...trunkFields - __typename - } - turbot { - ...controlTypeMetadataFields - __typename - } - __typename - } - turbot { - ...controlMetadataFields - __typename - } - __typename -} - -fragment controlMetadataFields on TurbotControlMetadata { - id - stateChangeTimestamp - versionId - __typename -} - -fragment controlTypeMetadataFields on TurbotControlTypeMetadata { - id - title - __typename -} - -fragment grantNotification on Grant { - roleName - groupName - type { - trunk { - ...trunkFields - __typename - } - turbot { - ...permissionMetadataFields - __typename - } - __typename - } - level { - turbot { - ...permissionMetadataFields - __typename - } - __typename - } - identity { - turbot { - ...resourceMetadataFields - __typename - } - __typename - } - turbot { - ...grantMetadataFields - __typename - } - __typename -} - -fragment notificationFields on Notification { - notificationType - icon - message - __typename -} - -fragment notificationMetadataFields on TurbotNotificationMetadata { - updateTimestamp - createTimestamp - id - processId - __typename -} - -fragment policySettingNotification on PolicySetting { - default - isCalculated - orphan - exception - precedence - template - templateInput - value - valueSource - note - validToTimestamp - resource { - turbot { - ...resourceMetadataFields - __typename - } - __typename - } - type { - icon - trunk { - ...trunkFields - __typename - } - turbot { - ...policyTypeMetadataFields - __typename - } - __typename - } - turbot { - ...policySettingMetadataFields - __typename - } - __typename -} - -fragment policySettingMetadataFields on TurbotPolicySettingMetadata { - id - timestamp - createTimestamp - updateTimestamp - versionId - __typename -} - -fragment policyTypeMetadataFields on TurbotPolicyTypeMetadata { - id - title - parentId - __typename -} - -fragment policyValueNotification on PolicyValue { - default - value - isCalculated - state - reason - precedence - lastProcess { - turbot { - id - __typename - } - __typename - } - setting { - resource { - turbot { - ...resourceMetadataFields - __typename - } - __typename - } - turbot { - ...policySettingMetadataFields - __typename - } - __typename - } - type { - ...policyTypeFields - trunk { - ...trunkFields - __typename - } - turbot { - ...policyTypeMetadataFields - __typename - } - __typename - } - turbot { - ...policyValueMetadataFields - __typename - } - __typename -} - -fragment policyTypeFields on PolicyType { - class - defaultTemplate - defaultTemplateInput - description - icon - readOnly - resolvedSchema - secret - secretLevel - targets - uri - __typename -} - -fragment policyValueMetadataFields on TurbotPolicyValueMetadata { - id - settingId - nextTickTimestamp - stateChangeTimestamp - createTimestamp - versionId - __typename -} - -fragment resourceNotification on Resource { - trunk { - ...trunkFields - __typename - } - type { - ...resourceTypeFields - trunk { - ...trunkFields - __typename - } - turbot { - ...resourceTypeMetadataFields - __typename - } - __typename - } - turbot { - ...resourceMetadataFields - __typename - } - object - __typename -} - -fragment resourceTypeFields on ResourceType { - description - icon - uri - __typename -} - -fragment resourceTypeMetadataFields on TurbotResourceTypeMetadata { - id - path - title - __typename -} - - -## Variables for the GraphQL query, specifying the filter for 'resource_deleted' notifications related to AWS EC2 snapshot resources, and pagination settings. - -{ - "filter": [ - "notificationType:resource_deleted resourceTypeId:'tmod:@turbot/aws-ec2#/resource/types/snapshot'", - "limit:300" - ], - "paging": "YOUR_PAGING_CURSOR_HERE" -} - -## Variables for the GraphQL query, specifying the filter for 'resource_deleted' notifications related to AWS EC2 snapshot resources by a specific actor, and pagination settings. - - - "filter": [ - "notificationType:resource_deleted actorIdentityId:YOUR_ACTOR_IDENTITY_ID resourceTypeId:'tmod:@turbot/aws-ec2#/resource/types/snapshot'", - "limit:300" - ], - "paging": "YOUR_PAGING_CURSOR_HERE" -} - -## Variables for the GraphQL query to fetch notifications for snapshot deletions by a specific actor, without pagination settings. - -{ - "filter":"notificationType:resource_deleted actorIdentityId:YOUR_ACTOR_IDENTITY_ID resourceTypeId:'tmod:@turbot/aws-ec2#/resource/types/snapshot'" -} \ No newline at end of file From a21cc725c46af0ab35f805c6b6b27b2b29eeb77b Mon Sep 17 00:00:00 2001 From: raj Date: Fri, 8 May 2026 17:33:41 +0530 Subject: [PATCH 6/8] Add --auto-detect-actor flag and workspace URL auto-read from credentials Auto-detect Turbot Identity ID via GraphQL query when --auto-detect-actor is passed. Auto-read workspace URL from credentials.yml so --workspace-url is no longer needed. Updated README case study and examples to use the simplified workflow. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../get-resource-activity-report/README.md | 126 +++++++++++++++++- .../fetch_resource_deletions.py | 58 ++++++-- 2 files changed, 170 insertions(+), 14 deletions(-) mode change 100644 => 100755 guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md index c39bc5d35..d822a7017 100644 --- a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md @@ -99,7 +99,16 @@ python fetch_resource_deletions.py --profile my-workspace --days 7 python fetch_resource_deletions.py --profile my-workspace --date 2026-05-07 --output may7-report.csv ``` -#### Different workspace with explicit actor ID +#### Auto-detect Turbot Identity and fetch snapshots + +```bash +python fetch_resource_deletions.py --profile my-workspace --date 2026-05-07 \ + --resource-type snapshot --auto-detect-actor +# Auto-detects the Turbot Identity ID and workspace URL from credentials.yml +# No need for --actor-id or --workspace-url +``` + +#### Explicit actor ID (if auto-detect is not desired) ```bash python fetch_resource_deletions.py --profile another-workspace --date 2026-05-07 \ @@ -180,10 +189,11 @@ time range (pick one): --until UNTIL End date exclusive (YYYY-MM-DD), use with --since --days DAYS Rolling window in days (default: 1) - --actor-id ACTOR_ID Turbot actor identity ID (auto-detected for known workspaces) + --actor-id ACTOR_ID Turbot actor identity ID (use with --auto-detect-actor or pass explicitly) + --auto-detect-actor Auto-detect Turbot Identity ID from the workspace via GraphQL --resource-type TYPE Resource type alias or full tmod URI (default: all types) --output OUTPUT Output CSV file path (default: auto-generated) - --workspace-url URL Workspace base URL (auto-detected for known workspaces) + --workspace-url URL Workspace base URL (auto-read from credentials.yml if omitted) ``` ### Time range behavior @@ -250,8 +260,116 @@ turbot workspace list --- +## Case study: Investigating snapshot deletions by Turbot + +A customer reported that EC2 snapshots were being deleted in their workspace. The console Resource Activities report timed out due to the workspace having millions of notifications. Here is the workflow used to investigate and produce a daily report. + +### Step 1 — Fetch snapshot deletions by Turbot for a specific day + +The simplest command — `--auto-detect-actor` queries the workspace to find the Turbot Identity ID automatically, and the workspace URL is read from `credentials.yml`: + +```bash +python fetch_resource_deletions.py \ + --profile my-workspace \ + --date 2026-05-07 \ + --resource-type snapshot \ + --auto-detect-actor +# Auto-detected Turbot Identity: 123456789012345 +# Output: my-workspace-resource-deleted-snapshot-2026-05-07.csv +``` + +This fetches all snapshot deletions by the Turbot automation identity on May 7, midnight-to-midnight UTC. + +### Step 2 — Broaden the scope to all resource types + +To check what else Turbot deleted on the same day, omit `--resource-type`: + +```bash +python fetch_resource_deletions.py \ + --profile my-workspace \ + --date 2026-05-07 \ + --auto-detect-actor +# Output: my-workspace-resource-deleted-all-types-2026-05-07.csv +``` + +A typical breakdown might look like: + +``` + 127 Object > Snapshot + 72 Object > Instance + 7 Object > Function Version + 5 Object > Subscription + 3 Object > Launch Template + 2 Object > Volume + 1 Object > Service + 217 TOTAL +``` + +### Step 3 — Generate a multi-day report + +```bash +for d in 01 02 03 04 05 06 07; do + python fetch_resource_deletions.py \ + --profile my-workspace \ + --date 2026-05-$d \ + --resource-type snapshot \ + --auto-detect-actor +done +``` + +Each file covers exactly midnight-to-midnight UTC with no overlap, making day-over-day comparison reliable. + +### Step 4 — Fetch a date range in a single CSV + +```bash +python fetch_resource_deletions.py \ + --profile my-workspace \ + --since 2026-05-01 --until 2026-05-08 \ + --resource-type snapshot \ + --auto-detect-actor +# Output: my-workspace-resource-deleted-snapshot-2026-05-01.csv +``` + +### Step 5 — Fetch deletions by all actors (not just Turbot) + +Omit `--auto-detect-actor` and `--actor-id` to see deletions by all actors. This helps determine if resources were deleted by Turbot automation, by users, or by external processes: + +```bash +python fetch_resource_deletions.py \ + --profile my-workspace \ + --date 2026-05-07 \ + --resource-type snapshot +``` + +The Actor column in the CSV will show who performed each deletion. + +### Step 6 — Use an explicit actor ID (alternative to auto-detect) + +If you already know the Turbot Identity ID (found via console > Permissions > Turbot Identity), you can pass it directly: + +```bash +python fetch_resource_deletions.py \ + --profile my-workspace \ + --date 2026-05-07 \ + --resource-type snapshot \ + --actor-id 123456789012345 +``` + +### Key findings from this investigation + +- The console Export CSV is capped at **5,000 rows** — this script has no such limit +- The console report times out on workspaces with millions of notifications — this script paginates reliably +- `--auto-detect-actor` queries the workspace for the Turbot Identity ID automatically — no need to look it up manually, and the workspace URL is auto-read from `credentials.yml` +- The Turbot Identity ID is **different per workspace** — do not reuse an ID from one workspace on another +- The `metadata.stats.total` in the GraphQL response is **approximate** and does not reflect timestamp filters — use the actual row count +- Without `--actor-id` or `--auto-detect-actor`, deletions by all actors are returned, including "Unidentified Identity" (typically AWS-side deletions not initiated by Guardrails) +- Without `--resource-type`, all resource types are fetched — useful for a full picture but requires a time boundary (`--date` or `--since`) to avoid fetching millions of rows + +--- + ## Notes - **metadata.stats.total is approximate** — the total count in the GraphQL response does not apply all filter conditions (particularly timestamp boundaries). The actual item count from pagination is the accurate number. -- **Turbot Identity ID** — the actor identity ID for the Turbot automation identity varies per workspace. Find it via the console under Permissions > Turbot Identity, or pass `--actor-id` explicitly. Omit it to fetch deletions by all actors. +- **Turbot Identity ID** — the actor identity ID for the Turbot automation identity varies per workspace. Use `--auto-detect-actor` to query it automatically, or find it via the console under Permissions > Turbot Identity and pass `--actor-id` explicitly. Omit both to fetch deletions by all actors. +- **Workspace URL auto-detection** — the workspace URL is automatically read from `~/.config/turbot/credentials.yml` when `--workspace-url` is not provided. This populates the Detail URL column in the CSV. - **Timestamps are UTC** — all `--date`, `--since`, and `--until` values are interpreted as UTC midnight boundaries. diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py old mode 100644 new mode 100755 index d5b21961b..b8b1ec485 --- a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py @@ -30,6 +30,8 @@ import sys from datetime import datetime, timedelta +import yaml + QUERY_FILE = os.path.join(os.path.dirname(__file__), "resource_deleted_by_turbot.graphql") PAGE_SIZE = 200 @@ -84,14 +86,15 @@ def resolve_resource_type(value): sys.exit(1) -def run_query(profile, variables): +def run_turbot_graphql(profile, query, variables=None): cmd = [ "turbot", "graphql", "--profile", profile, "--format", "json", - "--query", QUERY_FILE, - "--variables", json.dumps(variables), + "--query", query, ] + if variables: + cmd.extend(["--variables", json.dumps(variables)]) result = subprocess.run(cmd, capture_output=True, text=True, timeout=600) if result.returncode != 0: print(f"turbot CLI error: {result.stderr.strip()}", file=sys.stderr) @@ -99,6 +102,39 @@ def run_query(profile, variables): return json.loads(result.stdout) +def run_query(profile, variables): + return run_turbot_graphql(profile, QUERY_FILE, variables) + + +def detect_turbot_identity(profile): + query = ("{ resources(filter: \"resourceTypeId:'tmod:@turbot/turbot-iam" + "#/resource/types/turbotIdentity' limit:1\") " + "{ items { turbot { id title } } } }") + try: + data = run_turbot_graphql(profile, query) + items = data.get("resources", {}).get("items", []) + if items: + actor_id = items[0]["turbot"]["id"] + print(f"Auto-detected Turbot Identity: {actor_id}") + return actor_id + except (subprocess.TimeoutExpired, json.JSONDecodeError, KeyError): + pass + print("Warning: could not auto-detect Turbot Identity ID.", file=sys.stderr) + return None + + +def detect_workspace_url(profile): + try: + with open(os.path.expanduser("~/.config/turbot/credentials.yml")) as f: + creds = yaml.safe_load(f) + url = (creds.get(profile) or {}).get("workspace", "").rstrip("/") + if url: + return url + except (FileNotFoundError, AttributeError): + pass + return "" + + def build_filter(actor_id, resource_type_uri, date, since, until, days): parts = ["notificationType:resource_deleted"] if actor_id: @@ -191,23 +227,25 @@ def main(): time_group.add_argument("--until", help="End date exclusive (YYYY-MM-DD), use with --since") time_group.add_argument("--days", type=int, default=1, help="Rolling window in days (default: 1)") - parser.add_argument("--actor-id", help="Turbot actor identity ID (auto-detected for known workspaces)") + parser.add_argument("--actor-id", help="Turbot actor identity ID") + parser.add_argument("--auto-detect-actor", action="store_true", + help="Auto-detect Turbot Identity ID from the workspace") parser.add_argument("--resource-type", help="Resource type alias or tmod URI (default: all types)") parser.add_argument("--output", help="Output CSV file path (default: auto-generated)") - parser.add_argument("--workspace-url", help="Workspace base URL (auto-detected for known workspaces)") + parser.add_argument("--workspace-url", help="Workspace base URL (auto-read from credentials.yml if omitted)") args = parser.parse_args() - actor_id = args.actor_id or TURBOT_IDENTITY_IDS.get(args.profile) - workspace_url = args.workspace_url or WORKSPACE_URLS.get(args.profile, "") + workspace_url = args.workspace_url or WORKSPACE_URLS.get(args.profile) or detect_workspace_url(args.profile) resource_type_uri = resolve_resource_type(args.resource_type) + actor_id = args.actor_id or TURBOT_IDENTITY_IDS.get(args.profile) + if not actor_id and args.auto_detect_actor: + actor_id = detect_turbot_identity(args.profile) if not actor_id: print(f"Warning: no actor-id for profile '{args.profile}'. Fetching deletions by ALL actors.", file=sys.stderr) - if not workspace_url: - print(f"Warning: no workspace URL for profile '{args.profile}'. Detail URLs will be incomplete.", - file=sys.stderr) + print(f" Use --auto-detect-actor or --actor-id to filter by Turbot Identity.", file=sys.stderr) if not resource_type_uri and not args.date and not args.since: print("Error: fetching all resource types without a time boundary is unsafe (millions of rows).", From 42015eb62bcde01ce5ebc04f6e6bcfee796d0107 Mon Sep 17 00:00:00 2001 From: raj Date: Fri, 8 May 2026 17:41:52 +0530 Subject: [PATCH 7/8] Fix CodeQL clear-text logging alerts and add resource type lookup to README Separate workspace URL extraction from auth token construction in resource_activity_report.py to prevent CodeQL from tracing secrets through to print statements. Delete secret variables after use. Add resource type URI lookup command to README aliases section. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../get-resource-activity-report/README.md | 13 +++++++++---- .../fetch_resource_deletions.py | 12 ++++++------ .../resource_activity_report.py | 12 ++++++++---- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md index d822a7017..91285310d 100644 --- a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/README.md @@ -170,14 +170,19 @@ Short names you can use with `--resource-type` instead of full URIs: | `subscription` | `tmod:@turbot/aws-sns#/resource/types/subscription` | | `ecs-service` | `tmod:@turbot/aws-ecs#/resource/types/service` | -You can also pass any `tmod:@turbot/...` URI directly. +You can also pass any `tmod:@turbot/...` URI directly. To find the URI for a resource type not listed above, navigate to the resource type in the Guardrails console and copy the URI from the resource type details, or run: + +```bash +turbot graphql --profile my-workspace --query='{ resourceTypes(filter: "snapshot") { items { uri turbot { title } } } }' --format yaml +``` ### Command-line reference ``` usage: fetch_resource_deletions.py [-h] --profile PROFILE [--date DATE] [--since SINCE] [--until UNTIL] [--days DAYS] - [--actor-id ACTOR_ID] [--resource-type RESOURCE_TYPE] + [--actor-id ACTOR_ID] [--auto-detect-actor] + [--resource-type RESOURCE_TYPE] [--output OUTPUT] [--workspace-url WORKSPACE_URL] options: @@ -189,8 +194,8 @@ time range (pick one): --until UNTIL End date exclusive (YYYY-MM-DD), use with --since --days DAYS Rolling window in days (default: 1) - --actor-id ACTOR_ID Turbot actor identity ID (use with --auto-detect-actor or pass explicitly) - --auto-detect-actor Auto-detect Turbot Identity ID from the workspace via GraphQL + --actor-id ACTOR_ID Turbot actor identity ID (pass explicitly, or use --auto-detect-actor) + --auto-detect-actor Auto-detect Turbot Identity ID from the workspace via GraphQL query --resource-type TYPE Resource type alias or full tmod URI (default: all types) --output OUTPUT Output CSV file path (default: auto-generated) --workspace-url URL Workspace base URL (auto-read from credentials.yml if omitted) diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py index b8b1ec485..70cbbd930 100755 --- a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py @@ -124,15 +124,15 @@ def detect_turbot_identity(profile): def detect_workspace_url(profile): + creds_path = os.path.expanduser("~/.config/turbot/credentials.yml") try: - with open(os.path.expanduser("~/.config/turbot/credentials.yml")) as f: + with open(creds_path) as f: creds = yaml.safe_load(f) - url = (creds.get(profile) or {}).get("workspace", "").rstrip("/") - if url: - return url + profile_data = creds.get(profile) or {} + url = str(profile_data.get("workspace", "")).rstrip("/") + return url if url else "" except (FileNotFoundError, AttributeError): - pass - return "" + return "" def build_filter(actor_id, resource_type_uri, date, since, until, days): diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py index da46dd07e..05a9c1198 100644 --- a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/resource_activity_report.py @@ -161,12 +161,16 @@ def load_profile(profile_name): print(f"Error: Profile '{profile_name}' missing '{key}'") sys.exit(1) - workspace = profile["workspace"].rstrip("/") - auth_bytes = f"{profile['accessKey']}:{profile['secretKey']}".encode("utf-8") - auth_token = b64encode(auth_bytes).decode() + workspace = str(profile["workspace"]).rstrip("/") + endpoint = f"{workspace}/{GRAPHQL_PATH}" + + access_key = str(profile["accessKey"]) + secret_key = str(profile["secretKey"]) + auth_token = b64encode(f"{access_key}:{secret_key}".encode("utf-8")).decode() + del access_key, secret_key return workspace, { - "endpoint": f"{workspace}/{GRAPHQL_PATH}", + "endpoint": endpoint, "headers": { "Authorization": f"Basic {auth_token}", "Content-Type": "application/json", From d1426d4cd44e753ea38ccf82033fa3aed28d28c3 Mon Sep 17 00:00:00 2001 From: raj Date: Fri, 8 May 2026 18:13:19 +0530 Subject: [PATCH 8/8] Fix auto-detect-actor error handling to gracefully fall back on CLI errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Auto-detect now uses exit_on_error=False so transient CLI failures don't kill the script — it falls back to fetching by all actors. Error messages now include stdout when stderr is empty for better debugging. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../fetch_resource_deletions.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py index 70cbbd930..7dff1bda4 100755 --- a/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py +++ b/guardrails_utilities/python_utils/notifications/get-resource-activity-report/fetch_resource_deletions.py @@ -86,7 +86,7 @@ def resolve_resource_type(value): sys.exit(1) -def run_turbot_graphql(profile, query, variables=None): +def run_turbot_graphql(profile, query, variables=None, exit_on_error=True): cmd = [ "turbot", "graphql", "--profile", profile, @@ -97,8 +97,11 @@ def run_turbot_graphql(profile, query, variables=None): cmd.extend(["--variables", json.dumps(variables)]) result = subprocess.run(cmd, capture_output=True, text=True, timeout=600) if result.returncode != 0: - print(f"turbot CLI error: {result.stderr.strip()}", file=sys.stderr) - sys.exit(1) + err = result.stderr.strip() or result.stdout.strip() or "(no details)" + if exit_on_error: + print(f"turbot CLI error: {err}", file=sys.stderr) + sys.exit(1) + raise RuntimeError(err) return json.loads(result.stdout) @@ -111,15 +114,16 @@ def detect_turbot_identity(profile): "#/resource/types/turbotIdentity' limit:1\") " "{ items { turbot { id title } } } }") try: - data = run_turbot_graphql(profile, query) + data = run_turbot_graphql(profile, query, exit_on_error=False) items = data.get("resources", {}).get("items", []) if items: actor_id = items[0]["turbot"]["id"] print(f"Auto-detected Turbot Identity: {actor_id}") return actor_id - except (subprocess.TimeoutExpired, json.JSONDecodeError, KeyError): - pass - print("Warning: could not auto-detect Turbot Identity ID.", file=sys.stderr) + except Exception as e: + print(f"Warning: auto-detect failed: {e}", file=sys.stderr) + print("Warning: could not auto-detect Turbot Identity ID. Fetching deletions by ALL actors.", + file=sys.stderr) return None