feat: add CloudTrail S3 log source

api/v1/aws.go

@@ -100,20 +100,45 @@ func (aws AWS) ShouldExclude(configType, name string, tags map[string]string) bo
 }
 
 type CloudTrail struct {
-	Exclude []string `json:"exclude,omitempty"`
-	MaxAge  string   `json:"maxAge,omitempty"`
+	Exclude []string      `json:"exclude,omitempty"`
+	MaxAge  string        `json:"maxAge,omitempty"`
+	Source  string        `json:"source,omitempty"`
+	S3      *CloudTrailS3 `json:"s3,omitempty"`
 }
 
-func (c CloudTrail) GetMaxAge() time.Duration {
-	if c.MaxAge == "" {
-		return 7 * 24 * time.Hour
+// CloudTrailS3 configures CloudTrail log ingestion from an existing S3 bucket.
+type CloudTrailS3 struct {
+	Bucket       string `json:"bucket,omitempty"`
+	BucketRegion string `json:"bucketRegion,omitempty"`
+	Prefix       string `json:"prefix,omitempty"`
+}
+
+// SourceType resolves the CloudTrail event source while preserving API backward compatibility.
+func (c CloudTrail) SourceType() string {
+	switch strings.ToLower(c.Source) {
+	case "s3":
+		return "s3"
+	case "api", "":
+		if c.Source == "" && c.S3 != nil && c.S3.Bucket != "" {
+			return "s3"
+		}
+		return "api"
+	default:
+		return "api"
 	}
-	d, err := time.ParseDuration(c.MaxAge)
-	if err != nil {
+}
+
+// GetMaxAge returns the configured CloudTrail lookback window or the source-specific default.
+func (c CloudTrail) GetMaxAge() time.Duration {
+	if c.MaxAge != "" {
+		d, err := time.ParseDuration(c.MaxAge)
+		if err == nil {
+			return d
+		}
 		logger.Warnf("Invalid cloudtrail max age %s: %v", c.MaxAge, err)
-		return 7 * 24 * time.Hour
 	}
-	return d
+
+	return 7 * 24 * time.Hour
 }
 
 type CostReporting struct {

api/v1/aws_test.go

@@ -1,11 +1,35 @@
 package v1
 
 import (
+	"time"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
 
 var _ = Describe("AWS", func() {
+	DescribeTable("CloudTrail SourceType",
+		func(config CloudTrail, want string) {
+			Expect(config.SourceType()).To(Equal(want))
+		},
+		Entry("empty source without S3 uses api", CloudTrail{}, "api"),
+		Entry("empty source with S3 bucket uses s3", CloudTrail{S3: &CloudTrailS3{Bucket: "logs"}}, "s3"),
+		Entry("explicit api wins over S3 bucket", CloudTrail{Source: "api", S3: &CloudTrailS3{Bucket: "logs"}}, "api"),
+		Entry("explicit s3", CloudTrail{Source: "s3"}, "s3"),
+		Entry("unknown source falls back to api", CloudTrail{Source: "lake"}, "api"),
+	)
+
+	DescribeTable("CloudTrail GetMaxAge",
+		func(config CloudTrail, want time.Duration) {
+			Expect(config.GetMaxAge()).To(Equal(want))
+		},
+		Entry("api default", CloudTrail{}, 7*24*time.Hour),
+		Entry("s3 default", CloudTrail{Source: "s3"}, 7*24*time.Hour),
+		Entry("implicit s3 default", CloudTrail{S3: &CloudTrailS3{Bucket: "logs"}}, 7*24*time.Hour),
+		Entry("explicit overrides api", CloudTrail{MaxAge: "30m"}, 30*time.Minute),
+		Entry("explicit overrides s3", CloudTrail{Source: "s3", MaxAge: "1h"}, time.Hour),
+	)
+
 	DescribeTable("Includes",
 		func(config AWS, resource string, want bool) {
 			Expect(config.Includes(resource)).To(Equal(want))

chart/crds/configs.flanksource.com_scrapeconfigs.yaml

@@ -100,6 +100,19 @@ spec:
                           type: array
                         maxAge:
                           type: string
+                        s3:
+                          description: CloudTrailS3 configures CloudTrail log ingestion
+                            from an existing S3 bucket.
+                          properties:
+                            bucket:
+                              type: string
+                            bucketRegion:
+                              type: string
+                            prefix:
+                              type: string
+                          type: object
+                        source:
+                          type: string
                       type: object
                     compliance:
                       type: boolean

config/schemas/config_aws.schema.json

@@ -214,11 +214,33 @@
         },
         "maxAge": {
           "type": "string"
+        },
+        "source": {
+          "type": "string"
+        },
+        "s3": {
+          "$ref": "#/$defs/CloudTrailS3"
         }
       },
       "additionalProperties": false,
       "type": "object"
     },
+    "CloudTrailS3": {
+      "properties": {
+        "bucket": {
+          "type": "string"
+        },
+        "bucketRegion": {
+          "type": "string"
+        },
+        "prefix": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object",
+      "description": "CloudTrailS3 configures CloudTrail log ingestion from an existing S3 bucket."
+    },
     "ConfigFieldExclusion": {
       "properties": {
         "types": {

config/schemas/scrape_config.schema.json

@@ -913,11 +913,33 @@
         },
         "maxAge": {
           "type": "string"
+        },
+        "source": {
+          "type": "string"
+        },
+        "s3": {
+          "$ref": "#/$defs/CloudTrailS3"
         }
       },
       "additionalProperties": false,
       "type": "object"
     },
+    "CloudTrailS3": {
+      "properties": {
+        "bucket": {
+          "type": "string"
+        },
+        "bucketRegion": {
+          "type": "string"
+        },
+        "prefix": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object",
+      "description": "CloudTrailS3 configures CloudTrail log ingestion from an existing S3 bucket."
+    },
     "ConfigFieldExclusion": {
       "properties": {
         "types": {