diff --git a/internal/pkg/postprocessor/extractor/object_storage.go b/internal/pkg/postprocessor/extractor/object_storage.go new file mode 100644 index 00000000..3c023bbb --- /dev/null +++ b/internal/pkg/postprocessor/extractor/object_storage.go @@ -0,0 +1,36 @@ +package extractor + +import ( + "fmt" + "strings" + + "github.com/internetarchive/Zeno/internal/pkg/utils" + "github.com/internetarchive/Zeno/pkg/models" +) + +// All the supported object storage servers +var ObjectStorageServers = func() (s []string) { + s = append(s, s3CompatibleServers...) + s = append(s, azureServers...) + return s +}() + +// IsObjectStorage checks if the response is from an object storage server +func IsObjectStorage(URL *models.URL) bool { + return utils.StringContainsSliceElements(URL.GetResponse().Header.Get("Server"), ObjectStorageServers) && + strings.Contains(strings.ToLower(URL.GetResponse().Header.Get("Content-Type")), "/xml") // tricky match both application/xml and text/xml +} + +// ObjectStorage decides which helper to call based on the object storage server +func ObjectStorage(URL *models.URL) ([]*models.URL, error) { + defer URL.RewindBody() + + server := URL.GetResponse().Header.Get("Server") + if utils.StringContainsSliceElements(server, s3CompatibleServers) { + return s3Compatible(URL) + } else if utils.StringContainsSliceElements(server, azureServers) { + return azure(URL) + } else { + return nil, fmt.Errorf("unknown object storage server: %s", server) + } +} diff --git a/internal/pkg/postprocessor/extractor/object_storage_azure.go b/internal/pkg/postprocessor/extractor/object_storage_azure.go new file mode 100644 index 00000000..75f2e579 --- /dev/null +++ b/internal/pkg/postprocessor/extractor/object_storage_azure.go @@ -0,0 +1,78 @@ +package extractor + +import ( + "encoding/xml" + "fmt" + "strings" + + "github.com/internetarchive/Zeno/internal/pkg/log" + "github.com/internetarchive/Zeno/pkg/models" +) + +// Azure Blob Storage +var azureServers = []string{ + // Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0 + "Windows-Azure-Blob", + // Blob Service Version 1.0 Microsoft-HTTPAPI/2.0 + "Blob Service Version", + // emulator, https://github.com/Azure/Azurite + "Azurite-Blob", +} + +// AzureBlobEnumerationResults represents the XML structure of an Azure Blob Storage listing +// +type AzureBlobEnumerationResults struct { + XMLName xml.Name `xml:"EnumerationResults"` + Prefix string `xml:"Prefix"` + Marker string `xml:"Marker"` + Blobs []AzureBlob `xml:"Blobs>Blob"` + NextMarker string `xml:"NextMarker"` +} + +type AzureBlob struct { + Name string `xml:"Name"` // path/to/file.txt, no leading slash + LastModified string `xml:"Properties>Last-Modified"` + Size int64 `xml:"Properties>Content-Length"` +} + +var azureLogger = log.NewFieldedLogger(&log.Fields{ + "component": "postprocessor.extractor.object_storage_azure", +}) + +func azure(URL *models.URL) ([]*models.URL, error) { + defer URL.RewindBody() + + // Decode XML result + var result AzureBlobEnumerationResults + if err := xml.NewDecoder(URL.GetBody()).Decode(&result); err != nil { + return nil, fmt.Errorf("error decoding AzureBlobEnumerationResults XML: %w", err) + } + + reqURL := URL.GetRequest().URL + + var outlinks []string + + if result.NextMarker != "" { + nextURL := *reqURL + q := nextURL.Query() + q.Set("marker", result.NextMarker) + nextURL.RawQuery = q.Encode() + outlinks = append(outlinks, nextURL.String()) + } + + // Build base url for files + baseURL := *reqURL + baseURL.RawQuery = "" + baseURL.ForceQuery = false + + for _, blob := range result.Blobs { + if strings.HasPrefix(blob.Name, "/") { + azureLogger.Warn("invalid blob name: it starts with a leading slash", "blob_name", blob.Name) + continue + } + fileURL := baseURL.JoinPath(blob.Name) + outlinks = append(outlinks, fileURL.String()) + } + + return toURLs(outlinks), nil +} diff --git a/internal/pkg/postprocessor/extractor/object_storage_azure_test.go b/internal/pkg/postprocessor/extractor/object_storage_azure_test.go new file mode 100644 index 00000000..2d63e841 --- /dev/null +++ b/internal/pkg/postprocessor/extractor/object_storage_azure_test.go @@ -0,0 +1,74 @@ +package extractor + +import ( + "net/http" + "strings" + "testing" +) + +func TestAzure(t *testing.T) { + t.Run("Valid XML with single object and nextMarker", func(t *testing.T) { + xmlBody := ` + + + dir/azure_files/test_10.txt + 1 + + + dir/azure_files/test_100.txt + + Thu, 15 May 2025 08:02:20 GMT + Thu, 15 May 2025 08:02:20 GMT + 0x202A0424CF3AFA0 + 4 + text/plain + kZ0ReVbTE1xMaD/wITUvXA== + BlockBlob + unlocked + available + true + Hot + true + Thu, 15 May 2025 08:02:20 GMT + + + + dir/azure_files/test_100.txt +` + URLObj := buildTestObjectStorageURLObj("http://example.com/devstoreaccount1/zeno?restype=container&comp=list&maxresults=1", xmlBody, http.Header{"Server": []string{"Windows-Azure-Blob/1.0"}}) + outlinks, err := azure(URLObj) + outlinks2, err2 := ObjectStorage(URLObj) // indirectly call, for coverage testing + if err != nil || err2 != nil { + t.Fatalf("azure() returned unexpected error: %v, err2: %v", err, err2) + } + + if len(outlinks) != 2 || len(outlinks2) != 2 { + t.Fatalf("expected 2 outlinks, got %d and %d", len(outlinks), len(outlinks2)) + } + + expectedOutlinks := []string{ + "http://example.com/devstoreaccount1/zeno?comp=list&marker=dir%2Fazure_files%2Ftest_100.txt&maxresults=1&restype=container", + "http://example.com/devstoreaccount1/zeno/dir/azure_files/test_100.txt", + } + + for i, outlink := range outlinks { + outlink.Parse() + if outlink.String() != expectedOutlinks[i] { + t.Errorf("expected %s, got %s", expectedOutlinks[i], outlink.String()) + } + } + }) + + t.Run("invalid XML", func(t *testing.T) { + xmlBody := ` as the base for direct file links - baseStr := fmt.Sprintf("https://%s", reqURL.Host) - parsedBase, err := url.Parse(baseStr) - if err != nil { - return nil, fmt.Errorf("invalid base URL: %v", err) - } - - var outlinkStrings []string - - // Delegate to old style or new style - if listType != "2" { - // Old style S3 listing, uses marker - outlinkStrings = s3Legacy(reqURL, parsedBase, result) - } else { - // New style listing (list-type=2), uses continuation token and/or CommonPrefixes - outlinkStrings = s3V2(reqURL, parsedBase, result) - } - - // Convert from []string -> []*models.URL - var outlinks []*models.URL - for _, link := range outlinkStrings { - outlinks = append(outlinks, &models.URL{Raw: link}) - } - return outlinks, nil -} - // s3Legacy handles the old ListObjects style, which uses `marker` for pagination. -func s3Legacy(reqURL *url.URL, parsedBase *url.URL, result S3ListBucketResult) []string { +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html +func s3Legacy(reqURL, baseURL *url.URL, result S3ListBucketResult) []string { var outlinks []string // If there are objects in , create a "next page" URL using `marker` @@ -103,8 +55,7 @@ func s3Legacy(reqURL *url.URL, parsedBase *url.URL, result S3ListBucketResult) [ // Produce direct file links for each object for _, obj := range result.Contents { if obj.Size > 0 { - fileURL := *parsedBase - fileURL.Path += "/" + obj.Key + fileURL := baseURL.JoinPath(obj.Key) outlinks = append(outlinks, fileURL.String()) } } @@ -113,7 +64,8 @@ func s3Legacy(reqURL *url.URL, parsedBase *url.URL, result S3ListBucketResult) [ } // s3V2 handles the new ListObjectsV2 style, which uses `continuation-token` and can return CommonPrefixes. -func s3V2(reqURL *url.URL, parsedBase *url.URL, result S3ListBucketResult) []string { +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html +func s3V2(reqURL, baseURL *url.URL, result S3ListBucketResult) []string { var outlinks []string // If we have common prefixes => "subfolders" @@ -129,11 +81,11 @@ func s3V2(reqURL *url.URL, parsedBase *url.URL, result S3ListBucketResult) []str } } } else { + // Otherwise, we have actual objects in for _, obj := range result.Contents { if obj.Size > 0 { - fileURL := *parsedBase - fileURL.Path += "/" + obj.Key + fileURL := *baseURL.JoinPath(obj.Key) outlinks = append(outlinks, fileURL.String()) } } @@ -150,3 +102,37 @@ func s3V2(reqURL *url.URL, parsedBase *url.URL, result S3ListBucketResult) []str return outlinks } + +// s3Compatible decides which helper to call based on the query param: old style (no list-type=2) vs. new style (list-type=2) +func s3Compatible(URL *models.URL) ([]*models.URL, error) { + defer URL.RewindBody() + + // Decode XML result + var result S3ListBucketResult + if err := xml.NewDecoder(URL.GetBody()).Decode(&result); err != nil { + return nil, fmt.Errorf("error decoding S3ListBucketResult XML: %w", err) + } + + reqURL := URL.GetRequest().URL + listType := reqURL.Query().Get("list-type") + + // Prepare base url + baseURL := new(url.URL) + *baseURL = *reqURL + baseURL.RawQuery = "" + baseURL.ForceQuery = false + baseURL.Path = "/" + + var outlinks []string + + // Delegate to old style or new style + if listType != "2" { + // Old style S3 listing, uses marker + outlinks = s3Legacy(reqURL, baseURL, result) + } else { + // New style listing (list-type=2), uses continuation token and/or CommonPrefixes + outlinks = s3V2(reqURL, baseURL, result) + } + + return toURLs(outlinks), nil +} diff --git a/internal/pkg/postprocessor/extractor/object_storage_s3_test.go b/internal/pkg/postprocessor/extractor/object_storage_s3_test.go new file mode 100644 index 00000000..c0737859 --- /dev/null +++ b/internal/pkg/postprocessor/extractor/object_storage_s3_test.go @@ -0,0 +1,89 @@ +package extractor + +import ( + "net/http" + "strings" + "testing" +) + +func TestS3Compatible(t *testing.T) { + // This subtest shows a scenario of a valid XML with a single object, + // and list-type != 2 => "marker" logic should be used. + t.Run("Valid XML with single object, no list-type=2 => marker next link", func(t *testing.T) { + xmlBody := ` + + + file1.txt + 2021-01-01T12:00:00.000Z + 123 + + false +` + + URLObj := buildTestObjectStorageURLObj("https://example.com/?someparam=1", xmlBody, http.Header{"Server": []string{"AmazonS3"}}) + outlinks, err := s3Compatible(URLObj) + outlinks2, err2 := ObjectStorage(URLObj) // indirectly call, for coverage testing + if err != nil || err2 != nil { + t.Fatalf("S3() returned unexpected error: %v, err2: %v", err, err2) + } + + if len(outlinks) != 2 || len(outlinks2) != 2 { + t.Fatalf("expected 2 outlinks, got %d and %d", len(outlinks), len(outlinks2)) + } + expectedOutlinks := []string{ + "https://example.com/?marker=file1.txt&someparam=1", + "https://example.com/file1.txt", + } + for i, outlink := range outlinks { + if outlink.Raw != expectedOutlinks[i] { + t.Errorf("expected %s, got %s", expectedOutlinks[i], outlink.Raw) + } + } + }) + + // Another subtest example: common prefixes => subfolder links for list-type=2 + t.Run("Valid XML with common prefixes => subfolder links (list-type=2)", func(t *testing.T) { + xmlBody := ` + + false + + folder1/ + folder2/ + +` + + URLObj := buildTestObjectStorageURLObj("https://example.com/?list-type=2", xmlBody, http.Header{"Server": []string{"AmazonS3"}}) + outlinks, err := s3Compatible(URLObj) + outlinks2, err2 := ObjectStorage(URLObj) // indirectly call, for coverage testing + if err != nil || err2 != nil { + t.Fatalf("s3Compatible() returned unexpected error: %v, err2: %v", err, err2) + } + + if len(outlinks) != 2 || len(outlinks2) != 2 { + t.Fatalf("expected 2 outlinks, got %d and %d", len(outlinks), len(outlinks2)) + } + + if !strings.Contains(outlinks[0].Raw, "prefix=folder1%2F") { + t.Errorf("expected prefix=folder1/ in outlink, got %s", outlinks[0].Raw) + } + if !strings.Contains(outlinks[1].Raw, "prefix=folder2%2F") { + t.Errorf("expected prefix=folder2/ in outlink, got %s", outlinks[1].Raw) + } + }) + + // Example for invalid XML + t.Run("Invalid XML => error", func(t *testing.T) { + xmlBody := `XML` + + URLObj := buildTestObjectStorageURLObj("https://example.com/", xmlBody, http.Header{"Server": []string{"NotAnOSS"}}) + _, err := ObjectStorage(URLObj) + + if err == nil { + t.Fatalf("expected error for unknown object storage server, got none") + } + + if err.Error() != "unknown object storage server: NotAnOSS" { + t.Fatalf("expected error 'unknown object storage server: NotAnOSS', got %v", err) + } + }) +} diff --git a/internal/pkg/postprocessor/extractor/s3_test.go b/internal/pkg/postprocessor/extractor/s3_test.go deleted file mode 100644 index 54d7fc7a..00000000 --- a/internal/pkg/postprocessor/extractor/s3_test.go +++ /dev/null @@ -1,171 +0,0 @@ -package extractor - -import ( - "net/http" - "net/url" - "os" - "strings" - "testing" - - "github.com/internetarchive/Zeno/pkg/models" - "github.com/internetarchive/gowarc/pkg/spooledtempfile" -) - -// TestIsS3 checks the Server header for known S3 strings. -func TestIsS3(t *testing.T) { - tests := []struct { - name string - server string - want bool - }{ - {"AmazonS3", "AmazonS3", true}, - {"WasabiS3", "WasabiS3", true}, - {"AliyunOSS", "AliyunOSS", true}, - {"No match", "Apache", false}, - {"Partial match", "Amazon", false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Create a *models.URL with the response Server header set - URLObj := &models.URL{} - - URLObj.SetResponse(&http.Response{ - Header: http.Header{ - "Server": []string{tt.server}, - "Content-Type": []string{"text/xml"}, - }, - }) - - got := IsS3(URLObj) - if got != tt.want { - t.Errorf("IsS3(server=%q) = %v, want %v", tt.server, got, tt.want) - } - }) - } -} - -func TestS3(t *testing.T) { - // This subtest shows a scenario of a valid XML with a single object, - // and list-type != 2 => "marker" logic should be used. - t.Run("Valid XML with single object, no list-type=2 => marker next link", func(t *testing.T) { - xmlBody := ` - - - file1.txt - 2021-01-01T12:00:00.000Z - 123 - - false -` - - // Build an http.Request with a query param that is NOT list-type=2 - reqURL, _ := url.Parse("https://example.com/?someparam=1") - - // Create your models.URL instance. - URLObj := &models.URL{} - URLObj.SetRequest(&http.Request{URL: reqURL}) - - // Likewise, set the HTTP response header using SetResponse. - // We want to simulate an S3 server for these tests. - URLObj.SetResponse(&http.Response{ - Header: http.Header{ - "Server": []string{"AmazonS3"}, - }, - }) - - spooledTempFile := spooledtempfile.NewSpooledTempFile("test", os.TempDir(), 2048, false, -1) - spooledTempFile.Write([]byte(xmlBody)) - - URLObj.SetBody(spooledTempFile) - - outlinks, err := S3(URLObj) - if err != nil { - t.Fatalf("S3() returned unexpected error: %v", err) - } - - if len(outlinks) != 2 { - t.Fatalf("expected 2 outlinks, got %d", len(outlinks)) - } - expectedOutlinks := []string{ - "https://example.com/?marker=file1.txt&someparam=1", - "https://example.com/file1.txt", - } - for i, outlink := range outlinks { - if outlink.Raw != expectedOutlinks[i] { - t.Errorf("expected %s, got %s", expectedOutlinks[i], outlink.Raw) - } - } - }) - - // Another subtest example: common prefixes => subfolder links for list-type=2 - t.Run("Valid XML with common prefixes => subfolder links (list-type=2)", func(t *testing.T) { - xmlBody := ` - - false - - folder1/ - folder2/ - -` - - reqURL, _ := url.Parse("https://example.com/?list-type=2") - - URLObj := &models.URL{} - URLObj.SetRequest(&http.Request{URL: reqURL}) - URLObj.SetResponse(&http.Response{ - Header: http.Header{ - "Server": []string{"AmazonS3"}, - }, - }) - - spooledTempFile := spooledtempfile.NewSpooledTempFile("test", os.TempDir(), 2048, false, -1) - spooledTempFile.Write([]byte(xmlBody)) - - URLObj.SetBody(spooledTempFile) - - outlinks, err := S3(URLObj) - if err != nil { - t.Fatalf("S3() returned unexpected error: %v", err) - } - - if len(outlinks) != 2 { - t.Fatalf("expected 2 outlinks, got %d", len(outlinks)) - } - if !strings.Contains(outlinks[0].Raw, "prefix=folder1%2F") { - t.Errorf("expected prefix=folder1/ in outlink, got %s", outlinks[0].Raw) - } - if !strings.Contains(outlinks[1].Raw, "prefix=folder2%2F") { - t.Errorf("expected prefix=folder2/ in outlink, got %s", outlinks[1].Raw) - } - }) - - // Example for invalid XML - t.Run("Invalid XML => error", func(t *testing.T) { - xmlBody := ` []*models.URL +func toURLs(s []string) []*models.URL { + outlinks := make([]*models.URL, 0, len(s)) + for _, link := range s { + outlinks = append(outlinks, &models.URL{Raw: link}) + } + return outlinks +} diff --git a/internal/pkg/postprocessor/outlinks.go b/internal/pkg/postprocessor/outlinks.go index 21d0eb50..99fd622d 100644 --- a/internal/pkg/postprocessor/outlinks.go +++ b/internal/pkg/postprocessor/outlinks.go @@ -41,10 +41,10 @@ func extractOutlinks(item *models.Item) (outlinks []*models.URL, err error) { logger.Error("unable to extract outlinks", "extractor", "truthsocial.GenerateOutlinksURLsFromLookup", "err", err.Error(), "item", item.GetShortID(), "url", item.GetURL().String()) return outlinks, err } - case extractor.IsS3(item.GetURL()): - outlinks, err = extractor.S3(item.GetURL()) + case extractor.IsObjectStorage(item.GetURL()): + outlinks, err = extractor.ObjectStorage(item.GetURL()) if err != nil { - logger.Error("unable to extract outlinks from S3", "extractor", "S3", "err", err.Error(), "item", item.GetShortID(), "url", item.GetURL().String()) + logger.Error("unable to extract outlinks from ObjectStorage", "extractor", "ObjectStorage", "err", err.Error(), "item", item.GetShortID(), "url", item.GetURL().String()) return outlinks, err } case extractor.IsSitemapXML(item.GetURL()):