diff --git a/.github/workflows/validate-checksums.yml b/.github/workflows/validate-checksums.yml index c7d8937667..fd45c47568 100644 --- a/.github/workflows/validate-checksums.yml +++ b/.github/workflows/validate-checksums.yml @@ -60,6 +60,9 @@ jobs: echo echo "Please refer to the developer docs at docs/developer for more information." echo "-----------------------------------------------------------------------------" + echo "git status:" + git status -vv + echo "-----------------------------------------------------------------------------" exit 1 fi diff --git a/pkg/flatpak/flatpak.go b/pkg/flatpak/flatpak.go index 8d7543179e..cbb529cce2 100644 --- a/pkg/flatpak/flatpak.go +++ b/pkg/flatpak/flatpak.go @@ -2,6 +2,7 @@ package flatpak import ( "fmt" + "strings" "github.com/osbuild/images/pkg/container" "github.com/osbuild/images/pkg/ostree" @@ -33,11 +34,40 @@ func Resolve(source SourceSpec) (Spec, error) { } func ResolveAll(sources map[string][]SourceSpec) (map[string][]Spec, error) { + ociClients := make(map[string]*OCIRegistryIndex) + defer func() { + for _, c := range ociClients { + c.Close() + } + }() + flatpaks := make(map[string][]Spec, len(sources)) for name, srcList := range sources { specs := make([]Spec, len(srcList)) for i, src := range srcList { + if src.Registry.Type == REGISTRY_TYPE_OCI { + uri, found := strings.CutPrefix(src.Registry.URI, "oci+") + if !found { + return nil, fmt.Errorf("flatpak registry %q: missing oci+ prefix", src.Registry.URI) + } + idx, ok := ociClients[uri] + if !ok { + var err error + idx, err = NewOCIRegistryIndex(uri, "linux", "latest") + if err != nil { + return nil, err + } + ociClients[uri] = idx + } + res, err := src.Registry.queryOCIWithIndex(idx, src.Reference.String()) + if err != nil { + return nil, fmt.Errorf("failed to resolve flatpak: %w", err) + } + specs[i] = *res + continue + } + res, err := Resolve(src) if err != nil { return nil, fmt.Errorf("failed to resolve flatpak: %w", err) diff --git a/pkg/flatpak/oci_registry_index.go b/pkg/flatpak/oci_registry_index.go index 0ca9e9d06d..c1fb9463bf 100644 --- a/pkg/flatpak/oci_registry_index.go +++ b/pkg/flatpak/oci_registry_index.go @@ -6,6 +6,7 @@ import ( "net/http" "net/url" "strings" + "sync" "time" "github.com/osbuild/images/pkg/container" @@ -44,139 +45,175 @@ type ResponseImage struct { Labels map[string]string `json:"Labels"` } -// Deserialization of an OCI Manifest, only defines the fields necessary for us -// to get what we need; which is the digest of the config object. -type OCIManifest struct { - Config *struct { - Digest string `json:"digest,omitempty"` - } `json:"config,omitempty"` -} - type ResponseImageList struct{} -func QueryOCIRegistryIndex(uri, ref, os, tag string) (*container.Spec, error) { - res, err := fetchRegistryIndex(uri, os, tag) - if err != nil { - return nil, err - } +// OCIRegistryIndex is a session for querying one OCI registry's Flatpak static index +// (/index/static) for a fixed os/tag pair. The decoded JSON is cached on this instance +// until [OCIRegistryIndex.Close]. Callers should defer Close() once they are done +// issuing [OCIRegistryIndex.Query] calls. +// +// Manifest and config resolution for the digest-pinned image is delegated to +// [container.Resolver] from pkg/container ([container.NewBlockingResolver]). +type OCIRegistryIndex struct { + baseURI string + os string + tag string + + mu sync.Mutex + cacheKey string + root *ResponseRoot +} - for _, result := range res.Results { - for _, img := range result.Images { - if img.Labels["org.flatpak.ref"] == ref { - imageID, err := fetchDockerAPIConfigDigest(res.Registry, result.Name, img.Digest) - if err != nil { - return nil, err - } - - reg, _ := strings.CutPrefix(res.Registry, "https://") - - return &container.Spec{ - Source: fmt.Sprintf("%s%s", reg, result.Name), - Digest: img.Digest, - ImageID: imageID, - LocalName: result.Name, - }, nil - } - } +// NewOCIRegistryIndex constructs an index client for baseURI (https host or full origin +// without the oci+ scheme prefix), Flatpak index os and tag query parameters. +func NewOCIRegistryIndex(baseURI, os, tag string) (*OCIRegistryIndex, error) { + if baseURI == "" { + return nil, fmt.Errorf("flatpak oci registry: empty base URI") } - - return nil, fmt.Errorf("did not find image %q", ref) + return &OCIRegistryIndex{baseURI: baseURI, os: os, tag: tag}, nil } -func httpClient() (*http.Client, error) { - return &http.Client{ - Transport: http.DefaultTransport.(*http.Transport).Clone(), - Timeout: 300 * time.Second, - }, nil +// Close drops the cached decoded index. It must not be called concurrently with Query. +func (q *OCIRegistryIndex) Close() { + q.mu.Lock() + defer q.mu.Unlock() + q.cacheKey = "" + q.root = nil } -func fetchRegistryIndex(uri, os, tag string) (*ResponseRoot, error) { - client, err := httpClient() +// Query parses flatpakRef with [NewReferenceFromString] for architecture selection, +// loads the registry index (cached on this instance), finds the image by +// org.flatpak.ref, then resolves the digest-pinned image using [container.NewBlockingResolver] +// (pkg/container). +func (q *OCIRegistryIndex) Query(flatpakRef string) (*container.Spec, error) { + parsed, err := NewReferenceFromString(flatpakRef) if err != nil { return nil, err } - uri, err = url.JoinPath(uri, ENDPOINT_STATIC) + res, err := q.getResponseRoot() if err != nil { - return nil, fmt.Errorf("could not format URI: %w", err) + return nil, err } - req, err := http.NewRequest(http.MethodGet, uri, nil) + repoName, manifestDigest, err := findFlatpakInIndex(res, flatpakRef) if err != nil { return nil, err } - // The default set of query parameters that are always passed by - // flatpak. See [1]. - // - // [1]: https://github.com/flatpak/flatpak-oci-specs/blob/main/registry-index.md#appendix---usage-by-flatpak - q := req.URL.Query() - q.Set("label:org.flatpak.ref:exists", "1") - q.Set("os", os) - q.Set("tag", tag) - req.URL.RawQuery = q.Encode() + imageRef := ociImageRefFromIndexComponents(res.Registry, repoName, manifestDigest) - res, err := client.Do(req) + r := container.NewBlockingResolver(parsed.Arch) + spec, err := r.Resolve(container.SourceSpec{ + Source: imageRef, + Name: repoName, + TLSVerify: nil, + Local: false, + }) if err != nil { - return nil, err + return nil, fmt.Errorf("resolve flatpak container %q: %w", imageRef, err) } - defer res.Body.Close() + return &spec, nil +} - if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("registry index %q returned status: %s", uri, res.Status) +func (q *OCIRegistryIndex) buildIndexGETRequest() (*http.Request, string, error) { + indexURL, err := url.JoinPath(q.baseURI, ENDPOINT_STATIC) + if err != nil { + return nil, "", fmt.Errorf("could not format URI: %w", err) } - var root ResponseRoot - if err := json.NewDecoder(res.Body).Decode(&root); err != nil { - return nil, err + req, err := http.NewRequest(http.MethodGet, indexURL, nil) + if err != nil { + return nil, "", err } - return &root, nil + // The default set of query parameters that are always passed by + // flatpak. See [1]. + // + // [1]: https://github.com/flatpak/flatpak-oci-specs/blob/main/registry-index.md#appendix---usage-by-flatpak + query := req.URL.Query() + query.Set("label:org.flatpak.ref:exists", "1") + query.Set("os", q.os) + query.Set("tag", q.tag) + req.URL.RawQuery = query.Encode() + + return req, req.URL.String(), nil } -// Use the docker API provided by a registry to fetch the digest of the 'config' section of -// a manifest [1], [2]. -// [1]: https://distribution.github.io/distribution/spec/api/#pulling-an-image-manifest -// [2]: https://specs.opencontainers.org/image-spec/manifest/ -func fetchDockerAPIConfigDigest(registry, name, digest string) (string, error) { - client, err := httpClient() +func (q *OCIRegistryIndex) getResponseRoot() (*ResponseRoot, error) { + req, cacheKey, err := q.buildIndexGETRequest() if err != nil { - return "", err + return nil, err } - uri, err := url.JoinPath(registry, "v2", name, "manifests", digest) - if err != nil { - return "", fmt.Errorf("could not format URI: %w", err) + q.mu.Lock() + if q.cacheKey == cacheKey && q.root != nil { + root := q.root + q.mu.Unlock() + return root, nil } + q.mu.Unlock() - req, err := http.NewRequest(http.MethodGet, uri, nil) + client, err := httpClient() if err != nil { - return "", err + return nil, err } - req.Header.Set("Accept", "application/vnd.oci.image.manifest.v1+json") res, err := client.Do(req) if err != nil { - return "", err + return nil, err } defer res.Body.Close() if res.StatusCode != http.StatusOK { - return "", fmt.Errorf("docker api error %d %w %q", res.StatusCode, err, uri) + return nil, fmt.Errorf("registry index %q returned status: %s", cacheKey, res.Status) } - var manifest OCIManifest - if err := json.NewDecoder(res.Body).Decode(&manifest); err != nil { - return "", err + root := new(ResponseRoot) + if err := json.NewDecoder(res.Body).Decode(root); err != nil { + return nil, err } - if manifest.Config == nil { - return "", fmt.Errorf("manifest did not contain config") - } + q.mu.Lock() + q.cacheKey = cacheKey + q.root = root + q.mu.Unlock() + + return root, nil +} - if manifest.Config.Digest == "" { - return "", fmt.Errorf("config did not contain digest") +// findFlatpakInIndex returns the repository name and manifest digest for the first image +// whose org.flatpak.ref label equals wantRef. +func findFlatpakInIndex(res *ResponseRoot, wantRef string) (repoName, manifestDigest string, err error) { + for _, result := range res.Results { + for _, img := range result.Images { + if img == nil || img.Labels == nil { + continue + } + if img.Labels["org.flatpak.ref"] != wantRef { + continue + } + if res.Registry == "" { + return "", "", fmt.Errorf("registry index: found %q but Registry field was missing", wantRef) + } + return result.Name, img.Digest, nil + } } + return "", "", fmt.Errorf("did not find image %q", wantRef) +} - return manifest.Config.Digest, nil +// ociImageRefFromIndexComponents builds a docker-style reference host/repo@digest for +// [container.NewClient] / the blocking resolver. +func ociImageRefFromIndexComponents(registryURL, repoName, manifestDigest string) string { + host := strings.TrimPrefix(strings.TrimPrefix(registryURL, "https://"), "http://") + host = strings.TrimSuffix(host, "/") + repoPath := strings.TrimPrefix(repoName, "/") + return fmt.Sprintf("%s/%s@%s", host, repoPath, manifestDigest) +} + +func httpClient() (*http.Client, error) { + return &http.Client{ + Transport: http.DefaultTransport.(*http.Transport).Clone(), + Timeout: 300 * time.Second, + }, nil } diff --git a/pkg/flatpak/oci_registry_index_test.go b/pkg/flatpak/oci_registry_index_test.go new file mode 100644 index 0000000000..2ab85628cc --- /dev/null +++ b/pkg/flatpak/oci_registry_index_test.go @@ -0,0 +1,297 @@ +package flatpak + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +func TestOciImageRefFromIndexComponents(t *testing.T) { + tests := []struct { + name string + registry string + repo string + digest string + want string + }{ + { + name: "https_host_and_leading_slash_on_repo", + registry: "https://registry.example.com", + repo: "/flatpak/repo", + digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + want: "registry.example.com/flatpak/repo@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }, + { + name: "http_host_trailing_slash", + registry: "http://localhost:5000/", + repo: "library/foo", + digest: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + want: "localhost:5000/library/foo@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ociImageRefFromIndexComponents(tt.registry, tt.repo, tt.digest) + if got != tt.want { + t.Fatalf("got %q want %q", got, tt.want) + } + }) + } +} + +func TestFindFlatpakInIndex(t *testing.T) { + const wantRef = "app/org.test/x86_64/stable" + const dig = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + tests := []struct { + name string + root *ResponseRoot + ref string + wantRepo string + wantDigest string + errContains string + }{ + { + name: "match_skips_noise_rows", + root: &ResponseRoot{ + Registry: "https://registry.example.com", + Results: []ResponseRepository{ + { + Name: "/other", + Images: []*ResponseImage{ + {Labels: map[string]string{"org.flatpak.ref": "other"}, Digest: "sha256:1111"}, + }, + }, + { + Name: "/flatpak/want", + Images: []*ResponseImage{ + {Labels: map[string]string{"org.flatpak.ref": wantRef}, Digest: dig}, + }, + }, + }, + }, + ref: wantRef, + wantRepo: "/flatpak/want", + wantDigest: dig, + }, + { + name: "not_found", + root: &ResponseRoot{ + Registry: "https://r.example", + Results: []ResponseRepository{ + {Name: "/x", Images: []*ResponseImage{{Labels: map[string]string{"org.flatpak.ref": "other"}}}}, + }, + }, + ref: wantRef, + errContains: "did not find", + }, + { + name: "missing_registry", + root: &ResponseRoot{ + Registry: "", + Results: []ResponseRepository{ + {Name: "/x", Images: []*ResponseImage{{Labels: map[string]string{"org.flatpak.ref": wantRef}, Digest: dig}}}, + }, + }, + ref: wantRef, + errContains: "Registry field was missing", + }, + { + name: "nil_image_skipped", + root: &ResponseRoot{ + Registry: "https://r.example", + Results: []ResponseRepository{ + {Name: "/x", Images: []*ResponseImage{nil, {Digest: dig, Labels: map[string]string{"org.flatpak.ref": wantRef}}}}, + }, + }, + ref: wantRef, + wantRepo: "/x", + wantDigest: dig, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo, gotDig, err := findFlatpakInIndex(tt.root, tt.ref) + if tt.errContains != "" { + if err == nil || !strings.Contains(err.Error(), tt.errContains) { + t.Fatalf("err=%v want substring %q", err, tt.errContains) + } + return + } + if err != nil { + t.Fatal(err) + } + if repo != tt.wantRepo || gotDig != tt.wantDigest { + t.Fatalf("repo=%q digest=%q want repo=%q digest=%q", repo, gotDig, tt.wantRepo, tt.wantDigest) + } + }) + } +} + +func TestFetchRegistryIndex(t *testing.T) { + const wantRef = "app/org.test/x86_64/stable" + const wantDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + tests := []struct { + name string + status int + body string + os string + tag string + checkQuery bool + wantRef string + wantDigest string + wantRegistry string + wantRepo string + wantErr bool + errContains string + }{ + { + name: "query_params_and_decode_registry_first", + status: http.StatusOK, + body: fmt.Sprintf( + `{"Registry":"https://registry.example.com","Results":[{"Name":"/flatpak/test","Images":[{"Digest":%q,"Labels":{"org.flatpak.ref":%q}}]}]}`, + wantDigest, wantRef, + ), + os: "linux", + tag: "stable", + checkQuery: true, + wantRef: wantRef, + wantDigest: wantDigest, + wantRegistry: "https://registry.example.com", + wantRepo: "/flatpak/test", + }, + { + name: "decode_results_before_registry", + status: http.StatusOK, + body: fmt.Sprintf( + `{"Results":[{"Name":"/r","Images":[{"Digest":%q,"Labels":{"org.flatpak.ref":%q}}]}],"Registry":"https://r.example"}`, + "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "app/x", + ), + os: "linux", + tag: "latest", + checkQuery: false, + wantRef: "app/x", + wantDigest: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + wantRegistry: "https://r.example", + wantRepo: "/r", + }, + { + name: "http_not_ok", + status: http.StatusGone, + body: "", + os: "linux", + tag: "latest", + wantErr: true, + errContains: "410", + }, + { + name: "invalid_json", + status: http.StatusOK, + body: `{`, + os: "linux", + tag: "latest", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if tt.status != http.StatusOK { + http.Error(w, "gone", tt.status) + return + } + if tt.checkQuery { + if !strings.Contains(r.URL.Path, "/index/static") { + t.Errorf("path %q", r.URL.Path) + } + q := r.URL.Query() + if q.Get("label:org.flatpak.ref:exists") != "1" { + t.Errorf("label:org.flatpak.ref:exists: got %q", q.Get("label:org.flatpak.ref:exists")) + } + if q.Get("os") != tt.os { + t.Errorf("os: got %q want %q", q.Get("os"), tt.os) + } + if q.Get("tag") != tt.tag { + t.Errorf("tag: got %q want %q", q.Get("tag"), tt.tag) + } + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(tt.body)) + })) + defer ts.Close() + + idx, err := NewOCIRegistryIndex(ts.URL, tt.os, tt.tag) + if err != nil { + t.Fatal(err) + } + defer idx.Close() + + root, err := idx.getResponseRoot() + if tt.wantErr { + if err == nil { + t.Fatal("expected error") + } + if tt.errContains != "" && !strings.Contains(err.Error(), tt.errContains) { + t.Fatalf("err=%v want substring %q", err, tt.errContains) + } + return + } + if err != nil { + t.Fatal(err) + } + if root.Registry != tt.wantRegistry { + t.Errorf("Registry: got %q want %q", root.Registry, tt.wantRegistry) + } + repo, dig, err := findFlatpakInIndex(root, tt.wantRef) + if err != nil { + t.Fatal(err) + } + if repo != tt.wantRepo || dig != tt.wantDigest { + t.Fatalf("repo=%q digest=%q want repo=%q digest=%q", repo, dig, tt.wantRepo, tt.wantDigest) + } + }) + } +} + +func TestOCIRegistryIndex_cacheReusesGET(t *testing.T) { + const body = `{"Registry":"https://registry.example.com","Results":[]}` + var indexGets atomic.Int32 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "/index/static") { + indexGets.Add(1) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + defer ts.Close() + + idx, err := NewOCIRegistryIndex(ts.URL, "linux", "latest") + if err != nil { + t.Fatal(err) + } + defer idx.Close() + + if _, err := idx.getResponseRoot(); err != nil { + t.Fatal(err) + } + if _, err := idx.getResponseRoot(); err != nil { + t.Fatal(err) + } + if indexGets.Load() != 1 { + t.Fatalf("expected 1 index GET, got %d", indexGets.Load()) + } + + idx.Close() + if _, err := idx.getResponseRoot(); err != nil { + t.Fatal(err) + } + if indexGets.Load() != 2 { + t.Fatalf("after Close expected 2 index GETs, got %d", indexGets.Load()) + } +} diff --git a/pkg/flatpak/registry.go b/pkg/flatpak/registry.go index 649ea7df6e..b504858687 100644 --- a/pkg/flatpak/registry.go +++ b/pkg/flatpak/registry.go @@ -46,20 +46,24 @@ func (r *Registry) queryOCI(ref string) (*Spec, error) { panic("uri missing oci+ prefix") } - container, err := QueryOCIRegistryIndex( - uri, - ref, - "linux", - "latest", - ) - + idx, err := NewOCIRegistryIndex(uri, "linux", "latest") if err != nil { return nil, err } + defer idx.Close() - return &Spec{ - ContainerSpec: container, - }, nil + return r.queryOCIWithIndex(idx, ref) +} + +// queryOCIWithIndex resolves ref using an existing index client (shared across +// multiple refs in [ResolveAll]). idx must have been constructed from this +// registry's oci+ URI (without the prefix). +func (r *Registry) queryOCIWithIndex(idx *OCIRegistryIndex, ref string) (*Spec, error) { + containerSpec, err := idx.Query(ref) + if err != nil { + return nil, err + } + return &Spec{ContainerSpec: containerSpec}, nil } func (r *Registry) Query(ref string) (*Spec, error) {