Providing Repo Include Feature

pkg/kudoctl/cmd/repo_index.go

@@ -140,7 +140,7 @@ func (ri *repoIndexCmd) run() error {
 		if err != nil {
 			return err
 		}
-		merge(index, mergeIndex)
+		client.Merge(index, mergeIndex)
 	}
 
 	if err := index.WriteFile(ri.fs, target); err != nil {
@@ -150,20 +150,6 @@ func (ri *repoIndexCmd) run() error {
 	return nil
 }
 
-func merge(index *repo.IndexFile, mergeIndex *repo.IndexFile) {
-	// index is the master, any dups in the merged in index will have what is local replace those entries
-	for _, pvs := range mergeIndex.Entries {
-		for _, pv := range pvs {
-			err := index.AddPackageVersion(pv)
-			// this is most likely to be a duplicate pv, which we ignore (but will log at the right v)
-			if err != nil {
-				// todo: add verbose logging here
-				continue
-			}
-		}
-	}
-}
-
 func (ri *repoIndexCmd) mergeRepoConfig() (*repo.Configuration, error) {
 	if ri.mergeRepoName != "" {
 		return ri.repoConfig(ri.mergeRepoName)

pkg/kudoctl/cmd/repo_index_test.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"bytes"
+	"fmt"
 	"io/ioutil"
 	"path/filepath"
 	"testing"
@@ -92,7 +93,16 @@ func TestRepoIndexCmd_MergeIndex(t *testing.T) {
 	mergeFile, _ := repo.ParseIndexFile(mergeBytes)
 
 	resultBuf := &bytes.Buffer{}
-	merge(indexFile, mergeFile)
+
+	p, err := filepath.Abs("testdata/include-index")
+	assert.NoError(t, err)
+	config := &repo.Configuration{
+		URL: fmt.Sprintf("file://%s", p),
+	}
+	client, err := repo.NewClient(config)
+	assert.NoError(t, err)
+
+	client.Merge(indexFile, mergeFile)
 	if err := indexFile.Write(resultBuf); err != nil {
 		t.Fatal(err)
 	}

pkg/kudoctl/util/repo/index.go

@@ -22,6 +22,7 @@ const defaultURL = "http://localhost/"
 type IndexFile struct {
 	APIVersion string                     `json:"apiVersion"`
 	Entries    map[string]PackageVersions `json:"entries"`
+	Includes   []string                   `json:"includes,omitempty"`
 	Generated  *time.Time                 `json:"generated"`
 }
 

pkg/kudoctl/util/repo/repo_operator.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/spf13/afero"
 
+	"github.com/kudobuilder/kudo/pkg/kudoctl/clog"
 	"github.com/kudobuilder/kudo/pkg/kudoctl/http"
 	"github.com/kudobuilder/kudo/pkg/kudoctl/kudohome"
 )
@@ -50,20 +51,28 @@ func NewClient(conf *Configuration) (*Client, error) {
 
 // DownloadIndexFile fetches the index file from a repository.
 func (c *Client) DownloadIndexFile() (*IndexFile, error) {
-	var indexURL string
 	parsedURL, err := url.Parse(c.Config.URL)
 	if err != nil {
 		return nil, fmt.Errorf("parsing config url: %w", err)
 	}
-	parsedURL.Path = fmt.Sprintf("%s/index.yaml", strings.TrimSuffix(parsedURL.Path, "/"))
 
-	indexURL = parsedURL.String()
+	return c.downloadIndexFile(nil, parsedURL)
+}
+
+func (c *Client) downloadIndexFile(parent *IndexFile, url *url.URL) (*IndexFile, error) {
 	var resp *bytes.Buffer
-	if strings.HasPrefix(indexURL, "file:") {
-		b, _ := ioutil.ReadFile(parsedURL.Path)
+	var err error
+	// we need the index.yaml at the url provided
+	url.Path = fmt.Sprintf("%s/index.yaml", strings.TrimSuffix(url.Path, "/"))
+
+	if url.Scheme == "file" || strings.HasPrefix(url.String(), "file:") {
+		b, err := ioutil.ReadFile(url.Path)
+		if err != nil {
+			return nil, err
+		}
 		resp = bytes.NewBuffer(b)
 	} else {
-		resp, err = c.Client.Get(indexURL)
+		resp, err = c.Client.Get(url.String())
 	}
 	if err != nil {
 		return nil, fmt.Errorf("getting index url: %w", err)
@@ -75,5 +84,39 @@ func (c *Client) DownloadIndexFile() (*IndexFile, error) {
 	}
 
 	indexFile, err := ParseIndexFile(indexBytes)
+	//TODO (kensipe): err handling such that error in includes are ignored (but not root)
+	//TODO (kensipe): track which includes have happened so there are no repeats
+	for _, include := range indexFile.Includes {
+		iURL, err := url.Parse(include)
+		if err != nil {
+			clog.Printf("Unable to parse include url for %s", include)
+		}
+		nextIndex, err := c.downloadIndexFile(indexFile, iURL)
+		if err != nil {
+			return nil, err
+		}
+		if parent != nil {
+			c.Merge(parent, nextIndex)
+		} else {
+			c.Merge(indexFile, nextIndex)
+		}
+	}
+
 	return indexFile, err
 }
+
+// Merge combines the Entries of 2 index files.   The first index file is the master
+// the second is merged into the first.  Any duplicates are ignored.
+func (c *Client) Merge(index *IndexFile, mergeIndex *IndexFile) {
+	// index is the master, any dups in the merged in index will have what is local replace those entries
+	for _, pvs := range mergeIndex.Entries {
+		for _, pv := range pvs {
+			err := index.AddPackageVersion(pv)
+			// this is most likely to be a duplicate pv, which we ignore (but will log at the right v)
+			if err != nil {
+				// todo: add verbose logging here
+				continue
+			}
+		}
+	}
+}

pkg/kudoctl/util/repo/repo_test.go

@@ -42,3 +42,29 @@ func TestLoadRepositories(t *testing.T) {
 	assert.Equal(t, r.CurrentConfiguration().Name, Default.Name)
 	assert.Equal(t, r.CurrentConfiguration().URL, Default.URL)
 }
+
+func TestDownloadMultiRepo(t *testing.T) {
+
+	p, err := filepath.Abs("testdata/include-index")
+	assert.NoError(t, err)
+	config := &Configuration{
+		URL: fmt.Sprintf("file://%s", p),
+	}
+	client, err := NewClient(config)
+	assert.NoError(t, err)
+	index, err := client.DownloadIndexFile()
+	assert.NoError(t, err)
+	// mysql package only there, if include worked
+	assert.NotNil(t, index.Entries["mysql"])
+
+	// the merge for flink will have 1 dup that doesn't merge
+	flink, err := index.FindFirstMatch("flink", "", "0.3.0")
+	assert.NoError(t, err)
+	assert.Equal(t, "correct flink", flink.Description)
+
+	// and a new version that does merge
+	flink, err = index.FindFirstMatch("flink", "", "0.4.0")
+	assert.NoError(t, err)
+	assert.Equal(t, "0.4.0", flink.OperatorVersion)
+
+}

pkg/kudoctl/util/repo/testdata/include-index/index.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+entries:
+  flink:
+    - appVersion: 0.7.0
+      description: correct flink
+      digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd
+      maintainers:
+        - email: <runyontr@gmail.com>
+          name: Tom Runyon
+        - email: <kensipe@gmail.com>
+          name: Ken Sipe
+      name: flink
+      operatorVersion: 0.3.0
+      urls:
+        - http://kudo.dev/flink
+includes:
+  - ../included-repo
+generated: "2020-04-19T15:04:00Z"

pkg/kudoctl/util/repo/testdata/included-repo/index.yaml

@@ -0,0 +1,39 @@
+apiVersion: v1
+entries:
+  mysql:
+    - appVersion: "5.7"
+      digest: ad2451eaf68896490a2864afbb3fcd81e6fe3368e4bf001f8a23dc9cbcddf49a
+      maintainers:
+        - email: nick@dischord.org
+          name: Nick Jones
+      name: mysql
+      operatorVersion: 0.2.0
+      urls:
+        - https://kudo-repository.storage.googleapis.com/0.10.0/mysql-5.7_0.2.0.tgz
+  flink:
+    - appVersion: 0.7.0
+      description: wrong flink (should NOT be included)
+      digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd
+      maintainers:
+        - email: <runyontr@gmail.com>
+          name: Tom Runyon
+        - email: <kensipe@gmail.com>
+          name: Ken Sipe
+      name: flink
+      operatorVersion: 0.3.0
+    - appVersion: 0.8.0
+      description: this merges because the version doesn't exist in parent
+      digest: 0787a078e64c73064287751b833d63ca3d1d284b4f494ebf670443683d5b96dd
+      maintainers:
+        - email: <runyontr@gmail.com>
+          name: Tom Runyon
+        - email: <kensipe@gmail.com>
+          name: Ken Sipe
+      name: flink
+      operatorVersion: 0.4.0
+      urls:
+        - http://kudo.dev/flink
+
+generated: "2020-04-19T15:04:00Z"
+
+