diff --git a/Makefile b/Makefile index eb76618b..3408a045 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ install: gotestsum: $(gopath)/bin/gotestsum $(gopath)/bin/gotestsum: - go get gotest.tools/gotestsum + go install gotest.tools/gotestsum@latest go mod tidy build: install diff --git a/bramble.lock b/bramble.lock index 4778ded7..72f276c0 100644 --- a/bramble.lock +++ b/bramble.lock @@ -1,14 +1,14 @@ [URLHashes] - "basic_fetch_url https://brmbl.s3.amazonaws.com/busybox-x86_64.tar.gz" = "uw5ichj6dhcccmcts6p7jq6etzlh5baf" - "basic_fetch_url https://brmbl.s3.amazonaws.com/url_fetcher.tar.gz" = "p2vbvabkdqckjlm43rf7bfccdseizych" - "fetch_git https://github.com/maxmcd/bramble.git@v0" = "ebzl4qxuufyxuejtyujh7mdul4spjcgk" - "fetch_url http://s.minos.io/archive/bifrost/x86_64/git-2.10.2-1.tar.gz" = "omtz2dd5irsgvbg7eeksovceutw5abhb" - "fetch_url http://tarballs.nixos.org/stdenv-linux/x86_64/c5aabb0d603e2c1ea05f5a93b3be82437f5ebf31/bootstrap-tools.tar.xz" = "cspu7ndkuuq2f7t5vvgsmvdt3xnoypqb" - "fetch_url https://brmbl.s3.amazonaws.com/busybox-x86_64.tar.gz" = "uw5ichj6dhcccmcts6p7jq6etzlh5baf" - "fetch_url https://brmbl.s3.amazonaws.com/ca-certificates.crt" = "qucryjlyakz2x2asktkm6dtzpx4qz5rj" - "fetch_url https://brmbl.s3.amazonaws.com/file-links.tar.gz" = "rdk4ljksk75wjx7fflcv5vyc7a4dgbbt" - "fetch_url https://github.com/NixOS/patchelf/releases/download/0.13/patchelf-0.13.tar.bz2" = "qffzn3vcrs5tu4lb5vtm6eaplwfjgtth" - "fetch_url https://github.com/denoland/deno/releases/download/v1.14.0/deno-x86_64-unknown-linux-gnu.zip" = "k5fftqwt46izovwwttzh5ebmjiccxuhz" - "fetch_url https://golang.org/dl/go1.17.2.linux-amd64.tar.gz" = "fm3ujt23vqou74mr5fz5dyygzisllhnq" - "fetch_url https://maxmcd.com/" = "y6yjtwbeq54vudyqh573msqquk67lbd6" - "fetch_url https://ziglang.org/builds/zig-linux-x86_64-0.9.0-dev.946+6237dc0ab.tar.xz" = "3yhnkjln5ctjemvbvyf7uiff4523lway" +"basic_fetch_url https://brmbl.s3.amazonaws.com/busybox-x86_64.tar.gz" = "uw5ichj6dhcccmcts6p7jq6etzlh5baf" +"basic_fetch_url https://brmbl.s3.amazonaws.com/url_fetcher.tar.gz" = "p2vbvabkdqckjlm43rf7bfccdseizych" +"fetch_git https://github.com/maxmcd/bramble.git@v0" = "ebzl4qxuufyxuejtyujh7mdul4spjcgk" +"fetch_url http://s.minos.io/archive/bifrost/x86_64/git-2.10.2-1.tar.gz" = "omtz2dd5irsgvbg7eeksovceutw5abhb" +"fetch_url http://tarballs.nixos.org/stdenv-linux/x86_64/c5aabb0d603e2c1ea05f5a93b3be82437f5ebf31/bootstrap-tools.tar.xz" = "cspu7ndkuuq2f7t5vvgsmvdt3xnoypqb" +"fetch_url https://brmbl.s3.amazonaws.com/busybox-x86_64.tar.gz" = "uw5ichj6dhcccmcts6p7jq6etzlh5baf" +"fetch_url https://brmbl.s3.amazonaws.com/ca-certificates.crt" = "qucryjlyakz2x2asktkm6dtzpx4qz5rj" +"fetch_url https://brmbl.s3.amazonaws.com/file-links.tar.gz" = "rdk4ljksk75wjx7fflcv5vyc7a4dgbbt" +"fetch_url https://github.com/NixOS/patchelf/releases/download/0.13/patchelf-0.13.tar.bz2" = "qffzn3vcrs5tu4lb5vtm6eaplwfjgtth" +"fetch_url https://github.com/denoland/deno/releases/download/v1.14.0/deno-x86_64-unknown-linux-gnu.zip" = "k5fftqwt46izovwwttzh5ebmjiccxuhz" +"fetch_url https://golang.org/dl/go1.17.2.linux-amd64.tar.gz" = "fm3ujt23vqou74mr5fz5dyygzisllhnq" +"fetch_url https://maxmcd.com/" = "y6yjtwbeq54vudyqh573msqquk67lbd6" +"fetch_url https://ziglang.org/builds/zig-linux-x86_64-0.9.0-dev.946+6237dc0ab.tar.xz" = "3yhnkjln5ctjemvbvyf7uiff4523lway" diff --git a/bramble.toml b/bramble.toml index 279a399a..6b894c3e 100644 --- a/bramble.toml +++ b/bramble.toml @@ -3,4 +3,5 @@ name = "github.com/maxmcd/bramble" version = "0.0.3" [dependencies] +"github.com/brmbl/std" = "0.0.1" "github.com/maxmcd/busybox" = "0.0.2" diff --git a/cmd/replacer/main.go b/cmd/replacer/main.go index 7126f34f..00df18cf 100644 --- a/cmd/replacer/main.go +++ b/cmd/replacer/main.go @@ -6,8 +6,8 @@ import ( "io/ioutil" "os" - "github.com/maxmcd/bramble/pkg/reptar" "github.com/maxmcd/bramble/pkg/textreplace" + "github.com/maxmcd/reptar" "github.com/mholt/archiver/v3" ) @@ -28,7 +28,7 @@ func run() (err error) { if err != nil { return err } - if err := reptar.Reptar(location, f); err != nil { + if err := reptar.Archive(location, f); err != nil { return err } f.Seek(0, 0) diff --git a/cmd/reptar/main.go b/cmd/reptar/main.go deleted file mode 100644 index 722ab232..00000000 --- a/cmd/reptar/main.go +++ /dev/null @@ -1,43 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "io" - "os" - "strings" - - "github.com/maxmcd/bramble/pkg/reptar" -) - -func main() { - if err := run(os.Args); err != nil { - fmt.Println(err) - os.Exit(1) - } -} - -func run(args []string) error { - if len(args) < 3 { - return errors.New("reptar is run like: reptar outputfile.tar.gz ./files-to-package") - } - - var fn func(a string, b io.Writer) error - - switch { - case strings.HasSuffix(args[1], ".tar"): - fn = reptar.Reptar - case strings.HasSuffix(args[1], ".tar.gz"): - fn = reptar.GzipReptar - default: - return errors.New("archive name must end in .tar or .tar.gz") - } - - f, err := os.Create(args[1]) - if err != nil { - return err - } - defer f.Close() - - return fn(args[2], f) -} diff --git a/default.bramble b/default.bramble deleted file mode 100644 index ac42a690..00000000 --- a/default.bramble +++ /dev/null @@ -1,21 +0,0 @@ -load("github.com/maxmcd/bramble/tests/simple/simple") -load(nix_seed="github.com/maxmcd/bramble/lib/nix-seed") -load("github.com/maxmcd/bramble/lib") -load("github.com/maxmcd/bramble/tests/nested-sources/another-folder/nested") - - -def print_simple(): - return run(simple.simple(), "simple", hidden_paths=["/"]) - - -def bash(): - return run(nix_seed.stdenv(), "bash", read_only_paths=["./"]) - - -def all(): - return [ - lib.busybox(), - lib.git_fetcher(), - nested.nested(), - simple.simple(), - ] diff --git a/go.mod b/go.mod index eace4f96..1c46e202 100644 --- a/go.mod +++ b/go.mod @@ -10,14 +10,16 @@ require ( github.com/charmbracelet/lipgloss v0.4.0 github.com/containerd/console v1.0.3 github.com/julienschmidt/httprouter v1.3.0 + github.com/klauspost/pgzip v1.2.5 github.com/maxmcd/dag v0.0.0-20210909010249-5757e2034a95 + github.com/maxmcd/reptar v0.0.0-20220507012129-b7cb8d03dbe9 github.com/mholt/archiver/v3 v3.5.0 github.com/minio/sha256-simd v1.0.0 github.com/mitchellh/go-wordwrap v1.0.1 github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 github.com/opencontainers/runc v1.0.3 github.com/pkg/errors v0.9.1 - github.com/rhnvrm/simples3 v0.7.0 + github.com/rlmcpherson/s3gof3r v0.5.1-0.20170210004045-864ae0bf7cf2 github.com/stretchr/testify v1.7.0 github.com/urfave/cli/v2 v2.3.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.25.0 @@ -28,6 +30,7 @@ require ( go.starlark.net v0.0.0-20210901212718-87f333178d59 go.uber.org/zap v1.19.1 golang.org/x/mod v0.4.2 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac gotest.tools/v3 v3.0.3 // indirect ) diff --git a/go.sum b/go.sum index 376f999d..4fde8479 100644 --- a/go.sum +++ b/go.sum @@ -89,8 +89,9 @@ github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A= github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= @@ -111,6 +112,8 @@ github.com/maxmcd/archiver/v3 v3.3.2-0.20210923004632-06ef4f8f175b h1:Z+y8sx28U7 github.com/maxmcd/archiver/v3 v3.3.2-0.20210923004632-06ef4f8f175b/go.mod h1:2X2ALNbjYXnVYbkESAzRbpja4i0m4VNuFUQT7F69X/I= github.com/maxmcd/dag v0.0.0-20210909010249-5757e2034a95 h1:tBGx3z+FLew3L1BOuklkzm2UEIBcOZCgFa58VcJR9xk= github.com/maxmcd/dag v0.0.0-20210909010249-5757e2034a95/go.mod h1:pYGWUsNzYvkcdAOr2py4zGskjw/wdqSGUUwi8jwcikk= +github.com/maxmcd/reptar v0.0.0-20220507012129-b7cb8d03dbe9 h1:0Pj51hoeko8PuOlkLdNvhLBClYTomOm76gZHCCpxvUY= +github.com/maxmcd/reptar v0.0.0-20220507012129-b7cb8d03dbe9/go.mod h1:GPjLc3cTgkXFhW5v7bq6ejs6ELsj/ukIL4pR2bLW/Tc= github.com/maxmcd/starlark-go v0.0.0-20201021154825-b2f805d0d122 h1:5ntUQ6qQLi3wcHdxEVWud2Q5z11Pj6cQFA6OHbCvbRU= github.com/maxmcd/starlark-go v0.0.0-20201021154825-b2f805d0d122/go.mod h1:f0znQkUKRrkk36XxWbGjMqQM8wGv/xHBVE2qc3B5oFU= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= @@ -148,11 +151,11 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rhnvrm/simples3 v0.7.0 h1:KSEuKw0eGC5vltLW8ChLvjko+aUr0HbGet+bZHdwfMo= -github.com/rhnvrm/simples3 v0.7.0/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rlmcpherson/s3gof3r v0.5.1-0.20170210004045-864ae0bf7cf2 h1:O472FrMcTREIDyg5eBZhWDIQZit6GZ9gE6Ieq+k1Pbw= +github.com/rlmcpherson/s3gof3r v0.5.1-0.20170210004045-864ae0bf7cf2/go.mod h1:s7vv7SMDPInkitQMuZzH615G7yWHdrU2r/Go7Bo71Rs= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/schollz/progressbar/v2 v2.13.2/go.mod h1:6YZjqdthH6SCZKv2rqGryrxPtfmRB/DWZxSMfCXPyD8= @@ -223,6 +226,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -248,6 +252,8 @@ golang.org/x/term v0.0.0-20210422114643-f5beecf764ed h1:Ei4bQjjpYUsS4efOUz+5Nz++ golang.org/x/term v0.0.0-20210422114643-f5beecf764ed/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w= +golang.org/x/time v0.0.0-20220411224347-583f2d630306/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= diff --git a/internal/cacheclient/cache.go b/internal/cacheclient/cache.go deleted file mode 100644 index d38601f6..00000000 --- a/internal/cacheclient/cache.go +++ /dev/null @@ -1,113 +0,0 @@ -package cacheclient - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "os" - "strings" - - "github.com/maxmcd/bramble/internal/store" - "github.com/maxmcd/bramble/pkg/chunkedarchive" - "github.com/maxmcd/bramble/pkg/httpx" - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" -) - -type cacheClient interface { - PostChunk(context.Context, io.Reader) (string, error) - PostDerivation(context.Context, store.Derivation) (string, error) - PostOutput(context.Context, store.OutputRequestBody) error -} - -type Client struct { - host string - client *http.Client -} - -var _ cacheClient = new(Client) - -func New(host string) *Client { - return &Client{ - host: host, - client: &http.Client{ - Transport: otelhttp.NewTransport(http.DefaultTransport), - }, - } -} - -func (cc *Client) request(ctx context.Context, method, path, contentType string, body io.Reader, resp interface{}) (err error) { - url := fmt.Sprintf("%s/%s", - strings.TrimSuffix(cc.host, "/"), - strings.TrimPrefix(path, "/"), - ) - return httpx.Request(ctx, cc.client, method, url, contentType, body, resp) -} - -func (cc *Client) PostDerivation(ctx context.Context, drv store.Derivation) (filename string, err error) { - return filename, cc.request(ctx, - http.MethodPost, - "/derivation", - "application/json", - bytes.NewBuffer(drv.JSON()), - &filename) -} - -func (cc *Client) PostOutput(ctx context.Context, req store.OutputRequestBody) (err error) { - b, err := json.Marshal(req) - if err != nil { - return err - } - return cc.request(ctx, - http.MethodPost, - "/output", - "application/json", - bytes.NewBuffer(b), - nil) -} - -func (cc *Client) PostChunk(ctx context.Context, chunk io.Reader) (hash string, err error) { - return hash, cc.request(ctx, - http.MethodPost, - "/chunk", - "application/octet-stream", - chunk, - &hash) -} - -func (cc *Client) GetDerivation(ctx context.Context, filename string) (drv store.Derivation, exists bool, err error) { - err = cc.request(ctx, - http.MethodGet, - "/derivation/"+filename, - "", - nil, - drv) - if err == os.ErrNotExist { - return drv, false, nil - } - return drv, err == nil, err -} - -func (cc *Client) GetOutput(ctx context.Context, hash string) (output []chunkedarchive.TOCEntry, exists bool, err error) { - err = cc.request(ctx, - http.MethodGet, - "/output/"+hash, - "", - nil, - &output) - if err == os.ErrNotExist { - return nil, false, nil - } - return output, err == nil, err -} - -func (cc *Client) GetChunk(ctx context.Context, hash string, chunk io.Writer) (err error) { - return cc.request(ctx, - http.MethodGet, - "/chunk/"+hash, - "", - nil, - chunk) -} diff --git a/internal/command/bramble.go b/internal/command/bramble.go index 461fe0b7..b3d36935 100644 --- a/internal/command/bramble.go +++ b/internal/command/bramble.go @@ -25,6 +25,7 @@ func newBramble(wd string, bramblePath string) (b bramble, err error) { dependency.NewManager( filepath.Join(b.store.BramblePath, "var/dependencies"), "https://store.bramble.run", + nil, ), ) return b, nil diff --git a/internal/command/build.go b/internal/command/build.go index ef64e2a6..ce95fc6e 100644 --- a/internal/command/build.go +++ b/internal/command/build.go @@ -87,7 +87,8 @@ func (b bramble) runBuild(ctx context.Context, output project.ExecModuleOutput, err = output.WalkAndPatch(8, func(dep project.Dependency, drv project.Derivation) (addGraph *project.ExecModuleOutput, buildOutputs []project.BuildOutput, err error) { select { case <-ctx.Done(): - return + fmt.Println("context cancelled") + return nil, nil, context.Canceled default: } dependencies := []store.DerivationOutput{} @@ -107,7 +108,7 @@ func (b bramble) runBuild(ctx context.Context, output project.ExecModuleOutput, derivationDataLock.Unlock() source, err := b.store.StoreLocalSources(ctx, store.SourceFiles{ - ProjectLocation: b.project.Location(), + ProjectLocation: drv.Sources.ProjectLocation, Location: drv.Sources.Location, Files: drv.Sources.Files, }) // TODO: delete this if the build fails? diff --git a/internal/command/cli.go b/internal/command/cli.go index 7e89938d..8bc8188f 100644 --- a/internal/command/cli.go +++ b/internal/command/cli.go @@ -14,8 +14,11 @@ import ( "syscall" "time" + _ "net/http/pprof" + "github.com/maxmcd/bramble/internal/dependency" "github.com/maxmcd/bramble/internal/logger" + "github.com/maxmcd/bramble/internal/netcache" "github.com/maxmcd/bramble/internal/project" "github.com/maxmcd/bramble/internal/store" "github.com/maxmcd/bramble/internal/tracing" @@ -24,7 +27,6 @@ import ( "github.com/maxmcd/bramble/pkg/starutil" "github.com/mitchellh/go-wordwrap" "github.com/pkg/errors" - "github.com/rhnvrm/simples3" cli "github.com/urfave/cli/v2" "go.opentelemetry.io/otel/trace" ) @@ -352,7 +354,7 @@ their public functions with documentation. If an immediate subdirectory has a }, { Name: "publish", - UsageText: `bramble publish package [reference]`, + UsageText: `bramble publish `, Flags: []cli.Flag{ &cli.StringFlag{ Name: "url", @@ -378,57 +380,26 @@ their public functions with documentation. If an immediate subdirectory has a if len(args) > 2 { return errors.New("bramble publish takes at most two arguments") } - module := args[0] - reference := "" - if len(args) == 2 { - reference = args[1] - } + pkg := args[0] - if c.Bool("local") { - // TODO: add build cache handler to this server - s, err := store.NewStore("") - if err != nil { - return err - } - builder := dependency.Builder(filepath.Join(s.BramblePath, "var/dependencies"), - newBuilder(s), - dependency.DownloadGithubRepo, - ) - builtDerivations, err := builder(&dependency.Job{ - Package: module, - Reference: reference, - }) - if err != nil { - return err - } - if c.Bool("upload") { - var drvs []store.Derivation - for _, drvFilename := range builtDerivations { - drv, _, err := s.LoadDerivation(drvFilename) - if err != nil { - return errors.Wrap(err, "error loading derivation from store") - } - drvs = append(drvs, drv) - } - // TODO: replace with something generally usable - s3 := simples3.New("", - os.Getenv("DIGITALOCEAN_SPACES_ACCESS_ID"), - os.Getenv("DIGITALOCEAN_SPACES_SECRET_KEY")) - s3.SetEndpoint("nyc3.digitaloceanspaces.com") - cc := store.NewS3CacheClient(s3) - fmt.Printf("Uploading %d derivations\n", len(drvs)) - if err := s.UploadDerivationsToCache(c.Context, drvs, cc); err != nil { - return err - } - } - return nil + client, err := netcache.NewS3Cache(netcache.S3CacheOptions{ + AccessKeyID: os.Getenv("ACCESS_KEY_ID"), + SecretAccessKey: os.Getenv("SECRET_ACCESS_KEY"), + S3EndpointPrefix: "https://nyc3.digitaloceanspaces.com", + CDNEndpointPrefix: "https://store.bramble.run", + }) + if err != nil { + return errors.Wrap(err, "couldn't initialize cache client") } - url := "https://store.bramble.run" - if u := c.String("url"); u != "" { - url = u - } - return dependency.PostJob(c.Context, url, module, reference) + return publish(c.Context, publishOptions{ + pkg: pkg, + local: c.Bool("local"), + upload: c.Bool("upload"), + url: c.String("url"), + }, + dependency.DownloadGithubRepo, + client) }, }, { @@ -446,8 +417,14 @@ their public functions with documentation. If an immediate subdirectory has a if err != nil { return err } - parts := strings.Split(c.Args().First(), "@") - return b.project.AddDependency(types.Package{Version: parts[1], Name: parts[0]}) + cut := func(s, sep string) (before, after string, ok bool) { + if i := strings.Index(s, sep); i >= 0 { + return s[:i], s[i+len(sep):], true + } + return s, "", false + } + name, version, _ := cut(c.Args().First(), "@") + return b.project.AddDependency(c.Context, types.Package{Name: name, Version: version}) }, }, { @@ -543,6 +520,9 @@ func RunCLI() { panic("give me the stack") }() sandbox.Entrypoint() + + go func() { _ = http.ListenAndServe(":6060", nil) }() + defer tracing.Stop() // Patch cli lib to remove bool default diff --git a/internal/command/integration_test.go b/internal/command/integration_test.go index 06fc22b4..fd14e466 100644 --- a/internal/command/integration_test.go +++ b/internal/command/integration_test.go @@ -4,15 +4,16 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http/httptest" "os" "path/filepath" "strings" "testing" - "github.com/maxmcd/bramble/internal/cacheclient" "github.com/maxmcd/bramble/internal/config" "github.com/maxmcd/bramble/internal/dependency" + "github.com/maxmcd/bramble/internal/netcache" "github.com/maxmcd/bramble/internal/store" "github.com/maxmcd/bramble/internal/tracing" "github.com/maxmcd/bramble/pkg/fxt" @@ -85,17 +86,17 @@ func TestRun(t *testing.T) { for _, tt := range []test{ { name: "simple", - args: []string{"../../:print_simple"}, + args: []string{"../../tests:print_simple"}, checks: []check{noError()}, }, { name: "simple explicit", - args: []string{"../../:print_simple", "simple"}, + args: []string{"../../tests:print_simple", "echo", "simple"}, checks: []check{noError()}, }, { name: "sim", - args: []string{"../../:print_simple", "sim"}, + args: []string{"../../tests:print_simple", "foo"}, checks: []check{ errContains("executable file not found"), exitCodeIs(1), @@ -103,21 +104,21 @@ func TestRun(t *testing.T) { }, { name: "exit code", - args: []string{"../../:bash", "bash", "-c", "exit 2"}, + args: []string{"../../tests:ash", "ash", "-c", "exit 2"}, checks: []check{ exitCodeIs(2), }, }, { name: "write to readonly system", - args: []string{"../../:bash", "bash", "-c", "touch foo"}, + args: []string{"../../tests:ash", "ash", "-c", "touch foo"}, checks: []check{ exitCodeIs(1), }, }, { name: "weird exit code", - args: []string{"../../:bash", "bash", "-c", "exit 56"}, + args: []string{"../../tests:ash", "ash", "-c", "exit 56"}, checks: []check{ exitCodeIs(56), }, @@ -168,11 +169,15 @@ func TestBuildAllFunction(t *testing.T) { initIntegrationTest(t) app := cliApp(".") - if err := app.Run([]string{"bramble", "build", "github.com/maxmcd/bramble:all"}); err != nil { + if err := app.Run([]string{"bramble", "build", "github.com/maxmcd/bramble/tests:all"}); err != nil { t.Fatal(err) } } +func TestDepComplex(t *testing.T) { + +} + func TestDep(t *testing.T) { initIntegrationTest(t) @@ -199,6 +204,7 @@ func TestDep(t *testing.T) { name string pkg string files map[string]interface{} + installed map[string]config.Dependency errContains string install []string } @@ -206,25 +212,25 @@ func TestDep(t *testing.T) { {"first", "first", map[string]interface{}{ "./first/bramble.toml": config.Config{Package: config.Package{Name: "first", Version: "0.0.1"}}, "./first/default.bramble": "def first():\n print('print from first')", - }, "", nil}, + }, map[string]config.Dependency{}, "", nil}, {"second syntax err", "second", map[string]interface{}{ "./second/bramble.toml": config.Config{Package: config.Package{Name: "second", Version: "0.0.1"}}, "./second/default.bramble": "def first)", - }, "second/default.bramble:1:10", nil}, + }, map[string]config.Dependency{}, "second/default.bramble:1:10", nil}, {"third with load", "third", map[string]interface{}{ "./third/bramble.toml": config.Config{Package: config.Package{Name: "third", Version: "0.0.1"}}, "./third/default.bramble": "load('first')\ndef third():\n first.first()", - }, "", []string{"first@0.0.1"}}, + }, map[string]config.Dependency{"first": {Version: "0.0.1"}}, "", []string{"first@0.0.1"}}, {"fourth nested", "fourth", map[string]interface{}{ "./fourth/bramble.toml": config.Config{Package: config.Package{Name: "fourth", Version: "0.0.1"}}, "./fourth/default.bramble": "load('third')\ndef fourth():\n third.third()", "./fourth/nested/bramble.toml": config.Config{Package: config.Package{Name: "fourth/nested", Version: "0.0.1"}}, "./fourth/nested/default.bramble": "def nested():\n print('hello nested')", - }, "", []string{"third@0.0.1"}}, + }, map[string]config.Dependency{"third": {Version: "0.0.1"}}, "", []string{"third@0.0.1"}}, {"fifth with nested load", "fifth", map[string]interface{}{ "./fifth/bramble.toml": config.Config{Package: config.Package{Name: "fifth", Version: "0.0.1"}}, "./fifth/default.bramble": "load('fourth/nested')\ndef fifth():\n nested.nested()", - }, "", []string{"fourth/nested@0.0.1"}}, + }, map[string]config.Dependency{"fourth/nested": {Version: "0.0.1"}}, "", []string{"fourth/nested@0.0.1"}}, } { t.Run(tt.name, func(t *testing.T) { for path, file := range tt.files { @@ -240,9 +246,11 @@ func TestDep(t *testing.T) { } } test.ErrContains(t, func() error { + loc := filepath.Join(projectDir, tt.pkg) { - app := cliApp(filepath.Join(projectDir, tt.pkg)) + app := cliApp(loc) for _, toInstall := range tt.install { + fmt.Println(toInstall, "-------------") if err := app.Run([]string{"bramble", "add", toInstall}); err != nil { return err } @@ -250,6 +258,12 @@ func TestDep(t *testing.T) { if err := app.Run([]string{"bramble", "build", "--just-parse", "./..."}); err != nil { return err } + f, err := os.Open(loc + "/bramble.lock") + if err != nil { + fmt.Println(err) + } + io.Copy(os.Stdout, f) + } { app := cliApp(".") @@ -257,6 +271,14 @@ func TestDep(t *testing.T) { return err } } + { + cfg, err := config.ReadConfig(loc + "/bramble.toml") + if err != nil { + t.Fatal(err) + } + cfg.Render(os.Stdout) + require.Equal(t, cfg.Dependencies, tt.installed) + } return nil }(), tt.errContains) }) @@ -273,11 +295,11 @@ func TestStore_CacheServer(t *testing.T) { { test.SetEnv(t, "BRAMBLE_PATH", clientBramblePath) app := cliApp(".") - if err := app.Run([]string{"bramble", "build", "../../lib:busybox"}); err != nil { + if err := app.Run([]string{"bramble", "build", "../../tests/busybox:busybox"}); err != nil { t.Fatal(err) } } - + fmt.Println("build complete") { serverBramblePath := t.TempDir() s, err := store.NewStore(serverBramblePath) @@ -297,10 +319,9 @@ func TestStore_CacheServer(t *testing.T) { } drvs = append(drvs, drv) } - cc := cacheclient.New(server.URL) - // s3 := simples3.New("", "", "") - // s3.SetEndpoint("nyc3.digitaloceanspaces.com") - // cc := store.NewS3CacheClient(s3) + + cc := store.NewCacheClient(netcache.NewStdCache(server.URL)) + // cc := store.NewS3CacheClient("", "", "nyc3.digitaloceanspaces.com") if err := clientStore.UploadDerivationsToCache(ctx, drvs, cc); err != nil { t.Fatal(err) } @@ -321,20 +342,20 @@ func TestModuleCLIParsing(t *testing.T) { {"build ./...", "../../", false}, {"build tests", "../../", true}, {"build github.com/maxmcd/bramble/...", "../../", false}, - {"build ./lib", "../../", false}, + {"build ./tests", "../../", false}, {"build ./internal", "../../", true}, - {"build :all", "../../", true}, - {"build ./:all", "../../", false}, + {"build tests:all", "../../", true}, + {"build ./tests:all", "../../", false}, {"build github.com/maxmcd/bramble/tests/...", "../../", false}, {"build github.com/maxmcd/busybox/...", "../../", true}, // run {"run ./...", "../../", true}, - {"run tests", "../../", true}, + // {"run tests", "../../", true}, // re-add when we can support a remote lookup of "tests" {"run :print_simple", "../../", true}, - {"run ./:print_simple simple", "../../", false}, - {"run ./lib:git git", "../../", false}, + {"run ./tests:print_simple simple", "../../", false}, + {"run ./tests:ash ash", "../../", false}, {"run ./internal:foo foo", "../../", true}, - {"run github.com/maxmcd/bramble:print_simple simple", "../../", false}, + {"run github.com/maxmcd/bramble/tests:print_simple simple", "../../", false}, // {"run github.com/maxmcd/busybox:busybox ash", "../../", false}, // {"run github.com/maxmcd/busybox@0.0.1:busybox ash", "../../", false}, } diff --git a/internal/command/publish.go b/internal/command/publish.go new file mode 100644 index 00000000..1d8a18d4 --- /dev/null +++ b/internal/command/publish.go @@ -0,0 +1,75 @@ +package command + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/maxmcd/bramble/internal/dependency" + "github.com/maxmcd/bramble/internal/netcache" + "github.com/maxmcd/bramble/internal/store" + "github.com/maxmcd/bramble/internal/types" + "github.com/pkg/errors" +) + +type publishOptions struct { + pkg string + upload bool + local bool + url string +} + +func publish(ctx context.Context, opt publishOptions, dgr types.DownloadGithubRepo, cacheClient netcache.Client) error { + cc := store.NewCacheClient(cacheClient) + + // Regular behavior, publish the job to the CDN + if !opt.local { + url := "https://store.bramble.run" + if opt.url != "" { + url = opt.url + } + return dependency.PostJob(ctx, url, opt.pkg, "") + } + fmt.Println("Building package locally") + // Pull down and build a package locally + s, err := store.NewStore("") + if err != nil { + return err + } + builder := dependency.Builder( + filepath.Join(s.BramblePath, "var/dependencies"), + newBuilder(s), + dgr, + ) + + builtDerivations, packages, err := builder(ctx, opt.pkg) + if err != nil { + return err + } + if !opt.upload { + return nil + } + var drvs []store.Derivation + for _, drvFilename := range builtDerivations { + drv, _, err := s.LoadDerivation(drvFilename) + if err != nil { + return errors.Wrap(err, "error loading derivation from store") + } + drvs = append(drvs, drv) + } + fmt.Printf("Uploading %d derivations\n", len(drvs)) + if err := s.UploadDerivationsToCache(ctx, drvs, cc); err != nil { + return err + } + fmt.Println("Uploading packages") + depManager := dependency.NewManager( + filepath.Join(s.BramblePath, "var/dependencies"), "", + cacheClient, + ) + for _, pkg := range packages { + if err := depManager.UploadPackage(ctx, pkg); err != nil { + return err + } + } + return nil +} diff --git a/internal/command/publish_test.go b/internal/command/publish_test.go new file mode 100644 index 00000000..9dddbd17 --- /dev/null +++ b/internal/command/publish_test.go @@ -0,0 +1,61 @@ +package command + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/maxmcd/bramble/internal/netcache/netcachetest" +) + +var ( + lock string = `[URLHashes] +"basic_fetch_url https://brmbl.s3.amazonaws.com/busybox-x86_64.tar.gz" = "uw5ichj6dhcccmcts6p7jq6etzlh5baf" +"basic_fetch_url https://brmbl.s3.amazonaws.com/url_fetcher.tar.gz" = "p2vbvabkdqckjlm43rf7bfccdseizych" +"fetch_url https://brmbl.s3.amazonaws.com/busybox-x86_64.tar.gz" = "uw5ichj6dhcccmcts6p7jq6etzlh5baf"` + toml string = `[package] +name = "github.com/maxmcd/busybox" +version = "0.0.2"` + main string = `def busybox(): + b = derivation( + name="busybox-x86_64.tar.gz", + builder="fetch_url", + env={"url": "https://brmbl.s3.amazonaws.com/busybox-x86_64.tar.gz"}) + + script = """ + set -ex + # cachebust + $busybox_download/busybox-x86_64 mkdir $out/bin + $busybox_download/busybox-x86_64 cp $busybox_download/busybox-x86_64 $out/bin/busybox + cd $out/bin + for command in $(./busybox --list); do + ./busybox ln -s busybox $command + done + """ + + return derivation( + name="busybox", + builder=b.out + "/busybox-x86_64", + args=["sh", "-c", script], + env={"busybox_download": b, "PATH": b.out}, + )` +) + +func TestPublish(t *testing.T) { + cacheClient := netcachetest.StartMinio(t) + + if err := publish(context.Background(), publishOptions{ + pkg: "github.com/maxmcd/busybox", + upload: true, + local: true, + }, func(url, reference string) (location string, err error) { + location = t.TempDir() + _ = os.WriteFile(filepath.Join(location, "bramble.toml"), []byte(toml), 0755) + _ = os.WriteFile(filepath.Join(location, "bramble.lock"), []byte(lock), 0755) + _ = os.WriteFile(filepath.Join(location, "default.bramble"), []byte(main), 0755) + return location, nil + }, cacheClient); err != nil { + t.Fatal(err) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 807a2fe4..ca8cd221 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -135,7 +135,7 @@ func ParseConfig(r io.Reader) (cfg Config, err error) { return cfg, nil } -func ReadConfigs(dir string) (cfg Config, lockFile *LockFile, err error) { +func ReadConfigs(dir string) (cfg Config, lockFile *Lockfile, err error) { { bDotToml := filepath.Join(dir, "bramble.toml") cfg, err = ReadConfig(bDotToml) @@ -150,7 +150,7 @@ func ReadConfigs(dir string) (cfg Config, lockFile *LockFile, err error) { lockFileLocation := filepath.Join(dir, "bramble.lock") if !fileutil.FileExists(lockFileLocation) { // Don't read the lockfile if we don't have one - return cfg, &LockFile{}, err + return cfg, &Lockfile{}, err } f, err := os.Open(lockFileLocation) if err != nil { @@ -162,12 +162,9 @@ func ReadConfigs(dir string) (cfg Config, lockFile *LockFile, err error) { } } -func WriteLockfile(lockFile *LockFile, dir string) (err error) { +func WriteLockfile(lockFile *Lockfile, dir string) (err error) { lockFile.lock.Lock() defer lockFile.lock.Unlock() - if !lockFile.changed { - return nil - } // Get lock on lockfile done, err := getConfigLock(dir) @@ -183,58 +180,92 @@ func WriteLockfile(lockFile *LockFile, dir string) (err error) { } defer func() { _ = f.Close() }() - lf := LockFile{ + lf := Lockfile{ URLHashes: map[string]string{}, } if _, err := toml.DecodeReader(f, &lf); err != nil { return err } - if reflect.DeepEqual(lockFile.URLHashes, lf.URLHashes) { - return nil - } - _ = f.Truncate(0) _, _ = f.Seek(0, 0) - for url, hash := range lockFile.URLHashes { - if v, ok := lf.URLHashes[url]; ok && v != hash { - return errors.Errorf("found existing hash for %q with value %q not %q, not sure how to proceed", url, v, hash) + if !reflect.DeepEqual(lockFile.URLHashes, lf.URLHashes) { + for url, hash := range lockFile.URLHashes { + if v, ok := lf.URLHashes[url]; ok && v != hash { + return errors.Errorf("found existing hash for %q with value %q not %q, not sure how to proceed", url, v, hash) + } + lf.URLHashes[url] = hash } - lf.URLHashes[url] = hash } - - return toml.NewEncoder(f).Encode(&lf) + lf.Dependencies = lockFile.Dependencies + lockFile.Render(f) + return nil } -type LockFile struct { +type Lockfile struct { URLHashes map[string]string - changed bool lock sync.RWMutex + + Dependencies map[string]Dependency } -var _ types.LockfileWriter = new(LockFile) +func (lf *Lockfile) Render(w io.Writer) { + var keys []string + { + fmt.Fprintln(w, "[URLHashes]") + for key := range lf.URLHashes { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + fxt.Fprintfln(w, "%q = %q", key, lf.URLHashes[key]) + } + } + if len(lf.Dependencies) == 0 { + return + } + fmt.Fprintln(w) + { + fmt.Fprintln(w, "[Dependencies]") + var keys []string + for key := range lf.Dependencies { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + dep := lf.Dependencies[key] + fxt.Fprintfln(w, "%q = %q", key, dep.Version) + } + } +} + +var _ types.LockfileWriter = new(Lockfile) -func (l *LockFile) AddEntry(k, v string) error { - l.lock.Lock() - defer l.lock.Unlock() - oldV, found := l.URLHashes[k] +func (lf *Lockfile) AddEntry(k, v string) error { + lf.lock.Lock() + defer lf.lock.Unlock() + oldV, found := lf.URLHashes[k] if found && oldV != v { return errors.Errorf( "Existing lockfile entry found for %q, old hash %q does not equal new has value %q", k, oldV, v) } if !found { - if l.URLHashes == nil { - l.URLHashes = map[string]string{} + if lf.URLHashes == nil { + lf.URLHashes = map[string]string{} } - l.URLHashes[k] = v - l.changed = true + lf.URLHashes[k] = v } return nil } -func (l *LockFile) LookupEntry(k string) (v string, found bool) { - l.lock.RLock() - defer l.lock.RUnlock() - v, found = l.URLHashes[k] +func (lf *Lockfile) LookupEntry(k string) (v string, found bool) { + lf.lock.RLock() + defer lf.lock.RUnlock() + v, found = lf.URLHashes[k] return v, found } + +type ConfigAndLockfile struct { + Lockfile *Lockfile + Config Config +} diff --git a/internal/dependency/client.go b/internal/dependency/client.go new file mode 100644 index 00000000..c1fa624f --- /dev/null +++ b/internal/dependency/client.go @@ -0,0 +1,149 @@ +package dependency + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strings" + + "github.com/maxmcd/bramble/internal/config" + "github.com/maxmcd/bramble/internal/netcache" + "github.com/maxmcd/bramble/internal/types" + "github.com/maxmcd/bramble/pkg/httpx" + "github.com/maxmcd/reptar" + "github.com/pkg/errors" +) + +type dependencyClient struct { + client *http.Client + host string + cacheClient netcache.Client + dependencyDirectory dependencyDirectory +} + +func (dc *dependencyClient) request(ctx context.Context, method, path, contentType string, body io.Reader, resp interface{}) (err error) { + url := fmt.Sprintf("%s/%s", + strings.TrimSuffix(dc.host, "/"), + strings.TrimPrefix(path, "/"), + ) + return httpx.Request(ctx, dc.client, method, url, contentType, body, resp) +} + +func (dc *dependencyClient) postJob(ctx context.Context, job JobRequest) (id string, err error) { + b, err := json.Marshal(job) + if err != nil { + return "", err + } + return id, dc.request(ctx, + http.MethodPost, + "/job", + "application/json", + bytes.NewBuffer(b), + &id) +} + +func (dc *dependencyClient) getJob(ctx context.Context, id string) (job Job, err error) { + return job, dc.request(ctx, + http.MethodGet, + "/job/"+id, + "", + nil, + &job) +} + +func (dc *dependencyClient) getLogs(ctx context.Context, id string, out io.Writer) (err error) { + return dc.request(ctx, + http.MethodGet, + "/job/"+id+"/logs", + "", + nil, + out) +} + +func (dc *dependencyClient) getPackageVersions(ctx context.Context, name string) (vs []string, err error) { + r, err := dc.cacheClient.Get(ctx, "package/versions/"+name) + if err != nil { + return nil, err + } + return vs, json.NewDecoder(r).Decode(&vs) +} + +func possiblePackageVariants(name string) (variants []string) { + parts := strings.Split(name, "/") + for len(parts) > 0 { + n := strings.Join(parts, "/") + parts = parts[:len(parts)-1] + variants = append(variants, n) + } + return +} + +func (dc *dependencyClient) findPackageFromModuleName(ctx context.Context, name string) (n string, vs []string, err error) { + for _, n := range possiblePackageVariants(name) { + vs, err := dc.getPackageVersions(ctx, n) + if err != nil { + if err == os.ErrNotExist { + continue + } + return "", nil, err + } + return n, vs, nil + } + return "", nil, os.ErrNotExist +} + +func (dc *dependencyClient) getPackageSource(ctx context.Context, pkg types.Package, location string) (err error) { + r, err := dc.cacheClient.Get(ctx, "package/source/"+pkg.String()) + if err != nil { + return err + } + return reptar.Unarchive(r, location) +} + +func (dc *dependencyClient) getPackageConfig(ctx context.Context, pkg types.Package) (cfg config.ConfigAndLockfile, err error) { + r, err := dc.cacheClient.Get(ctx, "package/config/"+pkg.String()) + if err != nil { + return config.ConfigAndLockfile{}, nil + } + return cfg, json.NewDecoder(r).Decode(&cfg) +} + +func (dc *dependencyClient) uploadPackage(ctx context.Context, pkg types.Package) (err error) { + location := dc.dependencyDirectory.localPackageLocation(pkg) + { + sourcePath := "package/source/" + pkg.String() + if exists, err := dc.cacheClient.Exists(ctx, sourcePath); err != nil { + return err + } else if exists { + return errors.Errorf("a version of package %s already exists in the registry", pkg) + } + writer, err := dc.cacheClient.Put(ctx, sourcePath) + if err != nil { + return err + } + if err := reptar.Archive(location, writer); err != nil { + return err + } + if err := writer.Close(); err != nil { + return err + } + } + { + cfg, lockfile, err := config.ReadConfigs(location) + if err != nil { + return err + } + writer, err := dc.cacheClient.Put(ctx, "package/config/"+pkg.String()) + if err != nil { + return err + } + if err := json.NewEncoder(writer).Encode(config.ConfigAndLockfile{Config: cfg, Lockfile: lockfile}); err != nil { + return err + } + return writer.Close() + } +} diff --git a/internal/dependency/dependency.go b/internal/dependency/dependency.go index 83d8f9ac..db55af0b 100644 --- a/internal/dependency/dependency.go +++ b/internal/dependency/dependency.go @@ -15,35 +15,44 @@ import ( "time" "github.com/maxmcd/bramble/internal/config" + "github.com/maxmcd/bramble/internal/netcache" "github.com/maxmcd/bramble/internal/types" - "github.com/maxmcd/bramble/pkg/chunkedarchive" "github.com/maxmcd/bramble/pkg/fileutil" "github.com/maxmcd/bramble/pkg/httpx" "github.com/maxmcd/bramble/v/cmd/go/mvs" + "github.com/maxmcd/reptar" "github.com/pkg/errors" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "golang.org/x/mod/semver" ) type Manager struct { - dir dir - - dependencyClient *dependencyClient + dependencyDirectory dependencyDirectory + dependencyClient *dependencyClient } -func NewManager(dependencyDir string, packageHost string) *Manager { +func NewManager(dependencyDir string, packageHost string, cacheClient netcache.Client) *Manager { return &Manager{ - dir: dir(dependencyDir), - dependencyClient: &dependencyClient{host: packageHost, client: &http.Client{}}, + dependencyDirectory: dependencyDirectory(dependencyDir), + dependencyClient: &dependencyClient{ + host: packageHost, + cacheClient: cacheClient, + dependencyDirectory: dependencyDirectory(dependencyDir), + client: &http.Client{ + // For tracing + Transport: otelhttp.NewTransport(http.DefaultTransport), + }, + }, } } -type dir string +type dependencyDirectory string -func (dd dir) join(v ...string) string { +func (dd dependencyDirectory) join(v ...string) string { return filepath.Join(append([]string{string(dd)}, v...)...) } -func (dd dir) localPackageVersions(pkg string) ([]string, error) { +func (dd dependencyDirectory) localPackageVersions(pkg string) ([]string, error) { path := dd.join("src", pkg) searchGlob := fmt.Sprintf("%s*", path) matches, err := filepath.Glob(searchGlob) @@ -56,48 +65,27 @@ func (dd dir) localPackageVersions(pkg string) ([]string, error) { return matches, nil } -func (dd dir) localPackageLocation(pkg types.Package) (path string) { +func (dd dependencyDirectory) localPackageLocation(pkg types.Package) (path string) { return dd.join("src", pkg.String()) } +func (dm *Manager) UploadPackage(ctx context.Context, pkg types.Package) (err error) { + return dm.dependencyClient.uploadPackage(ctx, pkg) +} + func (dm *Manager) PackagePathOrDownload(ctx context.Context, pkg types.Package) (path string, err error) { - path = dm.dir.localPackageLocation(pkg) + path = dm.dependencyDirectory.localPackageLocation(pkg) if fileutil.DirExists(path) { return path, nil } - body, err := dm.dependencyClient.getPackageSource(ctx, pkg) - if err != nil { - if err == os.ErrNotExist { - return "", errors.Errorf("Package %q doesn't exist in the remote cache, do you need to publish it?", pkg) - } - return "", err - } - defer body.Close() if err := os.MkdirAll(path, 0755); err != nil { return "", err } - // Copy body to file, we can stream the unarchive if we figure out how to - // get the final size earlier and/or seek over http. - var name string - { - f, err := os.CreateTemp("", "") - if err != nil { - return "", err - } - name = f.Name() - _, _ = io.Copy(f, body) - if err := f.Close(); err != nil { - return "", err - } - } - if err := chunkedarchive.FileUnarchive(name, path); err != nil { - return "", errors.Wrap(err, "error unwrapping chunked archive") + if err := dm.dependencyClient.getPackageSource(ctx, pkg, path); err != nil { + _ = os.RemoveAll(path) + return "", err } - return path, os.RemoveAll(name) -} - -func (dm *Manager) FindPackage(name string) { - + return path, nil } func mvsVersionFromPackage(p types.Package) mvs.Version { @@ -123,36 +111,36 @@ func configVersions(cfg config.Config) (pkgs []types.Package) { } func (dm *Manager) existsLocally(pkg types.Package) bool { - return fileutil.PathExists(dm.dir.localPackageLocation(pkg)) + return fileutil.PathExists(dm.dependencyDirectory.localPackageLocation(pkg)) } func (dm *Manager) localPackageDependencies(pkg types.Package) (vs []types.Package, err error) { - cfg, err := config.ReadConfig(filepath.Join(dm.dir.localPackageLocation(pkg), "bramble.toml")) + cfg, err := config.ReadConfig(filepath.Join(dm.dependencyDirectory.localPackageLocation(pkg), "bramble.toml")) if err != nil { return nil, err } return configVersions(cfg), nil } -func (dm *Manager) CalculateConfigBuildlist(cfg config.Config) (config.Config, error) { +func (dm *Manager) CalculateConfigBuildlist(ctx context.Context, cfg config.Config) (map[string]config.Dependency, error) { versions, err := mvs.BuildList( mvsVersionFromPackage(types.Package{Name: cfg.Package.Name, Version: cfg.Package.Version}), - dm.reqs(cfg), + dm.reqs(ctx, cfg), ) if err != nil { - return config.Config{}, err + return nil, err } - cfg.Dependencies = make(map[string]config.Dependency) + buildList := make(map[string]config.Dependency) for _, version := range versions { v := packageFromMVSVersion(version) if v.Name == cfg.Package.Name { continue } // Support path overrides - cfg.Dependencies[v.Name] = config.Dependency{Version: v.Version} + buildList[v.Name] = config.Dependency{Version: v.Version} } - return cfg, nil + return buildList, nil } func (dm *Manager) remotePackageDependencies(ctx context.Context, m types.Package) (vs []types.Package, err error) { @@ -160,14 +148,14 @@ func (dm *Manager) remotePackageDependencies(ctx context.Context, m types.Packag if err != nil { return nil, err } - return configVersions(cfg), nil + return configVersions(cfg.Config), nil } func PostJob(ctx context.Context, url, pkg, reference string) (err error) { jr := JobRequest{Package: pkg, Reference: reference} dc := &dependencyClient{client: &http.Client{}, host: url} fmt.Println("Sending build to build server") - id, err := dc.postJob(context.Background(), jr) + id, err := dc.postJob(ctx, jr) if err != nil { return err } @@ -179,12 +167,12 @@ func PostJob(ctx context.Context, url, pkg, reference string) (err error) { return context.Canceled default: } - job, err := dc.getJob(context.Background(), id) + job, err := dc.getJob(ctx, id) if err != nil { return err } if job.Error != "" { - _ = dc.getLogs(context.Background(), id, os.Stdout) + _ = dc.getLogs(ctx, id, os.Stdout) return errors.Wrap(errors.New(job.ErrWithStack), "got error posting job") } if !job.End.IsZero() { @@ -196,17 +184,20 @@ func PostJob(ctx context.Context, url, pkg, reference string) (err error) { return nil } -func (dm *Manager) reqs(cfg config.Config) mvs.Reqs { - return dependencyManagerReqs{deps: dm, cfg: cfg} +func (dm *Manager) reqs(ctx context.Context, cfg config.Config) mvs.Reqs { + return dependencyManagerReqs{deps: dm, cfg: cfg, ctx: ctx} } type dependencyManagerReqs struct { deps *Manager cfg config.Config + ctx context.Context } var _ mvs.Reqs = dependencyManagerReqs{} +// Required returns the module versions explicitly required by m itself. +// The caller must not modify the returned list. func (r dependencyManagerReqs) Required(m mvs.Version) (versions []mvs.Version, err error) { p := packageFromMVSVersion(m) var pkgs []types.Package @@ -222,7 +213,7 @@ func (r dependencyManagerReqs) Required(m mvs.Version) (versions []mvs.Version, default: // TODO: tracing // TODO: cache this result locally? - pkgs, err = r.deps.remotePackageDependencies(context.Background(), p) + pkgs, err = r.deps.remotePackageDependencies(r.ctx, p) } if err != nil { return nil, errors.Wrap(err, "error fetching package") @@ -233,103 +224,38 @@ func (r dependencyManagerReqs) Required(m mvs.Version) (versions []mvs.Version, return } +// Max returns the maximum of v1 and v2 (it returns either v1 or v2). +// +// For all versions v, Max(v, "none") must be v, and for the target passed as +// the first argument to MVS functions, Max(target, v) must be target. +// +// Note that v1 < v2 can be written Max(v1, v2) != v1 and similarly v1 <= v2 can +// be written Max(v1, v2) == v2. func (r dependencyManagerReqs) Max(v1, v2 string) (o string) { - switch semver.Compare("v0."+v1, "v0."+v2) { - case -1: + if semver.Compare("v0."+v1, "v0."+v2) == -1 { return v2 - default: - return v1 } + return v1 } +// Upgrade returns the upgraded version of m, for use during an UpgradeAll +// operation. If m should be kept as is, Upgrade returns m. If m is not yet used +// in the build, then m.Version will be "none". More typically, m.Version will +// be the version required by some other module in the build. +// +// If no module version is available for the given path, Upgrade returns a +// non-nil error. func (r dependencyManagerReqs) Upgrade(m mvs.Version) (v mvs.Version, err error) { - panic("") - return + panic("unimplemented") } +// Previous returns the version of m.Path immediately prior to m.Version, or +// "none" if no such version is known. func (r dependencyManagerReqs) Previous(m mvs.Version) (v mvs.Version, err error) { - panic("") - return -} - -type dependencyClient struct { - client *http.Client - host string -} - -func (dc *dependencyClient) request(ctx context.Context, method, path, contentType string, body io.Reader, resp interface{}) (err error) { - url := fmt.Sprintf("%s/%s", - strings.TrimSuffix(dc.host, "/"), - strings.TrimPrefix(path, "/"), - ) - return httpx.Request(ctx, dc.client, method, url, contentType, body, resp) -} - -func (dc *dependencyClient) postJob(ctx context.Context, job JobRequest) (id string, err error) { - b, err := json.Marshal(job) - if err != nil { - return "", err - } - return id, dc.request(ctx, - http.MethodPost, - "/job", - "application/json", - bytes.NewBuffer(b), - &id) -} - -func (dc *dependencyClient) getJob(ctx context.Context, id string) (job Job, err error) { - return job, dc.request(ctx, - http.MethodGet, - "/job/"+id, - "", - nil, - &job) -} - -func (dc *dependencyClient) getLogs(ctx context.Context, id string, out io.Writer) (err error) { - return dc.request(ctx, - http.MethodGet, - "/job/"+id+"/logs", - "", - nil, - out) -} - -func (dc *dependencyClient) getPackageVersions(ctx context.Context, name string) (vs []string, err error) { - return vs, dc.request(ctx, - http.MethodGet, - "/package/"+name, - "", - nil, - &vs) -} - -func possiblePackageVariants(name string) (variants []string) { - parts := strings.Split(name, "/") - for len(parts) > 0 { - n := strings.Join(parts, "/") - parts = parts[:len(parts)-1] - variants = append(variants, n) - } - return + panic("unimplemented") } -func (dc *dependencyClient) findPackageFromModuleName(ctx context.Context, name string) (n string, vs []string, err error) { - for _, n := range possiblePackageVariants(name) { - vs, err := dc.getPackageVersions(ctx, n) - if err != nil { - if err == os.ErrNotExist { - continue - } - return "", nil, err - } - return n, vs, nil - } - return "", nil, os.ErrNotExist -} - -func (dd dir) findPackageFromModuleName(module string) (name string, vs []string, err error) { +func (dd dependencyDirectory) findPackageFromModuleName(module string) (name string, vs []string, err error) { for _, n := range possiblePackageVariants(module) { vs, err := dd.localPackageVersions(n) if err != nil { @@ -342,13 +268,30 @@ func (dd dir) findPackageFromModuleName(module string) (name string, vs []string } return "", nil, os.ErrNotExist } -func (dm *Manager) FindPackageFromModuleName(ctx context.Context, module string) (name string, vs []string, err error) { + +// FindPackageFromModuleName will search locally for a module, and if it's not +// found it will search remotely for a module. Passing version is optional, but +// if passed it will force a remote search if that version is not found locally +func (dm *Manager) FindPackageFromModuleName(ctx context.Context, module string, version string) (name string, vs []string, err error) { // Prefer local - name, vs, err = dm.dir.findPackageFromModuleName(module) + name, vs, err = dm.dependencyDirectory.findPackageFromModuleName(module) if err != nil && err != os.ErrNotExist { return "", nil, err } - if err == os.ErrNotExist { + matchingVersion := func() bool { + for _, v := range vs { + if v == version { + return true + } + } + return false + } + versionNotFound := false + if version != "" { + versionNotFound = !matchingVersion() + } + + if err == os.ErrNotExist || versionNotFound { name, vs, err = dm.dependencyClient.findPackageFromModuleName(ctx, module) } if err == os.ErrNotExist { @@ -357,36 +300,6 @@ func (dm *Manager) FindPackageFromModuleName(ctx context.Context, module string) return name, vs, err } -func (dc *dependencyClient) getPackageSource(ctx context.Context, pkg types.Package) (body io.ReadCloser, err error) { - if err := dc.request(ctx, - http.MethodGet, - "/package/source/"+pkg.String(), - "", - nil, &body); err != nil { - if err == os.ErrNotExist { - err = errors.Errorf("request to server could not find package %s", pkg) - } - return nil, err - } - return body, nil -} - -func (dc *dependencyClient) getPackageConfig(ctx context.Context, pkg types.Package) (cfg config.Config, err error) { - var buf bytes.Buffer - var w io.Writer = &buf - if err := dc.request(ctx, - http.MethodGet, - "/package/config/"+pkg.String(), - "", - nil, w); err != nil { - if err == os.ErrNotExist { - err = errors.Errorf("request to server could not find package %s", pkg) - } - return cfg, err - } - return config.ParseConfig(&buf) -} - func addDependencyMetadata(dependencyDir, pkg, version, src string, mapping map[string]map[string][]string) (err error) { srcs := filepath.Join(dependencyDir, "src") fileDest := filepath.Join(srcs, pkg+"@"+version) @@ -398,9 +311,9 @@ func addDependencyMetadata(dependencyDir, pkg, version, src string, mapping map[ // If the metadata is here we already have a record of the output mapping. // If we checked the src directory it might just be there as a dependency of // another nomad project - fmt.Println(metadataDest) if fileutil.PathExists(metadataDest) { - return errors.Errorf("version %s of package %q is already present on this server", version, pkg) + fmt.Println(pkg, version, "already exists locally, skipping writing to store") + return nil } if err := os.MkdirAll(fileDest, 0755); err != nil { @@ -429,7 +342,7 @@ func addDependencyMetadata(dependencyDir, pkg, version, src string, mapping map[ } func serverHandler(dependencyDir string, newBuilder types.NewBuilder, downloadGithubRepo func(url string, reference string) (location string, err error)) http.Handler { - dependencyDirectory := dir(dependencyDir) + dependencyDirectory := dependencyDirectory(dependencyDir) router := httpx.New() router.GET("/job/:id", func(c httpx.Context) error { @@ -453,27 +366,12 @@ func serverHandler(dependencyDir string, newBuilder types.NewBuilder, downloadGi // Run job go func() { - _, err := buildJob(job, dependencyDir, newBuilder, downloadGithubRepo) + _, _, err := buildJob(c.Request.Context(), job.Package, dependencyDir, newBuilder, downloadGithubRepo) jq.End(job.ID, err) }() return nil }) - // router.GET("/package/outputs/:platform/:name/:version", func(c httpx.Context) error { - // name := c.Params.ByName("name") - // path := filepath.Join(bramblePath, "var", platform, name) - // searchGlob := fmt.Sprintf("%s*", path) - // matches, err := filepath.Glob(searchGlob) - // if err != nil { - // return err - // } - // for i, match := range matches { - // matches[i] = strings.TrimPrefix(match, path+"@") - // } - // return json.NewEncoder(c.ResponseWriter).Encode(matches) - // }) - // This is hard because :name can have slashes... - // router.GET("/package/platform/:platform/:name_version/", func(c httpx.Context) error { return nil }) router.GET("/package/versions/*name", func(c httpx.Context) error { // TODO: Return all matches for cached derivation outputs that we have // as well? @@ -490,32 +388,43 @@ func serverHandler(dependencyDir string, newBuilder types.NewBuilder, downloadGi if !fileutil.DirExists(path) { return httpx.ErrNotFound(errors.New("can't find package")) } - return chunkedarchive.StreamArchive(c.ResponseWriter, path) + return reptar.Archive(path, c.ResponseWriter) }) router.GET("/package/config/*name_version", func(c httpx.Context) error { name := c.Params.ByName("name_version") - path := filepath.Join(dependencyDir, "src", name, "bramble.toml") - if !fileutil.FileExists(path) { + path := filepath.Join(dependencyDir, "src", name) + if !fileutil.DirExists(path) { return httpx.ErrNotFound(errors.New("can't find package")) } - f, err := os.Open(path) + cfg, lockfile, err := config.ReadConfigs(path) if err != nil { return err } - defer f.Close() - if _, err := io.Copy(c.ResponseWriter, f); err != nil { - return err - } - return nil + return json.NewEncoder(c.ResponseWriter).Encode( + config.ConfigAndLockfile{Config: cfg, Lockfile: lockfile}) }) return router } -func buildJob(job *Job, dependencyDir string, newBuilder types.NewBuilder, downloadGithubRepo func(url string, reference string) (location string, err error)) (builtDerivations []string, err error) { - loc, err := downloadGithubRepo(job.Package, job.Reference) +// TODO: this is begging to be something other than a heavily overloaded +// function +func buildJob( + + ctx context.Context, + repo string, + dependencyDir string, + newBuilder types.NewBuilder, + downloadGithubRepo func(url string, reference string) (location string, err error)) ( + + builtDerivations []string, + pkgs []types.Package, + err error, + +) { + loc, err := downloadGithubRepo(repo, "") if err != nil { - return nil, errors.Wrap(err, "error downloading git repo") + return nil, nil, errors.Wrap(err, "error downloading git repo") } builder, err := newBuilder(loc) if err != nil { @@ -527,20 +436,21 @@ func buildJob(job *Job, dependencyDir string, newBuilder types.NewBuilder, downl if err != nil { panic(loc + " - " + path) } - expectedPackageName := strings.TrimSuffix(job.Package+"/"+strings.Trim(strings.TrimPrefix(rel, "."), "/"), "/") + expectedPackageName := strings.TrimSuffix(repo+"/"+strings.Trim(strings.TrimPrefix(rel, "."), "/"), "/") if expectedPackageName != pkg.Name { - return nil, errors.Errorf("package name %q does not match the location the project was fetched from: %q", + return nil, nil, errors.Errorf("package name %q does not match the location the project was fetched from: %q", pkg.Name, expectedPackageName) } + pkgs = append(pkgs, pkg) } toRun := []func() error{} // Build each package in the repository for path, pkg := range packages { fmt.Println("Building package", path, pkg) - resp, err := builder.Build(context.Background(), path, []string{"./..."}, types.BuildOptions{Check: true}) + resp, err := builder.Build(ctx, path, []string{"./..."}, types.BuildOptions{Check: true}) if err != nil { - return nil, err + return nil, nil, err } for _, drvFilename := range resp.FinalHashMapping { builtDerivations = append(builtDerivations, drvFilename) @@ -560,19 +470,19 @@ func buildJob(job *Job, dependencyDir string, newBuilder types.NewBuilder, downl // before writing packages to the store for _, tr := range toRun { if err = tr(); err != nil { - return nil, err + return nil, nil, err } } - return builtDerivations, nil + return builtDerivations, pkgs, nil } func ServerHandler(dependencyDir string, newBuilder types.NewBuilder, dgr types.DownloadGithubRepo) http.Handler { return serverHandler(dependencyDir, newBuilder, dgr) } -func Builder(dependencyDir string, newBuilder types.NewBuilder, dgr types.DownloadGithubRepo) func(*Job) ([]string, error) { - return func(job *Job) ([]string, error) { - return buildJob(job, dependencyDir, newBuilder, dgr) +func Builder(dependencyDir string, newBuilder types.NewBuilder, dgr types.DownloadGithubRepo) func(context.Context, string) ([]string, []types.Package, error) { + return func(ctx context.Context, pkg string) ([]string, []types.Package, error) { + return buildJob(ctx, pkg, dependencyDir, newBuilder, dgr) } } @@ -589,6 +499,9 @@ func DownloadGithubRepo(url string, reference string) (location string, err erro git clone %s %s cd %s`, url, location, location) if reference != "" { + // TODO: remove, we should not allow fetches of git repos at references. + // Otherwise it would be difficult to stage upcoming changes on a public + // branch. script += fmt.Sprintf("\ngit checkout %s", reference) } // script += "\nrm -rf ./.git" diff --git a/internal/dependency/dependency_test.go b/internal/dependency/dependency_test.go index 24b20bae..e992ee31 100644 --- a/internal/dependency/dependency_test.go +++ b/internal/dependency/dependency_test.go @@ -13,6 +13,8 @@ import ( "testing" "github.com/maxmcd/bramble/internal/config" + "github.com/maxmcd/bramble/internal/netcache" + "github.com/maxmcd/bramble/internal/netcache/netcachetest" "github.com/maxmcd/bramble/internal/types" "github.com/maxmcd/bramble/pkg/fxt" "github.com/maxmcd/bramble/v/cmd/go/mvs" @@ -26,11 +28,11 @@ func pkg(m string, deps ...string) func() (string, []string) { } func testDepMgr(t *testing.T, deps ...func() (string, []string)) (config.Config, *Manager) { - dm := &Manager{dir: dir(t.TempDir())} + dm := &Manager{dependencyDirectory: dependencyDirectory(t.TempDir())} var returnedConfig config.Config for i, dep := range deps { pkg, deps := dep() - if err := os.MkdirAll(dm.dir.join("src", pkg), 0755); err != nil { + if err := os.MkdirAll(dm.dependencyDirectory.join("src", pkg), 0755); err != nil { t.Fatal(err) } parts := strings.Split(pkg, "@") @@ -46,7 +48,7 @@ func testDepMgr(t *testing.T, deps ...func() (string, []string)) (config.Config, name, version := parts[0], parts[1] cfg.Dependencies[name] = config.Dependency{Version: version} } - f, err := os.Create(dm.dir.join("src", pkg, "bramble.toml")) + f, err := os.Create(dm.dependencyDirectory.join("src", pkg, "bramble.toml")) if err != nil { t.Fatal(err) } @@ -83,7 +85,7 @@ func blogScenario(t *testing.T) (config.Config, *Manager) { func TestDMReqsRequired(t *testing.T) { cfg, dm := blogScenario(t) - reqs := dm.reqs(cfg) + reqs := dm.reqs(context.Background(), cfg) deps, err := reqs.Required(mvs.Version{ Name: "A@1", Version: "1.0", @@ -99,7 +101,8 @@ func TestDMReqsRequired(t *testing.T) { func TestDMReqs(t *testing.T) { cfg, dm := blogScenario(t) - vs, err := mvs.BuildList(mvs.Version{Name: "A@1", Version: "1.0"}, dm.reqs(cfg)) + vs, err := mvs.BuildList(mvs.Version{Name: "A@1", Version: "1.0"}, + dm.reqs(context.Background(), cfg)) if err != nil { t.Fatal(err) } @@ -125,7 +128,7 @@ func TestDMReqsUpgrade(t *testing.T) { vs, err := mvs.Upgrade( mvs.Version{Name: "A@1", Version: "1.0"}, - dm.reqs(cfg), + dm.reqs(context.Background(), cfg), mvs.Version{Name: "C@1", Version: "3.0"}, ) if err != nil { @@ -144,7 +147,7 @@ func TestDMReqsUpgrade(t *testing.T) { } func (dm *Manager) deleteHalfDeps(t *testing.T) { - list, err := filepath.Glob(dm.dir.join("src", "*")) + list, err := filepath.Glob(dm.dependencyDirectory.join("src", "*")) if err != nil { t.Fatal(err) } @@ -169,14 +172,16 @@ func TestDMReqsRemote(t *testing.T) { // partially present subset localDM.deleteHalfDeps(t) - server := httptest.NewServer(ServerHandler(string(remoteDM.dir), nil, nil)) + server := httptest.NewServer(ServerHandler(string(remoteDM.dependencyDirectory), nil, nil)) localDM.dependencyClient = &dependencyClient{ - client: &http.Client{}, - host: server.URL, + client: &http.Client{}, + host: server.URL, + cacheClient: netcache.NewStdCache(server.URL), } - vs, err := mvs.BuildList(mvs.Version{Name: "A@1", Version: "1.0"}, localDM.reqs(cfg)) + vs, err := mvs.BuildList(mvs.Version{Name: "A@1", Version: "1.0"}, + localDM.reqs(context.Background(), cfg)) if err != nil { t.Fatal(err) } @@ -197,11 +202,12 @@ func TestDMPathOrDownload(t *testing.T) { remoteCFG, remoteDM := blogScenario(t) _, localDM := testDepMgr(t) // no deps - server := httptest.NewServer(ServerHandler(string(remoteDM.dir), nil, nil)) + server := httptest.NewServer(ServerHandler(string(remoteDM.dependencyDirectory), nil, nil)) localDM.dependencyClient = &dependencyClient{ - client: &http.Client{}, - host: server.URL, + client: &http.Client{}, + host: server.URL, + cacheClient: netcache.NewStdCache(server.URL), } path, err := localDM.PackagePathOrDownload(context.Background(), types.Package{"A", "1.1.0"}) @@ -324,7 +330,8 @@ func (tb testBuilder) testGithubDownloader(url, reference string) (location stri return location, nil } -func TestPushJob(t *testing.T) { +func TestPushJobAndUpload(t *testing.T) { + ctx := context.Background() tb := testBuilder{ t: t, packages: map[string]types.Package{ @@ -338,33 +345,65 @@ func TestPushJob(t *testing.T) { }, }, } - + dependencyDir := t.TempDir() server := httptest.NewServer( - serverHandler(t.TempDir(), tb.NewBuilder, tb.testGithubDownloader), + serverHandler(dependencyDir, tb.NewBuilder, tb.testGithubDownloader), ) - if err := PostJob(context.Background(), server.URL, "x.y/z", ""); err != nil { + if err := PostJob(ctx, server.URL, "x.y/z", ""); err != nil { t.Fatal(err) } + dc := &dependencyClient{ - host: server.URL, - client: &http.Client{}, + host: server.URL, + client: &http.Client{}, + cacheClient: netcache.NewStdCache(server.URL), } for _, m := range tb.packages { { - cfg, err := dc.getPackageConfig(context.Background(), types.Package{Name: m.Name, Version: m.Version}) + cfg, err := dc.getPackageConfig(ctx, types.Package{Name: m.Name, Version: m.Version}) if err != nil { t.Fatal(err) } - assert.Equal(t, cfg.Package.Name, m.Name) - assert.Equal(t, cfg.Package.Version, m.Version) + assert.Equal(t, cfg.Config.Package.Name, m.Name) + assert.Equal(t, cfg.Config.Package.Version, m.Version) } { - body, err := dc.getPackageSource(context.Background(), types.Package{Name: m.Name, Version: m.Version}) + loc := t.TempDir() + err := dc.getPackageSource(ctx, types.Package{Name: m.Name, Version: m.Version}, loc) if err != nil { t.Fatal(err) } - _ = body } } + { + XYZ := types.Package{ + Name: "x.y/z", + Version: "2.0.0", + } + client := netcachetest.StartMinio(t) + manager := NewManager(dependencyDir, "", client) + if err := manager.UploadPackage(ctx, XYZ); err != nil { + t.Fatal(err) + } + + vs, err := manager.dependencyClient.getPackageVersions(ctx, "x.y/z") + if err != nil { + t.Fatal(err) + } + require.Equal(t, vs, []string{"2.0.0"}) + + otherManager := NewManager(t.TempDir(), "", client) + path, err := otherManager.PackagePathOrDownload(ctx, XYZ) + if err != nil { + t.Fatal(err) + } + cfg, _, err := config.ReadConfigs(path) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, cfg.Package.Name, XYZ.Name) + assert.Equal(t, cfg.Package.Version, XYZ.Version) + } + } diff --git a/internal/errs/errs.go b/internal/errs/errs.go new file mode 100644 index 00000000..59550800 --- /dev/null +++ b/internal/errs/errs.go @@ -0,0 +1,15 @@ +package errs + +import "fmt" + +type ErrModuleNotFoundInProject struct { + Module string +} + +func (e ErrModuleNotFoundInProject) Error() string { + return fmt.Sprintf("%q is not a dependency of this project, do you need to add it?", e.Module) +} +func (e ErrModuleNotFoundInProject) Is(err error) bool { + _, ok := err.(ErrModuleNotFoundInProject) + return ok +} diff --git a/internal/netcache/netcache.go b/internal/netcache/netcache.go new file mode 100644 index 00000000..53e3376a --- /dev/null +++ b/internal/netcache/netcache.go @@ -0,0 +1,440 @@ +package netcache + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "encoding/xml" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + stdpath "path" + "regexp" + "strings" + "unicode/utf8" + + "github.com/julienschmidt/httprouter" + "github.com/klauspost/pgzip" + "github.com/maxmcd/bramble/pkg/io2" + "github.com/maxmcd/bramble/pkg/url2" + "github.com/pkg/errors" + "github.com/rlmcpherson/s3gof3r" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" +) + +type Client interface { + Exists(ctx context.Context, path string) (bool, error) + Get(ctx context.Context, path string) (body io.ReadCloser, err error) + Put(ctx context.Context, path string) (writer io.WriteCloser, err error) +} + +type ErrNotFound struct{ path string } + +func (e ErrNotFound) Error() string { + return fmt.Sprintf("object %q not found in cache", e.path) +} + +type ErrFailedRequest struct { + isPut bool + path string + body string +} + +func (e ErrFailedRequest) Error() string { + template := "error while trying to read %q from cache" + if e.isPut { + template = "error while trying to write %q to cache" + } + out := fmt.Sprintf(template, e.path) + if e.body != "" { + out = ": " + e.body + } + return out +} + +func errUploading(path, body string) error { + return ErrFailedRequest{isPut: true, body: body, path: path} +} +func errFetching(path, body string) error { + return ErrFailedRequest{isPut: false, body: body, path: path} +} + +type requestLookup struct { + router *httprouter.Router +} + +func (r requestLookup) lookup(method, path string) (string, httprouter.Params) { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + handler, params, _ := r.router.Lookup(method, path) + if handler == nil { + return "", nil + } + recorder := httptest.NewRecorder() + handler(recorder, nil, params) + return recorder.Body.String(), params +} + +func newRequestLookup() requestLookup { + router := httprouter.New() + for _, route := range [][]string{ + {http.MethodGet, "/derivation/:filename"}, + {http.MethodGet, "/output/:hash"}, + {http.MethodPost, "/derivation/:filename"}, + {http.MethodPost, "/output/:hash"}, + {http.MethodGet, "/package/versions/*name"}, + {http.MethodGet, "/package/source/*name_version"}, + {http.MethodGet, "/package/config/*name_version"}, + } { + method, path := route[0], route[1] + handler := func(rw http.ResponseWriter, r *http.Request, p httprouter.Params) { + fmt.Fprint(rw, path) + } + router.Handle(method, path, handler) + } + + return requestLookup{router: router} +} + +type S3CacheOptions struct { + AccessKeyID string + SecretAccessKey string + S3EndpointPrefix string + CDNEndpointPrefix string + PathStyle bool +} + +func NewS3Cache(opt S3CacheOptions) (Client, error) { + keys := s3gof3r.Keys{ + AccessKey: opt.AccessKeyID, + SecretKey: opt.SecretAccessKey, + } + parsed, err := url.Parse(opt.S3EndpointPrefix) + if err != nil { + return nil, errors.Wrapf(err, "error pasing S3url parameter %s", opt.S3EndpointPrefix) + } + + // TODO: this must be set for the entire lifetime of the client/bucket. + // Should patch underlying lib to support explicit region. Although since + // we're not relying on this value for now this is not really issue, the + // value just needs to be set to something. + os.Setenv("AWS_REGION", " ") + + s3 := s3gof3r.New(parsed.Host, keys) + cc := &S3Cache{bucket: s3.Bucket("bramble")} + cc.bucket.Client = &http.Client{ + // For tracing + Transport: otelhttp.NewTransport(http.DefaultTransport), + } + cc.Scheme = parsed.Scheme + cc.S3url = opt.S3EndpointPrefix + cc.CDNPrefix = opt.CDNEndpointPrefix + cc.PathStyle = opt.PathStyle + + cc.requestLookup = newRequestLookup() + return cc, nil +} + +type S3Cache struct { + bucket *s3gof3r.Bucket + + CDNPrefix string + S3url string + + Scheme string + PathStyle bool + + requestLookup requestLookup +} + +func (c *S3Cache) putWriter(ctx context.Context, path string) (putWriter io.WriteCloser, err error) { + encodedPath := encodePath(path) + h := http.Header{} + h.Set("x-amz-acl", "public-read") + putWriter, err = c.bucket.PutWriter(encodedPath, h, &s3gof3r.Config{ + Client: http.DefaultClient, + Scheme: c.Scheme, + Md5Check: false, + PathStyle: c.PathStyle, + }) + if err != nil { + return nil, errUploading(path, err.Error()) + } + gzw := pgzip.NewWriter(putWriter) + return wrapperWriteCloser{ctx: ctx, path: path, writeCloser: io2.WriterMultiCloser(gzw, gzw, putWriter)}, nil +} + +func (c *S3Cache) cdnPrefix() string { + if c.CDNPrefix != "" { + return c.CDNPrefix + } + if c.PathStyle { + return url2.Join(c.S3url, "bramble") + } + return c.S3url +} + +func (c *S3Cache) Exists(ctx context.Context, path string) (exists bool, err error) { + matchingPath, _ := c.requestLookup.lookup(http.MethodGet, path) + if matchingPath == "" { + return false, errors.Errorf("request path %q doesn't match an expected route", path) + } + // TODO: error on routes that are not "existable"? + encodedPath := encodePath(path) + req, err := http.NewRequest(http.MethodGet, url2.Join(c.cdnPrefix(), encodedPath), nil) + if err != nil { + return false, errFetching(path, err.Error()) + } + req = req.WithContext(ctx) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return false, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return false, nil + } + if resp.StatusCode == http.StatusOK { + return true, nil + } + return false, errFetching(path, fmt.Sprintf("unexpected response code: %d", resp.StatusCode)) +} + +func (c *S3Cache) Get(ctx context.Context, path string) (body io.ReadCloser, err error) { + // TODO: cleanup, lots of it + matchingPath, params := c.requestLookup.lookup(http.MethodGet, path) + if matchingPath == "" { + return nil, errors.Errorf("request path %q doesn't match an expected route", path) + } + + if matchingPath == "/package/versions/*name" { + resp, err := http.Get(url2.Join(c.S3url, "bramble") + "?prefix=" + + stdpath.Join("/package/source", params.ByName("name"))) + if err != nil { + return nil, err + } + defer resp.Body.Close() + var result ListBucketResult + if err := xml.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + + versions := []string{} + for _, result := range result.Contents { + unescaped, _ := url.QueryUnescape(result.Key) + parts := strings.Split(unescaped, "@") + versions = append(versions, parts[1]) + } + var out bytes.Buffer + if err := json.NewEncoder(&out).Encode(versions); err != nil { + return nil, err + } + return io.NopCloser(&out), nil + } + encodedPath := encodePath(path) + req, err := http.NewRequest(http.MethodGet, url2.Join(c.cdnPrefix(), encodedPath), nil) + if err != nil { + return nil, errFetching(path, err.Error()) + } + req = req.WithContext(ctx) + resp, err := http.DefaultClient.Do(req) + if err := responseError(false, path, resp, err); err != nil { + return nil, err + } + gzr, err := pgzip.NewReader(resp.Body) + if err != nil { + return nil, err + } + return io2.ReaderMultiCloser(gzr, gzr, resp.Body), nil +} + +func (c *S3Cache) Put(ctx context.Context, path string) (writer io.WriteCloser, err error) { + return c.putWriter(ctx, path) +} + +var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$") + +// https://github.com/rhnvrm/simples3/blob/ad0419ef77c905b3909459f5eaaa4cefe2232981/simples3.go#L617 +func encodePath(pathName string) string { + if reservedObjectNames.MatchString(pathName) { + return pathName + } + var encodedPathname strings.Builder + for _, s := range pathName { + if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark) + encodedPathname.WriteRune(s) + continue + } + switch s { + case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark) + encodedPathname.WriteRune(s) + continue + default: + lenR := utf8.RuneLen(s) + if lenR < 0 { + // if utf8 cannot convert, return the same string as is + return pathName + } + u := make([]byte, lenR) + utf8.EncodeRune(u, s) + for _, r := range u { + hex := hex.EncodeToString([]byte{r}) + encodedPathname.WriteString("%" + strings.ToUpper(hex)) + } + } + } + return encodedPathname.String() +} + +type StdCache struct { + host string + client *http.Client +} + +func NewStdCache(host string) Client { + return &StdCache{host: host, client: &http.Client{ + // For tracing + Transport: otelhttp.NewTransport(http.DefaultTransport), + }} +} + +func responseError(isPut bool, path string, resp *http.Response, err error) error { + e := ErrFailedRequest{isPut: isPut, path: path} + if err != nil { + e.body = err.Error() + return e + } + if resp.StatusCode == http.StatusNotFound { + if resp.Body != nil { + resp.Body.Close() + } + return ErrNotFound{path} + } + if resp.StatusCode != http.StatusOK { + if resp.Body != nil { + var buf bytes.Buffer + _, _ = io.Copy(&buf, resp.Body) + resp.Body.Close() + e.body = buf.String() + fmt.Println(buf.String()) + } + fmt.Println("REQUEST ERROR", e) + return e + } + return nil +} + +func (cs *StdCache) Exists(ctx context.Context, path string) (exists bool, err error) { + req, err := http.NewRequest(http.MethodHead, url2.Join(cs.host, path), nil) + if err != nil { + return false, errFetching(path, err.Error()) + } + req = req.WithContext(ctx) + resp, err := cs.client.Do(req) + if err != nil { + return false, errFetching(path, err.Error()) + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return false, nil + } + if resp.StatusCode == http.StatusOK { + return true, nil + } + return false, errFetching(path, fmt.Sprintf("unexpected response code: %d", resp.StatusCode)) +} + +func (cs *StdCache) Get(ctx context.Context, path string) (body io.ReadCloser, err error) { + req, err := http.NewRequest(http.MethodGet, url2.Join(cs.host, path), nil) + if err != nil { + return nil, errFetching(path, err.Error()) + } + req = req.WithContext(ctx) + resp, err := cs.client.Do(req) + if err := responseError(false, path, resp, err); err != nil { + return nil, err + } + return resp.Body, nil +} + +func (cs *StdCache) Put(ctx context.Context, path string) (writer io.WriteCloser, err error) { + errChan := make(chan error) + pr, pw := io.Pipe() + req, err := http.NewRequest(http.MethodPost, url2.Join(cs.host, path), pr) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + go func() { + resp, err := cs.client.Do(req) + if err != nil { + _ = pr.CloseWithError(err) + } + errChan <- responseError(true, path, resp, err) + }() + return io2.WriterCloseFunc(pw, func() error { + pw.Close() + return <-errChan + }), nil +} + +type ListBucketResult struct { + XMLName xml.Name `xml:"ListBucketResult"` + Text string `xml:",chardata"` + Xmlns string `xml:"xmlns,attr"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + Marker string `xml:"Marker"` + MaxKeys string `xml:"MaxKeys"` + Delimiter string `xml:"Delimiter"` + IsTruncated string `xml:"IsTruncated"` + Contents []struct { + Text string `xml:",chardata"` + Key string `xml:"Key"` + LastModified string `xml:"LastModified"` + ETag string `xml:"ETag"` + Size string `xml:"Size"` + Owner struct { + Text string `xml:",chardata"` + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName"` + } `xml:"Owner"` + StorageClass string `xml:"StorageClass"` + } `xml:"Contents"` +} + +type wrapperWriteCloser struct { + ctx context.Context + path string + writeCloser io.WriteCloser +} + +func (wc wrapperWriteCloser) Write(b []byte) (n int, err error) { + select { + case <-wc.ctx.Done(): + return 0, context.Canceled + default: + } + n, err = wc.writeCloser.Write(b) + if err != nil { + return n, errUploading(wc.path, err.Error()) + } + return n, err +} + +func (wc wrapperWriteCloser) Close() (err error) { + select { + case <-wc.ctx.Done(): + return context.Canceled + default: + } + if err := wc.writeCloser.Close(); err != nil { + return &ErrFailedRequest{isPut: true, path: wc.path, body: err.Error()} + } + return nil +} diff --git a/internal/netcache/netcache_test.go b/internal/netcache/netcache_test.go new file mode 100644 index 00000000..bda8dd3d --- /dev/null +++ b/internal/netcache/netcache_test.go @@ -0,0 +1,59 @@ +package netcache + +import ( + "bytes" + "context" + "io" + "math/rand" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDO(t *testing.T) { + t.Skip("this test requires live credentials") + client, err := NewS3Cache(S3CacheOptions{ + AccessKeyID: os.Getenv("AWS_ACCESS_KEY_ID"), + SecretAccessKey: os.Getenv("AWS_SECRET_ACCESS_KEY"), + S3EndpointPrefix: "https://nyc3.digitaloceanspaces.com", + }) + if err != nil { + t.Fatal(err) + } + fakeFile := make([]byte, 1e7) + if _, err := rand.Read(fakeFile); err != nil { + t.Fatal(err) + } + + key := "test/testfile@+-$%^" + { + // put + writer, err := client.Put(context.Background(), key) + if err != nil { + t.Fatal(err) + } + if _, err := io.Copy(writer, bytes.NewBuffer(fakeFile)); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + } + + { + // get + reader, err := client.Get(context.Background(), key) + if err != nil { + t.Fatal(err) + } + buf := &bytes.Buffer{} + if _, err := io.Copy(buf, reader); err != nil { + t.Fatal(err) + } + if err := reader.Close(); err != nil { + t.Fatal(err) + } + require.Equal(t, fakeFile, buf.Bytes()) + } +} diff --git a/internal/netcache/netcachetest/minio.go b/internal/netcache/netcachetest/minio.go new file mode 100644 index 00000000..5f722544 --- /dev/null +++ b/internal/netcache/netcachetest/minio.go @@ -0,0 +1,151 @@ +package netcachetest + +import ( + "fmt" + "go/build" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/maxmcd/bramble/internal/netcache" + "github.com/maxmcd/bramble/pkg/fileutil" + "github.com/pkg/errors" +) + +// getFreePort asks the kernel for a free open port that is ready to use. +func getFreePort() (int, error) { + addr, err := net.ResolveTCPAddr("tcp", "localhost:0") + if err != nil { + return 0, err + } + + l, err := net.ListenTCP("tcp", addr) + if err != nil { + return 0, err + } + defer l.Close() + return l.Addr().(*net.TCPAddr).Port, nil +} + +func StartMinio(t *testing.T) netcache.Client { + minioBin := filepath.Join(build.Default.GOPATH, "bin", "minio") + mcBin := filepath.Join(build.Default.GOPATH, "bin", "mc") + for _, bin := range [][]string{ + {minioBin, "https://dl.min.io/server/minio/release/linux-amd64/minio"}, + {mcBin, "https://dl.min.io/client/mc/release/linux-amd64/mc"}, + } { + location, url := bin[0], bin[1] + if !fileutil.FileExists(location) { + resp, err := http.Get(url) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatal(fmt.Errorf("unexpected response code %s %s: %d", resp.Request.Method, resp.Request.URL, resp.StatusCode)) + } + f, err := os.Create(location) + if err != nil { + t.Fatal(err) + } + if _, err := io.Copy(f, resp.Body); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + if err := os.Chmod(location, 0755); err != nil { + t.Fatal(err) + } + } + } + stateDir := t.TempDir() + if err := os.Mkdir(filepath.Join(stateDir, "bramble"), 0755); err != nil { + t.Fatal(err) + } + fmt.Println("Minio state directory:", stateDir) + + addr, err := getFreePort() + if err != nil { + t.Fatal(err) + } + + // Start server with address and path to bucket for object state + cmd := exec.Command(minioBin, "server", + "--address", fmt.Sprintf(":%d", addr), + stateDir) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGKILL, + } + + cmd.Stderr = os.Stderr + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, "MINIO_ROOT_USER=root", "MINIO_ROOT_PASSWORD=password") + t.Cleanup(func() { + if cmd.Process != nil { + _ = cmd.Process.Kill() + } + }) + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + s3Addr := fmt.Sprint("http://localhost:", addr) + fmt.Println("Minio running at", s3Addr) + for { + if hasExited(cmd) { + t.Fatal("process exited unexpectedly") + } + resp, err := http.Get(s3Addr) + if resp != nil && resp.Body != nil { + resp.Body.Close() + } + if err == nil { + // We're up! + break + } + time.Sleep(time.Millisecond * 100) + } + + runCommand(t, mcBin, "alias", "set", "local", s3Addr, "root", "password") + runCommand(t, mcBin, "policy", "set", "download", "local/bramble") + + client, err := netcache.NewS3Cache(netcache.S3CacheOptions{ + SecretAccessKey: "password", + AccessKeyID: "root", + S3EndpointPrefix: s3Addr, + PathStyle: true, + }) + if err != nil { + t.Fatal(err) + } + return client +} + +func runCommand(t *testing.T, arguments ...string) { + for { + cmd := exec.Command(arguments[0], arguments[1:]...) + b, err := cmd.CombinedOutput() + if strings.Contains(string(b), "Server not initialized") { + time.Sleep(time.Millisecond * 10) + continue + } + if err != nil { + t.Fatal(errors.Wrap(err, fmt.Sprint(arguments, " ", string(b)))) + } + return + } +} + +func hasExited(cmd *exec.Cmd) bool { + if cmd.ProcessState != nil { + return cmd.ProcessState.Exited() + } + return false +} diff --git a/internal/netcache/netcachetest/minio_test.go b/internal/netcache/netcachetest/minio_test.go new file mode 100644 index 00000000..7652f0c1 --- /dev/null +++ b/internal/netcache/netcachetest/minio_test.go @@ -0,0 +1,51 @@ +package netcachetest + +import ( + "bytes" + "context" + "io" + "math/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMinio(t *testing.T) { + client := StartMinio(t) + + fakeFile := make([]byte, 1e7) + if _, err := rand.Read(fakeFile); err != nil { + t.Fatal(err) + } + + key := "output/testfile@+-$%^" + { + // put + writer, err := client.Put(context.Background(), key) + if err != nil { + t.Fatal(err) + } + if _, err := io.Copy(writer, bytes.NewBuffer(fakeFile)); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + } + + { + // get + reader, err := client.Get(context.Background(), key) + if err != nil { + t.Fatal(err) + } + buf := &bytes.Buffer{} + if _, err := io.Copy(buf, reader); err != nil { + t.Fatal(err) + } + if err := reader.Close(); err != nil { + t.Fatal(err) + } + require.Equal(t, fakeFile, buf.Bytes()) + } +} diff --git a/internal/project/derivation.go b/internal/project/derivation.go index 620e45f0..06198fe1 100644 --- a/internal/project/derivation.go +++ b/internal/project/derivation.go @@ -204,20 +204,22 @@ func isTopLevel(thread *starlark.Thread) bool { return thread.CallStack().At(2).Name == "" || thread.CallStack().At(2).Name == "" } -func (rt *runtime) derivationFunction(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { - if thread.Name != "repl" && isTopLevel(thread) { - return nil, errors.New("derivation call not within a function") - } - // Parse function arguments and assemble the basic derivation - drv, err := rt.newDerivationFromArgs(args, kwargs) - if err != nil { - return nil, err +func (rt *runtime) derivationFunction(projectPath string) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { + return func(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if thread.Name != "repl" && isTopLevel(thread) { + return nil, errors.New("derivation call not within a function") + } + // Parse function arguments and assemble the basic derivation + drv, err := rt.newDerivationFromArgs(args, kwargs, projectPath) + if err != nil { + return nil, err + } + rt.allDerivations[drv.hash()] = drv + return drv, nil } - rt.allDerivations[drv.hash()] = drv - return drv, nil } -func (rt *runtime) newDerivationFromArgs(args starlark.Tuple, kwargs []starlark.Tuple) (drv Derivation, err error) { +func (rt *runtime) newDerivationFromArgs(args starlark.Tuple, kwargs []starlark.Tuple, projectPath string) (drv Derivation, err error) { drv = Derivation{ Outputs: []string{"out"}, } @@ -264,7 +266,7 @@ func (rt *runtime) newDerivationFromArgs(args starlark.Tuple, kwargs []starlark. } for _, src := range drv.Sources.Files { - abs := filepath.Join(rt.project.location, src) + abs := filepath.Join(projectPath, src) if !fileutil.PathExists(abs) { return drv, errors.Errorf("Source file %q doesn't exit", abs) } diff --git a/internal/project/exec_module.go b/internal/project/exec_module.go index 751380ef..bcc4bf97 100644 --- a/internal/project/exec_module.go +++ b/internal/project/exec_module.go @@ -212,12 +212,13 @@ func (rt *runtime) execModule(ctx context.Context, module string) (globals starl // Add a placeholder to indicate "load in progress". rt.cache[module] = nil - path, err := rt.project.moduleToPath(module) + projectPath, path, err := rt.project.moduleToPath(ctx, module) if err != nil { return nil, err } + // FOR_SUBLOAD // Load and initialize the module in a new thread. - globals, err = rt.starlarkExecFile(rt.newThread(ctx, "module "+module), path) + globals, err = rt.starlarkExecFile(rt.newThread(ctx, "module "+module), path, projectPath) rt.cache[module] = &entry{globals: globals, err: err} return globals, err } diff --git a/internal/project/files_builtin.go b/internal/project/files_builtin.go index e70d1914..3d8c2e52 100644 --- a/internal/project/files_builtin.go +++ b/internal/project/files_builtin.go @@ -14,8 +14,9 @@ import ( ) type FilesList struct { - Files []string - Location string + Files []string + Location string + ProjectLocation string } var _ starlark.Value = new(FilesList) @@ -27,6 +28,7 @@ func (fl FilesList) Type() string { return "file_list" } func (fl FilesList) Truth() starlark.Bool { return true } type filesBuiltin struct { + // FOR_SUBLOAD projectLocation string } @@ -107,7 +109,8 @@ func (fb filesBuiltin) filesBuiltin(thread *starlark.Thread, fn *starlark.Builti relFileDirectory = fileDirectory } fl := FilesList{ - Location: relFileDirectory, + Location: relFileDirectory, + ProjectLocation: fb.projectLocation, } for f := range inclSet { if _, match := exclSet[f]; !match { diff --git a/internal/project/module.go b/internal/project/module.go index 5b5d2c02..5bf210af 100644 --- a/internal/project/module.go +++ b/internal/project/module.go @@ -7,8 +7,10 @@ import ( "strings" "github.com/maxmcd/bramble/internal/config" + "github.com/maxmcd/bramble/internal/errs" "github.com/maxmcd/bramble/internal/types" "github.com/maxmcd/bramble/pkg/fileutil" + "github.com/maxmcd/bramble/pkg/url2" "github.com/pkg/errors" "go.starlark.net/starlark" ) @@ -68,7 +70,6 @@ func (p *Project) moduleFromPath(path string) (thisModule string, err error) { type Module struct { Name string Function string - External bool } func (p *Project) ArgumentsToModules(ctx context.Context, args []string, allowExternal bool) (modules []Module, err error) { @@ -120,32 +121,84 @@ func (p *Project) scanForLoadNames() (moduleNames []string, err error) { return moduleNames, nil } +type packageModule struct { + projectPath string + relPath string + module string +} + // TODO: function that takes load() argument values and references the config and pulls down the needed version -func (p *Project) findOrDownloadModulePath(ctx context.Context, module string) (path string, err error) { - if strings.HasPrefix(module, p.config.Package.Name) { - path = module[len(p.config.Package.Name):] - path = filepath.Join(p.location, path) - return path, nil +func (p *Project) findOrDownloadModulePath(ctx context.Context, pkg string) (pm packageModule, err error) { + if strings.HasPrefix(pkg, p.config.Package.Name) { + path := pkg[len(p.config.Package.Name):] + + return packageModule{ + projectPath: p.location, + relPath: filepath.Clean(path), + module: p.config.Package.Name, + }, nil + } - cd, found := p.config.Dependencies[module] + name, dep, found := p.doesModulePackageExist(pkg) if !found { - return "", errors.Errorf("%q is not a dependency of this project, do you need to add it?", module) + return packageModule{}, errs.ErrModuleNotFoundInProject{Module: pkg} } - if cd.Path != "" { + + relPath, err := url2.Rel(name, pkg) + if err != nil { + return packageModule{}, errors.Wrapf(err, "error calculating relative module path between %s and %s", name, pkg) + } + + if dep.Path != "" { // TODO: Does this actually work // TODO: cd.Path must be relative? - return filepath.Join(p.location, cd.Path), nil + return packageModule{ + projectPath: filepath.Join(p.location, dep.Path), + relPath: relPath, + module: name, + }, nil } - path, err = p.dm.PackagePathOrDownload(ctx, types.Package{Name: module, Version: cd.Version}) - return path, err + projectPath, err := p.dm.PackagePathOrDownload(ctx, types.Package{Name: name, Version: dep.Version}) + if err != nil { + return packageModule{}, err + } + return packageModule{ + projectPath: projectPath, + relPath: relPath, + module: name, + }, nil } -func (p *Project) moduleInProject(module string) bool { - if strings.HasPrefix(module, p.config.Package.Name) { - return true +func (p *Project) doesModulePackageExist(pkg string) (name string, dep config.Dependency, found bool) { + if d, ok := p.config.Dependencies[pkg]; ok { + return pkg, d, ok + } + if d, ok := p.lockFile.Dependencies[pkg]; ok { + return pkg, d, ok + } + var longestName string + var longestDep config.Dependency + // No specific match, looking for sub-path match + for name, d := range p.config.Dependencies { + if strings.HasPrefix(pkg, name) { + if len(name) > len(longestName) { + longestName = name + longestDep = d + } + } + } + for name, d := range p.lockFile.Dependencies { + if strings.HasPrefix(pkg, name) { + if len(name) > len(longestName) { + longestName = name + longestDep = d + } + } } - _, found := p.config.Dependencies[module] - return found + if longestName == "" { + return "", config.Dependency{}, false + } + return longestName, longestDep, true } func (p *Project) filepathToModuleName(path string) (module string, err error) { @@ -188,33 +241,44 @@ func (p *Project) ParseModuleFuncArgument(ctx context.Context, name string, allo module.Name, err = p.moduleFromPath(module.Name) return } - if p.moduleInProject(module.Name) { - if _, err := p.findOrDownloadModulePath(ctx, module.Name); err != nil { - return Module{}, err - } - return module, nil - } else if allowExternal { - name, versions, err := p.dm.FindPackageFromModuleName(ctx, module.Name) - if err != nil { - return Module{}, err + pm, err := p.findOrDownloadModulePath(ctx, module.Name) + if err == nil { + module.Name = pm.module + if pm.relPath != "." { + module.Name += pm.relPath } - // TODO: pick newest version - p.config.Dependencies[name] = config.Dependency{Version: versions[0]} - if err := p.writeConfig(p.config); err != nil { - return Module{}, errors.Wrapf(err, "error saving config with new package: %s", types.Package{Name: name, Version: versions[0]}) - } - module.Name = name return module, nil } - return Module{}, errors.Errorf("%q is not a dependency of this project, do you need to add it?", module.Name) + if err != nil && !errors.Is(err, errs.ErrModuleNotFoundInProject{}) { + return Module{}, err + } + // If the module is not found and external is allow, continue to trying + // to fetch an external module that's not in the project + if !allowExternal { + return Module{}, err + } + name, versions, err := p.dm.FindPackageFromModuleName(ctx, module.Name, "") + if err != nil { + return Module{}, err + } + // TODO: pick newest version + p.config.Dependencies[name] = config.Dependency{Version: versions[0]} + if err := p.writeConfig(p.config); err != nil { + return Module{}, errors.Wrapf(err, "error saving config with new package: %s", types.Package{Name: name, Version: versions[0]}) + } + module.Name = name + return module, nil + } -func (p *Project) moduleToPath(module string) (path string, err error) { - path, err = p.findOrDownloadModulePath(context.Background(), module) +func (p *Project) moduleToPath(ctx context.Context, module string) (projectPath, path string, err error) { + pm, err := p.findOrDownloadModulePath(ctx, module) if err != nil { - return "", err + return "", "", err } + path = filepath.Join(pm.projectPath, pm.relPath) + // TODO could make these three syscalls just a single directory file list call directoryWithNameExists := fileutil.PathExists(path) var directoryHasDefaultDotBramble bool @@ -230,11 +294,11 @@ func (p *Project) moduleToPath(module string) (path string, err error) { case fileWithNameExists: path += BrambleExtension default: - return "", errors.Errorf("Module %q not found, %q is not a directory and %q does not exist", + return "", "", errors.Errorf("Module %q not found, %q is not a directory and %q does not exist", module, path, path+BrambleExtension) } - return path, nil + return pm.projectPath, path, nil } func (p *Project) FindAllModules(path string) (modules []string, err error) { diff --git a/internal/project/module_test.go b/internal/project/module_test.go index 0aed91a7..0f4bf98b 100644 --- a/internal/project/module_test.go +++ b/internal/project/module_test.go @@ -99,7 +99,7 @@ func TestProject_FindAllModules(t *testing.T) { }, { "../../", ".", - []string{"github.com/maxmcd/bramble/lib"}, + []string{"github.com/maxmcd/bramble/tests"}, []string{ "github.com/maxmcd/bramble/internal/project/testdata/circular/b", "github.com/maxmcd/bramble/internal/project/testdata/circular", @@ -226,6 +226,11 @@ func Test_parseModuleFuncArgument(t *testing.T) { arg: "github.com/maxmcd/bramble:all", wantModule: "github.com/maxmcd/bramble", wantFn: "all", + }, { + name: "full module name with subdirectory", + arg: "github.com/maxmcd/bramble/tests:all", + wantModule: "github.com/maxmcd/bramble/tests", + wantFn: "all", }, { name: "relative path to file with slash", arg: "./bar/main:other", @@ -293,8 +298,8 @@ func TestProject_BuildArgumentsToModules(t *testing.T) { wd: "./testdata/project", args: []string{"./..."}, wantModules: []Module{ - {"testproject/a", "", false}, - {"testproject", "", false}, + {"testproject/a", ""}, + {"testproject", ""}, }, wantErr: false, }, @@ -303,7 +308,7 @@ func TestProject_BuildArgumentsToModules(t *testing.T) { wd: ".", args: []string{"./testdata"}, wantModules: []Module{ - {"github.com/maxmcd/bramble/internal/project/testdata", "", false}, + {"github.com/maxmcd/bramble/internal/project/testdata", ""}, }, wantErr: false, }, diff --git a/internal/project/project.go b/internal/project/project.go index 20c063c5..5437ec6a 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -32,7 +32,7 @@ type Project struct { wd string - lockFile *config.LockFile + lockFile *config.Lockfile dm *dependency.Manager } @@ -174,22 +174,36 @@ func (p *Project) CalculateDependencies() (err error) { return nil } -func (p *Project) AddDependency(v types.Package) (err error) { +func (p *Project) AddDependency(ctx context.Context, v types.Package) (err error) { + name, vs, err := p.dm.FindPackageFromModuleName(ctx, v.Name, v.Version) + if err != nil { + return err + } + _, _ = name, vs + existing, found := p.config.Dependencies[v.Name] if found { - existing.Version = v.Version - p.config.Dependencies[v.Name] = existing + if v.Version != "" { + // TODO: validate that the version is well formed + existing.Version = v.Version + p.config.Dependencies[v.Name] = existing + } } else { p.config.Dependencies[v.Name] = config.Dependency{ Version: v.Version, } } - cfg, err := p.dm.CalculateConfigBuildlist(p.config) + buildList, err := p.dm.CalculateConfigBuildlist(ctx, p.config) if err != nil { return err } - cfg.Render(os.Stdout) - return p.writeConfig(cfg) + + p.lockFile.Dependencies = buildList + if err := p.WriteLockfile(); err != nil { + return err + } + + return p.writeConfig(p.config) } func (p *Project) writeConfig(cfg config.Config) (err error) { diff --git a/internal/project/project_test.go b/internal/project/project_test.go index aeda3570..447a6ddb 100644 --- a/internal/project/project_test.go +++ b/internal/project/project_test.go @@ -24,5 +24,6 @@ func TestNewProject(t *testing.T) { if err := p.WriteLockfile(); err != nil { t.Fatal(err) } + } } diff --git a/internal/project/runtime.go b/internal/project/runtime.go index cacded58..4ad0e5f0 100644 --- a/internal/project/runtime.go +++ b/internal/project/runtime.go @@ -19,9 +19,10 @@ func (p *Project) newRuntime(target string) *runtime { rt := &runtime{project: p} rt.allDerivations = map[string]Derivation{} rt.cache = map[string]*entry{} + + // Random internal key so that fetch built-ins can use the network rt.internalKey = rand.Int63() - // TODO: sys will be needed by this, what else? - derivation, err := rt.loadNativeDerivation(starlark.NewBuiltin("_derivation", rt.derivationFunction)) + derivation, err := rt.loadNativeDerivation(starlark.NewBuiltin("_derivation", rt.derivationFunction(p.location))) if err != nil { repl.PrintError(err) panic(err) @@ -29,6 +30,7 @@ func (p *Project) newRuntime(target string) *runtime { assertGlobals, _ := assert.LoadAssertModule() rt.predeclared = starlark.StringDict{ + // FOR_SUBLOAD can't use reference to project location "derivation": derivation, "test": starlark.NewBuiltin("test", rt.testBuiltin), "run": starlark.NewBuiltin("run", rt.runBuiltin), @@ -41,6 +43,24 @@ func (p *Project) newRuntime(target string) *runtime { return rt } +func (rt *runtime) predeclatedWithPath(projectPath string) starlark.StringDict { + derivation, err := rt.loadNativeDerivation(starlark.NewBuiltin("_derivation", rt.derivationFunction(projectPath))) + if err != nil { + repl.PrintError(err) + panic(err) + } + return starlark.StringDict{ + "derivation": derivation, + "test": rt.predeclared["test"], + "run": rt.predeclared["run"], + "assert": rt.predeclared["assert"], + "sys": rt.predeclared["sys"], + "files": starlark.NewBuiltin("files", filesBuiltin{ + projectLocation: projectPath, + }.filesBuiltin), + } +} + func (rt *runtime) newThread(ctx context.Context, name string) *starlark.Thread { thread := &starlark.Thread{ Name: name, @@ -82,7 +102,7 @@ func starlarkSys(target string) *starlarkstruct.Module { func (p *Project) REPL() { rt := p.newRuntime("") - repl.REPL(rt.newThread(context.Background(), "repl"), rt.predeclared) + repl.REPL(rt.newThread(context.TODO(), "repl"), rt.predeclared) } func (p *Project) relativePathFromConfig() string { @@ -103,12 +123,13 @@ type entry struct { err error } -func (rt *runtime) starlarkExecFile(thread *starlark.Thread, filename string) (globals starlark.StringDict, err error) { +func (rt *runtime) starlarkExecFile(thread *starlark.Thread, filename, projectPath string) (globals starlark.StringDict, err error) { prog, err := rt.sourceStarlarkProgram(filename) if err != nil { return } - g, err := prog.Init(thread, rt.predeclared) + // FOR_SUBLOAD + g, err := prog.Init(thread, rt.predeclatedWithPath(projectPath)) for name := range g { // no importing or calling of underscored methods if strings.HasPrefix(name, "_") { diff --git a/internal/project/srg.go b/internal/project/srg.go deleted file mode 100644 index 7dac6f1a..00000000 --- a/internal/project/srg.go +++ /dev/null @@ -1 +0,0 @@ -package project diff --git a/internal/project/testdata/project/bramble.lock b/internal/project/testdata/project/bramble.lock index ffdf9973..67c4dd14 100644 --- a/internal/project/testdata/project/bramble.lock +++ b/internal/project/testdata/project/bramble.lock @@ -1,2 +1,2 @@ [URLHashes] - foo = "bar" +"foo" = "bar" diff --git a/internal/project/walk_test.go b/internal/project/walk_test.go index 37a3d5e9..8fe8eb9b 100644 --- a/internal/project/walk_test.go +++ b/internal/project/walk_test.go @@ -59,7 +59,7 @@ func TestExecModuleAndWalk(t *testing.T) { project, err := NewProject(".") require.NoError(t, err) - module, err := project.ParseModuleFuncArgument(context.Background(), "github.com/maxmcd/bramble:all", false) + module, err := project.ParseModuleFuncArgument(context.Background(), "github.com/maxmcd/bramble/tests:all", false) if err != nil { t.Fatal(err) } diff --git a/internal/store/builder.go b/internal/store/builder.go index 21052ea1..07bd0084 100644 --- a/internal/store/builder.go +++ b/internal/store/builder.go @@ -21,10 +21,9 @@ import ( "github.com/maxmcd/bramble/internal/types" "github.com/maxmcd/bramble/pkg/fileutil" "github.com/maxmcd/bramble/pkg/hasher" - "github.com/maxmcd/bramble/pkg/reptar" "github.com/maxmcd/bramble/pkg/sandbox" "github.com/maxmcd/bramble/pkg/textreplace" - "github.com/maxmcd/bramble/v/untar" + "github.com/maxmcd/reptar" "github.com/mholt/archiver/v3" "github.com/pkg/errors" "go.opentelemetry.io/otel/attribute" @@ -52,6 +51,11 @@ type BuildDerivationOptions struct { } func (b *Builder) BuildDerivation(ctx context.Context, drv Derivation, opts BuildDerivationOptions) (builtDrv Derivation, didBuild bool, err error) { + select { + case <-ctx.Done(): + return Derivation{}, false, context.Canceled + default: + } var span trace.Span ctx, span = tracer.Start(ctx, "store.BuildDerivation") defer span.End() @@ -189,9 +193,9 @@ func (b *Builder) checkFetchDerivationHashes(drv Derivation, url string) error { // If we have a hash to validate, ensure it's valid if hash != "" && outputPath != hash { return errors.Errorf( - "Urlfetch content doesn't match with the existing hash. "+ + "Urlfetch for %q content doesn't match with the existing hash. "+ "Hash %q was provided by the output was %q", - hash, outputPath) + url, hash, outputPath) } // If we never had a hash to validate, add it to lockfile if hash == "" { @@ -229,7 +233,7 @@ func (b *Builder) fetchURLBuilder(ctx context.Context, drv Derivation, outputPat if err != nil { return errors.Wrap(err, "requires gzip-compressed body") } - if err = untar.Untar(r, outputPaths["out"]); err != nil { + if err = reptar.Unarchive(r, outputPaths["out"]); err != nil { return err } return nil @@ -487,7 +491,7 @@ func (s *Store) archiveAndScanOutputDirectory(ctx context.Context, tarOutput, ha // write the output files into an archive go func() { btpw := bufio.NewWriter(tarPipeWriter) - if err := reptar.Reptar(s.joinStorePath(storeFolder), btpw); err != nil { + if err := reptar.Archive(s.joinStorePath(storeFolder), btpw); err != nil { errChan <- err return } @@ -588,7 +592,7 @@ func (s *Store) hashNormalizedBuildOutput(location string, hash string) (err err errChan := make(chan error) resultChan := make(chan string) go func() { - if err := reptar.Reptar(location, pipeWriter); err != nil { + if err := reptar.Archive(location, pipeWriter); err != nil { errChan <- err } pipeWriter.Close() diff --git a/internal/store/cache_server.go b/internal/store/cache_server.go deleted file mode 100644 index 4416390c..00000000 --- a/internal/store/cache_server.go +++ /dev/null @@ -1,130 +0,0 @@ -package store - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "os" - - "github.com/maxmcd/bramble/pkg/chunkedarchive" - "github.com/maxmcd/bramble/pkg/httpx" - "github.com/pkg/errors" -) - -type storeHashFetcher struct { - store *Store -} - -var _ chunkedarchive.HashFetcher = new(storeHashFetcher) - -func (hf *storeHashFetcher) Lookup(hash string) (file io.ReadCloser, err error) { - return os.Open(hf.store.joinStorePath(hash)) -} - -type OutputRequestBody struct { - Output Output - TOC []chunkedarchive.TOCEntry -} - -// Uploads a derivation and all outputs -// Sources aren't uploaded -// Outputs are uploaded in 4mb body chunks -func (s *Store) CacheServer() http.Handler { - router := httpx.New() - router.GET("/derivation/:filename", func(c httpx.Context) (err error) { - f, err := os.Open(s.joinStorePath(c.Params.ByName("filename"))) - if err != nil { - return httpx.ErrNotFound(err) - } - defer f.Close() - _, err = io.Copy(c.ResponseWriter, f) - return err - }) - router.GET("/output/:hash", func(c httpx.Context) (err error) { - f, err := os.Open(s.joinStorePath(c.Params.ByName("hash"))) - if err != nil { - return httpx.ErrNotFound(err) - } - var toc []chunkedarchive.TOCEntry - if err := json.NewDecoder(f).Decode(&toc); err != nil { - // If the hash isn't a valid TOC then it's not an output - return httpx.ErrNotFound(err) - } - _, _ = f.Seek(0, 0) - _, err = io.Copy(c.ResponseWriter, f) - return err - }) - router.GET("/chunk/:hash", func(c httpx.Context) (err error) { - f, err := os.Open(s.joinStorePath(c.Params.ByName("hash"))) - if err != nil { - return httpx.ErrNotFound(err) - } - _, err = io.Copy(c.ResponseWriter, f) - return err - }) - - router.POST("/derivation", func(c httpx.Context) (err error) { - var drv Derivation - if err := json.NewDecoder(c.Request.Body).Decode(&drv); err != nil { - return httpx.ErrNotAcceptable(err) - } - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(drv); err != nil { - return err - } - filename, err := s.WriteDerivation(drv) - if err != nil { - return err - } - fmt.Fprint(c.ResponseWriter, filename) - return nil - }) - router.POST("/output", func(c httpx.Context) (err error) { - var req OutputRequestBody - if err := json.NewDecoder(c.Request.Body).Decode(&req); err != nil { - return httpx.ErrNotAcceptable(err) - } - - tempDir, err := os.MkdirTemp("", "") - if err != nil { - return err - } - defer os.RemoveAll(tempDir) - if err := chunkedarchive.Unarchive(req.TOC, &storeHashFetcher{store: s}, tempDir); err != nil { - return err - } - if err := s.hashNormalizedBuildOutput(tempDir, req.Output.Path); err != nil { - return err - } - f, err := os.Create(s.joinStorePath(req.Output.Path + ".output")) - if err != nil { - return err - } - if err := json.NewEncoder(f).Encode(req.TOC); err != nil { - return err - } - return nil - }) - router.POST("/chunk", func(c httpx.Context) (err error) { - hash, err := s.WriteBlob(c.Request.Body) - if err != nil { - return err - } - loc := s.joinStorePath(hash) - fi, err := os.Stat(loc) - if err != nil { - return err - } - if fi.Size() > 4e6 { - _ = os.Remove(loc) - return httpx.ErrNotAcceptable(errors.New("chunk size can't be larger than 4MB")) - } - - fmt.Fprint(c.ResponseWriter, hash) - return nil - }) - - return router -} diff --git a/internal/store/new_derivation.go b/internal/store/new_derivation.go index 305f2b0e..e3af4f79 100644 --- a/internal/store/new_derivation.go +++ b/internal/store/new_derivation.go @@ -7,7 +7,7 @@ import ( "github.com/maxmcd/bramble/pkg/fileutil" "github.com/maxmcd/bramble/pkg/hasher" - "github.com/maxmcd/bramble/pkg/reptar" + "github.com/maxmcd/reptar" "github.com/pkg/errors" ) @@ -81,7 +81,7 @@ func (s *Store) StoreLocalSources(ctx context.Context, sources SourceFiles) (out return } hshr := hasher.New() - if err = reptar.Reptar(tmpDir, hshr); err != nil { + if err = reptar.Archive(tmpDir, hshr); err != nil { return } storeLocation := s.joinStorePath(hshr.String()) diff --git a/internal/store/package_cache.go b/internal/store/package_cache.go new file mode 100644 index 00000000..415be37a --- /dev/null +++ b/internal/store/package_cache.go @@ -0,0 +1,152 @@ +package store + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + + "github.com/maxmcd/bramble/internal/netcache" + "github.com/maxmcd/bramble/pkg/fileutil" + "github.com/maxmcd/bramble/pkg/httpx" + "github.com/maxmcd/reptar" + "github.com/pkg/errors" +) + +func (s *Store) CacheServer() http.Handler { + router := httpx.New() + router.HEAD("/derivation/:filename", func(c httpx.Context) error { + if fileutil.FileExists(s.joinStorePath(c.Params.ByName("filename"))) { + c.ResponseWriter.WriteHeader(http.StatusOK) + } + c.ResponseWriter.WriteHeader(http.StatusNotFound) + return nil + }) + router.GET("/derivation/:filename", func(c httpx.Context) (err error) { + f, err := os.Open(s.joinStorePath(c.Params.ByName("filename"))) + if err != nil { + return httpx.ErrNotFound(err) + } + defer f.Close() + _, err = io.Copy(c.ResponseWriter, f) + return err + }) + router.HEAD("/output/:hash", func(c httpx.Context) error { + if fileutil.FileExists(s.joinStorePath(c.Params.ByName("hash"))) { + c.ResponseWriter.WriteHeader(http.StatusOK) + } + c.ResponseWriter.WriteHeader(http.StatusNotFound) + return nil + }) + router.GET("/output/:hash", func(c httpx.Context) (err error) { + output := s.joinStorePath(c.Params.ByName("hash")) + if !fileutil.DirExists(output) { + return httpx.ErrNotFound(errors.New("Output not found")) + } + if err := reptar.Archive(output, c.ResponseWriter); err != nil { + return httpx.ErrInternalServerError(err) + } + return nil + }) + router.POST("/derivation/:filename", func(c httpx.Context) (err error) { + var drv Derivation + if err := json.NewDecoder(c.Request.Body).Decode(&drv); err != nil { + return httpx.ErrNotAcceptable(err) + } + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(drv); err != nil { + return err + } + filename, err := s.WriteDerivation(drv) + if err != nil { + return err + } + fmt.Fprint(c.ResponseWriter, filename) + return nil + }) + router.POST("/output/:hash", func(c httpx.Context) (err error) { + hash := c.Params.ByName("hash") + tempDir, err := os.MkdirTemp("", "") + if err != nil { + return err + } + defer os.RemoveAll(tempDir) + + if err := reptar.Unarchive(c.Request.Body, tempDir); err != nil { + return httpx.ErrNotAcceptable(errors.Wrap(err, "error unarchiving request body")) + } + if err := s.hashNormalizedBuildOutput(tempDir, hash); err != nil { + return err + } + storeLocation := s.joinStorePath(hash) + if !fileutil.PathExists(storeLocation) { + return os.Rename(tempDir, storeLocation) + } + return nil + }) + + return router +} + +func NewCacheClient(client netcache.Client) CacheClient { + return CacheClient{client} +} + +type CacheClient struct { + client netcache.Client +} + +func (cc CacheClient) DerivationExists(ctx context.Context, drv Derivation) (bool, error) { + return cc.client.Exists(ctx, "derivation/"+drv.Filename()) +} + +func (cc CacheClient) PostDerivation(ctx context.Context, drv Derivation) error { + writer, err := cc.client.Put(ctx, "derivation/"+drv.Filename()) + if err != nil { + return errors.WithStack(err) + } + if _, err := writer.Write([]byte(drv.JSON())); err != nil { + return errors.WithStack(err) + } + return writer.Close() +} + +func (cc CacheClient) OutputExists(ctx context.Context, hash string) (bool, error) { + return cc.client.Exists(ctx, "output/"+hash) +} + +func (cc CacheClient) PostOutput(ctx context.Context, hash string, body io.Reader) error { + w, err := cc.client.Put(ctx, "output/"+hash) + if err != nil { + return errors.WithStack(err) + } + if _, err := io.Copy(w, body); err != nil { + return errors.WithStack(err) + } + return errors.WithStack(w.Close()) +} + +func (cc CacheClient) GetOutput(ctx context.Context, hash string) (body io.ReadCloser, exists bool, err error) { + w, err := cc.client.Get(ctx, "output/"+hash) + if err != nil { + if _, ok := err.(netcache.ErrNotFound); ok { + return nil, false, nil + } + return nil, false, errors.WithStack(err) + } + return w, true, nil +} + +func (cc CacheClient) GetDerivation(ctx context.Context, filename string) (drv Derivation, exists bool, err error) { + w, err := cc.client.Get(ctx, "derivation/"+filename) + if err != nil { + if _, ok := err.(netcache.ErrNotFound); ok { + return Derivation{}, false, nil + } + return Derivation{}, false, errors.WithStack(err) + } + return drv, true, json.NewDecoder(w).Decode(&drv) +} diff --git a/internal/store/store.go b/internal/store/store.go index 7301c0f4..3b4164c7 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -2,28 +2,23 @@ package store import ( "bufio" - "bytes" - "compress/gzip" "context" "encoding/json" "fmt" "io" "io/ioutil" - "net/http" "os" "path/filepath" - "runtime" - "sync" "github.com/maxmcd/bramble/internal/logger" "github.com/maxmcd/bramble/internal/tracing" - "github.com/maxmcd/bramble/pkg/chunkedarchive" "github.com/maxmcd/bramble/pkg/fileutil" "github.com/maxmcd/bramble/pkg/hasher" "github.com/maxmcd/bramble/pkg/sandbox" + "github.com/maxmcd/reptar" "github.com/pkg/errors" - "github.com/rhnvrm/simples3" "go.opentelemetry.io/otel/trace" + "golang.org/x/sync/errgroup" ) var ( @@ -320,12 +315,6 @@ func (s *Store) WriteDerivation(drv Derivation) (filename string, err error) { return filename, ioutil.WriteFile(fileLocation, drv.JSON(), 0644) } -type CacheClient interface { - PostChunk(context.Context, io.Reader) (string, error) - PostDerivation(context.Context, Derivation) (string, error) - PostOutput(context.Context, OutputRequestBody) error -} - func (s *Store) UploadDerivationsToCache(ctx context.Context, derivations []Derivation, cc CacheClient) (err error) { var span trace.Span ctx, span = tracer.Start(ctx, "store.UploadDerivationsToCache") @@ -334,191 +323,65 @@ func (s *Store) UploadDerivationsToCache(ctx context.Context, derivations []Deri ctx, cancel = context.WithCancel(ctx) defer cancel() - bodyWriter := chunkedarchive.NewParallelBodyWriter( - 8, - func(rc io.ReadCloser) (out []string, err error) { - buf := bufio.NewReader(rc) - for { - // TODO: hash the body before uploading to confirm it doesn't already exist - limited := io.LimitReader(buf, 4e6) - hash, err := cc.PostChunk(ctx, limited) + numParallel := 12 + sem := make(chan struct{}, numParallel) + + group, ctx := errgroup.WithContext(ctx) + + group.Go(func() error { + for _, d := range derivations { + drv := d // copy + select { + case sem <- struct{}{}: + case <-ctx.Done(): + return context.Canceled + } + + group.Go(func() error { + // Normalize them with the fixed prefix path + normalized, err := s.normalizeDerivation(drv) if err != nil { - return nil, err + return err } - out = append(out, hash) - fmt.Println("Finished uploading chunk", hash) - if _, err := buf.Peek(1); err != nil { - break - } - } - return out, rc.Close() - }, - ) - - uploaded := map[string]struct{}{} - errChan := make(chan error) - doneChan := make(chan struct{}) - sem := make(chan struct{}, runtime.NumCPU()) - var wg sync.WaitGroup - - // Loop through derivations - for _, drv := range derivations { - // Normalize them with the fixed prefix path - normalized, err := s.normalizeDerivation(drv) - if err != nil { - return err - } - // Upload, could confirm hash - if _, err := cc.PostDerivation(ctx, normalized); err != nil { - return err - } - // Loop through outputs and post them - for _, output := range normalized.Outputs { - if _, ok := uploaded[output.Path]; ok { - continue - } - uploaded[output.Path] = struct{}{} - wg.Add(1) - go func(output Output) { - // Limit parallelism - sem <- struct{}{} - defer func() { <-sem }() - // This will upload using the spawned queue in parallel - toc, err := chunkedarchive.Archive(bodyWriter, s.joinStorePath(output.Path)) + exists, err := cc.DerivationExists(ctx, normalized) if err != nil { - errChan <- err + return err + } + if !exists { + if err := cc.PostDerivation(ctx, normalized); err != nil { + return err + } } - if err := cc.PostOutput(ctx, OutputRequestBody{ - TOC: toc, - Output: output, - }); err != nil { - errChan <- err + // Loop through outputs and post them + for _, output := range normalized.Outputs { + if exists, err := cc.OutputExists(ctx, output.Path); err != nil { + return err + } else if exists { + continue + } + r, w := io.Pipe() + defer r.Close() + defer w.Close() + group.Go(func() error { + bufWriter := bufio.NewWriter(w) + if err := reptar.Archive(s.joinStorePath(output.Path), bufWriter); err != nil { + return err + } + if err := bufWriter.Flush(); err != nil { + return err + } + return w.Close() + }) + if err := cc.PostOutput(ctx, output.Path, r); err != nil { + return err + } } - wg.Done() - }(output) + <-sem + return nil + }) } - } - - go func() { - wg.Wait() - doneChan <- struct{}{} - }() - select { - case err := <-errChan: - return err - case <-doneChan: - return nil - } -} - -type S3CacheClient struct { - s3 *simples3.S3 -} - -type fakeSizeSeeker struct { - buf *bytes.Buffer - loc int64 -} - -func (fss fakeSizeSeeker) Seek(offset int64, whence int) (int64, error) { - switch whence { - case io.SeekStart: - fss.loc = offset - // Ignore seeks beyond loc - return fss.loc, nil - case io.SeekCurrent: - return 0, nil - case io.SeekEnd: - fss.loc -= offset - // Ignore seeks beyond loc - return int64(fss.buf.Len()) - fss.loc, nil - } - panic("unimplemented") -} -func (fss fakeSizeSeeker) Read(p []byte) (n int, err error) { return fss.buf.Read(p) } - -var _ CacheClient = new(S3CacheClient) - -func NewS3CacheClient(s3 *simples3.S3) CacheClient { - return &S3CacheClient{s3: s3} -} - -func fileUpload(s3 *simples3.S3, body *bytes.Buffer, ui simples3.UploadInput) error { - checkIt := "https://store.bramble.run/" + ui.ObjectKey - resp, err := http.Head(checkIt) - if err != nil { - return err - } - switch resp.StatusCode { - case http.StatusNotFound: - case http.StatusOK: return nil - default: - return errors.Errorf("unexpected response code %d for url %q", resp.StatusCode, checkIt) - } - // TODO: Could reduce memory overhead here - buf := &bytes.Buffer{} - w := gzip.NewWriter(buf) - if _, err := io.Copy(w, bytes.NewBuffer(body.Bytes())); err != nil { - return err - } - if err := w.Close(); err != nil { - return err - } - - ui.Body = fakeSizeSeeker{buf: body} - if _, err := s3.FileUpload(ui); err != nil { - return err - } - - ui.Body = fakeSizeSeeker{buf: buf} - ui.FileName += ".gz" - ui.ObjectKey += ".gz" - if _, err := s3.FileUpload(ui); err != nil { - return err - } - return nil -} - -func (cc *S3CacheClient) PostChunk(ctx context.Context, r io.Reader) (string, error) { - var buf bytes.Buffer - h := hasher.New() - tee := io.TeeReader(r, h) - if _, err := io.Copy(&buf, tee); err != nil { - return "", err - } - err := fileUpload(cc.s3, &buf, simples3.UploadInput{ - Bucket: "bramble", - ACL: "public-read", - ObjectKey: "chunk/" + h.String(), - FileName: h.String(), - ContentType: "application/octet-stream", - }) - return "", err -} -func (cc *S3CacheClient) PostDerivation(ctx context.Context, drv Derivation) (string, error) { - filename := drv.Filename() - err := fileUpload(cc.s3, bytes.NewBuffer([]byte(drv.JSON())), simples3.UploadInput{ - Bucket: "bramble", - ACL: "public-read", - ObjectKey: "derivation/" + drv.Filename(), - FileName: drv.Filename(), - ContentType: "application/json", - }) - return filename, err -} -func (cc *S3CacheClient) PostOutput(ctx context.Context, req OutputRequestBody) error { - buf := &bytes.Buffer{} - if err := json.NewEncoder(buf).Encode(req); err != nil { - return err - } - err := fileUpload(cc.s3, buf, simples3.UploadInput{ - Bucket: "bramble", - ACL: "public-read", - ObjectKey: "output/" + req.Output.Path, - FileName: req.Output.Path, - ContentType: "application/json", }) - return err + return group.Wait() } diff --git a/internal/types/bistringmap.go b/internal/types/bistringmap.go deleted file mode 100644 index e13f56a3..00000000 --- a/internal/types/bistringmap.go +++ /dev/null @@ -1,45 +0,0 @@ -package types - -import "sync" - -type BiStringMap struct { - s sync.RWMutex - forward map[string]string - inverse map[string]string -} - -// NewBiStringMap returns a an empty, mutable, BiStringMap -func NewBiStringMap() *BiStringMap { - return &BiStringMap{ - forward: make(map[string]string), - inverse: make(map[string]string), - } -} - -func (b *BiStringMap) Store(k, v string) { - b.s.Lock() - b.forward[k] = v - b.inverse[v] = k - b.s.Unlock() -} - -func (b *BiStringMap) Load(k string) (v string, exists bool) { - b.s.RLock() - v, exists = b.forward[k] - b.s.RUnlock() - return -} - -func (b *BiStringMap) StoreInverse(k, v string) { - b.s.Lock() - b.forward[v] = k - b.inverse[k] = v - b.s.Unlock() -} - -func (b *BiStringMap) LoadInverse(k string) (v string, exists bool) { - b.s.RLock() - v, exists = b.inverse[k] - b.s.RUnlock() - return -} diff --git a/lib/default.bramble b/lib/default.bramble deleted file mode 100644 index f3463a20..00000000 --- a/lib/default.bramble +++ /dev/null @@ -1,111 +0,0 @@ -""" -Lib provides various derivations to help build stuff -""" -load("github.com/maxmcd/bramble/lib/std") - - -def cacerts(): - """cacerts provides known certificate authority certificates to verify TLS connections""" - return derivation( - name="ca-certificates", - builder=busybox().out + "/bin/sh", - env=dict( - PATH=busybox().out + "/bin", - src=std.fetch_url("https://brmbl.s3.amazonaws.com/ca-certificates.crt"), - ), - args=[ - "-c", - """ - set -ex - cp -r $src/ca-certificates.crt $out - cp $out/ca-certificates.crt $out/ca-bundle.crt - """, - ], - ) - - -def git(): - b = std.fetch_url("http://s.minos.io/archive/bifrost/x86_64/git-2.10.2-1.tar.gz") - return derivation( - name="git", - builder=busybox().out + "/bin/sh", - env=dict(src=b, PATH=busybox().out + "/bin"), - args=[ - "-c", - """ - set -ex - cp -r $src/usr/* $out - mkdir test - cd test - $out/bin/git --version - """, - ], - ) - - -def git_fetcher(): - return derivation( - name="git_fetcher", - builder=busybox().out + "/bin/sh", - args=["-c", ""], # noop - env=dict( - git=git(), - PATH=git().out + "/bin", - GIT_EXEC_PATH=git().out + "/libexec/git-core", - GIT_SSL_CAINFO=cacerts().out + "/ca-certificates.crt", - ), - ) - - -# TODO: make reproducible -def git_test(): - return derivation( - "git-test", - "fetch_git", - env=dict( - url="https://github.com/maxmcd/bramble.git", reference="v0", cachebust=1 - ), - ) - - -def zig(): - b = std.fetch_url( - "https://ziglang.org/builds/zig-linux-x86_64-0.9.0-dev.946+6237dc0ab.tar.xz" - ) - return derivation( - name="zig", - builder=busybox().out + "/bin/sh", - env=dict(src=b, PATH=busybox().out + "/bin"), - args=[ - "-c", - """ - set -ex - cp -r $src/zig-linux-x86_64-0.9.0-dev.946+6237dc0ab/* $out - ls -lah $out/ - #$out/zig init-lib - mkdir $out/bin - cd $out/bin - ln -s ../zig ./zig - """, - ], - ) - - -def busybox(): - b = std.fetch_url("https://brmbl.s3.amazonaws.com/busybox-x86_64.tar.gz") - script = """ - set -ex - $busybox_download/busybox-x86_64 mkdir $out/bin - $busybox_download/busybox-x86_64 cp $busybox_download/busybox-x86_64 $out/bin/busybox - cd $out/bin - for command in $(./busybox --list); do - ./busybox ln -s busybox $command - done - """ - - return derivation( - name="busybox", - builder=b.out + "/busybox-x86_64", - args=["sh", "-c", script], - env={"busybox_download": b, "PATH": b.out}, - ) diff --git a/lib/deno/default.bramble b/lib/deno/default.bramble deleted file mode 100644 index aa41b064..00000000 --- a/lib/deno/default.bramble +++ /dev/null @@ -1,27 +0,0 @@ -load("github.com/maxmcd/bramble/lib/std") -load("github.com/maxmcd/bramble/lib/stdenv") - - -def deno(): - b = std.fetch_url( - "https://github.com/denoland/deno/releases/download/v1.14.0/deno-x86_64-unknown-linux-gnu.zip" - ) - return stdenv.std_derivation( - name="deno", - env=dict(src=b), - args=[ - "-c", - """ - set -ex - ls -lah $src - ls -lah $nix_seed - mkdir $out/bin - cp $src/deno $out/bin/deno - patchelf --remove-rpath $out/bin/deno - patchelf --set-interpreter $nix_seed/lib/ld-linux-x86-64.so.2 \ - --set-rpath $nix_seed/lib $out/bin/deno - - $out/bin/deno --version - """, - ], - ) diff --git a/lib/go/build.sh b/lib/go/build.sh deleted file mode 100644 index 64fffd26..00000000 --- a/lib/go/build.sh +++ /dev/null @@ -1,31 +0,0 @@ -set -ex - -export LD_LIBRARY_PATH=$stdenv/lib - -mkdir -p /var/tmp - -cp -r $go1_4/go $out -cd $out -include=$(pwd)/go/include -cd $out/go/src -ls -lah ./cmd/dist -export GO_LDFLAGS="-L $stdenv/lib -I $include -I $stdenv/include-glibc -I $stdenv/include" -export CC="gcc -L $stdenv/lib -I $include -I $stdenv/include-glibc -I $stdenv/include -Wl,-rpath=$stdenv/lib -Wl,--dynamic-linker=$stdenv/lib/ld-linux-x86-64.so.2 " - -export CGO_ENABLED="0" -sed -i 's/set -e/set -ex/g' ./make.bash - -bash ./make.bash - - -mkdir $out/bin -cd $out/bin -ln -s ../go/bin/go ./go -ln -s ../go/bin/gofmt ./gofmt - -# this works for a few things, but has trouble finding the network, resolve.conf -# syslog, no go in PATH, -# stat testdata/libmach8db: no such file or directory -# stat /Users/rsc/bin/xed: no such file or directory -# stat /usr/local/bin/arm-linux-elf-objdump: no such file or directory -# ../bin/go test all diff --git a/lib/go/default.bramble b/lib/go/default.bramble deleted file mode 100644 index 33d621f7..00000000 --- a/lib/go/default.bramble +++ /dev/null @@ -1,30 +0,0 @@ -load("github.com/maxmcd/bramble/lib/std") -load(nix_seed="github.com/maxmcd/bramble/lib/nix-seed") -load("github.com/maxmcd/bramble/lib") - - -def _bootstrap(): - go1_4 = std.fetch_url("https://dl.google.com/go/go1.4-bootstrap-20171003.tar.gz") - path = "%s/bin:%s/bin" % (nix_seed.stdenv(), lib.busybox()) - return derivation( - name="go-1.4", - builder=lib.busybox().out + "/bin/sh", - args=["./build.sh"], - sources=files(["./build.sh"]), - env=dict( - go1_4=go1_4, stdenv=nix_seed.stdenv(), busybox=lib.busybox(), PATH=path - ), - ) - - -def go_1_17(): - go_1_17 = std.fetch_url("https://golang.org/dl/go1.17.2.linux-amd64.tar.gz") - path = "%s/bin:%s/bin" % (nix_seed.stdenv(), lib.busybox()) - return derivation( - name="go-1.4", - builder=lib.busybox().out + "/bin/sh", - args=["-c", """"""], - env=dict( - go=go_1_17, stdenv=nix_seed.stdenv(), busybox=lib.busybox(), PATH=path - ), - ) diff --git a/lib/nix-seed/build_stdenv.sh b/lib/nix-seed/build_stdenv.sh deleted file mode 100644 index 59e7722d..00000000 --- a/lib/nix-seed/build_stdenv.sh +++ /dev/null @@ -1,32 +0,0 @@ -set -e - - -PATH=$PATH:$patchelf/bin:$busybox/bin - -echo $PATH - -cp -r $src/* $out -export LD_LIBRARY_PATH=$out/lib - -for filename in $out/bin/*; do - if readlink $filename | grep -q "coreutils"; then - # https://github.com/NixOS/patchelf/issues/96 - continue - fi - patchelf --remove-rpath $filename - patchelf --set-interpreter $out/lib/ld-linux-x86-64.so.2 \ - --set-rpath $out/lib $filename -done - -$out/bin/bash --help -$out/bin/coreutils --help - -for filename in $out/libexec/gcc/x86_64-unknown-linux-gnu/8.3.0/*; do - # ignore liblto - if echo $filename | grep -q "liblto"; then - continue - fi - patchelf --remove-rpath $filename - patchelf --set-interpreter $out/lib/ld-linux-x86-64.so.2 \ - --set-rpath $out/lib $filename -done diff --git a/lib/nix-seed/default.bramble b/lib/nix-seed/default.bramble deleted file mode 100644 index 8d8565d3..00000000 --- a/lib/nix-seed/default.bramble +++ /dev/null @@ -1,28 +0,0 @@ -load(static_patchelf="github.com/maxmcd/bramble/lib/static-patchelf") -load("github.com/maxmcd/bramble/lib/std") -load("github.com/maxmcd/bramble/lib") - - -def stdenv(): - - new_patchelf = std.fetch_url( - "https://github.com/NixOS/patchelf/releases/download/0.13/patchelf-0.13.tar.bz2" - ) - """ - the standard environment - """ - src = std.fetch_url( - "http://tarballs.nixos.org/stdenv-linux/x86_64/c5aabb0d603e2c1ea05f5a93b3be82437f5ebf31/bootstrap-tools.tar.xz" - ) - return derivation( - "stdenv", - builder=lib.busybox().out + "/bin/sh", - args=["./build_stdenv.sh"], - sources=files(["./build_stdenv.sh"]), - env=dict( - src=src, - patchelf=static_patchelf.patchelf(), - new_patchelf=new_patchelf, - busybox=lib.busybox(), - ), - ) diff --git a/lib/static-patchelf/Dockerfile b/lib/static-patchelf/Dockerfile deleted file mode 100644 index b14d7f5b..00000000 --- a/lib/static-patchelf/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM debian:10.5 - -RUN apt-get update && apt-get install -y build-essential wget - -ENV CFLAGS="--static" -ENV CXXFLAGS="--static" -WORKDIR /opt - -RUN wget https://github.com/NixOS/patchelf/releases/download/0.12/patchelf-0.12.tar.bz2 -RUN tar xf patchelf-0.12.tar.bz2 -WORKDIR /opt/patchelf-0.12.20200827.8d3a16e -RUN ./configure -RUN make -RUN make install -WORKDIR /usr/local/bin -RUN tar -czvf patchelf.tar.gz patchelf diff --git a/lib/static-patchelf/build.sh b/lib/static-patchelf/build.sh deleted file mode 100755 index 461700c3..00000000 --- a/lib/static-patchelf/build.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -set -Eeuxo pipefail -cd "$(dirname ${BASH_SOURCE[0]})" - -docker build -t static-patchelf . - - -id=$(docker create static-patchelf) -docker cp $id:/usr/local/bin/patchelf.tar.gz patchelf.tar.gz -docker rm -v $id diff --git a/lib/static-patchelf/default.bramble b/lib/static-patchelf/default.bramble deleted file mode 100644 index 96dcacd8..00000000 --- a/lib/static-patchelf/default.bramble +++ /dev/null @@ -1,36 +0,0 @@ -load("github.com/maxmcd/bramble/lib") - - -# def _upload(): -# s = os.session() -# s = s.setenv("AWS_PROFILE", "max") -# print( -# s.cmd( -# "aws s3 cp ./patchelf.tar.gz s3://brmbl/patchelf.tar.gz --acl public-read", -# ).output() -# ) - - -def patchelf(): - patch_dl = derivation( - name="patch_dl", - builder="fetch_url", - env={ - "url": "https://brmbl.s3.amazonaws.com/patchelf.tar.gz", - "hash": "icpfggjznz3jxnctxtcky55g7zhbsk4u", - }, - ) - - return derivation( - name="patchelf", - builder=lib.busybox().out + "/bin/sh", - args=[ - "-c", - """ - echo $out/bin - mkdir $out/bin - cp $patch_dl/patchelf $out/bin - """, - ], - env=dict(patch_dl=patch_dl, PATH=lib.busybox().out + "/bin"), - ) diff --git a/lib/stdenv/default.bramble b/lib/stdenv/default.bramble deleted file mode 100644 index 7a325951..00000000 --- a/lib/stdenv/default.bramble +++ /dev/null @@ -1,26 +0,0 @@ -"""the stdenv wooooo""" - -load(nix_seed="github.com/maxmcd/bramble/lib/nix-seed") -load("github.com/maxmcd/bramble/lib") -load(static_patchelf="github.com/maxmcd/bramble/lib/static-patchelf") - - -stdenv = nix_seed.stdenv - - -def std_derivation(name, builder=None, **kwargs): - nix = nix_seed.stdenv() - bb = lib.busybox() - - PATH = "{}/bin:{}/bin:{}bin".format(nix.out, bb.out, static_patchelf.patchelf()) - env = kwargs.get("env", {}) - kwargs.pop("env") - env.update(dict(PATH=PATH, nix_seed=nix.out, stdenv=nix.out, busybox=bb.out)) - if builder == None: - builder = nix.out + "/bin/bash" - return derivation( - name, - builder, - env=env, - **kwargs, - ) diff --git a/lib/zig/default.bramble b/lib/zig/default.bramble deleted file mode 100644 index 2fab8776..00000000 --- a/lib/zig/default.bramble +++ /dev/null @@ -1,18 +0,0 @@ -load("github.com/maxmcd/bramble/lib") - - -def hello(): - return derivation( - name="zig", - builder=lib.busybox().out + "/bin/sh", - args=[ - "-c", - """ - set -e - time zig build-exe hello.zig - ./hello - """, - ], - sources=files(["./hello.zig"]), - env=dict(PATH=lib.busybox().out + "/bin:" + lib.zig().out + "/bin"), - ) diff --git a/lib/zig/hello.zig b/lib/zig/hello.zig deleted file mode 100644 index 266bff31..00000000 --- a/lib/zig/hello.zig +++ /dev/null @@ -1,6 +0,0 @@ -const std = @import("std"); - -pub fn main() !void { - const stdout = std.io.getStdOut().writer(); - try stdout.print("Hello hello, {s}!\n", .{"world"}); -} diff --git a/notes/09-dependecies.md b/notes/09-dependecies.md index f85b11a4..a4c96287 100644 --- a/notes/09-dependecies.md +++ b/notes/09-dependecies.md @@ -140,3 +140,18 @@ Then what's the subset of these paths that have "github.com/a/b/c/d" as a valid ------- We must use the `...` path expansion because otherwise we have no other way do indicate the difference between "all things at this path" and "just the module at this path" since both of those things could have the same import path. ie: `foo.com/a/b` and `foo.com/a/b` could refer either to "all modules in the b folder" or the specific file "foo/com/a/b/default.bramble". We could call the first one `foo.com/a/b/...` + + +------ + +I think we're getting close to something that works, or is at least well defined and then might not work for complex reasons. + +We have a project. It pulls in dependency "a/b". + +If you `bramble install(???) 1/2` or `bramble install(???) 1/2/a/b` it will search for the longest module name that matches. So if there is also a module `1/2/a` that will be pulled by the second command. + +We basically assume import paths aren't overwritten within major version numbers. + +However, if we search for `bramble install(???) 1/2@v1` or `bramble install(???) 1/2/a/b@v1` or `bramble install(???) 1/2@v1.1.0` we search for compatible modules within that search path. We always look for major version module compatibility first, then search for feature/patch versions, so you can't use a more specific version to override the module search logic. + + diff --git a/notes/10-file-copying.md b/notes/10-source-file-copying.md similarity index 59% rename from notes/10-file-copying.md rename to notes/10-source-file-copying.md index 64f45113..0176e5b2 100644 --- a/notes/10-file-copying.md +++ b/notes/10-source-file-copying.md @@ -8,3 +8,9 @@ - when you build copy all the sources back into a tmpdir and run the build from there - if this is a problem we can optimize the use-case of minmially copying files within an active project - how does this look on the derivation? Inputsrc points to the store and has an entrypoint location? + +------------ + +- You could mount the sources into the container instead of copying files? Namespace filtering rules might be sufficient to show the correct files. +- For ^ you'd have to be able to re-assemble the source for uploads, could always re-compute though. +- Could have a cache hash like git, with mod time to confirm that a file doesn't need to be re-hashed. diff --git a/notes/25-dynamic-dependencies.md b/notes/25-dynamic-dependencies.md index 5a624184..bd982ecf 100644 --- a/notes/25-dynamic-dependencies.md +++ b/notes/25-dynamic-dependencies.md @@ -90,4 +90,12 @@ So now we: Ok, so regular build uses the network and re-generates the file if the sources have changed. Is there a way to track this? Maybe this goes in the lockfile? Yeah whatf do we do with the generate step? It's a derivation? It's a derivation that points at a generated file? How does that reference work? It's in the special value. So when a file changes, we re-generate. How do we know a file has changed? With generated file, we skip it, we just trust the generated file???? Yeah this is nasty, write it out... -Ah yes, ok, the problem is, how do we know that the input files have changed? +Ah yes, ok, the problem is, how do we know that the input files have changed? (add a hash to generated file) + +------------------- + +More thoughts on this: + +Maybe could just do recursive nix? Requirement is that the derivation that is calling bramble from within a derivation must not link to any dynamic dependencies in the output. Could allow people to have relatively full control of the build, but just aren't allowed to link out. Would be good for compiled languages, or similar. + +Tough that we don't know what the build graph is going to be in advance. Means we can't proactively fetch cache. Is there a way to persist what derivations were called on by that derivation? Maybe we just write it to the file as a non-hashable annotation? Oh yeah, could go in the output section. On a fresh build though we're still flying blind, maybe that's ok... not sure. diff --git a/notes/40-global-install.md b/notes/40-global-install.md new file mode 100644 index 00000000..2b108d4f --- /dev/null +++ b/notes/40-global-install.md @@ -0,0 +1,11 @@ +Thinking about the alias feature for projects. We could just maintain a `bramble.toml` that is used to configure the user's global commands. + +We could have a `$BRAMBLE_PATH/bramble.toml` and when a user did `bramble global alias github.com/maxmcd/busybox ash` we would add the alias to the global file. We would then also need to get `$BRAMBLE_PATH/bin` into a users path, probably, and then add shims to that folder to invoke bramble as that program. + +We could also support history of global installs this way, if desired, so that a user can roll back/forward their program versions. + +We'd be maintaining a `bramble.lock` for these aliases and tracking their sub-dependencies so that they're pinned at specific versions. + +Still need to think about "user global" vs "system global", but there are many more problems there. + + diff --git a/pkg/chunkedarchive/chunkedarchive.go b/pkg/chunkedarchive/chunkedarchive.go deleted file mode 100644 index 6cc5b89d..00000000 --- a/pkg/chunkedarchive/chunkedarchive.go +++ /dev/null @@ -1,312 +0,0 @@ -package chunkedarchive - -import ( - "archive/tar" - "fmt" - "io" - "os" - "path" - "path/filepath" - "strings" - "syscall" - "time" - - "github.com/pkg/errors" -) - -const ( - chunkSize int = 4e6 -) - -// TOCEntry is an entry in the file index's TOC (Table of Contents). -type TOCEntry struct { - // Name is the tar entry's name. It is the complete path - // stored in the tar file, not just the base name. - Name string `json:"name"` - - // Type is one of "dir", "reg", "symlink", "hardlink", "char", - // "block", "fifo" - Type string `json:"type"` - - // Size, for regular files, is the logical size of the file. - Size int64 `json:"size,omitempty"` - - // LinkName, for symlinks and hardlinks, is the link target. - LinkName string `json:"linkName,omitempty"` - - // Mode is the permission and mode bits. - Mode int64 `json:"mode,omitempty"` - - // DevMajor is the major device number for "char" and "block" types. - DevMajor int `json:"devMajor,omitempty"` - - // DevMinor is the major device number for "char" and "block" types. - DevMinor int `json:"devMinor,omitempty"` - - // NumLink is the number of entry names pointing to this entry. - // Zero means one name references this entry. - NumLink int - - // Xattrs are the extended attribute for the entry. - Xattrs map[string][]byte `json:"xattrs,omitempty"` - - // Body references hashes of body content - Body []string `json:"digest,omitempty"` -} - -type fileInfo struct{ e *TOCEntry } - -var _ os.FileInfo = fileInfo{} - -func (fi fileInfo) Name() string { return path.Base(fi.e.Name) } -func (fi fileInfo) IsDir() bool { return fi.e.Type == "dir" } -func (fi fileInfo) Size() int64 { return fi.e.Size } -func (fi fileInfo) ModTime() time.Time { return time.Time{} } -func (fi fileInfo) Sys() interface{} { return fi.e } -func (fi fileInfo) Mode() (m os.FileMode) { - m = os.FileMode(fi.e.Mode) & os.ModePerm - switch fi.e.Type { - case "dir": - m |= os.ModeDir - case "symlink": - m |= os.ModeSymlink - case "char": - m |= os.ModeDevice | os.ModeCharDevice - case "block": - m |= os.ModeDevice - case "fifo": - m |= os.ModeNamedPipe - } - // TODO: ModeSetuid, ModeSetgid, if/as needed. - return m -} - -type BodyWriter interface { - NewChunk(io.ReadCloser) (func() ([]string, error), error) -} - -func Archive(bw BodyWriter, location string) (toc []TOCEntry, err error) { - type entryPromise struct { - entry TOCEntry - promise func() ([]string, error) - } - queue := []entryPromise{} - - if err = filepath.Walk(location, func(path string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - var linkTarget string - if isSymlink(fi) { - var err error - linkTarget, err = os.Readlink(path) - if err != nil { - return fmt.Errorf("%s: readlink: %w", fi.Name(), err) - } - // TODO: convert from absolute to relative - } - // GNU Tar adds a slash to the end of directories, but Go removes them - if fi.IsDir() { - path += "/" - } - - // TODO: Could likely remove the tar dependency here pretty easily - hdr, err := tar.FileInfoHeader(fi, filepath.ToSlash(linkTarget)) - if err != nil { - return errors.WithStack(err) - } - - var xattrs map[string][]byte - if hdr.Xattrs != nil { - xattrs = make(map[string][]byte) - for k, v := range hdr.Xattrs { - xattrs[k] = []byte(v) - } - } - - ent := TOCEntry{ - Name: strings.TrimPrefix(path, location), - Size: fi.Size(), - Mode: hdr.Mode, - Xattrs: xattrs, - } - - switch hdr.Typeflag { - case tar.TypeLink: - // TODO: will this ever happen? File will just be read as a file? - ent.Type = "hardlink" - ent.LinkName = hdr.Linkname - case tar.TypeSymlink: - ent.Type = "symlink" - ent.LinkName = hdr.Linkname - case tar.TypeDir: - ent.Type = "dir" - case tar.TypeReg: - ent.Type = "reg" - ent.Size = hdr.Size - case tar.TypeFifo: - ent.Type = "fifo" - default: - return fmt.Errorf("unsupported input tar entry %q", hdr.Typeflag) - } - - if fi.IsDir() || hdr.Typeflag != tar.TypeReg || fi.Size() == 0 { - queue = append(queue, entryPromise{entry: ent}) - return nil // directories have no contents - } - var file io.ReadCloser - file, err = os.Open(path) - if err != nil { - return fmt.Errorf("%s: opening: %w", path, err) - } - promise, err := bw.NewChunk(file) - if err != nil { - return fmt.Errorf("%s: reading: %w", path, err) - } - queue = append(queue, entryPromise{entry: ent, promise: promise}) - return nil - }); err != nil { - return nil, errors.WithStack(err) - } - for _, ep := range queue { - if ep.promise != nil { - ep.entry.Body, err = ep.promise() - if err != nil { - return nil, err - } - } - toc = append(toc, ep.entry) - } - return -} - -type ParallelBodyWriter struct { - sem chan struct{} - cb func(io.ReadCloser) ([]string, error) -} - -var _ BodyWriter = new(ParallelBodyWriter) - -func NewParallelBodyWriter(numParallel int, cb func(io.ReadCloser) ([]string, error)) *ParallelBodyWriter { - bw := &ParallelBodyWriter{ - sem: make(chan struct{}, numParallel), - cb: cb, - } - return bw -} - -func (bw *ParallelBodyWriter) NewChunk(body io.ReadCloser) (func() ([]string, error), error) { - type result struct { - hashes []string - err error - } - // Take slot to do work - bw.sem <- struct{}{} - out := make(chan result) - go func() { - r := result{} - r.hashes, r.err = bw.cb(body) - out <- r - }() - return func() ([]string, error) { - r := <-out - // Free up slot - <-bw.sem - return r.hashes, r.err - }, nil -} - -type HashFetcher interface { - Lookup(hash string) (io.ReadCloser, error) -} - -func Unarchive(toc []TOCEntry, fetcher HashFetcher, location string) (err error) { - errChan := make(chan error) - type Chunk struct { - hash string - body io.ReadCloser - } - fetchchan := make(chan Chunk, 20) // arbitrary, just to not block - go func() { - for _, ent := range toc { - for _, hash := range ent.Body { - body, err := fetcher.Lookup(hash) - if err != nil { - errChan <- err - } - fetchchan <- Chunk{body: body, hash: hash} - } - } - }() - - madeDir := map[string]bool{} - for _, ent := range toc { - rel := filepath.FromSlash(ent.Name) - abs := filepath.Join(location, rel) - fi := fileInfo{&ent} - mode := fi.Mode() - switch mode & os.ModeType { - case os.ModeDir: - if err := os.MkdirAll(abs, 0755); err != nil { - return err - } - madeDir[abs] = true - case os.ModeSymlink: - if err := os.Symlink(ent.LinkName, abs); err != nil { - return err - } - case os.ModeNamedPipe: - if err := syscall.Mkfifo(abs, uint32(mode.Perm())); err != nil { - return err - } - default: - dir := filepath.Dir(abs) - if !madeDir[dir] { - if err := os.MkdirAll(dir, 0755); err != nil { - return err - } - madeDir[dir] = true - } - wf, err := os.OpenFile(abs, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode.Perm()) - if err != nil { - return errors.WithStack(err) - } - var n int64 - for _, hash := range ent.Body { - var chunk Chunk - select { - case err := <-errChan: - return err - case chunk = <-fetchchan: - } - if chunk.hash != hash { - panic("toc hashes must match") - } - i, err := io.Copy(wf, chunk.body) - if closeErr := chunk.body.Close(); closeErr != nil && err == nil { - err = closeErr - } - n += i - if err != nil { - return errors.Wrapf(err, "error writing to %s", abs) - } - } - if err := wf.Close(); err != nil { - return fmt.Errorf("error writing to %s: %v", abs, err) - } - if n != ent.Size { - return fmt.Errorf("only wrote %d bytes to %s; expected %d", n, abs, ent.Size) - } - } - } - - return -} - -func isSymlink(fi os.FileInfo) bool { - return fi.Mode()&os.ModeSymlink != 0 -} - -func makedev(major, minor int64) int { - return int(((major & 0xfff) << 8) | (minor & 0xff) | ((major &^ 0xfff) << 32) | ((minor & 0xfffff00) << 12)) -} diff --git a/pkg/chunkedarchive/file.go b/pkg/chunkedarchive/file.go deleted file mode 100644 index a5fbcca0..00000000 --- a/pkg/chunkedarchive/file.go +++ /dev/null @@ -1,29 +0,0 @@ -package chunkedarchive - -import ( - "io" - "os" -) - -func FileArchive(location string, archive string) error { - f, err := os.Create(archive) - if err != nil { - return err - } - if err := StreamArchive(f, location); err != nil { - return err - } - return f.Close() -} - -func FileUnarchive(archive, dest string) error { - f, err := os.Open(archive) - if err != nil { - return err - } - fi, err := f.Stat() - if err != nil { - return err - } - return StreamUnarchive(io.NewSectionReader(f, 0, fi.Size()), dest) -} diff --git a/pkg/chunkedarchive/stream.go b/pkg/chunkedarchive/stream.go deleted file mode 100644 index 94b5402d..00000000 --- a/pkg/chunkedarchive/stream.go +++ /dev/null @@ -1,268 +0,0 @@ -package chunkedarchive - -import ( - "archive/tar" - "bufio" - "bytes" - "compress/gzip" - "encoding/json" - "fmt" - "io" - "strconv" - "sync" - - "github.com/maxmcd/bramble/pkg/hasher" - "github.com/pkg/errors" -) - -const ( - // footerSize is the number of bytes in the chunkedarchive footer. - // - // The footer is an empty gzip stream with no compression and an Extra - // header of the form "%016xCHUNKA", where the 64 bit hex-encoded - // number is the offset to the gzip stream of JSON TOC. - // - // 47 comes from: - // - // 10 byte gzip header + - // 2 byte (LE16) length of extra, encoding 22 (16 hex digits + len("CHUNKA")) == "\x16\x00" + - // 22 bytes of extra (fmt.Sprintf("%016xCHUNKA", tocGzipOffset)) - // 5 byte flate header - // 8 byte gzip footer (two little endian uint32s: digest, size) - footerSize = 47 - tocTarName = "chunkedarchive.index.json" -) - -// StreamArchive and UnarchiveStream stick the chunked archive into a single -// file. So far this use case is just for local archiving and replacing bytes, -// but it implements some of the patterns present in crfs/stargz in case that is -// a future use-case for this format. For the time being we'll assume any -// network requests for files will download individual chunks from an index, but -// for now it seemed good to follow in this general direction over picking -// something arbitrary. -func StreamArchive(output io.Writer, location string) (err error) { - buf := bufio.NewWriter(output) - countW := &countWriter{w: buf} - - zw, _ := gzip.NewWriterLevel(countW, gzip.NoCompression) - tw := tar.NewWriter(zw) - bw := &tarBodyWriter{ - writer: tw, - buf: make([]byte, chunkSize), - } - toc, err := Archive(bw, location) - if err != nil { - return err - } - if err := tw.Close(); err != nil { - return err - } - if err := zw.Close(); err != nil { - return err - } - - tocOff := countW.n - // Write toc - { - zw, _ = gzip.NewWriterLevel(countW, gzip.NoCompression) - tocJSON, err := json.MarshalIndent(toc, "", " ") - if err != nil { - return err - } - tw := tar.NewWriter(zw) - if err := tw.WriteHeader(&tar.Header{ - Typeflag: tar.TypeReg, - Name: tocTarName, - Size: int64(len(tocJSON)), - }); err != nil { - return err - } - if _, err := tw.Write(tocJSON); err != nil { - return err - } - - if err := tw.Close(); err != nil { - return err - } - if err := zw.Close(); err != nil { - return err - } - } - - // And a little footer with pointer to the TOC gzip stream. - if _, err := countW.Write(footerBytes(tocOff)); err != nil { - return err - } - return buf.Flush() -} - -type tarBodyWriter struct { - writer *tar.Writer - buf []byte -} - -func (bw *tarBodyWriter) NewChunk(f io.ReadCloser) (func() ([]string, error), error) { - out := []string{} - defer f.Close() - for { - h := hasher.New() - tr := io.TeeReader(f, h) - - n, err := tr.Read(bw.buf) - if err != nil { - return nil, err - } - hash := h.String() - out = append(out, hash) - if err := bw.writer.WriteHeader(&tar.Header{ - Typeflag: tar.TypeReg, - Name: hash, - Size: int64(n), - }); err != nil { - return nil, err - } - - if _, err := bw.writer.Write(bw.buf[:n]); err != nil { - return nil, err - } - if n != chunkSize { - break - } - } - return func() ([]string, error) { return out, nil }, nil -} - -// footerBytes the 47 byte footer. -func footerBytes(tocOff int64) []byte { - buf := bytes.NewBuffer(make([]byte, 0, footerSize)) - gz, _ := gzip.NewWriterLevel(buf, gzip.NoCompression) - gz.Header.Extra = []byte(fmt.Sprintf("%016xCHUNKA", tocOff)) - gz.Close() - if buf.Len() != footerSize { - panic(fmt.Sprintf("footer buffer = %d, not %d", buf.Len(), footerSize)) - } - return buf.Bytes() -} - -// countWriter counts how many bytes have been written to its wrapped -// io.Writer. -type countWriter struct { - w io.Writer - n int64 -} - -func (cw *countWriter) Write(p []byte) (n int, err error) { - n, err = cw.w.Write(p) - cw.n += int64(n) - return -} - -func StreamUnarchive(sr *io.SectionReader, location string) (err error) { - // Pull TOC from footer - toc := []TOCEntry{} - { - if sr.Size() < footerSize { - return errors.Errorf("chunkedarchive size %d is smaller than the archive footer size", sr.Size()) - } - var footer [footerSize]byte - if _, err := sr.ReadAt(footer[:], sr.Size()-footerSize); err != nil { - return errors.Errorf("error reading footer: %v", err) - } - tocOff, ok := parseFooter(footer[:]) - if !ok { - return errors.Errorf("error parsing footer") - } - tocTargz := make([]byte, sr.Size()-tocOff-footerSize) - if _, err := sr.ReadAt(tocTargz, tocOff); err != nil { - return errors.Errorf("error reading %d byte TOC targz: %v", len(tocTargz), err) - } - zr, err := gzip.NewReader(bytes.NewReader(tocTargz)) - if err != nil { - return errors.Errorf("malformed TOC gzip header: %v", err) - } - zr.Multistream(false) - tr := tar.NewReader(zr) - h, err := tr.Next() - if err != nil { - return errors.Errorf("failed to find tar header in TOC gzip stream: %v", err) - } - if h.Name != tocTarName { - return errors.Errorf("TOC tar entry had name %q; expected %q", h.Name, tocTarName) - } - if err := json.NewDecoder(tr).Decode(&toc); err != nil { - return errors.Errorf("error decoding TOC JSON: %v", err) - } - } - // Seek to beginning of file and start writing files - if _, err := sr.Seek(0, 0); err != nil { - return err - } - gr, err := gzip.NewReader(sr) - if err != nil { - return err - } - return Unarchive(toc, &tarSerialHashFetcher{ - reader: tar.NewReader(gr), - wg: &sync.WaitGroup{}, - }, location) -} - -type tarSerialHashFetcher struct { - reader *tar.Reader - wg *sync.WaitGroup -} - -var _ HashFetcher = new(tarSerialHashFetcher) - -// wgLimitReader will call Done() on a waitgroup when reading is done -type wgLimitReader struct { - reader io.Reader - wg *sync.WaitGroup - done bool -} - -func (r *wgLimitReader) Read(p []byte) (n int, err error) { - n, err = r.reader.Read(p) - if err == io.EOF && !r.done { - r.done = true - r.wg.Done() - } - return n, err -} - -func (hf *tarSerialHashFetcher) Lookup(hash string) (io.ReadCloser, error) { - // Lookup will only read one chunk at a time and not proceed to read the - // next header until the current reader has been completely read - hf.wg.Wait() - th, err := hf.reader.Next() - if err != nil { - return nil, err - } - if hash != th.Name { - return nil, errors.New("tar header name and chunk hash do not match") - } - hf.wg.Add(1) - return io.NopCloser(&wgLimitReader{ - reader: io.LimitReader(hf.reader, th.Size), - wg: hf.wg, - }), nil -} - -func parseFooter(p []byte) (tocOffset int64, ok bool) { - if len(p) != footerSize { - return 0, false - } - zr, err := gzip.NewReader(bytes.NewReader(p)) - if err != nil { - return 0, false - } - extra := zr.Header.Extra - if len(extra) != 16+len("CHUNKA") { - return 0, false - } - if string(extra[16:]) != "CHUNKA" { - return 0, false - } - tocOffset, err = strconv.ParseInt(string(extra[:16]), 16, 64) - return tocOffset, err == nil -} diff --git a/pkg/chunkedarchive/stream_test.go b/pkg/chunkedarchive/stream_test.go deleted file mode 100644 index 176daee5..00000000 --- a/pkg/chunkedarchive/stream_test.go +++ /dev/null @@ -1,159 +0,0 @@ -package chunkedarchive - -import ( - "crypto/rand" - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "syscall" - "testing" - - "github.com/maxmcd/bramble/pkg/hasher" - "github.com/maxmcd/bramble/pkg/reptar" - "github.com/stretchr/testify/require" -) - -func TestArchive(t *testing.T) { - tests := []struct { - name string - in []entry - }{ - { - name: "one file", - in: entries(file("foo")), - }, - { - name: "files and dir", - in: entries( - dir("thing"), - file("foo"), - emptyFile("empty"), - ), - }, - { - name: "nested files", - in: entries( - dir("thing"), - file("foo"), - file("thing/bar"), - ), - }, - { - name: "unusual suspects", - in: entries( - dir("thing"), - file("thing/bar"), - symlink("thing/bar", "symlink"), - hardlink("thing/bar", "hardlink"), - fifo("thing/pipe"), - ), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - dir := t.TempDir() - for _, e := range tt.in { - if err := e(dir); err != nil { - t.Fatal(err) - } - } - f, err := os.CreateTemp("", "") - require.NoError(t, err) - f.Close() - t.Cleanup(func() { os.Remove(f.Name()) }) - - if err := FileArchive(dir, f.Name()); err != nil { - t.Fatal(err) - } - dir2 := t.TempDir() - - if err := FileUnarchive(f.Name(), dir2); err != nil { - t.Fatal(err) - } - - if reptarDir(t, dir) != reptarDir(t, dir2) { - { - cmd := "diff -qr " + dir + " " + dir2 - b, err := exec.Command("bash", "-c", cmd).CombinedOutput() - fmt.Println(string(b)) - if err != nil { - fmt.Println(cmd) - } - require.NoError(t, err) - } - cmd := "git diff --color=never --no-index " + dir + " " + dir2 - b, err := exec.Command("bash", "-c", cmd).CombinedOutput() - if err != nil { - fmt.Println(cmd) - fmt.Println(string(b)) - } - require.NoError(t, err) - } - }) - } -} - -func reptarDir(t *testing.T, location string) string { - h := hasher.New() - if err := reptar.Reptar(location, h); err != nil { - t.Fatal(err) - } - return h.String() -} - -type entry func(dir string) error - -func entries(v ...entry) []entry { - return v -} - -var j func(...string) string = filepath.Join - -func file(name string) func(dir string) error { - return func(dir string) error { - f, err := os.Create(j(dir, name)) - if err != nil { - return err - } - if _, err := io.CopyN(f, rand.Reader, 9e6); err != nil { - return err - } - return f.Close() - } -} - -func emptyFile(name string) func(dir string) error { - return func(dir string) error { - f, err := os.Create(j(dir, name)) - if err != nil { - return err - } - return f.Close() - } -} - -func dir(name string) func(dir string) error { - return func(dir string) error { - return os.Mkdir(j(dir, name), 0755) - } -} - -func hardlink(name, link string) func(dir string) error { - return func(dir string) error { - return os.Link(j(dir, name), j(dir, link)) - } -} - -func symlink(name, link string) func(dir string) error { - return func(dir string) error { - return os.Symlink(j(dir, name), j(dir, link)) - } -} - -func fifo(name string) func(dir string) error { - return func(dir string) error { - return syscall.Mkfifo(j(dir, name), 0755) - } -} diff --git a/pkg/fileutil/fileutil.go b/pkg/fileutil/fileutil.go index a0165646..aad78639 100644 --- a/pkg/fileutil/fileutil.go +++ b/pkg/fileutil/fileutil.go @@ -1,7 +1,6 @@ package fileutil import ( - "fmt" "io" "io/ioutil" "log" @@ -48,59 +47,6 @@ func CommonFilepathPrefix(paths []string) string { return string(c) } -func CP(wd string, paths ...string) (err error) { - if len(paths) == 1 { - return errors.New("copy takes at least two arguments") - } - absPaths := make([]string, 0, len(paths)) - for _, path := range paths { - if !filepath.IsAbs(path) { - absPaths = append(absPaths, filepath.Join(wd, path)) - } else { - absPaths = append(absPaths, path) - } - } - dest := absPaths[len(paths)-1] - // if dest exists and it's not a directory - if FileExists(dest) { - return errors.New("copy destination can't be a file that exists") - } - - toCopy := absPaths[:len(absPaths)-1] - - // "cp foo.txt bar.txt" or "cp ./foo ./bar" is a special case if it's just - // two paths and they don't exist yet - if len(toCopy) == 1 && !PathExists(dest) { - f := toCopy[0] - if IsDir(f) { - return errors.WithStack(CopyDirectory(f, dest)) - } - return errors.WithStack(CopyFile(f, dest)) - } - - // otherwise copy each listed file into a directory with the given name - for i, path := range toCopy { - // TODO: this should be Lstat. Do we need to add symlink support to CopyFile? - fi, err := os.Stat(path) - if err != nil { - return errors.Errorf("%q doesn't exist", paths[i]) - } - if fi.IsDir() { - destFolder := filepath.Join(dest, fi.Name()) - if err = CreateDirIfNotExists(destFolder, 0755); err != nil { - return err - } - err = CopyDirectory(path, filepath.Join(dest, fi.Name())) - } else { - err = CopyFile(path, filepath.Join(dest, fi.Name())) - } - if err != nil { - return errors.WithStack(err) - } - } - return nil -} - func ReplaceAll(filepath, old, new string) (err error) { f, err := os.Stat(filepath) if err != nil { @@ -134,11 +80,6 @@ func CopyDirectory(scrDir, dest string) error { return errors.WithStack(err) } - stat, ok := fileInfo.Sys().(*syscall.Stat_t) - _ = stat - if !ok { - return fmt.Errorf("failed to get raw syscall.Stat_t data for '%s'", sourcePath) - } switch fileInfo.Mode() & os.ModeType { case os.ModeSymlink: if err := CopySymLink(sourcePath, destPath); err != nil { @@ -157,6 +98,10 @@ func CopyDirectory(scrDir, dest string) error { } } + // stat, ok := fileInfo.Sys().(*syscall.Stat_t) + // if !ok { + // return fmt.Errorf("failed to get raw syscall.Stat_t data for '%s'", sourcePath) + // } // if err := os.Lchown(destPath, int(stat.Uid), int(stat.Gid)); err != nil { // return errors.WithStack(err) // } @@ -190,11 +135,6 @@ func CopyFilesByPath(prefix string, files []string, dest string) (err error) { return errors.Wrap(err, "error finding source file") } - stat, ok := fileInfo.Sys().(*syscall.Stat_t) - if !ok { - return errors.Errorf("failed to get raw syscall.Stat_t data for '%s'", file) - } - switch fileInfo.Mode() & os.ModeType { case os.ModeDir: if err := CreateDirIfNotExists(destPath, 0755); err != nil { @@ -209,10 +149,19 @@ func CopyFilesByPath(prefix string, files []string, dest string) (err error) { return errors.WithStack(err) } } - - if err := os.Lchown(destPath, int(stat.Uid), int(stat.Gid)); err != nil { - return errors.WithStack(err) - } + // TODO: Commenting this out (and the one in the function above) because + // we hit an issue where a file's group was `root`, but we can't write a + // file as the root group. Seems fine to leave this out, files should be + // greated with the default user and group of the of the current user, + // but just commenting for now. Who knows what the future holds. + // + // stat, ok := fileInfo.Sys().(*syscall.Stat_t) + // if !ok { + // return errors.Errorf("failed to get raw syscall.Stat_t data for '%s'", file) + // } + // if err := os.Lchown(destPath, int(stat.Uid), int(stat.Gid)); err != nil { + // return errors.WithStack(err) + // } // TODO: when does this happen??? isSymlink := fileInfo.Mode()&os.ModeSymlink != 0 diff --git a/pkg/httpx/httpx.go b/pkg/httpx/httpx.go index ca82a301..0abbf60a 100644 --- a/pkg/httpx/httpx.go +++ b/pkg/httpx/httpx.go @@ -37,6 +37,9 @@ type IM map[interface{}]interface{} func (err ErrHTTPResponse) Error() string { return err.err.Error() } func ErrNotFound(err error) error { return ErrHTTPResponse{err: err, code: http.StatusNotFound} } +func ErrInternalServerError(err error) error { + return ErrHTTPResponse{err: err, code: http.StatusInternalServerError} +} func ErrNotAcceptable(err error) error { return ErrHTTPResponse{err: err, code: http.StatusNotAcceptable} } diff --git a/pkg/io2/io2.go b/pkg/io2/io2.go new file mode 100644 index 00000000..b78d646e --- /dev/null +++ b/pkg/io2/io2.go @@ -0,0 +1,56 @@ +package io2 + +import "io" + +type writerMultiCloser struct { + io.Writer + closers []io.Closer +} + +func (w writerMultiCloser) Close() error { + for _, c := range w.closers { + if err := c.Close(); err != nil { + return err + } + } + return nil +} + +// WriterMultiCloser adds multiple closers to a writer. If close is called on +// the returned io.WriteCloser it will call all closers in order and return +// immediately if any errors are encountered +func WriterMultiCloser(w io.Writer, closers ...io.Closer) io.WriteCloser { + return writerMultiCloser{Writer: w, closers: closers} +} + +// ReaderMultiCloser adds multiple closers to a reader. If close is called on +// the returned io.ReadCloser it will call all closers in order and return +// immediately if any errors are encountered +func ReaderMultiCloser(r io.Reader, closers ...io.Closer) io.ReadCloser { + return readerMultiCloser{Reader: r, closers: closers} +} + +type readerMultiCloser struct { + io.Reader + closers []io.Closer +} + +func (w readerMultiCloser) Close() error { + for _, c := range w.closers { + if err := c.Close(); err != nil { + return err + } + } + return nil +} + +func WriterCloseFunc(w io.Writer, close func() error) io.WriteCloser { + return &writerCloseFunc{Writer: w, close: close} +} + +type writerCloseFunc struct { + io.Writer + close func() error +} + +func (w writerCloseFunc) Close() error { return w.close() } diff --git a/pkg/reptar/readme.md b/pkg/reptar/readme.md deleted file mode 100644 index 9175e242..00000000 --- a/pkg/reptar/readme.md +++ /dev/null @@ -1,6 +0,0 @@ -# Repatar - -TODO: write up a test suite with bramble that covers: - - relative paths everywhere - - correct symlink handling - - compatability to gnutar with the same inputs diff --git a/pkg/reptar/reptar.go b/pkg/reptar/reptar.go deleted file mode 100644 index af9b4e05..00000000 --- a/pkg/reptar/reptar.go +++ /dev/null @@ -1,120 +0,0 @@ -package reptar - -import ( - "archive/tar" - "compress/gzip" - "fmt" - "io" - "os" - "path/filepath" - "strings" - "time" -) - -var zeroTime time.Time - -// References: -// http://h2.jaguarpaw.co.uk/posts/reproducible-tar/ -// https://reproducible-builds.org/docs/archives/ - -// Reptar creates a tar of a location. Reptar stands for reproducible tar and is -// intended to replicate the following gnu tar command: -// -// tar - \ -// --sort=name \ -// --mtime="1970-01-01 00:00:00Z" \ -// --owner=0 --group=0 --numeric-owner \ -// --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime \ -// -cf -// -// This command is currently not complete and only works on very basic test -// cases. GNU Tar also adds padding to outputted files -func Reptar(location string, out io.Writer) (err error) { - // TODO: add our own null padding to match GNU Tar - // TODO: test with hardlinks - // TODO: confirm name sorting is identical in all cases - // TODO: disallow absolute paths - - tw := tar.NewWriter(out) - location = filepath.Clean(location) - if err = filepath.Walk(location, func(path string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - if location == path { - return nil - } - var linkTarget string - if isSymlink(fi) { - var err error - linkTarget, err = os.Readlink(path) - if err != nil { - return fmt.Errorf("%s: readlink: %w", fi.Name(), err) - } - // TODO: convert from absolute to relative - } - - // GNU Tar adds a slash to the end of directories, but Go removes them - if fi.IsDir() { - path += "/" - } - hdr, err := tar.FileInfoHeader(fi, filepath.ToSlash(linkTarget)) - if err != nil { - return err - } - - // Setting an explicit unix epoch using time.Date(1970, time.January..) - // resulted in zeros in the timestamp and not null, so we explicitly use - // a null time - hdr.ModTime = zeroTime - hdr.AccessTime = zeroTime - hdr.ChangeTime = zeroTime - - // It seems that both seeing these to 0 and using empty strings for - // Gname and Uname is required - hdr.Uid = 0 - hdr.Gid = 0 - hdr.Gname = "" - hdr.Uname = "" - - // pax format - hdr.Format = tar.FormatPAX - - hdr.Name = strings.TrimPrefix(path, location) - - if err = tw.WriteHeader(hdr); err != nil { - return fmt.Errorf("%s: writing header: %w", hdr.Name, err) - } - - if fi.IsDir() { - return nil // directories have no contents - } - if hdr.Typeflag == tar.TypeReg { - var file io.ReadCloser - file, err = os.Open(path) - if err != nil { - return fmt.Errorf("%s: opening: %w", path, err) - } - _, err := io.Copy(tw, file) - if err != nil { - return fmt.Errorf("%s: copying contents: %v", fi.Name(), err) - } - _ = file.Close() - } - return nil - }); err != nil { - return - } - return tw.Close() -} - -// GzipReptar just wraps reptar in gzip. -func GzipReptar(location string, out io.Writer) (err error) { - w := gzip.NewWriter(out) - defer w.Close() - return Reptar(location, w) -} - -func isSymlink(fi os.FileInfo) bool { - return fi.Mode()&os.ModeSymlink != 0 -} diff --git a/pkg/s3test/server.go b/pkg/s3test/server.go new file mode 100644 index 00000000..1539188e --- /dev/null +++ b/pkg/s3test/server.go @@ -0,0 +1,181 @@ +package s3test + +import ( + "crypto/md5" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "hash" + "io" + "net" + "net/http" + "net/http/httptest" + "net/http/httputil" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +type PostResponse struct { + ETag string + UploadID string `xml:"UploadId"` +} +type Server struct { + uploads map[string]*Upload + lock sync.Mutex + + server *httptest.Server + + objDir string +} + +func (s *Server) Hostname() string { + return s.server.Listener.Addr().String() +} + +func StartServer(t *testing.T, addr string) (s *Server) { + s = &Server{uploads: map[string]*Upload{}} + s.objDir = t.TempDir() + + s.server = httptest.NewUnstartedServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + if err := s.handler(rw, r); err != nil { + rw.WriteHeader(http.StatusInternalServerError) + fmt.Fprintln(rw, "error: "+err.Error()) + } + })) + listener, err := net.Listen("tcp", addr) + if err != nil { + t.Fatal(err) + } + s.server.Listener = listener + s.server.Start() + return s +} + +func (s *Server) handler(rw http.ResponseWriter, r *http.Request) (err error) { + b, _ := httputil.DumpRequest(r, false) + fmt.Println(string(b)) + uploadID := r.URL.Query().Get("uploadId") + switch r.Method { + case http.MethodGet: + loc := filepath.Join(s.objDir, strings.TrimPrefix(r.URL.Path, "/bramble")) + f, err := os.Open(loc) + if err == os.ErrNotExist { + rw.WriteHeader(http.StatusNotFound) + fmt.Fprintln(rw, "not found") + return nil + } + if err != nil { + return err + } + if _, err := io.Copy(rw, f); err != nil { + return err + } + return f.Close() + case http.MethodPost: + var pr PostResponse + if uploadID == "" { + // New + pr.UploadID, _ = s.newUpload(strings.TrimPrefix(r.URL.Path, "/bramble")) + } else { + // Existing + s.lock.Lock() + upload, found := s.uploads[uploadID] + s.lock.Unlock() + if !found { + return errors.New("upload not found ") + } + if err := upload.file.Close(); err != nil { + return err + } + pr.UploadID = uploadID + pr.ETag = fmt.Sprintf("%x", upload.md5OfParts.Sum(nil)) + } + + if err := xml.NewEncoder(rw).Encode(pr); err != nil { + return err + } + + case http.MethodPut: + if uploadID == "" { + return errors.New("no upload id") + } + partNumberStr := r.URL.Query().Get("partNumber") + hasPartNumber := partNumberStr != "" + var partNumber int + if hasPartNumber { + partNumber, err = strconv.Atoi(partNumberStr) + if err != nil { + panic(err) + } + } + s.lock.Lock() + upload, found := s.uploads[uploadID] + s.lock.Unlock() + if !found { + return errors.New("upload not found ") + } + if hasPartNumber { + upload.waitForPart(partNumber) + defer upload.incrementPartNumber() + } + + m := md5.New() + _, _ = io.Copy(io.MultiWriter(m, upload.file), r.Body) + upload.md5OfParts.Write(m.Sum(nil)) + rw.Header().Add("etag", fmt.Sprintf(`"%s"`, hex.EncodeToString(m.Sum(nil)))) + default: + panic("") + } + return nil +} + +func (s *Server) newUpload(path string) (id string, u *Upload) { + u = &Upload{} + id = fmt.Sprint(time.Now().UnixNano()) + objPath := filepath.Join(s.objDir, path) + _ = os.MkdirAll(filepath.Dir(objPath), 0755) + f, err := os.Create(objPath) + if err != nil { + panic(err) + } + u.file = f + + u.partNumber = 1 + u.md5OfParts = md5.New() + s.lock.Lock() + s.uploads[id] = u + s.lock.Unlock() + return +} + +type Upload struct { + md5OfParts hash.Hash + partNumber int + lock sync.Mutex + + file *os.File +} + +func (u *Upload) waitForPart(partNumber int) { + defer u.lock.Unlock() + for { + u.lock.Lock() + if u.partNumber == partNumber { + return + } + u.lock.Unlock() + time.Sleep(time.Millisecond * 5) + } +} + +func (u *Upload) incrementPartNumber() { + u.lock.Lock() + u.partNumber++ + u.lock.Unlock() +} diff --git a/pkg/s3test/server_test.go b/pkg/s3test/server_test.go new file mode 100644 index 00000000..e66bf86f --- /dev/null +++ b/pkg/s3test/server_test.go @@ -0,0 +1,60 @@ +package s3test + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "math/rand" + "net/http" + "os" + "testing" + + "github.com/rlmcpherson/s3gof3r" + "github.com/stretchr/testify/assert" +) + +func setACL(h http.Header, acl string) http.Header { + h.Set("x-amz-acl", acl) + return h +} + +func TestFoo(t *testing.T) { + s := StartServer(t, "127.0.0.1:8910") + host := s.Hostname() + + keys := s3gof3r.Keys{ + AccessKey: "a", + SecretKey: "b", + } + os.Setenv("AWS_REGION", " ") + s3 := s3gof3r.New(host, keys) + bucket := s3.Bucket("bramble") + + w, err := bucket.PutWriter("/output/foo", setACL(http.Header{}, "public-read"), &s3gof3r.Config{ + Client: http.DefaultClient, + Scheme: "http", // for this test + Md5Check: false, + PathStyle: true, // for this test + Concurrency: 12, // for this test + }) + if err != nil { + t.Fatal(err) + } + contents := make([]byte, 1e8) + if _, err := rand.Read(contents); err != nil { + t.Fatal(err) + } + _, _ = io.Copy(w, ioutil.NopCloser(bytes.NewBuffer(contents))) + if err := w.Close(); err != nil { + t.Fatal(err) + } + + resp, err := http.Get(fmt.Sprintf("http://%s/bramble/%s", s.Hostname(), "output/foo")) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + assert.Equal(t, b, contents) +} diff --git a/pkg/sandbox/sandbox.go b/pkg/sandbox/sandbox.go index d310936b..9eedd544 100644 --- a/pkg/sandbox/sandbox.go +++ b/pkg/sandbox/sandbox.go @@ -65,6 +65,12 @@ func (ee ExitError) Error() string { // Run runs the sandbox until execution has been completed func (s Sandbox) Run(ctx context.Context) (err error) { + select { + case <-ctx.Done(): + return context.Canceled + default: + } + container, err := newContainer(s) if err != nil { return err diff --git a/pkg/starutil/errors.go b/pkg/starutil/errors.go index 043e2962..00f3c839 100644 --- a/pkg/starutil/errors.go +++ b/pkg/starutil/errors.go @@ -3,11 +3,14 @@ package starutil import ( "bufio" "fmt" + "io" "os" "strings" "github.com/pkg/errors" + "go.starlark.net/resolve" "go.starlark.net/starlark" + "go.starlark.net/syntax" ) type ErrIncorrectType struct { @@ -27,6 +30,19 @@ func (err ErrUnhashable) Error() string { func AnnotateError(err error) string { sb := new(strings.Builder) + if err, ok := errors.Cause(err).(resolve.ErrorList); ok { + word := "errors" + if len(err) == 1 { + word = "error" + } + fmt.Fprintf(sb, "%d %s while resolving:\n", len(err), word) + for _, e := range err { + fmt.Fprintf(sb, "error: %s\n", e.Msg) + fmt.Fprintf(sb, " %s:\n", e.Pos) + lineWithArrow(sb, sourceLine(e.Pos.Filename(), e.Pos.Line), e.Pos, true) + } + return sb.String() + } if err, ok := errors.Cause(err).(*starlark.EvalError); ok { if len(err.CallStack) > 0 && err.CallStack.At(0).Pos.Filename() == "assert.star" { err.CallStack.Pop() @@ -45,14 +61,22 @@ func callStackString(stack starlark.CallStack) string { out := new(strings.Builder) fmt.Fprintf(out, "traceback (most recent call last):\n") - for _, fr := range stack { + for i, fr := range stack { fmt.Fprintf(out, " %s: in %s\n", fr.Pos, fr.Name) line := sourceLine(fr.Pos.Filename(), fr.Pos.Line) - fmt.Fprintf(out, " %s\n", strings.TrimSpace(line)) + lineWithArrow(out, line, fr.Pos, i == len(stack)-1) } return out.String() } +func lineWithArrow(out io.Writer, line string, pos syntax.Position, showArrow bool) { + trimmed := strings.TrimSpace(line) + fmt.Fprintf(out, " %s\n", trimmed) + if showArrow { + fmt.Fprintf(out, " %s^\n", strings.Repeat(" ", int(pos.Col)-(len(line)-len(trimmed)))) + } +} + func sourceLine(path string, lineNumber int32) string { f, err := os.Open(path) if err != nil { diff --git a/pkg/url2/url2.go b/pkg/url2/url2.go new file mode 100644 index 00000000..8958f80d --- /dev/null +++ b/pkg/url2/url2.go @@ -0,0 +1,51 @@ +package url2 + +import ( + "net/url" + "path" + "path/filepath" +) + +// Join is a url-aware path.Join, it will try and parse the first element as a +// valid url and then join any subsequent paths. If the first element errors +// when attempting to parse the passed elements will be joined with path.Join +func Join(elem ...string) string { + if len(elem) == 0 { + return "" + } + u, err := url.Parse(elem[0]) + if err != nil { + return path.Join(elem...) + } + pathElems := append([]string{u.Path}, elem[1:]...) + u.Path = path.Join(pathElems...) + // Preserve trailing slash if passed + if lastCharOfLastElem(pathElems) == "/" { + u.Path += "/" + } + return u.String() +} + +func Rel(basepath, targetpath string) (string, error) { + u, err := url.Parse(basepath) + if err != nil { + return filepath.Rel(basepath, targetpath) + } + u.Path, err = filepath.Rel(basepath, targetpath) + // Preserve trailing slash if passed + if err == nil && string(targetpath[len(u.Path)-1]) == "/" { + u.Path += "/" + } + return u.String(), nil +} + +func lastCharOfLastElem(elems []string) string { + if len(elems) == 0 { + return "" + } + last := elems[len(elems)-1] + if len(last) == 0 { + return "" + } + return string(last[len(last)-1]) +} diff --git a/pkg/url2/url2_test.go b/pkg/url2/url2_test.go new file mode 100644 index 00000000..c3423dbe --- /dev/null +++ b/pkg/url2/url2_test.go @@ -0,0 +1,28 @@ +package url2 + +import ( + "testing" +) + +func TestJoin(t *testing.T) { + tests := []struct { + args []string + want string + }{ + {[]string{"http://example.com/", "/foo"}, "http://example.com/foo"}, + {[]string{"example.com/", "/foo"}, "example.com/foo"}, + {[]string{"example.com/", "/foo/"}, "example.com/foo/"}, + {[]string{"example.com//////", "/foo/"}, "example.com/foo/"}, + {[]string{"foo://example.com//////", "/foo/"}, "foo://example.com/foo/"}, + + // TODO: strange things can happen with weird input, rethink + {[]string{"????://example.com//////", "/foo/"}, "/foo/????://example.com//////"}, + } + for _, tt := range tests { + t.Run(tt.want, func(t *testing.T) { + if got := Join(tt.args...); got != tt.want { + t.Errorf("Join() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/readme.md b/readme.md index adf288f5..07e8f67b 100644 --- a/readme.md +++ b/readme.md @@ -20,6 +20,7 @@ - [`bramble shell`](#bramble-shell) - [`bramble gc`](#bramble-gc) - [Dependencies](#dependencies) + - [Adding a dependency](#adding-a-dependency) - [Config language](#config-language) - [.bramble, default.bramble and the load() statement](#bramble-defaultbramble-and-the-load-statement) - [derivation()](#derivation) @@ -317,6 +318,57 @@ Lib provides various derivations to help build stuff ### Dependencies +Bramble dependencies are packages. Packages are collections of modules. All bramble dependencies must be served from version control repositories. Git is currently the only VCS supported, others will be added. + +Package paths use the location of the VCS repository. If you want to add a dependency that's at "https://github.com/maxmcd/bramble.git" you can use the package path "github.com/maxmcd/bramble". In a `load()` statement or `run` command you can also reference a specific module within a package by appending the path: "github.com/maxmcd/bramble/lib". + +Bramble uses Go's [Minimal Version Selection](https://research.swtch.com/vgo-mvs) (MVS) to select dependency versions. Package versions use [Semantic Versioning](https://semver.org/). + +Dependencies should follow the following rules to ensure compatibility with the broader ecosystem: + +1. Semantic versioning rules must be followed. Code changes within a Major version should not break API compatibility. Minor and Patch version increases should correspond with feature additions and bugfixes respectively. +2. The import path structure within a repository cannot be overwritten without a major version change. (TODO: explain) + +#### Adding a dependency + +Dependencies can be added to a current project with the `bramble add` command. The following formats are supported as arguments to `bramble add`. + +| Argument | Outcome | +| ---------------------------------------- | ------------------------------------------------------------------- | +| `github.com/maxmcd/busybox` | Adds the most recent version of busybox that we can find | +| `github.com/maxmcd/busybox@v1` | Adds the the highest available version with the prefix `1.` | +| `github.com/maxmcd/busybox@v1.1` | Adds the the highest available version with the prefix `1.1` | +| `github.com/maxmcd/busybox@v1.1.0` | Adds the specific version `1.1.0` will fail if it's not found | +| `github.com/maxmcd/busybox@v1.1.0-pre.1` | Adds the specific version `1.1.0-pre.1` will fail if it's not found | + +Dependencies are added to `bramble.toml`. Here are various valid dependency configurations and their meaning: + +```toml +[package] +name = "github.com/maxmcd/bramble" +version = "0.0.3" + + +[dependencies] +# Just a regular dependency and version +"github.com/maxmcd/busybox" = "0.0.2" +# If we need to include two major versions of a package we add a suffix of `@v` +# followed by the major version number. +"github.com/maxmcd/busybox@v1" = "1.0.0" +# We can point at sub-packages within this package. The path must be valid and +# the version must match the version in the bramble.toml of the sub-package. +"github.com/maxmcd/bramble/lib/foo" = {version="1.0.0", path="./lib/foo"} +# We can also point to sub-packages by version and they will just be treated +# like regular dependencies. +"github.com/maxmcd/bramble/lib/bar" = "1.0.0" +``` + +Checksums and sub-versions are added to `bramble.lock`. + +Load statements must include a major version number if it's included in the `bramble.toml`, like so: +```python +load("github.com/maxmcd/busybox@v1/foo/bar") +``` ### Config language diff --git a/tests/basic.bramble b/tests/basic.bramble index 9f07d74b..b0fe76a8 100644 --- a/tests/basic.bramble +++ b/tests/basic.bramble @@ -12,14 +12,14 @@ def link_test(): ) -load(nix_seed="github.com/maxmcd/bramble/lib/nix-seed") +load(busybox="github.com/maxmcd/bramble/tests/busybox") def self_reference(): return derivation( name="self-reference", - builder=nix_seed.stdenv().out + "/bin/bash", - env={"stdenv": nix_seed.stdenv(), "PATH": nix_seed.stdenv().out + "/bin"}, + builder=busybox.busybox().out + "/bin/ash", + env={"PATH": busybox.busybox().out + "/bin"}, args=[ "-c", """ diff --git a/tests/busybox/default.bramble b/tests/busybox/default.bramble new file mode 100644 index 00000000..b41d4566 --- /dev/null +++ b/tests/busybox/default.bramble @@ -0,0 +1,21 @@ +load("github.com/maxmcd/bramble/tests/util") + + +def busybox(): + b = util.fetch_url("https://brmbl.s3.amazonaws.com/busybox-x86_64.tar.gz") + script = """ + set -ex + $busybox_download/busybox-x86_64 mkdir $out/bin + $busybox_download/busybox-x86_64 cp $busybox_download/busybox-x86_64 $out/bin/busybox + cd $out/bin + for command in $(./busybox --list); do + ./busybox ln -s busybox $command + done + """ + + return derivation( + name="busybox", + builder=b.out + "/busybox-x86_64", + args=["sh", "-c", script], + env={"busybox_download": b, "PATH": b.out}, + ) diff --git a/tests/default.bramble b/tests/default.bramble new file mode 100644 index 00000000..6708b392 --- /dev/null +++ b/tests/default.bramble @@ -0,0 +1,14 @@ +load("github.com/maxmcd/bramble/tests/busybox") + + +def all(): + return [busybox.busybox()] + + +def print_simple(): + return run(busybox.busybox(), "echo", ["simple"], hidden_paths=["/"]) + + +def ash(): + # TODO: relative filepaths + return run(busybox.busybox(), "ash", read_only_paths=["../"]) diff --git a/tests/main_test.bramble b/tests/main_test.bramble index 0198aecc..d30a1b1d 100644 --- a/tests/main_test.bramble +++ b/tests/main_test.bramble @@ -1,10 +1,10 @@ load("github.com/maxmcd/bramble/tests/nested-sources/another-folder/nested") -load("github.com/maxmcd/bramble/lib") +load("github.com/maxmcd/bramble/tests/busybox") def test_nested(): - nested.nested() + return nested.nested() def test_busybox(): - lib.busybox() + return busybox.busybox() diff --git a/tests/nested-sources/another-folder/nested.bramble b/tests/nested-sources/another-folder/nested.bramble index 2839a68d..95286c93 100644 --- a/tests/nested-sources/another-folder/nested.bramble +++ b/tests/nested-sources/another-folder/nested.bramble @@ -1,8 +1,8 @@ -load("github.com/maxmcd/bramble/lib") +load("github.com/maxmcd/bramble/tests/busybox") def nested(): - bb = lib.busybox() + bb = busybox.busybox() return [_nested1(bb), _nested2(bb), _nested3(bb)] diff --git a/tests/simple/simple.bramble b/tests/simple/simple.bramble deleted file mode 100644 index c3cde54f..00000000 --- a/tests/simple/simple.bramble +++ /dev/null @@ -1,13 +0,0 @@ -load(nix_seed="github.com/maxmcd/bramble/lib/nix-seed") - - -def simple(): - simple = derivation( - name="simple", - builder=nix_seed.stdenv().out + "/bin/bash", - env={"stdenv": nix_seed.stdenv(), "PATH": nix_seed.stdenv().out + "/bin"}, - args=["./simple_builder.sh"], - sources=files(["./simple.c", "simple_builder.sh"]), - ) - test(simple, ["simple"]) - return simple diff --git a/tests/simple/simple.c b/tests/simple/simple.c deleted file mode 100644 index 53000867..00000000 --- a/tests/simple/simple.c +++ /dev/null @@ -1,4 +0,0 @@ -int main() { - puts("Simple!"); - return 0; -} diff --git a/tests/simple/simple_builder.sh b/tests/simple/simple_builder.sh deleted file mode 100644 index 91f460ca..00000000 --- a/tests/simple/simple_builder.sh +++ /dev/null @@ -1,15 +0,0 @@ -set -uxe - -export PATH="$stdenv/bin" - -mkdir -p $out/bin -gcc \ - -L "$stdenv/lib" \ - -I "$stdenv/include" \ - -ffile-prefix-map=OLD=NEW \ - -Wl,--rpath="$stdenv/lib" \ - -Wl,--dynamic-linker="$stdenv/lib/ld-linux-x86-64.so.2" \ - -o $out/bin/simple ./simple.c - -$out/bin/simple -patchelf --print-interpreter --print-soname --print-rpath $out/bin/simple diff --git a/tests/tutorial/default.bramble b/tests/tutorial/default.bramble index 3b949c0c..8d614e35 100644 --- a/tests/tutorial/default.bramble +++ b/tests/tutorial/default.bramble @@ -1,21 +1,21 @@ -load("github.com/maxmcd/bramble/lib/stdenv") -load("github.com/maxmcd/bramble/lib") -load("github.com/maxmcd/bramble/lib/std") +# load("github.com/maxmcd/bramble/lib/stdenv") +# load("github.com/maxmcd/bramble/lib") +# load("github.com/maxmcd/bramble/lib/std") -def fetch_a_url(): - return std.fetch_url("https://maxmcd.com/") +# def fetch_a_url(): +# return std.fetch_url("https://maxmcd.com/") -def step_1(): - bash = "%s/bin/bash" % stdenv.stdenv() - return stdenv.std_derivation( - "step_1", - builder=bash, - env=dict(bash=bash), - sources=files(["./step1.sh"]), - args=["./step1.sh"], - ) +# def step_1(): +# bash = "%s/bin/bash" % stdenv.stdenv() +# return stdenv.std_derivation( +# "step_1", +# builder=bash, +# env=dict(bash=bash), +# sources=files(["./step1.sh"]), +# args=["./step1.sh"], +# ) # def step_2(): diff --git a/lib/std/default.bramble b/tests/util.bramble similarity index 92% rename from lib/std/default.bramble rename to tests/util.bramble index d6a7a0a9..d1467ebb 100644 --- a/lib/std/default.bramble +++ b/tests/util.bramble @@ -1,6 +1,6 @@ -"""std defines some standard bramble helper functions""" +# Copied from std def fetch_url(url=None): """ fetch_url is a wrapper around the "fetch_url" builder that creates a diff --git a/v/untar/untar.go b/v/untar/untar.go deleted file mode 100644 index ca29aad4..00000000 --- a/v/untar/untar.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2017 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. -package untar - -import ( - "archive/tar" - "fmt" - "io" - "log" - "os" - "path" - "path/filepath" - "strings" - "time" -) - -// Untar reads the gzip-compressed tar file from r and writes it into dir. -func Untar(r io.Reader, dir string) error { - return untar(r, dir) -} - -func untar(r io.Reader, dir string) (err error) { - t0 := time.Now() - nFiles := 0 - madeDir := map[string]bool{} - tr := tar.NewReader(r) - loggedChtimesError := false - for { - f, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - log.Printf("tar reading error: %v", err) - return fmt.Errorf("tar error: %v", err) - } - // if !validRelPath(f.Name) { - // return fmt.Errorf("tar contained invalid name error %q", f.Name) - // } - rel := filepath.FromSlash(f.Name) - abs := filepath.Join(dir, rel) - - fi := f.FileInfo() - mode := fi.Mode() - switch { - case mode.IsRegular(): - // Make the directory. This is redundant because it should - // already be made by a directory entry in the tar - // beforehand. Thus, don't check for errors; the next - // write will fail with the same error. - dir := filepath.Dir(abs) - if !madeDir[dir] { - if err := os.MkdirAll(filepath.Dir(abs), 0755); err != nil { - return err - } - madeDir[dir] = true - } - wf, err := os.OpenFile(abs, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode.Perm()) - if err != nil { - return err - } - n, err := io.Copy(wf, tr) - if closeErr := wf.Close(); closeErr != nil && err == nil { - err = closeErr - } - if err != nil { - return fmt.Errorf("error writing to %s: %v", abs, err) - } - if n != f.Size { - return fmt.Errorf("only wrote %d bytes to %s; expected %d", n, abs, f.Size) - } - modTime := f.ModTime - if modTime.After(t0) { - // Clamp modtimes at system time. See - // golang.org/issue/19062 when clock on - // buildlet was behind the gitmirror server - // doing the git-archive. - modTime = t0 - } - if !modTime.IsZero() { - if err := os.Chtimes(abs, modTime, modTime); err != nil && !loggedChtimesError { - // benign error. Gerrit doesn't even set the - // modtime in these, and we don't end up relying - // on it anywhere (the gomote push command relies - // on digests only), so this is a little pointless - // for now. - log.Printf("error changing modtime: %v (further Chtimes errors suppressed)", err) - loggedChtimesError = true // once is enough - } - } - nFiles++ - case mode.IsDir(): - if err := os.MkdirAll(abs, 0755); err != nil { - return err - } - madeDir[abs] = true - default: - return fmt.Errorf("tar file entry %s contained unsupported file type %v", f.Name, mode) - } - } - return nil -} - -func validRelativeDir(dir string) bool { - if strings.Contains(dir, `\`) || path.IsAbs(dir) { - return false - } - dir = path.Clean(dir) - if strings.HasPrefix(dir, "../") || strings.HasSuffix(dir, "/..") || dir == ".." { - return false - } - return true -} - -func validRelPath(p string) bool { - if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") { - return false - } - return true -}