diff --git a/api/services/control/control.pb.go b/api/services/control/control.pb.go index e071d351bbfa..0fb42c69d829 100644 --- a/api/services/control/control.pb.go +++ b/api/services/control/control.pb.go @@ -410,6 +410,7 @@ type SolveRequest struct { EnableSessionExporter bool `protobuf:"varint,14,opt,name=EnableSessionExporter,proto3" json:"EnableSessionExporter,omitempty"` SourcePolicySession string `protobuf:"bytes,15,opt,name=SourcePolicySession,proto3" json:"SourcePolicySession,omitempty"` CompatibilityVersion int64 `protobuf:"varint,16,opt,name=CompatibilityVersion,proto3" json:"CompatibilityVersion,omitempty"` + ProxyNetwork bool `protobuf:"varint,17,opt,name=ProxyNetwork,proto3" json:"ProxyNetwork,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -556,6 +557,13 @@ func (x *SolveRequest) GetCompatibilityVersion() int64 { return 0 } +func (x *SolveRequest) GetProxyNetwork() bool { + if x != nil { + return x.ProxyNetwork + } + return false +} + type CacheOptions struct { state protoimpl.MessageState `protogen:"open.v1"` // ExportRefDeprecated is deprecated in favor or the new Exports since BuildKit v0.4.0. @@ -2066,7 +2074,7 @@ const file_github_com_moby_buildkit_api_services_control_control_proto_rawDesc = " \x01(\tR\n" + "RecordType\x12\x16\n" + "\x06Shared\x18\v \x01(\bR\x06Shared\x12\x18\n" + - "\aParents\x18\f \x03(\tR\aParents\"\xda\b\n" + + "\aParents\x18\f \x03(\tR\aParents\"\xfe\b\n" + "\fSolveRequest\x12\x10\n" + "\x03Ref\x18\x01 \x01(\tR\x03Ref\x12.\n" + "\n" + @@ -2086,7 +2094,8 @@ const file_github_com_moby_buildkit_api_services_control_control_proto_rawDesc = "\tExporters\x18\r \x03(\v2\x1a.moby.buildkit.v1.ExporterR\tExporters\x124\n" + "\x15EnableSessionExporter\x18\x0e \x01(\bR\x15EnableSessionExporter\x120\n" + "\x13SourcePolicySession\x18\x0f \x01(\tR\x13SourcePolicySession\x122\n" + - "\x14CompatibilityVersion\x18\x10 \x01(\x03R\x14CompatibilityVersion\x1aJ\n" + + "\x14CompatibilityVersion\x18\x10 \x01(\x03R\x14CompatibilityVersion\x12\"\n" + + "\fProxyNetwork\x18\x11 \x01(\bR\fProxyNetwork\x1aJ\n" + "\x1cExporterAttrsDeprecatedEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1a@\n" + diff --git a/api/services/control/control.proto b/api/services/control/control.proto index 4ec3ac89c9a8..0c5cdb17e81b 100644 --- a/api/services/control/control.proto +++ b/api/services/control/control.proto @@ -78,6 +78,7 @@ message SolveRequest { bool EnableSessionExporter = 14; string SourcePolicySession = 15; int64 CompatibilityVersion = 16; + bool ProxyNetwork = 17; } message CacheOptions { diff --git a/api/services/control/control_vtproto.pb.go b/api/services/control/control_vtproto.pb.go index 031b05231d80..2feee6ed38c7 100644 --- a/api/services/control/control_vtproto.pb.go +++ b/api/services/control/control_vtproto.pb.go @@ -144,6 +144,7 @@ func (m *SolveRequest) CloneVT() *SolveRequest { r.EnableSessionExporter = m.EnableSessionExporter r.SourcePolicySession = m.SourcePolicySession r.CompatibilityVersion = m.CompatibilityVersion + r.ProxyNetwork = m.ProxyNetwork if rhs := m.ExporterAttrsDeprecated; rhs != nil { tmpContainer := make(map[string]string, len(rhs)) for k, v := range rhs { @@ -1047,6 +1048,9 @@ func (this *SolveRequest) EqualVT(that *SolveRequest) bool { if this.CompatibilityVersion != that.CompatibilityVersion { return false } + if this.ProxyNetwork != that.ProxyNetwork { + return false + } return string(this.unknownFields) == string(that.unknownFields) } @@ -2253,6 +2257,18 @@ func (m *SolveRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.ProxyNetwork { + i-- + if m.ProxyNetwork { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x88 + } if m.CompatibilityVersion != 0 { i = protohelpers.EncodeVarint(dAtA, i, uint64(m.CompatibilityVersion)) i-- @@ -4188,6 +4204,9 @@ func (m *SolveRequest) SizeVT() (n int) { if m.CompatibilityVersion != 0 { n += 2 + protohelpers.SizeOfVarint(uint64(m.CompatibilityVersion)) } + if m.ProxyNetwork { + n += 3 + } n += len(m.unknownFields) return n } @@ -6359,6 +6378,26 @@ func (m *SolveRequest) UnmarshalVT(dAtA []byte) error { break } } + case 17: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ProxyNetwork", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.ProxyNetwork = bool(v != 0) default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/client/client_test.go b/client/client_test.go index 09a63a2e2bf4..f216096f3236 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -158,6 +158,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){ testUlimit, testCgroupParent, testNetworkMode, + testProxyNetworkNoRootless, testFrontendMetadataReturn, testFrontendUseSolveResults, testSSHMount, diff --git a/client/policy_test.go b/client/policy_test.go index 7925eef7aef4..71081fb475bb 100644 --- a/client/policy_test.go +++ b/client/policy_test.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "hash" + "net" "net/http" "net/http/httptest" "os" @@ -15,6 +16,7 @@ import ( "runtime" "slices" "strings" + "sync/atomic" "testing" "time" @@ -24,6 +26,8 @@ import ( "github.com/moby/buildkit/client/llb/sourceresolver" gateway "github.com/moby/buildkit/frontend/gateway/client" pb "github.com/moby/buildkit/frontend/gateway/pb" + solvererrdefs "github.com/moby/buildkit/solver/errdefs" + provenancetypes "github.com/moby/buildkit/solver/llbsolver/provenance/types" opspb "github.com/moby/buildkit/solver/pb" sourcepolicypb "github.com/moby/buildkit/sourcepolicy/pb" "github.com/moby/buildkit/sourcepolicy/policysession" @@ -37,6 +41,260 @@ import ( "github.com/stretchr/testify/require" ) +func testProxyNetworkNoRootless(t *testing.T, sb integration.Sandbox) { + integration.SkipOnPlatform(t, "windows") + + ctx := sb.Context() + c, err := New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + payload := []byte("buildkit proxy ok\n") + convertedPayload := []byte("buildkit proxy converted ok\n") + httpSrv, httpURL := newProxyReachableHTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/allowed": + _, _ = w.Write(payload) + case "/bar": + _, _ = w.Write(convertedPayload) + default: + http.NotFound(w, r) + return + } + })) + defer httpSrv.Close() + var leakHit atomic.Int32 + leakSrv, leakURL := newProxyReachableHTTPServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + leakHit.Add(1) + _, _ = w.Write([]byte("host namespace leak\n")) + })) + defer leakSrv.Close() + _, leakPort, err := net.SplitHostPort(strings.TrimPrefix(leakURL, "http://")) + require.NoError(t, err) + + st := llb.Image("alpine:latest"). + Run(llb.Shlexf(`sh -c 'wget -q -O- %s/allowed | grep "buildkit proxy ok"'`, httpURL)). + Root(). + Run(llb.Shlex(`sh -c '! wget -S -O- https://buildkit-ca-test.invalid/denied 2>/tmp/wget.log; grep "HTTP/1.1 502 Bad Gateway" /tmp/wget.log'`)). + Root(). + Run(llb.Shlex(`sh -c 'unset HTTP_PROXY HTTPS_PROXY http_proxy https_proxy ALL_PROXY all_proxy NO_PROXY no_proxy; ! wget -T 2 -q -O- http://1.1.1.1/'`)). + Root(). + Run(llb.Shlexf(`sh -c 'proxy=${HTTP_PROXY#http://}; host=${proxy%%:*}; unset HTTP_PROXY HTTPS_PROXY http_proxy https_proxy ALL_PROXY all_proxy NO_PROXY no_proxy; ! wget -T 2 -q -O- http://$host:%s/'`, leakPort)). + Root(). + Run(llb.Shlex(`sh -c 'grep "buildkit proxy CA begin" /etc/ssl/certs/ca-certificates.crt'`)). + Root(). + Run(llb.Shlex(`sh -c '! grep "buildkit proxy CA begin" /etc/ssl/certs/ca-certificates.crt'`), llb.Network(llb.NetModeNone)) + + def, err := st.Marshal(ctx) + require.NoError(t, err) + _, err = c.Solve(ctx, def, SolveOpt{ + ProxyNetwork: true, + }, nil) + require.NoError(t, err) + require.Equal(t, int32(0), leakHit.Load()) + + var checked atomic.Int32 + denyProvider := policysession.NewPolicyProvider(func(ctx context.Context, req *policysession.CheckPolicyRequest) (*policysession.DecisionResponse, *pb.ResolveSourceMetaRequest, error) { + if req.Source.Source.Identifier != httpURL+"/allowed" { + return &policysession.DecisionResponse{ + Action: sourcepolicypb.PolicyAction_ALLOW, + }, nil, nil + } + checked.Add(1) + return &policysession.DecisionResponse{ + Action: sourcepolicypb.PolicyAction_DENY, + }, nil, nil + }) + + deny := llb.Image("alpine:latest"). + Run(llb.Shlexf(`wget -q -O- %s/allowed`, httpURL), llb.IgnoreCache) + def, err = deny.Marshal(ctx) + require.NoError(t, err) + _, err = c.Solve(ctx, def, SolveOpt{ + ProxyNetwork: true, + SourcePolicyProvider: denyProvider, + }, nil) + require.Error(t, err) + require.Equal(t, int32(1), checked.Load()) + + destDir := t.TempDir() + withProvenance := llb.Image("alpine:latest"). + Run(llb.Shlexf(`sh -c 'wget -q -O /out/proxy-material %s/allowed'`, httpURL)). + AddMount("/out", llb.Scratch()) + def, err = withProvenance.Marshal(ctx) + require.NoError(t, err) + materialURL := httpURL + "/allowed" + statusCh := make(chan *SolveStatus) + logsCh := make(chan string, 1) + go func() { + var b strings.Builder + for st := range statusCh { + for _, l := range st.Logs { + b.Write(l.Data) + } + } + logsCh <- b.String() + }() + _, err = c.Solve(ctx, def, SolveOpt{ + ProxyNetwork: true, + FrontendAttrs: map[string]string{ + "attest:provenance": "mode=max,version=v1", + }, + Exports: []ExportEntry{{ + Type: ExporterLocal, + OutputDir: destDir, + }}, + }, statusCh) + require.NoError(t, err) + logOutput := <-logsCh + require.Contains(t, logOutput, "proxy network requests:\n- GET "+materialURL) + + dt, err := os.ReadFile(filepath.Join(destDir, "proxy-material")) + require.NoError(t, err) + require.Equal(t, payload, dt) + + provDt, err := os.ReadFile(filepath.Join(destDir, "provenance.json")) + require.NoError(t, err) + var stmt struct { + intoto.StatementHeader + Predicate provenancetypes.ProvenancePredicateSLSA1 `json:"predicate"` + } + require.NoError(t, json.Unmarshal(provDt, &stmt)) + foundMaterial := false + expectedDigest := digest.FromBytes(payload) + for _, m := range stmt.Predicate.BuildDefinition.ResolvedDependencies { + if m.URI == materialURL { + foundMaterial = true + require.Equal(t, expectedDigest.Hex(), m.Digest["sha256"]) + } + } + require.True(t, foundMaterial, "expected to find %q in %+v", materialURL, stmt.Predicate.BuildDefinition.ResolvedDependencies) + require.False(t, stmt.Predicate.RunDetails.Metadata.Hermetic) + require.True(t, stmt.Predicate.RunDetails.Metadata.Completeness.ResolvedDependencies) + require.NotNil(t, stmt.Predicate.RunDetails.Metadata.BuildKitMetadata.Network) + require.Equal(t, "proxy", stmt.Predicate.RunDetails.Metadata.BuildKitMetadata.Network.Mode) + + convertDestDir := t.TempDir() + convertFooURL := httpURL + "/foo" + convertBarURL := httpURL + "/bar" + convert := llb.Image("alpine:latest"). + Run(llb.Shlexf(`sh -c 'wget -q -O /out/proxy-material %s'`, convertFooURL)). + AddMount("/out", llb.Scratch()) + def, err = convert.Marshal(ctx) + require.NoError(t, err) + _, err = c.Solve(ctx, def, SolveOpt{ + ProxyNetwork: true, + SourcePolicy: &sourcepolicypb.Policy{ + Rules: []*sourcepolicypb.Rule{ + { + Action: sourcepolicypb.PolicyAction_CONVERT, + Selector: &sourcepolicypb.Selector{ + Identifier: convertFooURL, + }, + Updates: &sourcepolicypb.Update{ + Identifier: convertBarURL, + }, + }, + }, + }, + FrontendAttrs: map[string]string{ + "attest:provenance": "mode=max,version=v1", + }, + Exports: []ExportEntry{{ + Type: ExporterLocal, + OutputDir: convertDestDir, + }}, + }, nil) + require.NoError(t, err) + + dt, err = os.ReadFile(filepath.Join(convertDestDir, "proxy-material")) + require.NoError(t, err) + require.Equal(t, convertedPayload, dt) + + provDt, err = os.ReadFile(filepath.Join(convertDestDir, "provenance.json")) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(provDt, &stmt)) + foundMaterial = false + expectedDigest = digest.FromBytes(convertedPayload) + for _, m := range stmt.Predicate.BuildDefinition.ResolvedDependencies { + require.NotEqual(t, convertFooURL, m.URI) + if m.URI == convertBarURL { + foundMaterial = true + require.Equal(t, expectedDigest.Hex(), m.Digest["sha256"]) + } + } + require.True(t, foundMaterial, "expected to find %q in %+v", convertBarURL, stmt.Predicate.BuildDefinition.ResolvedDependencies) + + strict := llb.Image("alpine:latest"). + Run(llb.Shlexf(`sh -c 'wget -q -O- %s/missing || true'`, httpURL)). + AddMount("/out", llb.Scratch()) + def, err = strict.Marshal(ctx) + require.NoError(t, err) + _, err = c.Solve(ctx, def, SolveOpt{ + ProxyNetwork: true, + FrontendAttrs: map[string]string{ + "attest:provenance": "mode=max,version=v1,complete-materials=true", + }, + Exports: []ExportEntry{{ + Type: ExporterLocal, + OutputDir: t.TempDir(), + }}, + }, nil) + require.Error(t, err) + require.ErrorContains(t, err, "provenance materials are incomplete") + require.ErrorContains(t, err, "/missing") + var materialsErr *solvererrdefs.ProvenanceMaterialsIncompleteError + require.ErrorAs(t, err, &materialsErr) + require.Len(t, materialsErr.Incomplete, 1) + require.Equal(t, httpURL+"/missing", materialsErr.Incomplete[0].Uri) + require.Equal(t, "unsuccessful_response", materialsErr.Incomplete[0].Reason) +} + +func newProxyReachableHTTPServer(t *testing.T, handler http.Handler) (*httptest.Server, string) { + t.Helper() + var lc net.ListenConfig + ln, err := lc.Listen(t.Context(), "tcp4", "0.0.0.0:0") + require.NoError(t, err) + srv := httptest.NewUnstartedServer(handler) + srv.Listener = ln + srv.Start() + _, port, err := net.SplitHostPort(ln.Addr().String()) + require.NoError(t, err) + return srv, "http://" + net.JoinHostPort(testHostIP(t), port) +} + +func testHostIP(t *testing.T) string { + t.Helper() + conn, err := (&net.Dialer{}).DialContext(t.Context(), "udp4", "8.8.8.8:80") + if err == nil { + defer conn.Close() + if addr, ok := conn.LocalAddr().(*net.UDPAddr); ok && !addr.IP.IsLoopback() { + return addr.IP.String() + } + } + ifaces, err := net.Interfaces() + require.NoError(t, err) + for _, iface := range ifaces { + if iface.Flags&net.FlagUp == 0 || iface.Flags&net.FlagLoopback != 0 { + continue + } + addrs, err := iface.Addrs() + require.NoError(t, err) + for _, addr := range addrs { + ipnet, ok := addr.(*net.IPNet) + if !ok { + continue + } + ip := ipnet.IP.To4() + if ip != nil && !ip.IsLoopback() { + return ip.String() + } + } + } + t.Fatal("could not find non-loopback host IP for proxy integration test") + return "" +} + func testSourcePolicySession(t *testing.T, sb integration.Sandbox) { requiresLinux(t) diff --git a/client/solve.go b/client/solve.go index da13d015c175..3bef8846b91e 100644 --- a/client/solve.go +++ b/client/solve.go @@ -53,6 +53,7 @@ type SolveOpt struct { Internal bool SourcePolicy *spb.Policy SourcePolicyProvider session.Attachable + ProxyNetwork bool Ref string } @@ -309,6 +310,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG Internal: opt.Internal, CompatibilityVersion: int64(opt.CompatibilityVersion), SourcePolicy: opt.SourcePolicy, + ProxyNetwork: opt.ProxyNetwork, } if opt.SourcePolicyProvider != nil { sopt.SourcePolicySession = s.ID() diff --git a/cmd/buildctl/build.go b/cmd/buildctl/build.go index def59db1238f..a09084d3a20d 100644 --- a/cmd/buildctl/build.go +++ b/cmd/buildctl/build.go @@ -103,6 +103,10 @@ var buildCommand = cli.Command{ Name: "source-policy-file", Usage: "Read source policy file from a JSON file", }, + cli.BoolFlag{ + Name: "proxy-network", + Usage: "Run build with proxy network enforcement", + }, cli.StringFlag{ Name: "ref-file", Usage: "Write build ref to a file", @@ -243,7 +247,6 @@ func buildAction(clicontext *cli.Context) error { } srcPol = &srcPolStruct } - eg, ctx := errgroup.WithContext(bccommon.CommandContext(clicontext)) ref := identity.NewID() @@ -259,6 +262,7 @@ func buildAction(clicontext *cli.Context) error { Session: attachable, AllowedEntitlements: clicontext.StringSlice("allow"), SourcePolicy: srcPol, + ProxyNetwork: clicontext.Bool("proxy-network"), Ref: ref, } diff --git a/cmd/buildkitd/config/config.go b/cmd/buildkitd/config/config.go index 28822baea445..194e30a68376 100644 --- a/cmd/buildkitd/config/config.go +++ b/cmd/buildkitd/config/config.go @@ -19,6 +19,9 @@ type Config struct { // Entitlements e.g. security.insecure, network.host, device Entitlements []string `toml:"insecure-entitlements"` + // ProxyNetwork enables proxy network enforcement for all builds. + ProxyNetwork bool `toml:"proxyNetwork"` + // LogFormat is the format of the logs. It can be "json" or "text". Log LogConfig `toml:"log"` diff --git a/cmd/buildkitd/config/load_test.go b/cmd/buildkitd/config/load_test.go index b576351f287d..5a2768e67b3d 100644 --- a/cmd/buildkitd/config/load_test.go +++ b/cmd/buildkitd/config/load_test.go @@ -14,6 +14,7 @@ root = "/foo/bar" debug=true trace=true insecure-entitlements = ["security.insecure"] +proxyNetwork = true [gc] enabled=true @@ -85,6 +86,7 @@ searchDomains=["example.com"] require.Equal(t, true, cfg.Debug) require.Equal(t, true, cfg.Trace) require.Equal(t, "security.insecure", cfg.Entitlements[0]) + require.True(t, cfg.ProxyNetwork) require.Equal(t, "buildkit.sock", cfg.GRPC.Address[0]) require.Equal(t, "debug.sock", cfg.GRPC.DebugAddress) diff --git a/cmd/buildkitd/main.go b/cmd/buildkitd/main.go index 829ff6beb1c8..af69decde867 100644 --- a/cmd/buildkitd/main.go +++ b/cmd/buildkitd/main.go @@ -224,6 +224,10 @@ func main() { Name: "allow-insecure-entitlement", Usage: "allows insecure entitlements e.g. network.host, security.insecure, device", }, + cli.BoolFlag{ + Name: "proxy-network", + Usage: "enable proxy network enforcement for all builds", + }, cli.StringFlag{ Name: "otel-socket-path", Usage: "OTEL collector trace socket path", @@ -647,6 +651,9 @@ func applyMainFlags(c *cli.Context, cfg *config.Config, warnings *[]string) erro // override values from config cfg.Entitlements = c.StringSlice("allow-insecure-entitlement") } + if c.IsSet("proxy-network") { + cfg.ProxyNetwork = c.Bool("proxy-network") + } if c.IsSet("debugaddr") { cfg.GRPC.DebugAddress = c.String("debugaddr") @@ -928,6 +935,7 @@ func newController(ctx context.Context, c *cli.Context, cfg *config.Config) (*co LeaseManager: w.LeaseManager(), ContentStore: w.ContentStore(), HistoryConfig: cfg.History, + ProxyNetwork: cfg.ProxyNetwork, GarbageCollect: w.GarbageCollect, GracefulStop: ctx.Done(), ProvenanceEnv: provenanceEnv, diff --git a/cmd/buildkitd/main_test.go b/cmd/buildkitd/main_test.go new file mode 100644 index 000000000000..a98efdb47a33 --- /dev/null +++ b/cmd/buildkitd/main_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "flag" + "testing" + + "github.com/moby/buildkit/cmd/buildkitd/config" + "github.com/stretchr/testify/require" + "github.com/urfave/cli" +) + +func TestApplyMainFlagsProxyNetwork(t *testing.T) { + fs := flag.NewFlagSet("buildkitd", flag.ContinueOnError) + fs.Bool("proxy-network", false, "") + require.NoError(t, fs.Set("proxy-network", "true")) + + cfg := config.Config{} + err := applyMainFlags(cli.NewContext(cli.NewApp(), fs, nil), &cfg, nil) + require.NoError(t, err) + require.True(t, cfg.ProxyNetwork) +} + +func TestApplyMainFlagsProxyNetworkOverridesConfig(t *testing.T) { + fs := flag.NewFlagSet("buildkitd", flag.ContinueOnError) + fs.Bool("proxy-network", false, "") + require.NoError(t, fs.Set("proxy-network", "false")) + + cfg := config.Config{ProxyNetwork: true} + err := applyMainFlags(cli.NewContext(cli.NewApp(), fs, nil), &cfg, nil) + require.NoError(t, err) + require.False(t, cfg.ProxyNetwork) +} diff --git a/control/control.go b/control/control.go index caf6f045d746..8a34b62b7dc8 100644 --- a/control/control.go +++ b/control/control.go @@ -75,6 +75,7 @@ type Opt struct { LeaseManager *leaseutil.Manager ContentStore *containerdsnapshot.Store HistoryConfig *config.HistoryConfig + ProxyNetwork bool GarbageCollect func(context.Context) error GracefulStop <-chan struct{} ProvenanceEnv map[string]any @@ -118,6 +119,7 @@ func NewController(opt Opt) (*Controller, error) { SessionManager: opt.SessionManager, Entitlements: opt.Entitlements, HistoryQueue: hq, + ProxyNetwork: opt.ProxyNetwork, ProvenanceEnv: opt.ProvenanceEnv, }) if err != nil { @@ -552,7 +554,7 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* Exporters: expis, CacheExporters: cacheExporters, EnableSessionExporter: req.EnableSessionExporter, - }, entitlementsFromPB(req.Entitlements), procs, req.Internal, req.SourcePolicy, req.SourcePolicySession) + }, entitlementsFromPB(req.Entitlements), procs, req.Internal, req.SourcePolicy, req.SourcePolicySession, req.ProxyNetwork) if err != nil { return nil, err } diff --git a/docs/buildkitd.toml.md b/docs/buildkitd.toml.md index a1d62f5033d4..ce7656412e21 100644 --- a/docs/buildkitd.toml.md +++ b/docs/buildkitd.toml.md @@ -17,6 +17,9 @@ Note that some configuration options are only useful in edge cases. root = "/var/lib/buildkit" # insecure-entitlements allows insecure entitlements, disabled by default. insecure-entitlements = [ "network.host", "security.insecure", "device" ] +# proxyNetwork enables proxy network enforcement for all builds, disabled by default. +# It can also be enabled with buildkitd --proxy-network. +proxyNetwork = true # provenanceEnvDir is the directory where extra config is loaded that is added # to the provenance of builds: # slsa v0.2: invocation.environment.* diff --git a/docs/reference/buildctl.md b/docs/reference/buildctl.md index ede9679ab255..f68125feff85 100644 --- a/docs/reference/buildctl.md +++ b/docs/reference/buildctl.md @@ -77,6 +77,7 @@ OPTIONS: --ssh value Allow forwarding SSH agent or a raw Unix socket to the builder. Format default|[=[,raw=false]|[,]] --metadata-file value Output build metadata (e.g., image digest) to a file as JSON --source-policy-file value Read source policy file from a JSON file + --proxy-network Run build with proxy network enforcement --ref-file value Write build ref to a file --registry-auth-tlscontext value Overwrite TLS configuration when authenticating with registries, e.g. --registry-auth-tlscontext host=https://myserver:2376,insecure=false,ca=/path/to/my/ca.crt,cert=/path/to/my/cert.crt,key=/path/to/my/key.crt --debug-json-cache-metrics value Where to output json cache metrics, use 'stdout' or 'stderr' for standard (error) output. diff --git a/executor/containerdexecutor/executor.go b/executor/containerdexecutor/executor.go index f6d06acddf82..43a5261d28d5 100644 --- a/executor/containerdexecutor/executor.go +++ b/executor/containerdexecutor/executor.go @@ -165,11 +165,22 @@ func (w *containerdExecutor) Run(ctx context.Context, id string, root executor.M return nil, err } - namespace, err := provider.New(ctx, meta.Hostname) + namespace, err := provider.New(ctx, meta.Hostname, network.NamespaceOptions{ + ProxyPolicy: meta.ProxyPolicy, + ProxyCapture: meta.ProxyCapture, + }) if err != nil { return nil, err } defer namespace.Close() + if proxyNS, ok := namespace.(network.ProxyNamespace); ok { + meta.Env = append(meta.Env, proxyNS.ProxyEnv()...) + cleanProxyCA, err := executor.InjectProxyCA(details.rootfsPath, proxyNS.ProxyCACert()) + if err != nil { + return nil, err + } + defer cleanProxyCA() + } spec, releaseSpec, err := w.createOCISpec(ctx, id, resolvConf, hostsFile, namespace, mounts, meta, details) if err != nil { diff --git a/executor/executor.go b/executor/executor.go index aa011e27d700..07ecbc6975a7 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -9,6 +9,7 @@ import ( "github.com/containerd/containerd/v2/core/mount" resourcestypes "github.com/moby/buildkit/executor/resources/types" "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/util/network" "github.com/moby/sys/user" ) @@ -27,6 +28,8 @@ type Meta struct { NetMode pb.NetMode SecurityMode pb.SecurityMode ValidExitCodes []int + ProxyPolicy network.ProxyPolicy + ProxyCapture *network.ProxyCapture RemoveMountStubsRecursive bool } diff --git a/executor/proxyca_linux.go b/executor/proxyca_linux.go new file mode 100644 index 000000000000..a596634a4b02 --- /dev/null +++ b/executor/proxyca_linux.go @@ -0,0 +1,200 @@ +//go:build linux + +package executor + +import ( + "bytes" + "crypto/sha256" + "crypto/x509" + "encoding/pem" + "os" + "path/filepath" + "syscall" + + "github.com/containerd/continuity/fs" + "github.com/pkg/errors" +) + +var linuxSystemCertFiles = []string{ + "/etc/ssl/certs/ca-certificates.crt", + "/etc/pki/tls/certs/ca-bundle.crt", + "/etc/ssl/ca-bundle.pem", + "/etc/pki/tls/cacert.pem", + "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", + "/etc/ssl/cert.pem", +} + +var ( + proxyCABegin = []byte("\n# buildkit proxy CA begin\n") + proxyCAEnd = []byte("# buildkit proxy CA end\n") +) + +// InjectProxyCA appends caPEM to the rootfs trust bundle used by common Linux +// TLS stacks and returns a cleanup that removes only the injected CA. +func InjectProxyCA(rootfsPath string, caPEM []byte) (func() error, error) { + if len(caPEM) == 0 { + return func() error { return nil }, nil + } + cert, err := firstCertificate(caPEM) + if err != nil { + return nil, err + } + certSum := sha256.Sum256(cert.Raw) + + var bundle string + for _, name := range linuxSystemCertFiles { + p, err := fs.RootPath(rootfsPath, name) + if err != nil { + return nil, errors.Wrapf(err, "failed to resolve certificate bundle %s", name) + } + if st, err := os.Stat(p); err == nil && !st.IsDir() { + bundle = p + break + } + } + if bundle == "" { + return func() error { return nil }, nil + } + + original, err := os.ReadFile(bundle) + if err != nil { + return nil, errors.WithStack(err) + } + if containsCertificate(original, certSum) { + return func() error { return nil }, nil + } + st, err := os.Stat(bundle) + if err != nil { + return nil, errors.WithStack(err) + } + next := append([]byte{}, original...) + if len(next) > 0 && next[len(next)-1] != '\n' { + next = append(next, '\n') + } + next = append(next, proxyCABegin...) + next = append(next, caPEM...) + if len(next) > 0 && next[len(next)-1] != '\n' { + next = append(next, '\n') + } + next = append(next, proxyCAEnd...) + if err := writeCertBundle(bundle, next, st); err != nil { + return nil, err + } + + return func() error { + current, err := os.ReadFile(bundle) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return errors.WithStack(err) + } + cleaned := removeInjectedCA(current, certSum) + if bytes.Equal(current, cleaned) { + return nil + } + st, err := os.Stat(bundle) + if err != nil { + return errors.WithStack(err) + } + return writeCertBundle(bundle, cleaned, st) + }, nil +} + +func firstCertificate(dt []byte) (*x509.Certificate, error) { + for { + block, rest := pem.Decode(dt) + if block == nil { + return nil, errors.New("proxy CA PEM does not contain a certificate") + } + dt = rest + if block.Type != "CERTIFICATE" { + continue + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, errors.WithStack(err) + } + return cert, nil + } +} + +func containsCertificate(dt []byte, sum [sha256.Size]byte) bool { + for { + block, rest := pem.Decode(dt) + if block == nil { + return false + } + dt = rest + if block.Type != "CERTIFICATE" { + continue + } + if sha256.Sum256(block.Bytes) == sum { + return true + } + } +} + +func removeInjectedCA(dt []byte, sum [sha256.Size]byte) []byte { + begin := bytes.Index(dt, proxyCABegin) + if begin >= 0 { + end := bytes.Index(dt[begin+len(proxyCABegin):], proxyCAEnd) + if end >= 0 { + end += begin + len(proxyCABegin) + len(proxyCAEnd) + block := dt[begin:end] + if containsCertificate(block, sum) { + out := append([]byte{}, dt[:begin]...) + out = append(out, dt[end:]...) + return out + } + } + } + + var out []byte + for len(dt) > 0 { + idx := bytes.Index(dt, []byte("-----BEGIN ")) + if idx < 0 { + out = append(out, dt...) + break + } + out = append(out, dt[:idx]...) + block, rest := pem.Decode(dt[idx:]) + if block == nil { + out = append(out, dt[idx:]...) + break + } + consumed := len(dt[idx:]) - len(rest) + if block.Type != "CERTIFICATE" || sha256.Sum256(block.Bytes) != sum { + out = append(out, dt[idx:idx+consumed]...) + } + dt = rest + } + return out +} + +func writeCertBundle(path string, dt []byte, st os.FileInfo) error { + tmp, err := os.CreateTemp(filepath.Dir(path), ".buildkit-ca-*") + if err != nil { + return errors.WithStack(err) + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + if _, err := tmp.Write(dt); err != nil { + tmp.Close() + return errors.WithStack(err) + } + if err := tmp.Chmod(st.Mode()); err != nil { + tmp.Close() + return errors.WithStack(err) + } + if sys, ok := st.Sys().(*syscall.Stat_t); ok { + if err := tmp.Chown(int(sys.Uid), int(sys.Gid)); err != nil { + tmp.Close() + return errors.WithStack(err) + } + } + if err := tmp.Close(); err != nil { + return errors.WithStack(err) + } + return errors.WithStack(os.Rename(tmpName, path)) +} diff --git a/executor/proxyca_linux_test.go b/executor/proxyca_linux_test.go new file mode 100644 index 000000000000..643527e83734 --- /dev/null +++ b/executor/proxyca_linux_test.go @@ -0,0 +1,60 @@ +//go:build linux + +package executor + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestInjectProxyCACleanupPreservesContainerChanges(t *testing.T) { + rootfs := t.TempDir() + bundle := filepath.Join(rootfs, "etc/ssl/certs/ca-certificates.crt") + require.NoError(t, os.MkdirAll(filepath.Dir(bundle), 0o755)) + original := []byte("original bundle\n") + require.NoError(t, os.WriteFile(bundle, original, 0o644)) + + caPEM := testCertPEM(t) + cleanup, err := InjectProxyCA(rootfs, caPEM) + require.NoError(t, err) + + dt, err := os.ReadFile(bundle) + require.NoError(t, err) + require.Contains(t, string(dt), string(caPEM)) + + require.NoError(t, os.WriteFile(bundle, append(dt, []byte("container change\n")...), 0o644)) + require.NoError(t, cleanup()) + + dt, err = os.ReadFile(bundle) + require.NoError(t, err) + require.NotContains(t, string(dt), string(caPEM)) + require.Contains(t, string(dt), string(original)) + require.Contains(t, string(dt), "container change\n") +} + +func testCertPEM(t *testing.T) []byte { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test buildkit proxy"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + require.NoError(t, err) + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} diff --git a/executor/proxyca_unsupported.go b/executor/proxyca_unsupported.go new file mode 100644 index 000000000000..29871ab52031 --- /dev/null +++ b/executor/proxyca_unsupported.go @@ -0,0 +1,8 @@ +//go:build !linux + +package executor + +// InjectProxyCA is only implemented for Linux rootfs layouts. +func InjectProxyCA(rootfsPath string, caPEM []byte) (func() error, error) { + return func() error { return nil }, nil +} diff --git a/executor/runcexecutor/executor.go b/executor/runcexecutor/executor.go index c4e5f8fd0674..e031c0c30baf 100644 --- a/executor/runcexecutor/executor.go +++ b/executor/runcexecutor/executor.go @@ -187,10 +187,16 @@ func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount, if !ok { return nil, errors.Errorf("unknown network mode %s", meta.NetMode) } - namespace, err := provider.New(ctx, meta.Hostname) + namespace, err := provider.New(ctx, meta.Hostname, network.NamespaceOptions{ + ProxyPolicy: meta.ProxyPolicy, + ProxyCapture: meta.ProxyCapture, + }) if err != nil { return nil, err } + if proxyNS, ok := namespace.(network.ProxyNamespace); ok { + meta.Env = append(meta.Env, proxyNS.ProxyEnv()...) + } doReleaseNetwork := true defer func() { if doReleaseNetwork { @@ -254,6 +260,13 @@ func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount, defer mount.Unmount(rootFSPath, 0) defer executor.MountStubsCleaner(context.WithoutCancel(ctx), rootFSPath, mounts, meta.RemoveMountStubsRecursive)() + if proxyNS, ok := namespace.(network.ProxyNamespace); ok { + cleanProxyCA, err := executor.InjectProxyCA(rootFSPath, proxyNS.ProxyCACert()) + if err != nil { + return nil, err + } + defer cleanProxyCA() + } uid, gid, sgids, err := oci.GetUser(rootFSPath, meta.User) if err != nil { diff --git a/go.mod b/go.mod index 178ee34c704c..29b5e561bcca 100644 --- a/go.mod +++ b/go.mod @@ -87,6 +87,7 @@ require ( github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab github.com/urfave/cli v1.22.17 github.com/vishvananda/netlink v1.3.1 + github.com/vishvananda/netns v0.0.5 go.etcd.io/bbolt v1.4.3 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.68.0 @@ -219,7 +220,6 @@ require ( github.com/transparency-dev/formats v0.0.0-20251017110053-404c0d5b696c // indirect github.com/transparency-dev/merkle v0.0.2 // indirect github.com/vbatts/tar-split v0.12.2 // indirect - github.com/vishvananda/netns v0.0.5 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel/metric v1.43.0 // indirect diff --git a/solver/errdefs/errdefs.pb.go b/solver/errdefs/errdefs.pb.go index bb72cefc1229..807d71f239ed 100644 --- a/solver/errdefs/errdefs.pb.go +++ b/solver/errdefs/errdefs.pb.go @@ -514,6 +514,134 @@ func (x *ContentCache) GetIndex() int64 { return 0 } +type ProvenanceMaterialsIncomplete struct { + state protoimpl.MessageState `protogen:"open.v1"` + Incomplete []*ProvenanceMaterialIncomplete `protobuf:"bytes,1,rep,name=incomplete,proto3" json:"incomplete,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProvenanceMaterialsIncomplete) Reset() { + *x = ProvenanceMaterialsIncomplete{} + mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProvenanceMaterialsIncomplete) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProvenanceMaterialsIncomplete) ProtoMessage() {} + +func (x *ProvenanceMaterialsIncomplete) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProvenanceMaterialsIncomplete.ProtoReflect.Descriptor instead. +func (*ProvenanceMaterialsIncomplete) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{9} +} + +func (x *ProvenanceMaterialsIncomplete) GetIncomplete() []*ProvenanceMaterialIncomplete { + if x != nil { + return x.Incomplete + } + return nil +} + +type ProvenanceMaterialIncomplete struct { + state protoimpl.MessageState `protogen:"open.v1"` + Op string `protobuf:"bytes,1,opt,name=op,proto3" json:"op,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Method string `protobuf:"bytes,3,opt,name=method,proto3" json:"method,omitempty"` + Uri string `protobuf:"bytes,4,opt,name=uri,proto3" json:"uri,omitempty"` + FinalUri string `protobuf:"bytes,5,opt,name=final_uri,json=finalUri,proto3" json:"final_uri,omitempty"` + Reason string `protobuf:"bytes,6,opt,name=reason,proto3" json:"reason,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProvenanceMaterialIncomplete) Reset() { + *x = ProvenanceMaterialIncomplete{} + mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProvenanceMaterialIncomplete) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProvenanceMaterialIncomplete) ProtoMessage() {} + +func (x *ProvenanceMaterialIncomplete) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProvenanceMaterialIncomplete.ProtoReflect.Descriptor instead. +func (*ProvenanceMaterialIncomplete) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP(), []int{10} +} + +func (x *ProvenanceMaterialIncomplete) GetOp() string { + if x != nil { + return x.Op + } + return "" +} + +func (x *ProvenanceMaterialIncomplete) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *ProvenanceMaterialIncomplete) GetMethod() string { + if x != nil { + return x.Method + } + return "" +} + +func (x *ProvenanceMaterialIncomplete) GetUri() string { + if x != nil { + return x.Uri + } + return "" +} + +func (x *ProvenanceMaterialIncomplete) GetFinalUri() string { + if x != nil { + return x.FinalUri + } + return "" +} + +func (x *ProvenanceMaterialIncomplete) GetReason() string { + if x != nil { + return x.Reason + } + return "" +} + var File_github_com_moby_buildkit_solver_errdefs_errdefs_proto protoreflect.FileDescriptor const file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDesc = "" + @@ -550,7 +678,18 @@ const file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDesc = "" + "FileAction\x12\x14\n" + "\x05index\x18\x01 \x01(\x03R\x05index\"$\n" + "\fContentCache\x12\x14\n" + - "\x05index\x18\x01 \x01(\x03R\x05indexB)Z'github.com/moby/buildkit/solver/errdefsb\x06proto3" + "\x05index\x18\x01 \x01(\x03R\x05index\"f\n" + + "\x1dProvenanceMaterialsIncomplete\x12E\n" + + "\n" + + "incomplete\x18\x01 \x03(\v2%.errdefs.ProvenanceMaterialIncompleteR\n" + + "incomplete\"\xa1\x01\n" + + "\x1cProvenanceMaterialIncomplete\x12\x0e\n" + + "\x02op\x18\x01 \x01(\tR\x02op\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x12\x16\n" + + "\x06method\x18\x03 \x01(\tR\x06method\x12\x10\n" + + "\x03uri\x18\x04 \x01(\tR\x03uri\x12\x1b\n" + + "\tfinal_uri\x18\x05 \x01(\tR\bfinalUri\x12\x16\n" + + "\x06reason\x18\x06 \x01(\tR\x06reasonB)Z'github.com/moby/buildkit/solver/errdefsb\x06proto3" var ( file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescOnce sync.Once @@ -564,34 +703,37 @@ func file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescGZIP() [] return file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDescData } -var file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_msgTypes = make([]protoimpl.MessageInfo, 12) var file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_goTypes = []any{ - (*Vertex)(nil), // 0: errdefs.Vertex - (*Source)(nil), // 1: errdefs.Source - (*Frontend)(nil), // 2: errdefs.Frontend - (*FrontendCap)(nil), // 3: errdefs.FrontendCap - (*CompatibilityFeature)(nil), // 4: errdefs.CompatibilityFeature - (*Subrequest)(nil), // 5: errdefs.Subrequest - (*Solve)(nil), // 6: errdefs.Solve - (*FileAction)(nil), // 7: errdefs.FileAction - (*ContentCache)(nil), // 8: errdefs.ContentCache - nil, // 9: errdefs.Solve.DescriptionEntry - (*pb.SourceInfo)(nil), // 10: pb.SourceInfo - (*pb.Range)(nil), // 11: pb.Range - (*pb.Op)(nil), // 12: pb.Op + (*Vertex)(nil), // 0: errdefs.Vertex + (*Source)(nil), // 1: errdefs.Source + (*Frontend)(nil), // 2: errdefs.Frontend + (*FrontendCap)(nil), // 3: errdefs.FrontendCap + (*CompatibilityFeature)(nil), // 4: errdefs.CompatibilityFeature + (*Subrequest)(nil), // 5: errdefs.Subrequest + (*Solve)(nil), // 6: errdefs.Solve + (*FileAction)(nil), // 7: errdefs.FileAction + (*ContentCache)(nil), // 8: errdefs.ContentCache + (*ProvenanceMaterialsIncomplete)(nil), // 9: errdefs.ProvenanceMaterialsIncomplete + (*ProvenanceMaterialIncomplete)(nil), // 10: errdefs.ProvenanceMaterialIncomplete + nil, // 11: errdefs.Solve.DescriptionEntry + (*pb.SourceInfo)(nil), // 12: pb.SourceInfo + (*pb.Range)(nil), // 13: pb.Range + (*pb.Op)(nil), // 14: pb.Op } var file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_depIdxs = []int32{ - 10, // 0: errdefs.Source.info:type_name -> pb.SourceInfo - 11, // 1: errdefs.Source.ranges:type_name -> pb.Range - 12, // 2: errdefs.Solve.op:type_name -> pb.Op + 12, // 0: errdefs.Source.info:type_name -> pb.SourceInfo + 13, // 1: errdefs.Source.ranges:type_name -> pb.Range + 14, // 2: errdefs.Solve.op:type_name -> pb.Op 7, // 3: errdefs.Solve.file:type_name -> errdefs.FileAction 8, // 4: errdefs.Solve.cache:type_name -> errdefs.ContentCache - 9, // 5: errdefs.Solve.description:type_name -> errdefs.Solve.DescriptionEntry - 6, // [6:6] is the sub-list for method output_type - 6, // [6:6] is the sub-list for method input_type - 6, // [6:6] is the sub-list for extension type_name - 6, // [6:6] is the sub-list for extension extendee - 0, // [0:6] is the sub-list for field type_name + 11, // 5: errdefs.Solve.description:type_name -> errdefs.Solve.DescriptionEntry + 10, // 6: errdefs.ProvenanceMaterialsIncomplete.incomplete:type_name -> errdefs.ProvenanceMaterialIncomplete + 7, // [7:7] is the sub-list for method output_type + 7, // [7:7] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name } func init() { file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_init() } @@ -609,7 +751,7 @@ func file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDesc), len(file_github_com_moby_buildkit_solver_errdefs_errdefs_proto_rawDesc)), NumEnums: 0, - NumMessages: 10, + NumMessages: 12, NumExtensions: 0, NumServices: 0, }, diff --git a/solver/errdefs/errdefs.proto b/solver/errdefs/errdefs.proto index c03b9c75ef38..69a4a161f962 100644 --- a/solver/errdefs/errdefs.proto +++ b/solver/errdefs/errdefs.proto @@ -55,3 +55,16 @@ message ContentCache { // Original index of result that failed the slow cache calculation. int64 index = 1; } + +message ProvenanceMaterialsIncomplete { + repeated ProvenanceMaterialIncomplete incomplete = 1; +} + +message ProvenanceMaterialIncomplete { + string op = 1; + string name = 2; + string method = 3; + string uri = 4; + string final_uri = 5; + string reason = 6; +} diff --git a/solver/errdefs/errdefs_vtproto.pb.go b/solver/errdefs/errdefs_vtproto.pb.go index b88a9ad6040d..18421bc1d3f8 100644 --- a/solver/errdefs/errdefs_vtproto.pb.go +++ b/solver/errdefs/errdefs_vtproto.pb.go @@ -220,6 +220,51 @@ func (m *ContentCache) CloneMessageVT() proto.Message { return m.CloneVT() } +func (m *ProvenanceMaterialsIncomplete) CloneVT() *ProvenanceMaterialsIncomplete { + if m == nil { + return (*ProvenanceMaterialsIncomplete)(nil) + } + r := new(ProvenanceMaterialsIncomplete) + if rhs := m.Incomplete; rhs != nil { + tmpContainer := make([]*ProvenanceMaterialIncomplete, len(rhs)) + for k, v := range rhs { + tmpContainer[k] = v.CloneVT() + } + r.Incomplete = tmpContainer + } + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *ProvenanceMaterialsIncomplete) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (m *ProvenanceMaterialIncomplete) CloneVT() *ProvenanceMaterialIncomplete { + if m == nil { + return (*ProvenanceMaterialIncomplete)(nil) + } + r := new(ProvenanceMaterialIncomplete) + r.Op = m.Op + r.Name = m.Name + r.Method = m.Method + r.Uri = m.Uri + r.FinalUri = m.FinalUri + r.Reason = m.Reason + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *ProvenanceMaterialIncomplete) CloneMessageVT() proto.Message { + return m.CloneVT() +} + func (this *Vertex) EqualVT(that *Vertex) bool { if this == that { return true @@ -504,6 +549,73 @@ func (this *ContentCache) EqualMessageVT(thatMsg proto.Message) bool { } return this.EqualVT(that) } +func (this *ProvenanceMaterialsIncomplete) EqualVT(that *ProvenanceMaterialsIncomplete) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if len(this.Incomplete) != len(that.Incomplete) { + return false + } + for i, vx := range this.Incomplete { + vy := that.Incomplete[i] + if p, q := vx, vy; p != q { + if p == nil { + p = &ProvenanceMaterialIncomplete{} + } + if q == nil { + q = &ProvenanceMaterialIncomplete{} + } + if !p.EqualVT(q) { + return false + } + } + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *ProvenanceMaterialsIncomplete) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*ProvenanceMaterialsIncomplete) + if !ok { + return false + } + return this.EqualVT(that) +} +func (this *ProvenanceMaterialIncomplete) EqualVT(that *ProvenanceMaterialIncomplete) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if this.Op != that.Op { + return false + } + if this.Name != that.Name { + return false + } + if this.Method != that.Method { + return false + } + if this.Uri != that.Uri { + return false + } + if this.FinalUri != that.FinalUri { + return false + } + if this.Reason != that.Reason { + return false + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *ProvenanceMaterialIncomplete) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*ProvenanceMaterialIncomplete) + if !ok { + return false + } + return this.EqualVT(that) +} func (m *Vertex) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -982,6 +1094,126 @@ func (m *ContentCache) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *ProvenanceMaterialsIncomplete) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ProvenanceMaterialsIncomplete) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *ProvenanceMaterialsIncomplete) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Incomplete) > 0 { + for iNdEx := len(m.Incomplete) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Incomplete[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *ProvenanceMaterialIncomplete) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ProvenanceMaterialIncomplete) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *ProvenanceMaterialIncomplete) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Reason) > 0 { + i -= len(m.Reason) + copy(dAtA[i:], m.Reason) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Reason))) + i-- + dAtA[i] = 0x32 + } + if len(m.FinalUri) > 0 { + i -= len(m.FinalUri) + copy(dAtA[i:], m.FinalUri) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.FinalUri))) + i-- + dAtA[i] = 0x2a + } + if len(m.Uri) > 0 { + i -= len(m.Uri) + copy(dAtA[i:], m.Uri) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Uri))) + i-- + dAtA[i] = 0x22 + } + if len(m.Method) > 0 { + i -= len(m.Method) + copy(dAtA[i:], m.Method) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Method))) + i-- + dAtA[i] = 0x1a + } + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0x12 + } + if len(m.Op) > 0 { + i -= len(m.Op) + copy(dAtA[i:], m.Op) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Op))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *Vertex) SizeVT() (n int) { if m == nil { return 0 @@ -1170,6 +1402,56 @@ func (m *ContentCache) SizeVT() (n int) { return n } +func (m *ProvenanceMaterialsIncomplete) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Incomplete) > 0 { + for _, e := range m.Incomplete { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *ProvenanceMaterialIncomplete) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Op) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Name) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Method) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Uri) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.FinalUri) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Reason) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + func (m *Vertex) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -2257,3 +2539,331 @@ func (m *ContentCache) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *ProvenanceMaterialsIncomplete) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ProvenanceMaterialsIncomplete: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ProvenanceMaterialsIncomplete: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Incomplete", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Incomplete = append(m.Incomplete, &ProvenanceMaterialIncomplete{}) + if err := m.Incomplete[len(m.Incomplete)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ProvenanceMaterialIncomplete) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ProvenanceMaterialIncomplete: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ProvenanceMaterialIncomplete: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Op", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Op = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Method", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Method = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Uri", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Uri = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FinalUri", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.FinalUri = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Reason", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Reason = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/solver/errdefs/provenance.go b/solver/errdefs/provenance.go new file mode 100644 index 000000000000..16a51d51472b --- /dev/null +++ b/solver/errdefs/provenance.go @@ -0,0 +1,47 @@ +package errdefs + +import ( + "github.com/containerd/typeurl/v2" + "github.com/moby/buildkit/util/grpcerrors" + "github.com/pkg/errors" +) + +func init() { + typeurl.Register((*ProvenanceMaterialsIncomplete)(nil), "github.com/moby/buildkit", "errdefs.ProvenanceMaterialsIncomplete+json") +} + +type ProvenanceMaterialsIncompleteError struct { + *ProvenanceMaterialsIncomplete + error +} + +func (e *ProvenanceMaterialsIncompleteError) Unwrap() error { + return e.error +} + +func (e *ProvenanceMaterialsIncompleteError) ToProto() grpcerrors.TypedErrorProto { + return e.ProvenanceMaterialsIncomplete +} + +func (p *ProvenanceMaterialsIncomplete) WrapError(err error) error { + return &ProvenanceMaterialsIncompleteError{ + error: err, + ProvenanceMaterialsIncomplete: p, + } +} + +func WithProvenanceMaterialsIncomplete(err error, incomplete []*ProvenanceMaterialIncomplete) error { + if err == nil { + return nil + } + return &ProvenanceMaterialsIncompleteError{ + error: err, + ProvenanceMaterialsIncomplete: &ProvenanceMaterialsIncomplete{ + Incomplete: incomplete, + }, + } +} + +func NewProvenanceMaterialsIncomplete(incomplete []*ProvenanceMaterialIncomplete) error { + return WithProvenanceMaterialsIncomplete(errors.New("provenance materials are incomplete"), incomplete) +} diff --git a/solver/errdefs/provenance_test.go b/solver/errdefs/provenance_test.go new file mode 100644 index 000000000000..7e31cfeaf926 --- /dev/null +++ b/solver/errdefs/provenance_test.go @@ -0,0 +1,28 @@ +package errdefs + +import ( + "testing" + + "github.com/moby/buildkit/util/grpcerrors" + "github.com/stretchr/testify/require" +) + +func TestProvenanceMaterialsIncompleteRoundTrip(t *testing.T) { + err := NewProvenanceMaterialsIncomplete([]*ProvenanceMaterialIncomplete{ + { + Op: "sha256:abc", + Name: "curl https://example.com/missing", + Method: "GET", + Uri: "https://example.com/missing", + Reason: "unsuccessful_response", + }, + }) + + decoded := grpcerrors.FromGRPC(grpcerrors.ToGRPC(t.Context(), err)) + var pe *ProvenanceMaterialsIncompleteError + require.ErrorAs(t, decoded, &pe) + require.Len(t, pe.Incomplete, 1) + require.Equal(t, "https://example.com/missing", pe.Incomplete[0].Uri) + require.Equal(t, "unsuccessful_response", pe.Incomplete[0].Reason) + require.ErrorContains(t, decoded, "provenance materials are incomplete") +} diff --git a/solver/llbsolver/bridge.go b/solver/llbsolver/bridge.go index d6f2da5975b1..5955ff6d1a94 100644 --- a/solver/llbsolver/bridge.go +++ b/solver/llbsolver/bridge.go @@ -40,6 +40,7 @@ type llbBridge struct { cmsMu sync.Mutex sm *session.Manager provenanceStore *provenanceStore + proxyNetwork bool executorOnce sync.Once executorErr error @@ -139,8 +140,7 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp b.cmsMu.Unlock() } dpc := &detectPrunedCacheID{} - - edge, err := Load(ctx, def, b.policy(polEngine), dpc.Load, ValidateEntitlements(ent, w.CDIManager()), WithCacheSources(cms), NormalizeRuntimePlatforms(), WithValidateCaps()) + edge, err := loadWithProxyNetwork(ctx, def, b.policy(polEngine), b.proxyNetwork, dpc.Load, ValidateEntitlements(ent, w.CDIManager()), WithCacheSources(cms), NormalizeRuntimePlatforms(), WithValidateCaps()) if err != nil { return nil, errors.Wrap(err, "failed to load LLB") } @@ -167,11 +167,22 @@ func (b *llbBridge) policy(engine *sourcepolicy.Engine) SourcePolicyEvaluator { } } -func (b *llbBridge) validateEntitlements(p executor.ProcessInfo) error { +func (b *llbBridge) validateEntitlements(p *executor.ProcessInfo) error { ent, err := loadEntitlements(b.builder) if err != nil { return err } + if b.proxyNetwork { + switch p.Meta.NetMode { + case pb.NetMode_UNSET: + p.Meta.NetMode = pb.NetMode_PROXY + case pb.NetMode_NONE, pb.NetMode_PROXY: + default: + return errors.Errorf("network mode %s is not allowed when proxy network is enabled", p.Meta.NetMode) + } + } else if p.Meta.NetMode == pb.NetMode_PROXY { + return errors.Errorf("network mode %s requires proxy network to be enabled for the build", p.Meta.NetMode) + } v := entitlements.Values{ NetworkHost: p.Meta.NetMode == pb.NetMode_HOST, SecurityInsecure: p.Meta.SecurityMode == pb.SecurityMode_INSECURE, @@ -180,9 +191,16 @@ func (b *llbBridge) validateEntitlements(p executor.ProcessInfo) error { } func (b *llbBridge) Run(ctx context.Context, id string, rootfs executor.Mount, mounts []executor.Mount, process executor.ProcessInfo, started chan<- struct{}) (resourcestypes.Recorder, error) { - if err := b.validateEntitlements(process); err != nil { + if err := b.validateEntitlements(&process); err != nil { + return nil, err + } + policy, err := b.ProxyPolicy() + if err != nil { return nil, err } + if policy != nil { + process.Meta.ProxyPolicy = policy + } if err := b.loadExecutor(); err != nil { return nil, err @@ -191,9 +209,16 @@ func (b *llbBridge) Run(ctx context.Context, id string, rootfs executor.Mount, m } func (b *llbBridge) Exec(ctx context.Context, id string, process executor.ProcessInfo) error { - if err := b.validateEntitlements(process); err != nil { + if err := b.validateEntitlements(&process); err != nil { return err } + policy, err := b.ProxyPolicy() + if err != nil { + return err + } + if policy != nil { + process.Meta.ProxyPolicy = policy + } if err := b.loadExecutor(); err != nil { return err diff --git a/solver/llbsolver/network.go b/solver/llbsolver/network.go new file mode 100644 index 000000000000..dbf7358274fa --- /dev/null +++ b/solver/llbsolver/network.go @@ -0,0 +1,54 @@ +package llbsolver + +import ( + "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/sourcepolicy" + spb "github.com/moby/buildkit/sourcepolicy/pb" + "github.com/moby/buildkit/util/network" + "github.com/pkg/errors" +) + +func setProxyNetwork(op *pb.Op, proxyNetwork bool) error { + exec := op.GetExec() + if exec == nil { + return nil + } + if !proxyNetwork { + if exec.Network == pb.NetMode_PROXY { + return errors.Errorf("network mode %s requires proxy network to be enabled for the build", exec.Network) + } + return nil + } + switch exec.Network { + case pb.NetMode_UNSET: + exec.Network = pb.NetMode_PROXY + case pb.NetMode_NONE, pb.NetMode_PROXY: + return nil + default: + return errors.Errorf("network mode %s is not allowed when proxy network is enabled", exec.Network) + } + return nil +} + +func (b *provenanceBridge) ProxyPolicy() (network.ProxyPolicy, error) { + return b.llbBridge.ProxyPolicy() +} + +func (b *llbBridge) ProxyPolicy() (network.ProxyPolicy, error) { + srcPol, err := loadSourcePolicy(b.builder) + if err != nil { + return nil, err + } + policySession, err := loadSourcePolicySession(b.builder) + if err != nil { + return nil, err + } + if (srcPol == nil || len(srcPol.Rules) == 0) && policySession == "" { + return nil, nil + } + var policies []*spb.Policy + if srcPol != nil { + policies = append(policies, srcPol) + } + return b.policy(sourcepolicy.NewEngine(policies)), nil +} diff --git a/solver/llbsolver/network_test.go b/solver/llbsolver/network_test.go new file mode 100644 index 000000000000..087738691cd7 --- /dev/null +++ b/solver/llbsolver/network_test.go @@ -0,0 +1,5 @@ +package llbsolver + +import "github.com/moby/buildkit/util/network" + +var _ network.ProxyPolicy = (*policyEvaluator)(nil) diff --git a/solver/llbsolver/ops/exec.go b/solver/llbsolver/ops/exec.go index 09b37702cf8f..f0b95928de24 100644 --- a/solver/llbsolver/ops/exec.go +++ b/solver/llbsolver/ops/exec.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "io" "os" "path" "runtime" @@ -24,6 +25,7 @@ import ( "github.com/moby/buildkit/solver/llbsolver/ops/opsutils" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/cachedigest" + "github.com/moby/buildkit/util/network" "github.com/moby/buildkit/util/progress/logs" utilsystem "github.com/moby/buildkit/util/system" "github.com/moby/buildkit/worker" @@ -48,6 +50,7 @@ type ExecOp struct { parallelism *semaphore.Weighted rec resourcestypes.Recorder digest digest.Digest + proxyCap *network.ProxyCapture } var _ solver.Op = &ExecOp{} @@ -495,12 +498,20 @@ func (e *ExecOp) Exec(ctx context.Context, jobCtx solver.JobContext, inputs []so } }() + if e.op.Network == pb.NetMode_PROXY { + e.proxyCap = network.NewProxyCapture() + meta.ProxyCapture = e.proxyCap + } + rec, execErr := e.exec.Run(ctx, "", p.Root, p.Mounts, executor.ProcessInfo{ Meta: meta, Stdin: nil, Stdout: stdout, Stderr: stderr, }, nil) + if e.proxyCap != nil { + logProxyRequests(stderr, e.proxyCap.Requests()) + } for i, out := range p.OutputRefs { if mutable, ok := out.Ref.(cache.MutableRef); ok { @@ -519,6 +530,20 @@ func (e *ExecOp) Exec(ctx context.Context, jobCtx solver.JobContext, inputs []so return results, errors.Wrapf(execErr, "process %q did not complete successfully", strings.Join(e.op.Meta.Args, " ")) } +func logProxyRequests(w io.Writer, requests []network.ProxyRequest) { + if len(requests) == 0 { + return + } + _, _ = fmt.Fprintln(w, "proxy network requests:") + for _, req := range requests { + if req.StatusCode != 0 { + _, _ = fmt.Fprintf(w, "- %s %s -> %d\n", req.Method, req.URL, req.StatusCode) + } else { + _, _ = fmt.Fprintf(w, "- %s %s\n", req.Method, req.URL) + } + } +} + func proxyEnvList(p *pb.ProxyEnv) []string { out := []string{} if v := p.HttpProxy; v != "" { @@ -589,3 +614,7 @@ func (e *ExecOp) Samples() (*resourcestypes.Samples, error) { } return e.rec.Samples() } + +func (e *ExecOp) ProxyCapture() *network.ProxyCapture { + return e.proxyCap +} diff --git a/solver/llbsolver/ops/exec_test.go b/solver/llbsolver/ops/exec_test.go index 03e1ac333d6f..5512c475da55 100644 --- a/solver/llbsolver/ops/exec_test.go +++ b/solver/llbsolver/ops/exec_test.go @@ -1,13 +1,17 @@ package ops import ( + "bytes" "context" + "net/http" + "strings" "testing" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/util/network" "github.com/pkg/errors" "github.com/stretchr/testify/require" ) @@ -35,6 +39,29 @@ func TestDedupePaths(t *testing.T) { require.Equal(t, []string{"/"}, res) } +func TestLogProxyRequests(t *testing.T) { + var buf bytes.Buffer + logProxyRequests(&buf, []network.ProxyRequest{ + {Method: "GET", URL: "https://example.com/file", StatusCode: http.StatusOK}, + {Method: "POST", URL: "https://xxxxx:xxxxx@example.com/token", StatusCode: http.StatusCreated}, + {Method: "GET", URL: "https://example.com/unknown-status"}, + }) + + require.Equal(t, strings.Join([]string{ + "proxy network requests:", + "- GET https://example.com/file -> 200", + "- POST https://xxxxx:xxxxx@example.com/token -> 201", + "- GET https://example.com/unknown-status", + "", + }, "\n"), buf.String()) +} + +func TestLogProxyRequestsEmpty(t *testing.T) { + var buf bytes.Buffer + logProxyRequests(&buf, nil) + require.Empty(t, buf.String()) +} + func TestExecOpCacheMap(t *testing.T) { type testCase struct { name string diff --git a/solver/llbsolver/provenance.go b/solver/llbsolver/provenance.go index e39d2f0a638b..ca93fffdc1de 100644 --- a/solver/llbsolver/provenance.go +++ b/solver/llbsolver/provenance.go @@ -343,6 +343,28 @@ func captureProvenance(ctx context.Context, res solver.CachedResultWithProvenanc if pr.Network != pb.NetMode_NONE { c.NetworkAccess = true } + if pr.Network == pb.NetMode_PROXY { + c.ProxyNetwork = true + proxyCap := op.ProxyCapture() + if proxyCap != nil { + for _, m := range proxyCap.Materials() { + c.AddHTTP(provenancetypes.HTTPSource{ + URL: m.URL, + Digest: m.Digest, + }) + } + for _, in := range proxyCap.Incomplete() { + c.IncompleteMaterials = true + c.ProxyIncomplete = append(c.ProxyIncomplete, provenancetypes.ProxyCaptureIncomplete{ + Op: op.Digest().String(), + Name: strings.Join(pr.Meta.Args, " "), + Method: in.Method, + URI: in.URL, + Reason: in.Reason, + }) + } + } + } samples, err := op.Samples() if err != nil { return err @@ -400,6 +422,14 @@ func NewProvenanceCreator(ctx context.Context, slsaVersion provenancetypes.Prove b, err := strconv.ParseBool(v) withUsage = err == nil && b } + completeMaterials := false + if v, ok := attrs["complete-materials"]; ok { + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse complete-materials flag %q", v) + } + completeMaterials = b + } pr, err := provenance.NewPredicate(cp) if err != nil { @@ -408,6 +438,9 @@ func NewProvenanceCreator(ctx context.Context, slsaVersion provenancetypes.Prove if pr.RunDetails.Metadata == nil { pr.RunDetails.Metadata = &provenancetypes.ProvenanceMetadataSLSA1{} } + if completeMaterials && !pr.RunDetails.Metadata.Completeness.ResolvedDependencies { + return nil, incompleteMaterialsError(cp) + } st := j.StartedTime() @@ -534,6 +567,75 @@ func scrubMinRequest(req *provenancetypes.Parameters) bool { return incomplete } +func incompleteMaterialsError(c *provenance.Capture) error { + var b strings.Builder + b.WriteString("provenance materials are incomplete\n\n") + b.WriteString("The build requested complete provenance materials, but not all dependencies could be captured.\n\n") + details := make([]*errdefs.ProvenanceMaterialIncomplete, 0, len(c.ProxyIncomplete)+len(c.Sources.Local)) + if len(c.ProxyIncomplete) == 0 && len(c.Sources.Local) == 0 { + return errdefs.WithProvenanceMaterialsIncomplete(errors.New(strings.TrimSpace(b.String())), details) + } + + if len(c.Sources.Local) > 0 { + b.WriteString("Uncaptured local sources:") + for _, l := range c.Sources.Local { + details = append(details, &errdefs.ProvenanceMaterialIncomplete{ + Name: l.Name, + Reason: "local_source", + }) + b.WriteString("\n - ") + if l.Name != "" { + b.WriteString(l.Name) + } else { + b.WriteString("") + } + b.WriteString("\n reason: local_source") + } + } + + if len(c.ProxyIncomplete) > 0 { + if len(c.Sources.Local) > 0 { + b.WriteString("\n\n") + } + b.WriteString("Uncaptured requests:") + } + for _, in := range c.ProxyIncomplete { + details = append(details, &errdefs.ProvenanceMaterialIncomplete{ + Op: in.Op, + Name: in.Name, + Method: in.Method, + Uri: in.URI, + FinalUri: in.FinalURI, + Reason: in.Reason, + }) + b.WriteString("\n - ") + if in.Name != "" { + b.WriteString(in.Name) + } else if in.Op != "" { + b.WriteString(in.Op) + } else { + b.WriteString(in.URI) + } + if in.Method != "" { + b.WriteString("\n method: ") + b.WriteString(in.Method) + } + if in.URI != "" { + b.WriteString("\n url: ") + b.WriteString(in.URI) + } + if in.FinalURI != "" { + b.WriteString("\n finalUrl: ") + b.WriteString(in.FinalURI) + } + if in.Reason != "" { + b.WriteString("\n reason: ") + b.WriteString(in.Reason) + } + } + return errdefs.WithProvenanceMaterialsIncomplete(errors.New(b.String()), details) +} + func (p *ProvenanceCreator) PredicateType() string { if p.slsaVersion == provenancetypes.ProvenanceSLSA02 { return slsa02.PredicateSLSAProvenance diff --git a/solver/llbsolver/provenance/capture.go b/solver/llbsolver/provenance/capture.go index a6f1eb7102fe..1d9a18eab2f2 100644 --- a/solver/llbsolver/provenance/capture.go +++ b/solver/llbsolver/provenance/capture.go @@ -19,7 +19,9 @@ type Capture struct { Request provenancetypes.Parameters Sources provenancetypes.Sources NetworkAccess bool + ProxyNetwork bool IncompleteMaterials bool + ProxyIncomplete []provenancetypes.ProxyCaptureIncomplete Samples map[digest.Digest]*resourcestypes.Samples } @@ -29,7 +31,9 @@ func (c *Capture) Clone() *Capture { } out := &Capture{ NetworkAccess: c.NetworkAccess, + ProxyNetwork: c.ProxyNetwork, IncompleteMaterials: c.IncompleteMaterials, + ProxyIncomplete: slices.Clone(c.ProxyIncomplete), } if req := c.Request.Clone(); req != nil { out.Request = *req @@ -78,9 +82,13 @@ func (c *Capture) Merge(c2 *Capture) error { if c2.NetworkAccess { c.NetworkAccess = true } + if c2.ProxyNetwork { + c.ProxyNetwork = true + } if c2.IncompleteMaterials { c.IncompleteMaterials = true } + c.ProxyIncomplete = append(c.ProxyIncomplete, c2.ProxyIncomplete...) return nil } @@ -106,6 +114,18 @@ func (c *Capture) Sort() { slices.SortFunc(c.Request.SSH, func(a, b *provenancetypes.SSH) int { return cmp.Compare(a.ID, b.ID) }) + slices.SortFunc(c.ProxyIncomplete, func(a, b provenancetypes.ProxyCaptureIncomplete) int { + if c := cmp.Compare(a.Op, b.Op); c != 0 { + return c + } + if c := cmp.Compare(a.URI, b.URI); c != 0 { + return c + } + if c := cmp.Compare(a.Method, b.Method); c != 0 { + return c + } + return cmp.Compare(a.Reason, b.Reason) + }) } // OptimizeImageSources filters out image sources by digest reference if same digest diff --git a/solver/llbsolver/provenance/predicate.go b/solver/llbsolver/provenance/predicate.go index 70e2761ac7d5..571c31dcc65c 100644 --- a/solver/llbsolver/provenance/predicate.go +++ b/solver/llbsolver/provenance/predicate.go @@ -2,6 +2,7 @@ package provenance import ( "maps" + "slices" "strings" "github.com/containerd/platforms" @@ -288,6 +289,16 @@ func NewPredicate(c *Capture) (*provenancetypes.ProvenancePredicateSLSA1, error) if len(vcs) > 0 { pr.RunDetails.Metadata.BuildKitMetadata.VCS = vcs } + if c.ProxyNetwork { + pr.RunDetails.Metadata.BuildKitMetadata.Network = &provenancetypes.NetworkMetadata{ + Mode: "proxy", + } + if len(c.ProxyIncomplete) > 0 { + pr.RunDetails.Metadata.BuildKitMetadata.Network.Proxy = &provenancetypes.ProxyNetworkMetadata{ + Incomplete: slices.Clone(c.ProxyIncomplete), + } + } + } return pr, nil } diff --git a/solver/llbsolver/provenance/predicate_test.go b/solver/llbsolver/provenance/predicate_test.go index 66a0b9bc51c9..961b42eba3bf 100644 --- a/solver/llbsolver/provenance/predicate_test.go +++ b/solver/llbsolver/provenance/predicate_test.go @@ -336,3 +336,29 @@ func TestNewPredicateKeepsContextSubdir(t *testing.T) { require.Equal(t, "", pr.BuildDefinition.ExternalParameters.ConfigSource.URI) require.Equal(t, "src", pr.BuildDefinition.ExternalParameters.Request.Args["contextsubdir"]) } + +func TestNewPredicateProxyNetworkMetadata(t *testing.T) { + t.Parallel() + + c := &Capture{ + ProxyNetwork: true, + IncompleteMaterials: true, + ProxyIncomplete: []provenancetypes.ProxyCaptureIncomplete{ + { + Op: "sha256:abc", + Name: "curl -X POST https://example.com/token", + Method: "POST", + URI: "https://example.com/token", + Reason: "method_not_materializable", + }, + }, + } + pr, err := NewPredicate(c) + require.NoError(t, err) + require.NotNil(t, pr.RunDetails.Metadata.BuildKitMetadata.Network) + require.Equal(t, "proxy", pr.RunDetails.Metadata.BuildKitMetadata.Network.Mode) + require.Len(t, pr.RunDetails.Metadata.BuildKitMetadata.Network.Proxy.Incomplete, 1) + require.Equal(t, "method_not_materializable", pr.RunDetails.Metadata.BuildKitMetadata.Network.Proxy.Incomplete[0].Reason) + require.False(t, pr.RunDetails.Metadata.Completeness.ResolvedDependencies) + require.False(t, pr.RunDetails.Metadata.Hermetic) +} diff --git a/solver/llbsolver/provenance/types/types.go b/solver/llbsolver/provenance/types/types.go index 546395178cbd..e5b5f3741c0c 100644 --- a/solver/llbsolver/provenance/types/types.go +++ b/solver/llbsolver/provenance/types/types.go @@ -349,6 +349,25 @@ type BuildKitMetadata struct { Source *Source `json:"source,omitempty"` Layers map[string][][]ocispecs.Descriptor `json:"layers,omitempty"` SysUsage []*resourcestypes.SysSample `json:"sysUsage,omitempty"` + Network *NetworkMetadata `json:"network,omitempty"` +} + +type NetworkMetadata struct { + Mode string `json:"mode,omitempty"` + Proxy *ProxyNetworkMetadata `json:"proxy,omitempty"` +} + +type ProxyNetworkMetadata struct { + Incomplete []ProxyCaptureIncomplete `json:"incomplete,omitempty"` +} + +type ProxyCaptureIncomplete struct { + Op string `json:"op,omitempty"` + Name string `json:"name,omitempty"` + Method string `json:"method,omitempty"` + URI string `json:"uri,omitempty"` + FinalURI string `json:"finalUri,omitempty"` + Reason string `json:"reason,omitempty"` } type BuildKitComplete struct { diff --git a/solver/llbsolver/provenance_test.go b/solver/llbsolver/provenance_test.go new file mode 100644 index 000000000000..15cfaf6f4191 --- /dev/null +++ b/solver/llbsolver/provenance_test.go @@ -0,0 +1,28 @@ +package llbsolver + +import ( + "testing" + + "github.com/moby/buildkit/solver/errdefs" + "github.com/moby/buildkit/solver/llbsolver/provenance" + provenancetypes "github.com/moby/buildkit/solver/llbsolver/provenance/types" + "github.com/stretchr/testify/require" +) + +func TestIncompleteMaterialsErrorIncludesLocalSources(t *testing.T) { + err := incompleteMaterialsError(&provenance.Capture{ + Sources: provenancetypes.Sources{ + Local: []provenancetypes.LocalSource{ + {Name: "context"}, + }, + }, + }) + + var materialsErr *errdefs.ProvenanceMaterialsIncompleteError + require.ErrorAs(t, err, &materialsErr) + require.Len(t, materialsErr.Incomplete, 1) + require.Equal(t, "context", materialsErr.Incomplete[0].Name) + require.Equal(t, "local_source", materialsErr.Incomplete[0].Reason) + require.ErrorContains(t, err, "Uncaptured local sources") + require.ErrorContains(t, err, "context") +} diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 26e1edbda643..a109521377f3 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -59,6 +59,7 @@ type Opt struct { WorkerController *worker.Controller HistoryQueue *history.Queue ResourceMonitor *resources.Monitor + ProxyNetwork bool ProvenanceEnv map[string]any } @@ -74,6 +75,7 @@ type Solver struct { entitlements []string history *history.Queue sysSampler *resources.Sampler[*resourcestypes.SysSample] + proxyNetwork bool provenanceEnv map[string]any provenanceStore *provenanceStore } @@ -105,6 +107,7 @@ func New(opt Opt) (*Solver, error) { sm: opt.SessionManager, entitlements: opt.Entitlements, history: opt.HistoryQueue, + proxyNetwork: opt.ProxyNetwork, provenanceEnv: opt.ProvenanceEnv, provenanceStore: newProvenanceStore(), } @@ -140,7 +143,13 @@ func (s *Solver) resolver() solver.ResolveOpFunc { } } -func (s *Solver) bridge(b solver.Builder) *provenanceBridge { +func (s *Solver) bridge(b solver.Builder, opts ...bridgeOpt) *provenanceBridge { + cfg := bridgeConfig{ + proxyNetwork: s.proxyNetwork, + } + for _, opt := range opts { + opt(&cfg) + } return &provenanceBridge{llbBridge: &llbBridge{ builder: b, frontends: s.frontends, @@ -150,14 +159,27 @@ func (s *Solver) bridge(b solver.Builder) *provenanceBridge { cms: map[string]solver.CacheManager{}, sm: s.sm, provenanceStore: s.provenanceStore, + proxyNetwork: cfg.proxyNetwork, }} } +type bridgeConfig struct { + proxyNetwork bool +} + +type bridgeOpt func(*bridgeConfig) + +func withBridgeProxyNetwork(proxyNetwork bool) bridgeOpt { + return func(cfg *bridgeConfig) { + cfg.proxyNetwork = proxyNetwork + } +} + func (s *Solver) Bridge(b solver.Builder) frontend.FrontendLLBBridge { return s.bridge(b) } -func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req frontend.SolveRequest, compatibilityVersion int, exp ExporterRequest, ent []entitlements.Entitlement, post []Processor, internal bool, srcPol *spb.Policy, policySession string) (_ *client.SolveResponse, err error) { +func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req frontend.SolveRequest, compatibilityVersion int, exp ExporterRequest, ent []entitlements.Entitlement, post []Processor, internal bool, srcPol *spb.Policy, policySession string, proxyNetwork bool) (_ *client.SolveResponse, err error) { hasNamedDockerfileContext := false for k := range req.FrontendOpt { if k == "context:dockerfile.v0" || strings.HasPrefix(k, "context:dockerfile.v0::") { @@ -227,7 +249,7 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro j.SessionID = sessionID - br := s.bridge(j) + br := s.bridge(j, withBridgeProxyNetwork(proxyNetwork || s.proxyNetwork)) defer br.releaseProvenanceRefs() rootReq := req.Clone() br.rootReq = &rootReq diff --git a/solver/llbsolver/vertex.go b/solver/llbsolver/vertex.go index ca839ccb622c..6b9f35217731 100644 --- a/solver/llbsolver/vertex.go +++ b/solver/llbsolver/vertex.go @@ -232,7 +232,7 @@ func (dpc *detectPrunedCacheID) Load(op *pb.Op, md *pb.OpMetadata, opt *solver.V } func Load(ctx context.Context, def *pb.Definition, polEngine SourcePolicyEvaluator, opts ...LoadOpt) (solver.Edge, error) { - return loadLLB(ctx, def, polEngine, func(dgst digest.Digest, op *op, load func(digest.Digest) (solver.Vertex, error)) (solver.Vertex, error) { + return loadLLB(ctx, def, polEngine, nil, func(dgst digest.Digest, op *op, load func(digest.Digest) (solver.Vertex, error)) (solver.Vertex, error) { vtx, err := newVertex(dgst, op.Op, op.Metadata, load, opts...) if err != nil { return nil, err @@ -241,7 +241,17 @@ func Load(ctx context.Context, def *pb.Definition, polEngine SourcePolicyEvaluat }) } -func newVertex(dgst digest.Digest, op *pb.Op, opMeta *pb.OpMetadata, load func(digest.Digest) (solver.Vertex, error), opts ...LoadOpt) (*vertex, error) { +func loadWithProxyNetwork(ctx context.Context, def *pb.Definition, polEngine SourcePolicyEvaluator, proxyNetwork bool, opts ...LoadOpt) (solver.Edge, error) { + return loadLLB(ctx, def, polEngine, &proxyNetwork, func(dgst digest.Digest, op *op, load func(digest.Digest) (solver.Vertex, error)) (solver.Vertex, error) { + vtx, err := newVertex(dgst, op.Op, op.Metadata, load, opts...) + if err != nil { + return nil, err + } + return vtx, nil + }) +} + +func vertexOptions(opMeta *pb.OpMetadata) solver.VertexOptions { opt := solver.VertexOptions{} if opMeta != nil { opt.IgnoreCache = opMeta.IgnoreCache @@ -251,6 +261,11 @@ func newVertex(dgst digest.Digest, op *pb.Op, opMeta *pb.OpMetadata, load func(d } opt.ProgressGroup = opMeta.ProgressGroup } + return opt +} + +func newVertex(dgst digest.Digest, op *pb.Op, opMeta *pb.OpMetadata, load func(digest.Digest) (solver.Vertex, error), opts ...LoadOpt) (*vertex, error) { + opt := vertexOptions(opMeta) for _, fn := range opts { if err := fn(op, opMeta, &opt); err != nil { return nil, err @@ -319,7 +334,7 @@ type op struct { // loadLLB loads LLB. // fn is executed sequentially. -func loadLLB(ctx context.Context, def *pb.Definition, polEngine SourcePolicyEvaluator, fn func(digest.Digest, *op, func(digest.Digest) (solver.Vertex, error)) (solver.Vertex, error)) (solver.Edge, error) { +func loadLLB(ctx context.Context, def *pb.Definition, polEngine SourcePolicyEvaluator, proxyNetwork *bool, fn func(digest.Digest, *op, func(digest.Digest) (solver.Vertex, error)) (solver.Vertex, error)) (solver.Edge, error) { if len(def.Def) == 0 { return solver.Edge{}, errors.New("invalid empty definition") } @@ -345,6 +360,14 @@ func loadLLB(ctx context.Context, def *pb.Definition, polEngine SourcePolicyEval lastDgst = dgst } + if proxyNetwork != nil { + for _, op := range allOps { + if err := setProxyNetwork(op.Op, *proxyNetwork); err != nil { + return solver.Edge{}, err + } + } + } + if polEngine != nil && len(sources) > 0 { var eg errgroup.Group for dgst := range sources { diff --git a/solver/llbsolver/vertex_test.go b/solver/llbsolver/vertex_test.go index 7aa68c990f44..ed5e293308e6 100644 --- a/solver/llbsolver/vertex_test.go +++ b/solver/llbsolver/vertex_test.go @@ -114,3 +114,95 @@ func TestIngestDigest(t *testing.T) { require.Equal(t, op1Digest, newDgst) } } + +func TestWithProxyNetworkAffectsVertexDigest(t *testing.T) { + def := proxyNetworkTestDefinition(t) + + defaultEdge, err := Load(t.Context(), def, nil) + require.NoError(t, err) + defaultOp, ok := defaultEdge.Vertex.Sys().(*pb.Op) + require.True(t, ok) + require.Equal(t, pb.NetMode_UNSET, defaultOp.GetExec().Network) + + proxyEdge, err := loadWithProxyNetwork(t.Context(), def, nil, true) + require.NoError(t, err) + proxyOp, ok := proxyEdge.Vertex.Sys().(*pb.Op) + require.True(t, ok) + require.Equal(t, pb.NetMode_PROXY, proxyOp.GetExec().Network) + + require.NotEqual(t, defaultEdge.Vertex.Digest(), proxyEdge.Vertex.Digest()) +} + +func TestNormalizeRuntimePlatformsDoesNotAffectVertexDigest(t *testing.T) { + def := proxyNetworkTestDefinition(t) + + defaultEdge, err := Load(t.Context(), def, nil) + require.NoError(t, err) + + normalizedEdge, err := Load(t.Context(), def, nil, NormalizeRuntimePlatforms()) + require.NoError(t, err) + normalizedOp, ok := normalizedEdge.Vertex.Sys().(*pb.Op) + require.True(t, ok) + require.NotNil(t, normalizedOp.Platform) + + require.Equal(t, defaultEdge.Vertex.Digest(), normalizedEdge.Vertex.Digest()) +} + +func TestWithProxyNetworkRejectsExplicitProxyWhenDisabled(t *testing.T) { + def := proxyNetworkTestDefinition(t, func(exec *pb.ExecOp) { + exec.Network = pb.NetMode_PROXY + }) + + _, err := loadWithProxyNetwork(t.Context(), def, nil, false) + require.Error(t, err) + require.ErrorContains(t, err, "requires proxy network to be enabled") +} + +func TestBridgeUsesDefaultProxyNetwork(t *testing.T) { + s := &Solver{proxyNetwork: true} + + br := s.bridge(nil) + + require.True(t, br.proxyNetwork) +} + +func proxyNetworkTestDefinition(t *testing.T, opts ...func(*pb.ExecOp)) *pb.Definition { + t.Helper() + source := &pb.Op{ + Op: &pb.Op_Source{ + Source: &pb.SourceOp{Identifier: "local://context"}, + }, + } + sourceDigest, sourceBytes := marshalTestOp(t, source) + + exec := &pb.Op{ + Inputs: []*pb.Input{{Digest: string(sourceDigest)}}, + Op: &pb.Op_Exec{ + Exec: &pb.ExecOp{ + Meta: &pb.Meta{Args: []string{"true"}}, + Mounts: []*pb.Mount{{ + Input: 0, + Dest: pb.RootMount, + }}, + }, + }, + } + for _, opt := range opts { + opt(exec.GetExec()) + } + execDigest, execBytes := marshalTestOp(t, exec) + + root := &pb.Op{ + Inputs: []*pb.Input{{Digest: string(execDigest)}}, + } + _, rootBytes := marshalTestOp(t, root) + + return &pb.Definition{Def: [][]byte{sourceBytes, execBytes, rootBytes}} +} + +func marshalTestOp(t *testing.T, op *pb.Op) (digest.Digest, []byte) { + t.Helper() + dt, err := op.Marshal() + require.NoError(t, err) + return digest.FromBytes(dt), dt +} diff --git a/solver/pb/caps.go b/solver/pb/caps.go index b63eab97297a..eaea7c81c7d0 100644 --- a/solver/pb/caps.go +++ b/solver/pb/caps.go @@ -57,6 +57,7 @@ const ( CapExecMetaBase apicaps.CapID = "exec.meta.base" CapExecMetaCgroupParent apicaps.CapID = "exec.meta.cgroup.parent" CapExecMetaNetwork apicaps.CapID = "exec.meta.network" + CapExecMetaNetworkProxy apicaps.CapID = "exec.meta.network.proxy" CapExecMetaProxy apicaps.CapID = "exec.meta.proxyenv" CapExecMetaSecurity apicaps.CapID = "exec.meta.security" CapExecMetaSecurityDeviceWhitelistV1 apicaps.CapID = "exec.meta.security.devices.v1" @@ -367,6 +368,12 @@ func init() { Status: apicaps.CapStatusExperimental, }) + Caps.Init(apicaps.Cap{ + ID: CapExecMetaNetworkProxy, + Enabled: true, + Status: apicaps.CapStatusExperimental, + }) + Caps.Init(apicaps.Cap{ ID: CapExecMetaSetsDefaultPath, Enabled: true, diff --git a/solver/pb/ops.pb.go b/solver/pb/ops.pb.go index a64211acd114..7355e960f422 100644 --- a/solver/pb/ops.pb.go +++ b/solver/pb/ops.pb.go @@ -30,6 +30,7 @@ const ( NetMode_UNSET NetMode = 0 // sandbox NetMode_HOST NetMode = 1 NetMode_NONE NetMode = 2 + NetMode_PROXY NetMode = 3 ) // Enum value maps for NetMode. @@ -38,11 +39,13 @@ var ( 0: "UNSET", 1: "HOST", 2: "NONE", + 3: "PROXY", } NetMode_value = map[string]int32{ "UNSET": 0, "HOST": 1, "NONE": 2, + "PROXY": 3, } ) @@ -3652,11 +3655,12 @@ const file_github_com_moby_buildkit_solver_pb_ops_proto_rawDesc = "" + "\x05input\x18\x01 \x01(\x03R\x05input\"\\\n" + "\x06DiffOp\x12(\n" + "\x05lower\x18\x01 \x01(\v2\x12.pb.LowerDiffInputR\x05lower\x12(\n" + - "\x05upper\x18\x02 \x01(\v2\x12.pb.UpperDiffInputR\x05upper*(\n" + + "\x05upper\x18\x02 \x01(\v2\x12.pb.UpperDiffInputR\x05upper*3\n" + "\aNetMode\x12\t\n" + "\x05UNSET\x10\x00\x12\b\n" + "\x04HOST\x10\x01\x12\b\n" + - "\x04NONE\x10\x02*)\n" + + "\x04NONE\x10\x02\x12\t\n" + + "\x05PROXY\x10\x03*)\n" + "\fSecurityMode\x12\v\n" + "\aSANDBOX\x10\x00\x12\f\n" + "\bINSECURE\x10\x01*@\n" + diff --git a/solver/pb/ops.proto b/solver/pb/ops.proto index 731123b6ce36..a155f8db1c94 100644 --- a/solver/pb/ops.proto +++ b/solver/pb/ops.proto @@ -82,6 +82,7 @@ enum NetMode { UNSET = 0; // sandbox HOST = 1; NONE = 2; + PROXY = 3; } enum SecurityMode { diff --git a/util/network/cniprovider/bridge.go b/util/network/cniprovider/bridge.go index 161bafa0913d..47607a8f33be 100644 --- a/util/network/cniprovider/bridge.go +++ b/util/network/cniprovider/bridge.go @@ -152,11 +152,11 @@ func NewBridge(opt Opt) (network.Provider, error) { cleanOldNamespaces(cp) - cp.nsPool = &cniPool{targetSize: opt.PoolSize, provider: cp} + cp.nsPool = newCNIPool(cp, opt.PoolSize) if err := cp.initNetwork(false); err != nil { return nil, err } - go cp.nsPool.fillPool(context.TODO()) + go cp.nsPool.Fill(context.TODO()) return cp, nil } diff --git a/util/network/cniprovider/cni.go b/util/network/cniprovider/cni.go index 24a8c2b4aca7..375ff6c39a58 100644 --- a/util/network/cniprovider/cni.go +++ b/util/network/cniprovider/cni.go @@ -5,8 +5,6 @@ import ( "os" "runtime" "strings" - "sync" - "time" cni "github.com/containerd/go-cni" "github.com/gofrs/flock" @@ -14,13 +12,12 @@ import ( "github.com/moby/buildkit/identity" "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/network" + "github.com/moby/buildkit/util/network/netpool" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "go.opentelemetry.io/otel/trace" ) -const aboveTargetGracePeriod = 5 * time.Minute - type Opt struct { Root string ConfigPath string @@ -68,18 +65,18 @@ func New(opt Opt) (network.Provider, error) { } cleanOldNamespaces(cp) - cp.nsPool = &cniPool{targetSize: opt.PoolSize, provider: cp} + cp.nsPool = newCNIPool(cp, opt.PoolSize) if err := cp.initNetwork(true); err != nil { return nil, err } - go cp.nsPool.fillPool(context.TODO()) + go cp.nsPool.Fill(context.TODO()) return cp, nil } type cniProvider struct { cni.CNI root string - nsPool *cniPool + nsPool *netpool.Pool[*cniNS] release func() error } @@ -91,7 +88,7 @@ func (c *cniProvider) initNetwork(lock bool) error { } defer unlock() } - ns, err := c.New(context.TODO(), "") + ns, err := c.New(context.TODO(), "", network.NamespaceOptions{}) if err != nil { return err } @@ -99,11 +96,16 @@ func (c *cniProvider) initNetwork(lock bool) error { } func (c *cniProvider) Close() error { - c.nsPool.close() + var err error + if e := c.nsPool.Close(); e != nil { + err = e + } if c.release != nil { - return c.release() + if e := c.release(); e != nil && err == nil { + err = e + } } - return nil + return err } func initLock() (func() error, error) { @@ -117,133 +119,37 @@ func initLock() (func() error, error) { return func() error { return nil }, nil } -type cniPool struct { - provider *cniProvider - mu sync.Mutex - targetSize int - actualSize int - // LIFO: Ordered least recently used to most recently used - available []*cniNS - closed bool -} - -func (pool *cniPool) close() { - bklog.L.Debugf("cleaning up cni pool") - - pool.mu.Lock() - pool.closed = true - defer pool.mu.Unlock() - for len(pool.available) > 0 { - _ = pool.available[0].release() - pool.available = pool.available[1:] - pool.actualSize-- - } -} - -func (pool *cniPool) fillPool(ctx context.Context) { - for { - pool.mu.Lock() - if pool.closed { - pool.mu.Unlock() - return - } - actualSize := pool.actualSize - pool.mu.Unlock() - if actualSize >= pool.targetSize { - return - } - ns, err := pool.getNew(ctx) - if err != nil { - bklog.G(ctx).Errorf("failed to create new network namespace while prefilling pool: %s", err) - return - } - pool.put(ns) - } -} - -func (pool *cniPool) get(ctx context.Context) (*cniNS, error) { - pool.mu.Lock() - if len(pool.available) > 0 { - ns := pool.available[len(pool.available)-1] - pool.available = pool.available[:len(pool.available)-1] - pool.mu.Unlock() - trace.SpanFromContext(ctx).AddEvent("returning network namespace from pool") - bklog.G(ctx).Debugf("returning network namespace %s from pool", ns.id) - return ns, nil - } - pool.mu.Unlock() - - return pool.getNew(ctx) -} - -func (pool *cniPool) getNew(ctx context.Context) (*cniNS, error) { - var ns *cniNS - fn := func(ctx context.Context) error { - var err error - ns, err = pool.provider.newNS(ctx, "") - return err - } - err := withDetachedNetNSIfAny(ctx, fn) - if err != nil { - return nil, err - } - ns.pool = pool - - pool.mu.Lock() - defer pool.mu.Unlock() - if pool.closed { - return nil, errors.New("cni pool is closed") - } - pool.actualSize++ - return ns, nil -} - -func (pool *cniPool) put(ns *cniNS) { - putTime := time.Now() - ns.lastUsed = putTime - - pool.mu.Lock() - defer pool.mu.Unlock() - if pool.closed { - _ = ns.release() - return - } - pool.available = append(pool.available, ns) - actualSize := pool.actualSize - - if actualSize > pool.targetSize { - // We have more network namespaces than our target number, so - // schedule a shrinking pass. - time.AfterFunc(aboveTargetGracePeriod, pool.cleanupToTargetSize) - } -} - -func (pool *cniPool) cleanupToTargetSize() { - var toRelease []*cniNS - defer func() { - for _, poolNS := range toRelease { - _ = poolNS.release() - } - }() - - pool.mu.Lock() - defer pool.mu.Unlock() - for pool.actualSize > pool.targetSize && - len(pool.available) > 0 && - time.Since(pool.available[0].lastUsed) >= aboveTargetGracePeriod { - bklog.L.Debugf("releasing network namespace %s since it was last used at %s", pool.available[0].id, pool.available[0].lastUsed) - toRelease = append(toRelease, pool.available[0]) - pool.available = pool.available[1:] - pool.actualSize-- - } +func newCNIPool(c *cniProvider, targetSize int) *netpool.Pool[*cniNS] { + var pool *netpool.Pool[*cniNS] + pool = netpool.New(netpool.Opt[*cniNS]{ + Name: "cni network namespace", + TargetSize: targetSize, + New: func(ctx context.Context) (*cniNS, error) { + var ns *cniNS + fn := func(ctx context.Context) error { + var err error + ns, err = c.newNS(ctx, "") + return err + } + if err := withDetachedNetNSIfAny(ctx, fn); err != nil { + return nil, err + } + ns.pool = pool + return ns, nil + }, + Release: func(ns *cniNS) error { + return ns.release() + }, + }) + return pool } -func (c *cniProvider) New(ctx context.Context, hostname string) (network.Namespace, error) { +func (c *cniProvider) New(ctx context.Context, hostname string, _ network.NamespaceOptions) (network.Namespace, error) { // We can't use the pool for namespaces that need a custom hostname. // We also avoid using it on windows because we don't have a cleanup // mechanism for Windows yet. if hostname == "" || runtime.GOOS == "windows" { - return c.nsPool.get(ctx) + return c.nsPool.Get(ctx) } var res network.Namespace fn := func(ctx context.Context) error { @@ -326,12 +232,11 @@ func (c *cniProvider) newNS(ctx context.Context, hostname string) (*cniNS, error } type cniNS struct { - pool *cniPool + pool *netpool.Pool[*cniNS] handle cni.CNI id string nativeID string opts []cni.NamespaceOpts - lastUsed time.Time vethName string canSample bool offsetSample *resourcestypes.NetworkSample @@ -349,7 +254,7 @@ func (ns *cniNS) Close() error { if ns.pool == nil { return ns.release() } - ns.pool.put(ns) + ns.pool.Put(ns) return nil } diff --git a/util/network/host.go b/util/network/host.go index 0bf74d7e6c2f..da7e6a1154cd 100644 --- a/util/network/host.go +++ b/util/network/host.go @@ -17,7 +17,7 @@ func NewHostProvider() Provider { type host struct { } -func (h *host) New(_ context.Context, hostname string) (Namespace, error) { +func (h *host) New(_ context.Context, hostname string, _ NamespaceOptions) (Namespace, error) { return &hostNS{}, nil } diff --git a/util/network/netpool/pool.go b/util/network/netpool/pool.go new file mode 100644 index 000000000000..819ddbdaadb4 --- /dev/null +++ b/util/network/netpool/pool.go @@ -0,0 +1,172 @@ +package netpool + +import ( + "context" + "sync" + "time" + + "github.com/moby/buildkit/util/bklog" + "github.com/pkg/errors" +) + +const aboveTargetGracePeriod = 5 * time.Minute + +type Opt[T any] struct { + Name string + TargetSize int + New func(context.Context) (T, error) + Release func(T) error +} + +type Pool[T any] struct { + name string + targetSize int + new func(context.Context) (T, error) + release func(T) error + + mu sync.Mutex + actualSize int + available []pooled[T] + closed bool +} + +type pooled[T any] struct { + value T + lastUsed time.Time +} + +func New[T any](opt Opt[T]) *Pool[T] { + name := opt.Name + if name == "" { + name = "network namespace" + } + return &Pool[T]{ + name: name, + targetSize: opt.TargetSize, + new: opt.New, + release: opt.Release, + } +} + +func (p *Pool[T]) Close() error { + bklog.L.Debugf("cleaning up %s pool", p.name) + + p.mu.Lock() + p.closed = true + available := p.available + p.available = nil + p.actualSize -= len(available) + p.mu.Unlock() + + var err error + for _, v := range available { + if e := p.release(v.value); e != nil && err == nil { + err = e + } + } + return err +} + +func (p *Pool[T]) Fill(ctx context.Context) { + for { + p.mu.Lock() + if p.closed { + p.mu.Unlock() + return + } + actualSize := p.actualSize + p.mu.Unlock() + if actualSize >= p.targetSize { + return + } + v, err := p.getNew(ctx) + if err != nil { + bklog.G(ctx).Errorf("failed to create new %s while prefilling pool: %s", p.name, err) + return + } + p.Put(v) + } +} + +func (p *Pool[T]) Get(ctx context.Context) (T, error) { + p.mu.Lock() + if p.closed { + p.mu.Unlock() + var zero T + return zero, errors.Errorf("%s pool is closed", p.name) + } + if len(p.available) > 0 { + v := p.available[len(p.available)-1].value + p.available = p.available[:len(p.available)-1] + p.mu.Unlock() + return v, nil + } + p.mu.Unlock() + + return p.getNew(ctx) +} + +func (p *Pool[T]) Put(v T) { + putTime := time.Now() + + p.mu.Lock() + if p.closed { + p.actualSize-- + p.mu.Unlock() + _ = p.release(v) + return + } + p.available = append(p.available, pooled[T]{value: v, lastUsed: putTime}) + actualSize := p.actualSize + p.mu.Unlock() + + if actualSize > p.targetSize { + time.AfterFunc(aboveTargetGracePeriod, p.cleanupToTargetSize) + } +} + +func (p *Pool[T]) Discard(v T) error { + p.mu.Lock() + p.actualSize-- + p.mu.Unlock() + return p.release(v) +} + +func (p *Pool[T]) getNew(ctx context.Context) (T, error) { + v, err := p.new(ctx) + if err != nil { + return v, err + } + + p.mu.Lock() + if p.closed { + p.mu.Unlock() + if e := p.release(v); e != nil { + return v, e + } + var zero T + return zero, errors.Errorf("%s pool is closed", p.name) + } + p.actualSize++ + p.mu.Unlock() + return v, nil +} + +func (p *Pool[T]) cleanupToTargetSize() { + var toRelease []T + defer func() { + for _, v := range toRelease { + _ = p.release(v) + } + }() + + p.mu.Lock() + defer p.mu.Unlock() + for p.actualSize > p.targetSize && + len(p.available) > 0 && + time.Since(p.available[0].lastUsed) >= aboveTargetGracePeriod { + toRelease = append(toRelease, p.available[0].value) + p.available = p.available[1:] + p.actualSize-- + } +} diff --git a/util/network/netpool/pool_test.go b/util/network/netpool/pool_test.go new file mode 100644 index 000000000000..919a33c6c0c5 --- /dev/null +++ b/util/network/netpool/pool_test.go @@ -0,0 +1,60 @@ +package netpool + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPoolReusesReturnedValue(t *testing.T) { + var next int + p := New(Opt[int]{ + Name: "test", + TargetSize: 1, + New: func(context.Context) (int, error) { + next++ + return next, nil + }, + Release: func(int) error { + return nil + }, + }) + + v1, err := p.Get(t.Context()) + require.NoError(t, err) + p.Put(v1) + + v2, err := p.Get(t.Context()) + require.NoError(t, err) + require.Equal(t, v1, v2) + require.Equal(t, 1, next) + require.NoError(t, p.Discard(v2)) +} + +func TestPoolCloseReleasesAvailableAndReturnedValues(t *testing.T) { + var next int + var released []int + p := New(Opt[int]{ + Name: "test", + TargetSize: 1, + New: func(context.Context) (int, error) { + next++ + return next, nil + }, + Release: func(v int) error { + released = append(released, v) + return nil + }, + }) + + v, err := p.Get(t.Context()) + require.NoError(t, err) + require.NoError(t, p.Close()) + p.Put(v) + require.Equal(t, []int{v}, released) + + _, err = p.Get(t.Context()) + require.Error(t, err) + require.Equal(t, 1, next) +} diff --git a/util/network/netproviders/network.go b/util/network/netproviders/network.go index 4564782bd88e..18e888a4244a 100644 --- a/util/network/netproviders/network.go +++ b/util/network/netproviders/network.go @@ -7,6 +7,7 @@ import ( "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/network" "github.com/moby/buildkit/util/network/cniprovider" + "github.com/moby/buildkit/util/network/proxyprovider" "github.com/pkg/errors" ) @@ -67,6 +68,16 @@ func Providers(opt Opt) (providers map[pb.NetMode]network.Provider, resolvedMode pb.NetMode_UNSET: defaultProvider, pb.NetMode_NONE: network.NewNoneProvider(), } + if proxyprovider.Supported() { + proxyProvider, err := proxyprovider.New(proxyprovider.Opt{ + Root: opt.CNI.Root, + PoolSize: opt.CNI.PoolSize, + }) + if err != nil { + return nil, resolvedMode, err + } + providers[pb.NetMode_PROXY] = proxyProvider + } if hostProvider, ok := getHostProvider(); ok { providers[pb.NetMode_HOST] = hostProvider diff --git a/util/network/network.go b/util/network/network.go index c7b812043142..8b5d45c6e7ce 100644 --- a/util/network/network.go +++ b/util/network/network.go @@ -11,7 +11,12 @@ import ( // Provider interface for Network type Provider interface { io.Closer - New(ctx context.Context, hostname string) (Namespace, error) + New(ctx context.Context, hostname string, opt NamespaceOptions) (Namespace, error) +} + +type NamespaceOptions struct { + ProxyPolicy ProxyPolicy + ProxyCapture *ProxyCapture } // Namespace of network for workers diff --git a/util/network/none.go b/util/network/none.go index 253ce1b3e0de..b65c3268c8e9 100644 --- a/util/network/none.go +++ b/util/network/none.go @@ -14,7 +14,7 @@ func NewNoneProvider() Provider { type none struct { } -func (h *none) New(_ context.Context, hostname string) (Namespace, error) { +func (h *none) New(_ context.Context, hostname string, _ NamespaceOptions) (Namespace, error) { return &noneNS{}, nil } diff --git a/util/network/proxy.go b/util/network/proxy.go new file mode 100644 index 000000000000..698526cbbd6d --- /dev/null +++ b/util/network/proxy.go @@ -0,0 +1,141 @@ +package network + +import ( + "context" + "slices" + "sync" + + "github.com/moby/buildkit/solver/pb" + digest "github.com/opencontainers/go-digest" +) + +// ProxyPolicy authorizes requests made through a BuildKit-owned exec proxy. +type ProxyPolicy interface { + Evaluate(context.Context, *pb.Op) (bool, error) +} + +// ProxyNamespace is implemented by network namespaces that expose an internal +// HTTP(S) proxy to the container. +type ProxyNamespace interface { + ProxyEnv() []string + ProxyCACert() []byte +} + +type ProxyMaterial struct { + URL string + Digest digest.Digest +} + +type ProxyRequest struct { + Method string + URL string + RedirectTarget string + StatusCode int +} + +type ProxyIncomplete struct { + Method string + URL string + Reason string +} + +type ProxyCapture struct { + mu sync.Mutex + requests []ProxyRequest + materials []ProxyMaterial + incomplete []ProxyIncomplete +} + +func NewProxyCapture() *ProxyCapture { + return &ProxyCapture{} +} + +func (c *ProxyCapture) AddMaterial(m ProxyMaterial) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + c.materials = append(c.materials, m) +} + +func (c *ProxyCapture) AddRequest(r ProxyRequest) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + c.requests = append(c.requests, r) +} + +func (c *ProxyCapture) AddIncomplete(in ProxyIncomplete) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + c.incomplete = append(c.incomplete, in) +} + +func (c *ProxyCapture) Materials() []ProxyMaterial { + if c == nil { + return nil + } + c.mu.Lock() + defer c.mu.Unlock() + out := slices.Clone(c.materials) + redirects := map[string]string{} + digests := map[string]digest.Digest{} + for _, m := range out { + digests[m.URL] = m.Digest + } + for _, r := range c.requests { + if r.RedirectTarget != "" && r.URL != r.RedirectTarget { + redirects[r.URL] = r.RedirectTarget + } + } + for { + added := false + for from, to := range redirects { + if _, ok := digests[from]; ok { + continue + } + dgst, ok := digests[to] + if !ok { + continue + } + digests[from] = dgst + out = append(out, ProxyMaterial{ + URL: from, + Digest: dgst, + }) + added = true + } + if !added { + break + } + } + return out +} + +func (c *ProxyCapture) Requests() []ProxyRequest { + if c == nil { + return nil + } + c.mu.Lock() + defer c.mu.Unlock() + out := make([]ProxyRequest, len(c.requests)) + copy(out, c.requests) + return out +} + +func (c *ProxyCapture) Incomplete() []ProxyIncomplete { + if c == nil { + return nil + } + c.mu.Lock() + defer c.mu.Unlock() + out := make([]ProxyIncomplete, len(c.incomplete)) + copy(out, c.incomplete) + return out +} diff --git a/util/network/proxyprovider/provider_linux.go b/util/network/proxyprovider/provider_linux.go new file mode 100644 index 000000000000..7ed4e57bb65e --- /dev/null +++ b/util/network/proxyprovider/provider_linux.go @@ -0,0 +1,870 @@ +//go:build linux + +package proxyprovider + +import ( + "bufio" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "encoding/pem" + "fmt" + "hash" + "io" + "math/big" + "net" + "net/http" + neturl "net/url" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/containerd/containerd/v2/pkg/oci" + resourcestypes "github.com/moby/buildkit/executor/resources/types" + "github.com/moby/buildkit/identity" + "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/util/network" + "github.com/moby/buildkit/util/network/netpool" + "github.com/moby/buildkit/util/urlutil" + digest "github.com/opencontainers/go-digest" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" + "golang.org/x/sys/unix" +) + +type Opt struct { + Root string + PoolSize int +} + +func Supported() bool { + return true +} + +func New(opt Opt) (network.Provider, error) { + certPEM, ca, key, err := newCA() + if err != nil { + return nil, err + } + p := &provider{ + root: opt.Root, + caPEM: certPEM, + ca: ca, + caKey: key, + certs: map[string]*tls.Certificate{}, + client: &http.Transport{ + Proxy: nil, + DisableCompression: true, + }, + } + p.pool = netpool.New(netpool.Opt[*proxyNS]{ + Name: "proxy network namespace", + TargetSize: opt.PoolSize, + New: p.newNS, + Release: func(ns *proxyNS) error { + return ns.release() + }, + }) + go p.pool.Fill(context.TODO()) + return p, nil +} + +type provider struct { + root string + next atomic.Uint32 + pool *netpool.Pool[*proxyNS] + + caPEM []byte + ca *x509.Certificate + caKey *rsa.PrivateKey + + certsMu sync.Mutex + certs map[string]*tls.Certificate + client *http.Transport +} + +func (p *provider) Close() error { + err := p.pool.Close() + p.client.CloseIdleConnections() + return err +} + +func (p *provider) New(ctx context.Context, hostname string, opt network.NamespaceOptions) (_ network.Namespace, retErr error) { + ns, err := p.pool.Get(ctx) + if err != nil { + return nil, err + } + defer func() { + if retErr != nil { + _ = p.pool.Discard(ns) + } + }() + if err := ns.startProxy(ctx, opt.ProxyPolicy, opt.ProxyCapture); err != nil { + return nil, err + } + return ns, nil +} + +func (p *provider) newNS(ctx context.Context) (_ *proxyNS, retErr error) { + n := p.next.Add(1) + id := identity.NewID() + nsPath, err := createNetNS(p.root, id+"-exec") + if err != nil { + return nil, err + } + proxyNSPath, err := createNetNS(p.root, id+"-proxy") + if err != nil { + _ = unmountNetNS(nsPath) + _ = deleteNetNS(nsPath) + return nil, err + } + ns := &proxyNS{ + provider: p, + nsPath: nsPath, + proxyNSPath: proxyNSPath, + hostName: ifName("bkpxh", n), + ctrName: ifName("bkpxc", n), + hostIP: proxyHostIP(n), + ctrIP: proxyContainerIP(n), + prefix: proxyPrefix(), + } + defer func() { + if retErr != nil { + _ = ns.Close() + } + }() + if err := ns.setupVeth(); err != nil { + return nil, err + } + return ns, nil +} + +type proxyNS struct { + provider *provider + nsPath string + proxyNSPath string + hostName string + ctrName string + hostIP net.IP + ctrIP net.IP + prefix int + + server *http.Server + ln net.Listener +} + +func (n *proxyNS) Set(s *specs.Spec) error { + return oci.WithLinuxNamespace(specs.LinuxNamespace{ + Type: specs.NetworkNamespace, + Path: n.nsPath, + })(nil, nil, nil, s) +} + +func (n *proxyNS) Close() error { + if err := n.stopProxy(); err != nil { + if n.provider != nil && n.provider.pool != nil { + _ = n.provider.pool.Discard(n) + } + return err + } + if n.provider != nil && n.provider.pool != nil { + n.provider.pool.Put(n) + return nil + } + return n.release() +} + +func (n *proxyNS) stopProxy() error { + var err error + if n.server != nil { + if e := n.server.Close(); e != nil && !errors.Is(e, http.ErrServerClosed) { + err = errors.WithStack(e) + } + } + n.server = nil + n.ln = nil + return err +} + +func (n *proxyNS) release() error { + var err error + if e := n.stopProxy(); e != nil { + err = e + } + if e := n.deleteVeth(); e != nil && err == nil { + err = e + } + if e := unmountNetNS(n.nsPath); e != nil && err == nil { + err = e + } + if e := deleteNetNS(n.nsPath); e != nil && err == nil { + err = e + } + if e := unmountNetNS(n.proxyNSPath); e != nil && err == nil { + err = e + } + if e := deleteNetNS(n.proxyNSPath); e != nil && err == nil { + err = e + } + return err +} + +func (n *proxyNS) Sample() (*resourcestypes.NetworkSample, error) { + return nil, nil +} + +func (n *proxyNS) ProxyEnv() []string { + proxy := "http://" + n.ln.Addr().String() + noProxy := "127.0.0.1,localhost,::1" + return []string{ + "HTTP_PROXY=" + proxy, + "HTTPS_PROXY=" + proxy, + "http_proxy=" + proxy, + "https_proxy=" + proxy, + "NO_PROXY=" + noProxy, + "no_proxy=" + noProxy, + } +} + +func (n *proxyNS) ProxyCACert() []byte { + return n.provider.caPEM +} + +func (n *proxyNS) setupVeth() error { + veth := &netlink.Veth{ + LinkAttrs: netlink.LinkAttrs{Name: n.hostName}, + PeerName: n.ctrName, + } + if err := netlink.LinkAdd(veth); err != nil { + return errors.WithStack(err) + } + host, err := netlink.LinkByName(n.hostName) + if err != nil { + return errors.WithStack(err) + } + peer, err := netlink.LinkByName(n.ctrName) + if err != nil { + return errors.WithStack(err) + } + target, err := netns.GetFromPath(n.nsPath) + if err != nil { + return errors.WithStack(err) + } + defer target.Close() + proxyTarget, err := netns.GetFromPath(n.proxyNSPath) + if err != nil { + return errors.WithStack(err) + } + defer proxyTarget.Close() + if err := netlink.LinkSetNsFd(host, int(proxyTarget)); err != nil { + return errors.WithStack(err) + } + if err := netlink.LinkSetNsFd(peer, int(target)); err != nil { + return errors.WithStack(err) + } + ph, err := netlink.NewHandleAt(proxyTarget) + if err != nil { + return errors.WithStack(err) + } + defer ph.Close() + host, err = ph.LinkByName(n.hostName) + if err != nil { + return errors.WithStack(err) + } + hostAddr := &netlink.Addr{IPNet: &net.IPNet{IP: n.hostIP, Mask: net.CIDRMask(n.prefix, 32)}} + if err := ph.AddrAdd(host, hostAddr); err != nil { + return errors.WithStack(err) + } + if err := ph.LinkSetUp(host); err != nil { + return errors.WithStack(err) + } + proxyLO, err := ph.LinkByName("lo") + if err != nil { + return errors.WithStack(err) + } + if err := ph.LinkSetUp(proxyLO); err != nil { + return errors.WithStack(err) + } + h, err := netlink.NewHandleAt(target) + if err != nil { + return errors.WithStack(err) + } + defer h.Close() + peer, err = h.LinkByName(n.ctrName) + if err != nil { + return errors.WithStack(err) + } + if err := h.LinkSetName(peer, "eth0"); err != nil { + return errors.WithStack(err) + } + peer, err = h.LinkByName("eth0") + if err != nil { + return errors.WithStack(err) + } + ctrAddr := &netlink.Addr{IPNet: &net.IPNet{IP: n.ctrIP, Mask: net.CIDRMask(n.prefix, 32)}} + if err := h.AddrAdd(peer, ctrAddr); err != nil { + return errors.WithStack(err) + } + if err := h.LinkSetUp(peer); err != nil { + return errors.WithStack(err) + } + lo, err := h.LinkByName("lo") + if err != nil { + return errors.WithStack(err) + } + return errors.WithStack(h.LinkSetUp(lo)) +} + +func (n *proxyNS) startProxy(ctx context.Context, policy network.ProxyPolicy, capture *network.ProxyCapture) error { + ln, err := listenInNetNS(ctx, n.proxyNSPath, "tcp4", net.JoinHostPort(n.hostIP.String(), "0")) + if err != nil { + return errors.WithStack(err) + } + n.ln = ln + handler := &proxyHandler{ + provider: n.provider, + policy: policy, + capture: capture, + } + n.server = &http.Server{ + Handler: handler, + ReadHeaderTimeout: 30 * time.Second, + } + go func() { + _ = n.server.Serve(ln) + }() + return nil +} + +func listenInNetNS(ctx context.Context, nsPath, networkName, address string) (net.Listener, error) { + var ln net.Listener + if err := withNetNS(nsPath, func() error { + l, err := (&net.ListenConfig{}).Listen(ctx, networkName, address) + if err != nil { + return errors.WithStack(err) + } + ln = l + return nil + }); err != nil { + return nil, err + } + return ln, nil +} + +func withNetNS(nsPath string, fn func() error) (retErr error) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + orig, err := netns.Get() + if err != nil { + return errors.WithStack(err) + } + defer orig.Close() + + target, err := netns.GetFromPath(nsPath) + if err != nil { + return errors.WithStack(err) + } + defer target.Close() + + if err := netns.Set(target); err != nil { + return errors.WithStack(err) + } + defer func() { + if err := netns.Set(orig); err != nil && retErr == nil { + retErr = errors.WithStack(err) + } + }() + + return fn() +} + +func (n *proxyNS) deleteVeth() error { + target, err := netns.GetFromPath(n.proxyNSPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return errors.WithStack(err) + } + defer target.Close() + h, err := netlink.NewHandleAt(target) + if err != nil { + return errors.WithStack(err) + } + defer h.Close() + link, err := h.LinkByName(n.hostName) + if err != nil { + var linkNotFound netlink.LinkNotFoundError + if errors.As(err, &linkNotFound) { + return nil + } + return errors.WithStack(err) + } + return errors.WithStack(h.LinkDel(link)) +} + +type proxyHandler struct { + provider *provider + policy network.ProxyPolicy + capture *network.ProxyCapture +} + +func (h *proxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodConnect { + h.handleConnect(w, r) + return + } + if !r.URL.IsAbs() { + r.URL.Scheme = "http" + r.URL.Host = r.Host + } + if target, err := h.check(r.Context(), r.Method, r.URL.String()); err != nil { + http.Error(w, "Forbidden", http.StatusForbidden) + return + } else if target != nil { + r.URL = target + r.Host = target.Host + } + resp, err := h.roundTrip(r) + if err != nil { + h.recordRequest(r, http.StatusBadGateway, "") + h.recordIncomplete(r, "upstream_error") + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + defer resp.Body.Close() + h.recordRequest(r, resp.StatusCode, finalURL(r, resp)) + copyHeader(w.Header(), resp.Header) + w.WriteHeader(resp.StatusCode) + tracker := newProxyBodyTracker(resp.Body) + _, copyErr := io.Copy(w, tracker) + h.recordResponse(r, resp, tracker, copyErr) +} + +func (h *proxyHandler) handleConnect(w http.ResponseWriter, r *http.Request) { + hj, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "hijacking unsupported", http.StatusInternalServerError) + return + } + conn, _, err := hj.Hijack() + if err != nil { + return + } + defer conn.Close() + if _, err := io.WriteString(conn, "HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil { + return + } + host := stripPort(r.Host) + cert, err := h.provider.certForHost(host) + if err != nil { + return + } + tlsConn := tls.Server(conn, &tls.Config{ + Certificates: []tls.Certificate{*cert}, + NextProtos: []string{"http/1.1"}, + }) + defer tlsConn.Close() + if err := tlsConn.HandshakeContext(r.Context()); err != nil { + return + } + br := bufio.NewReader(tlsConn) + for { + req, err := http.ReadRequest(br) + if err != nil { + return + } + req.URL.Scheme = "https" + req.URL.Host = r.Host + req.RequestURI = "" + if target, err := h.check(req.Context(), req.Method, req.URL.String()); err != nil { + _ = req.Body.Close() + _, _ = io.WriteString(tlsConn, "HTTP/1.1 403 Forbidden\r\nContent-Length: 10\r\nConnection: close\r\n\r\nForbidden\n") + return + } else if target != nil { + req.URL = target + req.Host = target.Host + } + resp, err := h.roundTrip(req) + if err != nil { + _ = req.Body.Close() + h.recordRequest(req, http.StatusBadGateway, "") + h.recordIncomplete(req, "upstream_error") + _, _ = fmt.Fprintf(tlsConn, "HTTP/1.1 502 Bad Gateway\r\nContent-Length: %d\r\nConnection: close\r\n\r\n%s", len(err.Error())+1, err.Error()+"\n") + return + } + h.recordRequest(req, resp.StatusCode, finalURL(req, resp)) + // Response.Write uses resp.Proto for the status line. In the MITM + // path, resp describes the upstream fetch, so align it to the + // client-facing request we intercepted. + resp.Proto = req.Proto + resp.ProtoMajor = req.ProtoMajor + resp.ProtoMinor = req.ProtoMinor + resp.Close = resp.Close || req.Close || !req.ProtoAtLeast(1, 1) + tracker := newProxyBodyTracker(resp.Body) + resp.Body = tracker + if err := resp.Write(tlsConn); err != nil { + h.recordResponse(req, resp, tracker, err) + resp.Body.Close() + return + } + resp.Body.Close() + h.recordResponse(req, resp, tracker, nil) + if resp.Close || req.Close { + return + } + } +} + +type proxyBodyTracker struct { + body io.ReadCloser + hash hash.Hash + readErr error +} + +func newProxyBodyTracker(body io.ReadCloser) *proxyBodyTracker { + return &proxyBodyTracker{ + body: body, + hash: sha256.New(), + } +} + +func (t *proxyBodyTracker) Read(p []byte) (int, error) { + n, err := t.body.Read(p) + if n > 0 { + _, _ = t.hash.Write(p[:n]) + } + if err != nil && !errors.Is(err, io.EOF) && t.readErr == nil { + t.readErr = err + } + return n, err +} + +func (t *proxyBodyTracker) Close() error { + return t.body.Close() +} + +func (t *proxyBodyTracker) Digest() digest.Digest { + return digest.NewDigestFromHex(string(digest.SHA256), hex.EncodeToString(t.hash.Sum(nil))) +} + +func (h *proxyHandler) recordResponse(req *http.Request, resp *http.Response, tracker *proxyBodyTracker, copyErr error) { + if h.capture == nil { + return + } + reason := proxyIncompleteReason(req, resp, tracker, copyErr) + if reason != "" { + h.recordIncomplete(req, reason) + return + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return + } + h.capture.AddMaterial(network.ProxyMaterial{ + URL: captureURL(req.URL.String()), + Digest: tracker.Digest(), + }) +} + +func (h *proxyHandler) recordRequest(req *http.Request, statusCode int, redirectTarget string) { + if h.capture == nil { + return + } + h.capture.AddRequest(network.ProxyRequest{ + Method: req.Method, + URL: captureURL(req.URL.String()), + RedirectTarget: captureURL(redirectTarget), + StatusCode: statusCode, + }) +} + +func (h *proxyHandler) recordIncomplete(req *http.Request, reason string) { + if h.capture == nil { + return + } + h.capture.AddIncomplete(network.ProxyIncomplete{ + Method: req.Method, + URL: captureURL(req.URL.String()), + Reason: reason, + }) +} + +func proxyIncompleteReason(req *http.Request, resp *http.Response, tracker *proxyBodyTracker, copyErr error) string { + if req.Method != http.MethodGet { + return "method_not_materializable" + } + if req.Header.Get("Range") != "" || resp.StatusCode == http.StatusPartialContent { + return "partial_response" + } + if resp.Uncompressed { + return "response_transformed" + } + if copyErr != nil || tracker.readErr != nil { + return "body_read_failed" + } + if resp.StatusCode >= 400 { + return "unsuccessful_response" + } + return "" +} + +func finalURL(req *http.Request, resp *http.Response) string { + location := resp.Header.Get("Location") + if location == "" { + return "" + } + u, err := req.URL.Parse(location) + if err != nil { + return location + } + return u.String() +} + +func redactURL(s string) string { + if s == "" { + return "" + } + return urlutil.RedactCredentials(s) +} + +func captureURL(s string) string { + if s == "" { + return "" + } + u, err := neturl.Parse(s) + if err == nil && u.IsAbs() { + port := u.Port() + if (u.Scheme == "http" && port == "80") || (u.Scheme == "https" && port == "443") { + host := u.Hostname() + if strings.Contains(host, ":") { + host = "[" + host + "]" + } + u.Host = host + } + return redactURL(u.String()) + } + return redactURL(s) +} + +func (h *proxyHandler) roundTrip(r *http.Request) (*http.Response, error) { + stripProxyHeaders(r.Header) + r.Header.Del("Accept-Encoding") + r.RequestURI = "" + return h.provider.client.RoundTrip(r.WithContext(context.WithoutCancel(r.Context()))) +} + +func (h *proxyHandler) check(ctx context.Context, method, rawURL string) (*neturl.URL, error) { + if h.policy == nil { + return nil, nil + } + redactedURL := redactURL(rawURL) + op := &pb.Op{ + Op: &pb.Op_Source{ + Source: &pb.SourceOp{ + Identifier: redactedURL, + }, + }, + } + if _, err := h.policy.Evaluate(ctx, op); err != nil { + return nil, err + } + source := op.GetSource() + target := source.Identifier + converted := target != redactedURL || len(source.Attrs) != 0 + if !converted { + return nil, nil + } + if method != http.MethodGet { + return nil, errors.Errorf("source policy converted proxy request %q, but conversion is only supported for GET", redactedURL) + } + if len(source.Attrs) != 0 { + return nil, errors.Errorf("source policy converted proxy request %q with attrs, but proxy conversion only supports URL updates", redactedURL) + } + u, err := neturl.Parse(target) + if err != nil { + return nil, errors.Wrapf(err, "error parsing converted proxy request URL %q", redactURL(target)) + } + if !u.IsAbs() || (u.Scheme != "http" && u.Scheme != "https") { + return nil, errors.Errorf("source policy converted proxy request to unsupported URL %q", redactURL(target)) + } + return u, nil +} + +func newCA() ([]byte, *x509.Certificate, *rsa.PrivateKey, error) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, nil, errors.WithStack(err) + } + now := time.Now() + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "BuildKit exec proxy"}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(24 * time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + return nil, nil, nil, errors.WithStack(err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + return nil, nil, nil, errors.WithStack(err) + } + pemBytes := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + return pemBytes, cert, key, nil +} + +func (p *provider) certForHost(host string) (*tls.Certificate, error) { + p.certsMu.Lock() + defer p.certsMu.Unlock() + if cert, ok := p.certs[host]; ok { + return cert, nil + } + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, errors.WithStack(err) + } + serialBytes := sha256.Sum256([]byte(host + time.Now().String())) + serial := new(big.Int).SetBytes(serialBytes[:]) + tmpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: host}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + if ip := net.ParseIP(host); ip != nil { + tmpl.IPAddresses = []net.IP{ip} + } else { + tmpl.DNSNames = []string{host} + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, p.ca, &key.PublicKey, p.caKey) + if err != nil { + return nil, errors.WithStack(err) + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) + cert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return nil, errors.WithStack(err) + } + p.certs[host] = &cert + return &cert, nil +} + +func createNetNS(root, id string) (_ string, err error) { + nsPath := filepath.Join(root, "net/proxy", id) + if err := os.MkdirAll(filepath.Dir(nsPath), 0700); err != nil { + return "", errors.WithStack(err) + } + f, err := os.Create(nsPath) + if err != nil { + return "", errors.WithStack(err) + } + if err := f.Close(); err != nil { + return "", errors.WithStack(err) + } + defer func() { + if err != nil { + _ = deleteNetNS(nsPath) + } + }() + errCh := make(chan error, 1) + go func() { + defer close(errCh) + runtimeLockOSThread() + if err := syscall.Unshare(syscall.CLONE_NEWNET); err != nil { + errCh <- errors.WithStack(err) + return + } + if err := syscall.Mount(fmt.Sprintf("/proc/self/task/%d/ns/net", syscall.Gettid()), nsPath, "", syscall.MS_BIND, ""); err != nil { + errCh <- errors.WithStack(err) + return + } + }() + if err := <-errCh; err != nil { + return "", err + } + return nsPath, nil +} + +func unmountNetNS(nsPath string) error { + if err := unix.Unmount(nsPath, unix.MNT_DETACH); err != nil { + if !errors.Is(err, syscall.EINVAL) && !errors.Is(err, syscall.ENOENT) { + return errors.Wrap(err, "error unmounting network namespace") + } + } + return nil +} + +func deleteNetNS(nsPath string) error { + if err := os.Remove(nsPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return errors.Wrapf(err, "error removing network namespace %s", nsPath) + } + return nil +} + +func runtimeLockOSThread() { + runtime.LockOSThread() +} + +func proxyHostIP(n uint32) net.IP { + return proxyIP(n, 1) +} + +func proxyContainerIP(n uint32) net.IP { + return proxyIP(n, 2) +} + +func proxyPrefix() int { + return 30 +} + +func proxyIP(n uint32, offset byte) net.IP { + block := n % 16384 + return net.IPv4(10, 89, byte(block/64), byte((block%64)*4)+offset) +} + +func ifName(prefix string, n uint32) string { + return prefix + strconv.FormatUint(uint64(n%1000000000), 10) +} + +func stripPort(hostport string) string { + host, _, err := net.SplitHostPort(hostport) + if err == nil { + return host + } + return strings.Trim(hostport, "[]") +} + +func stripProxyHeaders(h http.Header) { + for _, k := range []string{"Connection", "Keep-Alive", "Proxy-Authenticate", "Proxy-Authorization", "Proxy-Connection", "Te", "Trailer", "Transfer-Encoding", "Upgrade"} { + h.Del(k) + } +} + +func copyHeader(dst, src http.Header) { + for k, vv := range src { + for _, v := range vv { + dst.Add(k, v) + } + } +} diff --git a/util/network/proxyprovider/provider_linux_test.go b/util/network/proxyprovider/provider_linux_test.go new file mode 100644 index 000000000000..c272b06dc4a4 --- /dev/null +++ b/util/network/proxyprovider/provider_linux_test.go @@ -0,0 +1,333 @@ +//go:build linux + +package proxyprovider + +import ( + "compress/gzip" + "context" + "crypto/x509" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/sourcepolicy" + spb "github.com/moby/buildkit/sourcepolicy/pb" + "github.com/moby/buildkit/util/network" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestProxyHandlerCapturesGetMaterial(t *testing.T) { + methodCh := make(chan string, 1) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + methodCh <- r.Method + _, _ = w.Write([]byte("proxy material")) + })) + t.Cleanup(upstream.Close) + + capture := network.NewProxyCapture() + handler := newTestProxyHandler(t, capture) + resp := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, upstream.URL+"/file", nil) + + handler.ServeHTTP(resp, req) + + require.Equal(t, http.StatusOK, resp.Code) + require.Equal(t, "proxy material", resp.Body.String()) + require.Equal(t, http.MethodGet, <-methodCh) + requests := capture.Requests() + require.Len(t, requests, 1) + require.Equal(t, http.MethodGet, requests[0].Method) + require.Equal(t, upstream.URL+"/file", requests[0].URL) + require.Equal(t, http.StatusOK, requests[0].StatusCode) + materials := capture.Materials() + require.Len(t, materials, 1) + require.Equal(t, upstream.URL+"/file", materials[0].URL) + require.Equal(t, "sha256:e352b3ec84adb842606c6d3638ac7466f5580f8617607ae6e0955f12130dd369", materials[0].Digest.String()) + require.Empty(t, capture.Incomplete()) +} + +func TestProxyHandlerDisablesUpstreamResponseTransforms(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.Header.Values("Accept-Encoding")) + w.Header().Set("Content-Encoding", "gzip") + zw := gzip.NewWriter(w) + _, _ = zw.Write([]byte("compressed proxy material")) + assert.NoError(t, zw.Close()) + })) + t.Cleanup(upstream.Close) + + capture := network.NewProxyCapture() + handler := newTestProxyHandler(t, capture) + resp := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, upstream.URL+"/file", nil) + req.Header.Set("Accept-Encoding", "gzip") + + handler.ServeHTTP(resp, req) + + require.Equal(t, http.StatusOK, resp.Code) + require.Equal(t, "gzip", resp.Header().Get("Content-Encoding")) + require.Empty(t, capture.Incomplete()) + require.Len(t, capture.Materials(), 1) +} + +func TestProxyHandlerRoundTripIgnoresClientContextCancel(t *testing.T) { + upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodGet, r.Method) + _, _ = w.Write([]byte("ok")) + })) + t.Cleanup(upstream.Close) + + pool := x509.NewCertPool() + pool.AddCert(upstream.Certificate()) + handler := newTestProxyHandler(t, nil) + handler.provider.client.TLSClientConfig = upstream.Client().Transport.(*http.Transport).TLSClientConfig.Clone() + handler.provider.client.TLSClientConfig.RootCAs = pool + + ctx, cancel := context.WithCancelCause(t.Context()) + cancel(context.Canceled) + req := httptest.NewRequest(http.MethodGet, upstream.URL, nil).WithContext(ctx) + + resp, err := handler.roundTrip(req) + require.NoError(t, err) + resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) +} + +func TestProxyHandlerMarksPostIncomplete(t *testing.T) { + methodCh := make(chan string, 1) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + methodCh <- r.Method + _, _ = w.Write([]byte("ok")) + })) + t.Cleanup(upstream.Close) + + capture := network.NewProxyCapture() + handler := newTestProxyHandler(t, capture) + resp := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, upstream.URL+"/token", nil) + + handler.ServeHTTP(resp, req) + + require.Equal(t, http.StatusOK, resp.Code) + require.Equal(t, http.MethodPost, <-methodCh) + require.Empty(t, capture.Materials()) + incomplete := capture.Incomplete() + require.Len(t, incomplete, 1) + require.Equal(t, http.MethodPost, incomplete[0].Method) + require.Equal(t, upstream.URL+"/token", incomplete[0].URL) + require.Equal(t, "method_not_materializable", incomplete[0].Reason) +} + +func TestProxyHandlerCapturesRedirectMaterialAlias(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/redirect": + http.Redirect(w, r, "/next", http.StatusFound) + case "/next": + _, _ = w.Write([]byte("redirect material")) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(upstream.Close) + + capture := network.NewProxyCapture() + handler := newTestProxyHandler(t, capture) + resp := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, upstream.URL+"/redirect", nil) + + handler.ServeHTTP(resp, req) + + require.Equal(t, http.StatusFound, resp.Code) + require.Empty(t, capture.Materials()) + resp = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, upstream.URL+"/next", nil) + + handler.ServeHTTP(resp, req) + + require.Equal(t, http.StatusOK, resp.Code) + expectedDigest := "sha256:230b890186495c4878036c4393de6137ca1a3d0e51899ea6402eaef3320a9e9b" + requests := capture.Requests() + require.Len(t, requests, 2) + require.Equal(t, upstream.URL+"/redirect", requests[0].URL) + require.Equal(t, upstream.URL+"/next", requests[0].RedirectTarget) + require.Equal(t, http.StatusFound, requests[0].StatusCode) + require.Equal(t, upstream.URL+"/next", requests[1].URL) + require.Empty(t, requests[1].RedirectTarget) + require.Equal(t, http.StatusOK, requests[1].StatusCode) + materials := capture.Materials() + require.Len(t, materials, 2) + require.Equal(t, upstream.URL+"/next", materials[0].URL) + require.Equal(t, expectedDigest, materials[0].Digest.String()) + require.Equal(t, upstream.URL+"/redirect", materials[1].URL) + require.Equal(t, expectedDigest, materials[1].Digest.String()) + require.Empty(t, capture.Incomplete()) +} + +func TestProxyHandlerRedactsCapturedCredentials(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("secret ok")) + })) + t.Cleanup(upstream.Close) + + capture := network.NewProxyCapture() + handler := newTestProxyHandler(t, capture) + resp := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, strings.Replace(upstream.URL, "http://", "http://user:pass@", 1)+"/file", nil) + + handler.ServeHTTP(resp, req) + + require.Equal(t, http.StatusOK, resp.Code) + materials := capture.Materials() + require.Len(t, materials, 1) + require.NotContains(t, materials[0].URL, "user") + require.NotContains(t, materials[0].URL, "pass") + require.Contains(t, materials[0].URL, "xxxxx:xxxxx@") + requests := capture.Requests() + require.Len(t, requests, 1) + require.NotContains(t, requests[0].URL, "user") + require.NotContains(t, requests[0].URL, "pass") + require.Contains(t, requests[0].URL, "xxxxx:xxxxx@") + require.Equal(t, http.StatusOK, requests[0].StatusCode) +} + +func TestCaptureURLNormalizesDefaultPort(t *testing.T) { + require.Equal(t, + "https://dl-cdn.alpinelinux.org/alpine/v3.23/main/aarch64/APKINDEX.tar.gz", + captureURL("https://dl-cdn.alpinelinux.org:443/alpine/v3.23/main/aarch64/APKINDEX.tar.gz"), + ) + require.Equal(t, + "http://example.com/file", + captureURL("http://example.com:80/file"), + ) + require.Equal(t, + "https://example.com:8443/file", + captureURL("https://example.com:8443/file"), + ) + require.Equal(t, + "https://xxxxx:xxxxx@example.com/file", + captureURL("https://user:pass@example.com:443/file"), + ) + require.Equal(t, + "https://[2001:db8::1]/file", + captureURL("https://[2001:db8::1]:443/file"), + ) + require.Equal(t, + "https://[2001:db8::1]:8443/file", + captureURL("https://[2001:db8::1]:8443/file"), + ) +} + +func TestProxyHandlerAppliesPolicyConvert(t *testing.T) { + original := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("original upstream should not receive converted request") + })) + t.Cleanup(original.Close) + mirrorMethodCh := make(chan string, 1) + mirror := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mirrorMethodCh <- r.Method + _, _ = w.Write([]byte("mirror material")) + })) + t.Cleanup(mirror.Close) + + capture := network.NewProxyCapture() + handler := newTestProxyHandler(t, capture) + handler.policy = proxyPolicyFunc(func(_ context.Context, op *pb.Op) (bool, error) { + require.Equal(t, original.URL+"/file", op.GetSource().Identifier) + op.GetSource().Identifier = mirror.URL + "/file" + return true, nil + }) + resp := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, original.URL+"/file", nil) + + handler.ServeHTTP(resp, req) + + require.Equal(t, http.StatusOK, resp.Code) + require.Equal(t, "mirror material", resp.Body.String()) + require.Equal(t, http.MethodGet, <-mirrorMethodCh) + requests := capture.Requests() + require.Len(t, requests, 1) + require.Equal(t, mirror.URL+"/file", requests[0].URL) + require.Equal(t, http.StatusOK, requests[0].StatusCode) + materials := capture.Materials() + require.Len(t, materials, 1) + require.Equal(t, mirror.URL+"/file", materials[0].URL) + require.Empty(t, capture.Incomplete()) +} + +func TestProxyHandlerPolicyRedactsCredentialsInErrors(t *testing.T) { + handler := newTestProxyHandler(t, nil) + handler.policy = enginePolicyEvaluator{engine: sourcepolicy.NewEngine([]*spb.Policy{ + { + Rules: []*spb.Rule{ + { + Action: spb.PolicyAction_DENY, + Selector: &spb.Selector{ + Identifier: "https://*", + }, + }, + }, + }, + })} + + _, err := handler.check(t.Context(), http.MethodGet, "https://user:pass@example.com/path") + require.ErrorIs(t, err, sourcepolicy.ErrSourceDenied) + require.NotContains(t, err.Error(), "user") + require.NotContains(t, err.Error(), "pass") + require.Contains(t, err.Error(), "https://xxxxx:xxxxx@example.com/path") +} + +func TestProxyHandlerRejectsConvertedNonGetRequest(t *testing.T) { + handler := newTestProxyHandler(t, nil) + handler.policy = proxyPolicyFunc(func(_ context.Context, op *pb.Op) (bool, error) { + op.GetSource().Identifier = "https://mirror.example.com/file" + return true, nil + }) + + _, err := handler.check(t.Context(), http.MethodPost, "https://example.com/file") + require.Error(t, err) + require.Contains(t, err.Error(), "conversion is only supported for GET") +} + +func TestProxyHandlerRejectsConvertedAttrs(t *testing.T) { + handler := newTestProxyHandler(t, nil) + handler.policy = proxyPolicyFunc(func(_ context.Context, op *pb.Op) (bool, error) { + op.GetSource().Identifier = "https://mirror.example.com/file" + op.GetSource().Attrs = map[string]string{ + pb.AttrHTTPChecksum: "sha256:6e4b94fc270e708e1068be28bd3551dc6917a4fc5a61293d51bb36e6b75c4b53", + } + return true, nil + }) + + _, err := handler.check(t.Context(), http.MethodGet, "https://example.com/file") + require.Error(t, err) + require.Contains(t, err.Error(), "proxy conversion only supports URL updates") +} + +type proxyPolicyFunc func(context.Context, *pb.Op) (bool, error) + +func (f proxyPolicyFunc) Evaluate(ctx context.Context, op *pb.Op) (bool, error) { + return f(ctx, op) +} + +type enginePolicyEvaluator struct { + engine *sourcepolicy.Engine +} + +func (e enginePolicyEvaluator) Evaluate(ctx context.Context, op *pb.Op) (bool, error) { + return e.engine.Evaluate(ctx, op.GetSource()) +} + +func newTestProxyHandler(t *testing.T, capture *network.ProxyCapture) *proxyHandler { + t.Helper() + tr := http.DefaultTransport.(*http.Transport).Clone() + tr.DisableCompression = true + t.Cleanup(tr.CloseIdleConnections) + return &proxyHandler{ + provider: &provider{client: tr}, + capture: capture, + } +} diff --git a/util/network/proxyprovider/provider_unsupported.go b/util/network/proxyprovider/provider_unsupported.go new file mode 100644 index 000000000000..b9b138c376a7 --- /dev/null +++ b/util/network/proxyprovider/provider_unsupported.go @@ -0,0 +1,21 @@ +//go:build !linux + +package proxyprovider + +import ( + "github.com/moby/buildkit/util/network" + "github.com/pkg/errors" +) + +type Opt struct { + Root string + PoolSize int +} + +func Supported() bool { + return false +} + +func New(opt Opt) (network.Provider, error) { + return nil, errors.New("proxy network provider is only supported on linux") +} diff --git a/worker/base/worker.go b/worker/base/worker.go index 8fcb472317b8..5ea26aeb0604 100644 --- a/worker/base/worker.go +++ b/worker/base/worker.go @@ -21,6 +21,7 @@ import ( "github.com/moby/buildkit/client/llb/sourceresolver" "github.com/moby/buildkit/executor" "github.com/moby/buildkit/executor/resources" + resourcestypes "github.com/moby/buildkit/executor/resources/types" "github.com/moby/buildkit/exporter" imageexporter "github.com/moby/buildkit/exporter/containerimage" localexporter "github.com/moby/buildkit/exporter/local" @@ -354,13 +355,59 @@ func (w *Worker) CacheManager() cache.Manager { return w.CacheMgr } +type proxyPolicyProvider interface { + ProxyPolicy() (network.ProxyPolicy, error) +} + +type proxyPolicyExecutor struct { + executor.Executor + provider proxyPolicyProvider +} + +func (e *proxyPolicyExecutor) Run(ctx context.Context, id string, rootfs executor.Mount, mounts []executor.Mount, process executor.ProcessInfo, started chan<- struct{}) (resourcestypes.Recorder, error) { + if process.Meta.NetMode == pb.NetMode_PROXY { + policy, err := e.proxyPolicy() + if err != nil { + return nil, err + } + process.Meta.ProxyPolicy = policy + } + return e.Executor.Run(ctx, id, rootfs, mounts, process, started) +} + +func (e *proxyPolicyExecutor) Exec(ctx context.Context, id string, process executor.ProcessInfo) error { + if process.Meta.NetMode == pb.NetMode_PROXY { + policy, err := e.proxyPolicy() + if err != nil { + return err + } + process.Meta.ProxyPolicy = policy + } + return e.Executor.Exec(ctx, id, process) +} + +func (e *proxyPolicyExecutor) proxyPolicy() (network.ProxyPolicy, error) { + policy, err := e.provider.ProxyPolicy() + if err != nil { + return nil, err + } + return policy, nil +} + func (w *Worker) ResolveOp(v solver.Vertex, s frontend.FrontendLLBBridge, sm *session.Manager) (solver.Op, error) { if baseOp, ok := v.Sys().(*pb.Op); ok { switch op := baseOp.Op.(type) { case *pb.Op_Source: return ops.NewSourceOp(v, op, baseOp.Platform, w.SourceManager, w.ParallelismSem, sm, w) case *pb.Op_Exec: - return ops.NewExecOp(v, op, baseOp.Platform, w.CacheMgr, w.ParallelismSem, sm, w.WorkerOpt.Executor, w) + exec := w.WorkerOpt.Executor + if op.Exec != nil && op.Exec.Network == pb.NetMode_PROXY { + provider, ok := s.(proxyPolicyProvider) + if ok { + exec = &proxyPolicyExecutor{Executor: exec, provider: provider} + } + } + return ops.NewExecOp(v, op, baseOp.Platform, w.CacheMgr, w.ParallelismSem, sm, exec, w) case *pb.Op_File: return ops.NewFileOp(v, op, w.CacheMgr, w.ParallelismSem, w) case *pb.Op_Build: