diff --git a/README.md b/README.md index f64eaec4..f0dc14cb 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,9 @@ cleanroom exec --expose 15432:5432 -- postgres # Local HTTPS: usually https://buildkite.cleanroom.localhost:8143 cleanroom exec --expose-https buildkite:3000 -- mise exec -- npm run dev + +# Or load HTTPS route names from expose.https in cleanroom.yaml +cleanroom exec --expose-https -- mise exec -- npm run dev ``` The installer starts the Cleanroom daemon. It does not install the macOS @@ -106,6 +109,24 @@ sudo cleanroom dns install listener is started by `--expose-https` and `cleanroom expose` while exposures are active. +Configured HTTPS routes can enumerate full local hostnames or use a shared base: + +```yaml +expose: + https: + base: "{sandbox_id}.cleanroom.localhost" + routes: + - port: 3000 + hosts: + - "{base}" + - "*.{base}" + - "*.*.{base}" +``` + +`cleanroom dns install` manages `cleanroom.localhost`. Routes under another +local suffix, such as `{sandbox_id}.localhost`, also work when that suffix +resolves to the Cleanroom DNS listener. + Run Docker inside the microVM when the workload needs it: ```yaml diff --git a/docs/networking.md b/docs/networking.md index e9e08327..e496eadb 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -69,6 +69,29 @@ The URL is usually: https://buildkite.cleanroom.localhost:8143 ``` +For projects that need several hostnames or wildcard labels, declare the routes +in `cleanroom.yaml` and pass `--expose-https` without a value: + +```yaml +expose: + https: + base: "{sandbox_id}.cleanroom.localhost" + routes: + - port: 3000 + hosts: + - "{base}" + - "*.{base}" + - "*.*.{base}" +``` + +```bash +cleanroom exec --expose-https -- npm run dev +``` + +`cleanroom dns install` manages `cleanroom.localhost`. Routes under another +local suffix, such as `{sandbox_id}.localhost`, also work when that suffix +resolves to the Cleanroom DNS listener. + For an existing sandbox, use: ```bash diff --git a/examples/multi-host-routing/README.md b/examples/multi-host-routing/README.md index eb635b3b..a9fd4437 100644 --- a/examples/multi-host-routing/README.md +++ b/examples/multi-host-routing/README.md @@ -42,14 +42,13 @@ cd examples/multi-host-routing mise exec -- cleanroom policy validate ``` -Start the example with three exact HTTPS hosts pointing at the same guest port: +Start the example with the HTTPS hosts declared in `cleanroom.yaml` pointing at +the same guest port: ```bash mise exec -- cleanroom exec \ --backend darwin-vz \ - --expose-https example:80 \ - --expose-https example-app:80 \ - --expose-https example-s3:80 \ + --expose-https \ -- sh -lc 'cd /workspace/examples/multi-host-routing && sh ./start.sh' ``` @@ -65,8 +64,8 @@ If Cleanroom chooses a different HTTPS listener port, pass that port to ## What This Exercises -- exact route registration for hosts covered by the existing - `*.cleanroom.localhost` TLS certificate +- configured exact route registration for hosts covered by the local Cleanroom + certificate authority - guest-side host-based virtual hosting in `nginx` - preserved `Host`, `X-Forwarded-Host`, `X-Forwarded-Proto`, `X-Forwarded-Port`, and `X-Forwarded-For` diff --git a/examples/multi-host-routing/cleanroom.yaml b/examples/multi-host-routing/cleanroom.yaml index 209ccf95..50845b7b 100644 --- a/examples/multi-host-routing/cleanroom.yaml +++ b/examples/multi-host-routing/cleanroom.yaml @@ -1,5 +1,14 @@ version: 1 +expose: + https: + routes: + - port: 80 + hosts: + - example.cleanroom.localhost + - example-app.cleanroom.localhost + - example-s3.cleanroom.localhost + sandbox: image: ref: ghcr.io/buildkite/cleanroom-base/debian@sha256:28c3f638fabe1ed780f87b82cfb0c6dda2549c86b9e4edbe519e8250243411c5 diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 95b90729..7b43a633 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -36,6 +36,7 @@ const ( type policyLoader interface { LoadAndCompile(cwd string) (*policy.CompiledPolicy, string, error) LoadRepository(cwd string) (policy.RepositoryConfig, string, error) + LoadExpose(cwd string) (policy.ExposeConfig, string, error) } type runtimeContext struct { @@ -129,6 +130,7 @@ type hasExitCode interface { } func Run(args []string, version string) (runErr error) { + args = normalizeBareExposeHTTPSArgs(args) runtimeCtx := &runtimeContext{ Stdout: os.Stdout, Stderr: os.Stderr, diff --git a/internal/cli/configured_exposure.go b/internal/cli/configured_exposure.go new file mode 100644 index 00000000..959bcb66 --- /dev/null +++ b/internal/cli/configured_exposure.go @@ -0,0 +1,152 @@ +package cli + +import ( + "errors" + "fmt" + "strings" + + "github.com/buildkite/cleanroom/internal/exposure" + cleanroomv1 "github.com/buildkite/cleanroom/internal/gen/cleanroom/v1" + "github.com/buildkite/cleanroom/internal/policy" +) + +const configuredHTTPSPreflightSandboxID = "cr-preflight" + +func normalizeBareExposeHTTPSArgs(args []string) []string { + if len(args) == 0 { + return nil + } + out := append([]string(nil), args...) + for i, arg := range out { + if arg == "--" { + break + } + if arg != "--expose-https" { + continue + } + if i+1 >= len(out) || out[i+1] == "--" || strings.HasPrefix(out[i+1], "-") { + out[i] = "--expose-https=" + configuredHTTPSExposureSpec + } + } + return out +} + +func resolveRequestedExposures(ctx *runtimeContext, cwd, sandboxID string, requested []*cleanroomv1.PortExposure) ([]*cleanroomv1.PortExposure, error) { + if !hasConfiguredHTTPSExposure(requested) { + return requested, nil + } + cfg, err := loadConfiguredHTTPSExposure(ctx, cwd) + if err != nil { + return nil, err + } + configured, err := expandConfiguredHTTPSExposures(cfg, sandboxID) + if err != nil { + return nil, err + } + + resolved := make([]*cleanroomv1.PortExposure, 0, len(requested)+len(configured)) + for _, req := range requested { + if isConfiguredHTTPSExposure(req) { + resolved = append(resolved, configured...) + continue + } + resolved = append(resolved, req) + } + return resolved, nil +} + +func prevalidateConfiguredExposures(ctx *runtimeContext, cwd string, requested []*cleanroomv1.PortExposure) error { + if !hasConfiguredHTTPSExposure(requested) { + return nil + } + cfg, err := loadConfiguredHTTPSExposure(ctx, cwd) + if err != nil { + return err + } + _, err = expandConfiguredHTTPSExposures(cfg, configuredHTTPSPreflightSandboxID) + return err +} + +func loadConfiguredHTTPSExposure(ctx *runtimeContext, cwd string) (policy.ExposeHTTPSConfig, error) { + if ctx == nil || ctx.Loader == nil { + return policy.ExposeHTTPSConfig{}, errors.New("configured --expose-https requires a policy loader") + } + cfg, _, err := ctx.Loader.LoadExpose(cwd) + if err != nil { + if errors.Is(err, policy.ErrPolicyNotFound) { + return policy.ExposeHTTPSConfig{}, errors.New("configured --expose-https requires expose.https in cleanroom.yaml") + } + return policy.ExposeHTTPSConfig{}, err + } + return cfg.HTTPS, nil +} + +func hasConfiguredHTTPSExposure(requested []*cleanroomv1.PortExposure) bool { + for _, req := range requested { + if isConfiguredHTTPSExposure(req) { + return true + } + } + return false +} + +func isConfiguredHTTPSExposure(req *cleanroomv1.PortExposure) bool { + return req != nil && + strings.TrimSpace(req.GetProtocol()) == exposureProtocolHTTPS && + strings.TrimSpace(req.GetName()) == configuredHTTPSExposureSpec && + req.GetGuestPort() == 0 +} + +func expandConfiguredHTTPSExposures(cfg policy.ExposeHTTPSConfig, sandboxID string) ([]*cleanroomv1.PortExposure, error) { + sandboxID = strings.TrimSpace(strings.ToLower(sandboxID)) + if sandboxID == "" { + return nil, errors.New("configured --expose-https requires a sandbox id") + } + if cfg.IsZero() { + return nil, errors.New("configured --expose-https requires expose.https in cleanroom.yaml") + } + base := strings.TrimSpace(strings.ToLower(cfg.Base)) + if base != "" { + base = expandExposeTemplate(base, sandboxID, "") + base = strings.TrimSpace(strings.ToLower(base)) + } + if strings.TrimSpace(cfg.Base) != "" && base == "" { + return nil, errors.New("expose.https.base expanded to an empty host") + } + + exposures := make([]*cleanroomv1.PortExposure, 0) + for i, route := range cfg.Routes { + if route.Port < 1 || route.Port > 65535 { + return nil, fmt.Errorf("expose.https.routes[%d].port must be in range 1-65535", i) + } + if len(route.Hosts) == 0 { + return nil, fmt.Errorf("expose.https.routes[%d].hosts must include at least one host", i) + } + for j, host := range route.Hosts { + if strings.Contains(host, "{base}") && base == "" { + return nil, fmt.Errorf("expose.https.routes[%d].hosts[%d] uses {base} but expose.https.base is empty", i, j) + } + host = expandExposeTemplate(host, sandboxID, base) + host = strings.TrimSpace(strings.ToLower(host)) + if host == "" { + return nil, fmt.Errorf("expose.https.routes[%d].hosts[%d] expanded to an empty host", i, j) + } + if err := exposure.ValidateHTTPSRouteName(host); err != nil { + return nil, fmt.Errorf("expose.https.routes[%d].hosts[%d] is invalid: %w", i, j, err) + } + exposures = append(exposures, &cleanroomv1.PortExposure{ + Protocol: exposureProtocolHTTPS, + GuestPort: int32(route.Port), + Name: host, + }) + } + } + return exposures, nil +} + +func expandExposeTemplate(value, sandboxID, base string) string { + value = strings.ReplaceAll(value, "{sandbox_id}", sandboxID) + value = strings.ReplaceAll(value, "{container_id}", sandboxID) + value = strings.ReplaceAll(value, "{base}", base) + return value +} diff --git a/internal/cli/configured_exposure_test.go b/internal/cli/configured_exposure_test.go new file mode 100644 index 00000000..12f46105 --- /dev/null +++ b/internal/cli/configured_exposure_test.go @@ -0,0 +1,180 @@ +package cli + +import ( + "errors" + "reflect" + "strings" + "testing" + + cleanroomv1 "github.com/buildkite/cleanroom/internal/gen/cleanroom/v1" + "github.com/buildkite/cleanroom/internal/policy" +) + +func TestNormalizeBareExposeHTTPSArgs(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + args []string + want []string + }{ + { + name: "bare before command separator", + args: []string{"exec", "--expose-https", "--", "npm", "run", "dev"}, + want: []string{"exec", "--expose-https=" + configuredHTTPSExposureSpec, "--", "npm", "run", "dev"}, + }, + { + name: "bare at end", + args: []string{"create", "--expose-https"}, + want: []string{"create", "--expose-https=" + configuredHTTPSExposureSpec}, + }, + { + name: "explicit value", + args: []string{"create", "--expose-https", "buildkite:3000"}, + want: []string{"create", "--expose-https", "buildkite:3000"}, + }, + { + name: "guest arg after command separator", + args: []string{"exec", "--", "tool", "--expose-https"}, + want: []string{"exec", "--", "tool", "--expose-https"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := normalizeBareExposeHTTPSArgs(tt.args) + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("unexpected args: got %v want %v", got, tt.want) + } + if len(tt.args) > 0 && &got[0] == &tt.args[0] { + t.Fatal("expected normalized args to be a copy") + } + }) + } +} + +func TestExpandConfiguredHTTPSExposures(t *testing.T) { + t.Parallel() + + got, err := expandConfiguredHTTPSExposures(policy.ExposeHTTPSConfig{ + Base: "{sandbox_id}.localhost", + Routes: []policy.ExposeHTTPSRoute{{ + Port: 3000, + Hosts: []string{"{base}", "*.{base}", "*.*.{base}"}, + }}, + }, "Cr-123") + if err != nil { + t.Fatalf("expandConfiguredHTTPSExposures returned error: %v", err) + } + + want := []*cleanroomv1.PortExposure{ + {Protocol: exposureProtocolHTTPS, GuestPort: 3000, Name: "cr-123.localhost"}, + {Protocol: exposureProtocolHTTPS, GuestPort: 3000, Name: "*.cr-123.localhost"}, + {Protocol: exposureProtocolHTTPS, GuestPort: 3000, Name: "*.*.cr-123.localhost"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("unexpected exposures: got %v want %v", got, want) + } +} + +func TestExpandConfiguredHTTPSExposuresValidatesExpandedHosts(t *testing.T) { + t.Parallel() + + _, err := expandConfiguredHTTPSExposures(policy.ExposeHTTPSConfig{ + Base: "{sandbox_id}.localhost", + Routes: []policy.ExposeHTTPSRoute{{ + Port: 3000, + Hosts: []string{"bad_{base}"}, + }}, + }, "sandbox-1") + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "expose.https.routes[0].hosts[0] is invalid") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestResolveRequestedExposuresLoadsConfiguredHTTPS(t *testing.T) { + t.Parallel() + + loader := &configuredExposureLoader{ + cfg: policy.ExposeConfig{HTTPS: policy.ExposeHTTPSConfig{ + Base: "{container_id}.localhost", + Routes: []policy.ExposeHTTPSRoute{{ + Port: 3000, + Hosts: []string{"{base}", "*.{base}"}, + }}, + }}, + } + got, err := resolveRequestedExposures(&runtimeContext{Loader: loader}, "/repo", "sandbox-1", []*cleanroomv1.PortExposure{ + {Protocol: exposureProtocolTCP, HostPort: 5432, GuestPort: 5432}, + {Protocol: exposureProtocolHTTPS, Name: configuredHTTPSExposureSpec}, + }) + if err != nil { + t.Fatalf("resolveRequestedExposures returned error: %v", err) + } + if got, want := loader.cwd, "/repo"; got != want { + t.Fatalf("unexpected loader cwd: got %q want %q", got, want) + } + if got, want := len(got), 3; got != want { + t.Fatalf("unexpected exposure count: got %d want %d", got, want) + } + if got, want := got[1].GetName(), "sandbox-1.localhost"; got != want { + t.Fatalf("unexpected first configured host: got %q want %q", got, want) + } + if got, want := got[2].GetName(), "*.sandbox-1.localhost"; got != want { + t.Fatalf("unexpected second configured host: got %q want %q", got, want) + } +} + +func TestResolveRequestedExposuresRequiresConfiguredHTTPS(t *testing.T) { + t.Parallel() + + _, err := resolveRequestedExposures(&runtimeContext{Loader: &configuredExposureLoader{}}, "/repo", "sandbox-1", []*cleanroomv1.PortExposure{ + {Protocol: exposureProtocolHTTPS, Name: configuredHTTPSExposureSpec}, + }) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "requires expose.https") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestPrevalidateConfiguredExposuresRequiresConfiguredHTTPS(t *testing.T) { + t.Parallel() + + loader := &configuredExposureLoader{} + err := prevalidateConfiguredExposures(&runtimeContext{Loader: loader}, "/repo", []*cleanroomv1.PortExposure{ + {Protocol: exposureProtocolHTTPS, Name: configuredHTTPSExposureSpec}, + }) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "requires expose.https") { + t.Fatalf("unexpected error: %v", err) + } + if got, want := loader.cwd, "/repo"; got != want { + t.Fatalf("unexpected loader cwd: got %q want %q", got, want) + } +} + +type configuredExposureLoader struct { + cfg policy.ExposeConfig + cwd string +} + +func (l *configuredExposureLoader) LoadAndCompile(string) (*policy.CompiledPolicy, string, error) { + return nil, "", errors.New("LoadAndCompile should not be called") +} + +func (l *configuredExposureLoader) LoadRepository(string) (policy.RepositoryConfig, string, error) { + return policy.RepositoryConfig{}, "", errors.New("LoadRepository should not be called") +} + +func (l *configuredExposureLoader) LoadExpose(cwd string) (policy.ExposeConfig, string, error) { + l.cwd = cwd + return l.cfg, "/repo/cleanroom.yaml", nil +} diff --git a/internal/cli/console.go b/internal/cli/console.go index 8e9304cb..4ae74523 100644 --- a/internal/cli/console.go +++ b/internal/cli/console.go @@ -27,7 +27,7 @@ type ConsoleCommand struct { DangerouslyAllowAll bool `name:"dangerously-allow-all" help:"Disable network egress filtering for a newly created sandbox"` Env []string `short:"e" name:"env" help:"Set guest environment variables; use KEY to inherit from the local environment or KEY=VALUE to set an explicit value"` Expose []string `name:"expose" help:"Expose raw TCP as or :"` - ExposeHTTPS []string `name:"expose-https" help:"Expose HTTPS as [name:] under cleanroom.localhost"` + ExposeHTTPS []string `name:"expose-https" help:"Expose HTTPS as [name:], or configured expose.https routes when omitted"` PrintSandboxID bool `name:"print-sandbox-id" help:"Print resolved sandbox_id= to stderr before attaching"` LaunchSeconds int64 `help:"VM boot/guest-agent readiness timeout in seconds"` @@ -82,11 +82,6 @@ func (c *ConsoleCommand) Run(ctx *runtimeContext) (runErr error) { logger := newClientLogger() - host := c.resolvedHost(ctx.Config) - client, err := c.connect(ctx) - if err != nil { - return err - } cwd, err := resolveCWD(ctx.CWD, c.Chdir) if err != nil { return err @@ -99,6 +94,14 @@ func (c *ConsoleCommand) Run(ctx *runtimeContext) (runErr error) { if err != nil { return err } + if err := prevalidateRequestedExposures(ctx, cwd, exposures); err != nil { + return err + } + host := c.resolvedHost(ctx.Config) + client, err := c.connect(ctx) + if err != nil { + return err + } if err := validateWorkspaceCopyOutBeforeExecution(rootCtx, ctx, client, cwd, c.Chdir, c.In, c.LaunchSeconds, c.workspaceCopyFlags); err != nil { return err } @@ -169,6 +172,10 @@ func (c *ConsoleCommand) Run(ctx *runtimeContext) (runErr error) { } return joinExecutionAndWorkspaceCopyOutError(executionErr, copyOutErr) } + exposures, err = resolveRequestedExposures(ctx, cwd, sandboxID, exposures) + if err != nil { + return err + } if c.preAttach != nil { if err := c.preAttach(rootCtx, client, sandboxID); err != nil { return err diff --git a/internal/cli/dns.go b/internal/cli/dns.go index dc9a1e71..199b24f7 100644 --- a/internal/cli/dns.go +++ b/internal/cli/dns.go @@ -452,7 +452,7 @@ func exposureCertificateTrustStatus(certPath string) (bool, string) { if err != nil { return false, err.Error() } - if err := runExposureTrustSecurity("verify-cert", "-c", certPath, "-p", "ssl", "-n", "buildkite."+exposure.Domain, "-L", "-k", keychain); err != nil { + if err := runExposureTrustSecurity("verify-cert", "-c", certPath, "-p", "ssl", "-L", "-k", keychain); err != nil { return false, err.Error() } return true, "trusted" diff --git a/internal/cli/dns_test.go b/internal/cli/dns_test.go index a819df0b..40ecdb65 100644 --- a/internal/cli/dns_test.go +++ b/internal/cli/dns_test.go @@ -49,7 +49,7 @@ func TestResolverFileContentRejectsNonIPNameserver(t *testing.T) { } } -func TestDNSInstallTrustsLeafCertificateInInvokingUserKeychain(t *testing.T) { +func TestDNSInstallTrustsCertificateAuthorityInInvokingUserKeychain(t *testing.T) { home := t.TempDir() resolverPath := filepath.Join(t.TempDir(), "resolver", exposure.Domain) var calls [][]string @@ -88,11 +88,8 @@ func TestDNSInstallTrustsLeafCertificateInInvokingUserKeychain(t *testing.T) { if err != nil { t.Fatalf("parse generated certificate: %v", err) } - if cert.IsCA { - t.Fatal("expected generated exposure certificate not to be a CA") - } - if err := cert.VerifyHostname("buildkite." + exposure.Domain); err != nil { - t.Fatalf("expected generated certificate to verify exposure hostname: %v", err) + if !cert.IsCA { + t.Fatal("expected generated exposure certificate to be a CA") } if strings.Contains(strings.Join(calls[0], " "), "System.keychain") { t.Fatalf("did not expect install to trust the system keychain: %v", calls[0]) @@ -250,7 +247,6 @@ func TestDNSStatusReportsCertificateTrust(t *testing.T) { "sudo", "-u", "lachlan", "security", "verify-cert", "-c", cert.CertPath, "-p", "ssl", - "-n", "buildkite." + exposure.Domain, "-L", "-k", keychain, }} diff --git a/internal/cli/doctor_test.go b/internal/cli/doctor_test.go index 2b83dd83..18f921f7 100644 --- a/internal/cli/doctor_test.go +++ b/internal/cli/doctor_test.go @@ -98,6 +98,10 @@ func (doctorFailingLoader) LoadRepository(string) (policy.RepositoryConfig, stri return policy.RepositoryConfig{}, "", errors.New("policy unavailable") } +func (doctorFailingLoader) LoadExpose(string) (policy.ExposeConfig, string, error) { + return policy.ExposeConfig{}, "", errors.New("policy unavailable") +} + type doctorStaticLoader struct{} func (doctorStaticLoader) LoadAndCompile(cwd string) (*policy.CompiledPolicy, string, error) { @@ -108,6 +112,10 @@ func (doctorStaticLoader) LoadRepository(string) (policy.RepositoryConfig, strin return policy.RepositoryConfig{}, "", nil } +func (doctorStaticLoader) LoadExpose(string) (policy.ExposeConfig, string, error) { + return policy.ExposeConfig{}, "", nil +} + func TestDoctorCommandJSONIncludesCapabilities(t *testing.T) { t.Setenv("CLEANROOM_GITHUB_TOKEN", "ghp_testtoken") t.Setenv("CLEANROOM_GITLAB_TOKEN", "") diff --git a/internal/cli/exec.go b/internal/cli/exec.go index 6ad33bb1..88220e33 100644 --- a/internal/cli/exec.go +++ b/internal/cli/exec.go @@ -29,7 +29,7 @@ type ExecCommand struct { DangerouslyAllowAll bool `name:"dangerously-allow-all" help:"Disable network egress filtering for a newly created sandbox"` Env []string `short:"e" name:"env" help:"Set guest environment variables; use KEY to inherit from the local environment or KEY=VALUE to set an explicit value"` Expose []string `name:"expose" help:"Expose raw TCP as or :"` - ExposeHTTPS []string `name:"expose-https" help:"Expose HTTPS as [name:] under cleanroom.localhost"` + ExposeHTTPS []string `name:"expose-https" help:"Expose HTTPS as [name:], or configured expose.https routes when omitted"` TTY bool `name:"tty" help:"Allocate a tty and attach through the interactive transport; stdout and stderr merge into a single stream"` NoStdin bool `short:"n" name:"no-stdin" aliases:"stdin-eof" help:"Close stdin immediately instead of attaching it"` PrintSandboxID bool `name:"print-sandbox-id" help:"Print resolved sandbox_id= to stderr before streaming output"` @@ -87,11 +87,6 @@ func (e *ExecCommand) Run(ctx *runtimeContext) (runErr error) { logger := newClientLogger() - host := e.resolvedHost(ctx.Config) - client, err := e.connect(ctx) - if err != nil { - return err - } cwd, err := resolveCWD(ctx.CWD, e.Chdir) if err != nil { return err @@ -104,6 +99,14 @@ func (e *ExecCommand) Run(ctx *runtimeContext) (runErr error) { if err != nil { return err } + if err := prevalidateRequestedExposures(ctx, cwd, exposures); err != nil { + return err + } + host := e.resolvedHost(ctx.Config) + client, err := e.connect(ctx) + if err != nil { + return err + } if err := validateWorkspaceCopyOutBeforeExecution(rootCtx, ctx, client, cwd, e.Chdir, e.In, e.LaunchSeconds, e.workspaceCopyFlags); err != nil { return err } @@ -198,6 +201,10 @@ func (e *ExecCommand) Run(ctx *runtimeContext) (runErr error) { } return joinExecutionAndWorkspaceCopyOutError(executionErr, copyOutErr) } + exposures, err = resolveRequestedExposures(ctx, cwd, sandboxID, exposures) + if err != nil { + return err + } exposureManager, exposed, err := startClientExposures(rootCtx, client, sandboxID, exposures) if err != nil { return err diff --git a/internal/cli/exec_integration_test.go b/internal/cli/exec_integration_test.go index 24584bed..d801b016 100644 --- a/internal/cli/exec_integration_test.go +++ b/internal/cli/exec_integration_test.go @@ -160,6 +160,10 @@ func (integrationLoader) LoadRepository(_ string) (policy.RepositoryConfig, stri return policy.RepositoryConfig{}, "/repo/cleanroom.yaml", nil } +func (integrationLoader) LoadExpose(_ string) (policy.ExposeConfig, string, error) { + return policy.ExposeConfig{}, "/repo/cleanroom.yaml", nil +} + type failingLoader struct{} func (failingLoader) LoadAndCompile(_ string) (*policy.CompiledPolicy, string, error) { @@ -170,6 +174,10 @@ func (failingLoader) LoadRepository(_ string) (policy.RepositoryConfig, string, return policy.RepositoryConfig{}, "/repo/cleanroom.yaml", nil } +func (failingLoader) LoadExpose(_ string) (policy.ExposeConfig, string, error) { + return policy.ExposeConfig{}, "/repo/cleanroom.yaml", nil +} + type execOutcome struct { err error stdout string diff --git a/internal/cli/expose.go b/internal/cli/expose.go index 35ac7695..358934ea 100644 --- a/internal/cli/expose.go +++ b/internal/cli/expose.go @@ -9,7 +9,7 @@ type ExposeCommand struct { clientFlags SandboxID string `name:"in" aliases:"sandbox-id" required:"" help:"Sandbox ID to expose"` Expose []string `name:"expose" help:"Expose raw TCP as or :"` - ExposeHTTPS []string `name:"expose-https" help:"Expose HTTPS as [name:] under cleanroom.localhost"` + ExposeHTTPS []string `name:"expose-https" help:"Expose HTTPS as [name:], or configured expose.https routes when omitted"` } type PortForwardCommand struct { @@ -23,6 +23,10 @@ func (c *ExposeCommand) Run(ctx *runtimeContext) error { if err != nil { return err } + exposures, err = resolveRequestedExposures(ctx, ctx.CWD, c.SandboxID, exposures) + if err != nil { + return err + } return runForegroundClientExposures(ctx, c.clientFlags, c.SandboxID, exposures) } diff --git a/internal/cli/exposure_flags.go b/internal/cli/exposure_flags.go index 7152a0e0..e438d15c 100644 --- a/internal/cli/exposure_flags.go +++ b/internal/cli/exposure_flags.go @@ -13,6 +13,8 @@ import ( const ( exposureProtocolTCP = "tcp" exposureProtocolHTTPS = "https" + + configuredHTTPSExposureSpec = "__cleanroom_configured_https__" ) func parseExposureFlags(tcpSpecs, httpsSpecs []string) ([]*cleanroomv1.PortExposure, error) { @@ -93,7 +95,13 @@ func parseTCPExposureSpec(spec string) (*cleanroomv1.PortExposure, error) { func parseHTTPSExposureSpec(spec string) (*cleanroomv1.PortExposure, error) { spec = strings.TrimSpace(spec) if spec == "" { - return nil, errors.New("invalid --expose-https value: empty exposure") + spec = configuredHTTPSExposureSpec + } + if spec == configuredHTTPSExposureSpec { + return &cleanroomv1.PortExposure{ + Protocol: exposureProtocolHTTPS, + Name: configuredHTTPSExposureSpec, + }, nil } name := "" diff --git a/internal/cli/exposure_flags_test.go b/internal/cli/exposure_flags_test.go index 1f662433..bcf88dfd 100644 --- a/internal/cli/exposure_flags_test.go +++ b/internal/cli/exposure_flags_test.go @@ -10,12 +10,12 @@ func TestParseExposureFlags(t *testing.T) { exposures, err := parseExposureFlags( []string{"5432", "15432:5432"}, - []string{"buildkite:3000", "3001"}, + []string{"buildkite:3000", "3001", ""}, ) if err != nil { t.Fatalf("parseExposureFlags returned error: %v", err) } - if got, want := len(exposures), 4; got != want { + if got, want := len(exposures), 5; got != want { t.Fatalf("unexpected exposure count: got %d want %d", got, want) } if got := exposures[0]; got.GetProtocol() != exposureProtocolTCP || got.GetHostPort() != 5432 || got.GetGuestPort() != 5432 { @@ -30,6 +30,9 @@ func TestParseExposureFlags(t *testing.T) { if got := exposures[3]; got.GetProtocol() != exposureProtocolHTTPS || got.GetName() != "" || got.GetGuestPort() != 3001 { t.Fatalf("unexpected fourth exposure: %#v", got) } + if got := exposures[4]; got.GetProtocol() != exposureProtocolHTTPS || got.GetName() != configuredHTTPSExposureSpec || got.GetGuestPort() != 0 { + t.Fatalf("unexpected configured exposure: %#v", got) + } } func TestParseExposureFlagsRejectsInvalidSpecs(t *testing.T) { @@ -45,7 +48,6 @@ func TestParseExposureFlagsRejectsInvalidSpecs(t *testing.T) { {name: "tcp too many parts", tcpSpecs: []string{"a:b:c"}, want: "expected "}, {name: "tcp bad port", tcpSpecs: []string{"buildkite:5432"}, want: "host port"}, {name: "tcp out of range", tcpSpecs: []string{"70000"}, want: "out of range"}, - {name: "https empty", httpsSpec: []string{""}, want: "empty exposure"}, {name: "https bad name", httpsSpec: []string{"Buildkite:3000"}, want: "lowercase"}, {name: "https too many parts", httpsSpec: []string{"buildkite:https:3000"}, want: "expected [name:]"}, {name: "https bad port", httpsSpec: []string{"buildkite:port"}, want: "port must be numeric"}, diff --git a/internal/cli/local_exposure.go b/internal/cli/local_exposure.go index fac6050a..7b5fcc02 100644 --- a/internal/cli/local_exposure.go +++ b/internal/cli/local_exposure.go @@ -32,14 +32,7 @@ func startClientExposures(ctx context.Context, client *controlclient.Client, san } manager := exposure.NewManager(exposure.Config{}) - needsDNS := false - for _, req := range requested { - if req != nil && strings.TrimSpace(req.GetProtocol()) == exposureProtocolHTTPS { - needsDNS = true - break - } - } - if needsDNS { + if hasHTTPSExposure(requested) { if err := manager.StartDNS(ctx); err != nil { _ = manager.Close() return nil, nil, err @@ -66,6 +59,26 @@ func startClientExposures(ctx context.Context, client *controlclient.Client, san return manager, registered, nil } +func prevalidateRequestedExposures(ctx *runtimeContext, cwd string, requested []*cleanroomv1.PortExposure) error { + if err := prevalidateConfiguredExposures(ctx, cwd, requested); err != nil { + return err + } + if !hasHTTPSExposure(requested) { + return nil + } + _, err := exposure.EnsureRuntimeCertificateAuthority(exposure.Domain, "") + return err +} + +func hasHTTPSExposure(requested []*cleanroomv1.PortExposure) bool { + for _, req := range requested { + if req != nil && strings.TrimSpace(req.GetProtocol()) == exposureProtocolHTTPS { + return true + } + } + return false +} + func ensureSandboxPortDialSupported(ctx context.Context, client *controlclient.Client, sandboxID string) error { resp, err := client.GetSandbox(ctx, &cleanroomv1.GetSandboxRequest{SandboxId: sandboxID}) if err != nil { diff --git a/internal/cli/local_exposure_test.go b/internal/cli/local_exposure_test.go index 0495855c..e79a51f9 100644 --- a/internal/cli/local_exposure_test.go +++ b/internal/cli/local_exposure_test.go @@ -3,6 +3,8 @@ package cli import ( "context" "net/http/httptest" + "os" + "path/filepath" "strings" "testing" @@ -11,6 +13,7 @@ import ( "github.com/buildkite/cleanroom/internal/controlserver" "github.com/buildkite/cleanroom/internal/controlservice" "github.com/buildkite/cleanroom/internal/endpoint" + "github.com/buildkite/cleanroom/internal/exposure" cleanroomv1 "github.com/buildkite/cleanroom/internal/gen/cleanroom/v1" "github.com/buildkite/cleanroom/internal/runtimeconfig" ) @@ -57,6 +60,35 @@ func TestStartClientExposuresRejectsUnsupportedSandboxPortDial(t *testing.T) { } } +func TestPrevalidateRequestedExposuresChecksHTTPSCertificate(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + tlsDir, err := exposure.DefaultTLSDir() + if err != nil { + t.Fatalf("DefaultTLSDir returned error: %v", err) + } + if err := os.MkdirAll(tlsDir, 0o700); err != nil { + t.Fatalf("create TLS dir: %v", err) + } + if err := os.WriteFile(filepath.Join(tlsDir, exposure.LocalCertificateFilename), []byte("not a certificate"), 0o644); err != nil { + t.Fatalf("write invalid certificate: %v", err) + } + if err := os.WriteFile(filepath.Join(tlsDir, exposure.LocalCertificateKeyFilename), []byte("not a key"), 0o600); err != nil { + t.Fatalf("write invalid key: %v", err) + } + + err = prevalidateRequestedExposures(&runtimeContext{Loader: failingLoader{}}, t.TempDir(), []*cleanroomv1.PortExposure{{ + Protocol: exposureProtocolHTTPS, + Name: "buildkite", + GuestPort: 3000, + }}) + if err == nil { + t.Fatal("expected HTTPS exposure prevalidation to fail on invalid local certificate") + } + if !strings.Contains(err.Error(), "certificate PEM") { + t.Fatalf("unexpected prevalidation error: %v", err) + } +} + func newLocalExposureTestClient(t *testing.T, adapter backend.Adapter) (*controlclient.Client, string) { t.Helper() service := &controlservice.Service{ diff --git a/internal/cli/policy_validate_test.go b/internal/cli/policy_validate_test.go index affb91ad..4e20dca2 100644 --- a/internal/cli/policy_validate_test.go +++ b/internal/cli/policy_validate_test.go @@ -24,6 +24,10 @@ func (l *policyValidateLoader) LoadRepository(string) (policy.RepositoryConfig, return policy.RepositoryConfig{}, "", nil } +func (l *policyValidateLoader) LoadExpose(string) (policy.ExposeConfig, string, error) { + return policy.ExposeConfig{}, "", nil +} + func TestPolicyValidateCommandRunJSON(t *testing.T) { t.Parallel() diff --git a/internal/cli/repository_test.go b/internal/cli/repository_test.go index a3a1da19..68f1df61 100644 --- a/internal/cli/repository_test.go +++ b/internal/cli/repository_test.go @@ -33,6 +33,10 @@ func (l repositoryIntegrationLoader) LoadRepository(_ string) (policy.Repository return l.repository, "/repo/cleanroom.yaml", nil } +func (l repositoryIntegrationLoader) LoadExpose(_ string) (policy.ExposeConfig, string, error) { + return policy.ExposeConfig{}, "/repo/cleanroom.yaml", nil +} + func (repositoryNotFoundLoader) LoadAndCompile(_ string) (*policy.CompiledPolicy, string, error) { return &policy.CompiledPolicy{ Version: 1, @@ -46,6 +50,10 @@ func (repositoryNotFoundLoader) LoadRepository(_ string) (policy.RepositoryConfi return policy.RepositoryConfig{}, "", fmt.Errorf("%w: expected /tmp/cleanroom.yaml or /tmp/.buildkite/cleanroom.yaml", policy.ErrPolicyNotFound) } +func (repositoryNotFoundLoader) LoadExpose(_ string) (policy.ExposeConfig, string, error) { + return policy.ExposeConfig{}, "", fmt.Errorf("%w: expected /tmp/cleanroom.yaml or /tmp/.buildkite/cleanroom.yaml", policy.ErrPolicyNotFound) +} + func initGitRepository(t *testing.T, remoteURL string) string { t.Helper() diff --git a/internal/cli/sandbox.go b/internal/cli/sandbox.go index fb49e4ef..b4fae50b 100644 --- a/internal/cli/sandbox.go +++ b/internal/cli/sandbox.go @@ -31,7 +31,7 @@ type SandboxCreateCommand struct { Docker bool `help:"Enable the guest Docker service for this repo-agnostic sandbox"` DangerouslyAllowAll bool `name:"dangerously-allow-all" help:"Disable network egress filtering for this repo-agnostic sandbox"` Expose []string `name:"expose" help:"Expose raw TCP as or :"` - ExposeHTTPS []string `name:"expose-https" help:"Expose HTTPS as [name:] under cleanroom.localhost"` + ExposeHTTPS []string `name:"expose-https" help:"Expose HTTPS as [name:], or configured expose.https routes when omitted"` LaunchSeconds int64 `help:"VM boot/guest-agent readiness timeout in seconds"` JSON bool `help:"Print sandbox as JSON"` } @@ -64,7 +64,7 @@ type CreateCommand struct { workspaceCopyInFlags DangerouslyAllowAll bool `name:"dangerously-allow-all" help:"Disable network egress filtering for a newly created sandbox"` Expose []string `name:"expose" help:"Expose raw TCP as or :"` - ExposeHTTPS []string `name:"expose-https" help:"Expose HTTPS as [name:] under cleanroom.localhost"` + ExposeHTTPS []string `name:"expose-https" help:"Expose HTTPS as [name:], or configured expose.https routes when omitted"` LaunchSeconds int64 `help:"VM boot/guest-agent readiness timeout in seconds"` JSON bool `help:"Print sandbox as JSON"` } @@ -296,6 +296,9 @@ func runSandboxCreate(ctx *runtimeContext, connectFlags clientFlags, backend, fr if err != nil { return err } + if err := prevalidateRequestedExposures(ctx, ctx.CWD, exposures); err != nil { + return err + } resolvedHost := connectFlags.resolvedHost(ctx.Config) client, err := connectFlags.connect(ctx) if err != nil { @@ -344,6 +347,11 @@ func runSandboxCreate(ctx *runtimeContext, connectFlags clientFlags, backend, fr if sandboxID == "" { return errors.New("create sandbox: response missing sandbox id") } + exposures, err = resolveRequestedExposures(ctx, ctx.CWD, sandboxID, exposures) + if err != nil { + _ = writeSandboxID(os.Stderr, sandboxID) + return err + } if outputJSON { enc := json.NewEncoder(ctx.Stdout) @@ -374,13 +382,16 @@ func (c *CreateCommand) Run(ctx *runtimeContext) error { if err != nil { return err } - host := c.resolvedHost(ctx.Config) - client, err := c.connect(ctx) + + cwd, err := resolveCWD(ctx.CWD, c.Chdir) if err != nil { return err } - - cwd, err := resolveCWD(ctx.CWD, c.Chdir) + if err := prevalidateRequestedExposures(ctx, cwd, exposures); err != nil { + return err + } + host := c.resolvedHost(ctx.Config) + client, err := c.connect(ctx) if err != nil { return err } @@ -404,6 +415,11 @@ func (c *CreateCommand) Run(ctx *runtimeContext) error { if c.CopyIn && repository != nil { warnWorkspaceBindingError(ctx, recordGitWorkspaceBinding(sandboxID, repository, toRepositoryCheckout(repository), repositoryLocalChangesFiles(localChanges), "copy-in")) } + exposures, err = resolveRequestedExposures(ctx, cwd, sandboxID, exposures) + if err != nil { + _ = writeSandboxID(os.Stderr, sandboxID) + return err + } if c.JSON { enc := json.NewEncoder(ctx.Stdout) enc.SetIndent("", " ") diff --git a/internal/cli/sandbox_create_integration_test.go b/internal/cli/sandbox_create_integration_test.go index 38b5d28b..0acb9903 100644 --- a/internal/cli/sandbox_create_integration_test.go +++ b/internal/cli/sandbox_create_integration_test.go @@ -138,6 +138,33 @@ func TestTopLevelCreateIntegrationReportsExposureSetupFailureAfterCreate(t *test } } +func TestTopLevelCreatePrevalidatesConfiguredExposureBeforeCreate(t *testing.T) { + cwd := t.TempDir() + + outcome := runCreateAliasWithCapture(CreateCommand{ + Chdir: cwd, + ExposeHTTPS: []string{""}, + }, runtimeContext{ + CWD: cwd, + Loader: &configuredExposureLoader{}, + }) + if outcome.cause != nil { + t.Fatalf("capture failure: %v", outcome.cause) + } + if outcome.err == nil { + t.Fatal("expected CreateCommand.Run to fail before creating a sandbox") + } + if !strings.Contains(outcome.err.Error(), "requires expose.https") { + t.Fatalf("unexpected CreateCommand.Run error: %v", outcome.err) + } + if strings.TrimSpace(outcome.stdout) != "" { + t.Fatalf("expected no sandbox id on stdout, got %q", outcome.stdout) + } + if strings.Contains(outcome.stderr, "sandbox_id=") { + t.Fatalf("expected no sandbox id on stderr before create, got %q", outcome.stderr) + } +} + func TestSandboxCreateIntegrationDangerouslyAllowAllSetsAllowNetworkDefault(t *testing.T) { restore := stubPolicyUpdateResolver(t, func(_ context.Context, source string) (string, error) { if got, want := source, defaultBumpRefSource; got != want { diff --git a/internal/exposure/manager.go b/internal/exposure/manager.go index 7e1ced25..059b763d 100644 --- a/internal/exposure/manager.go +++ b/internal/exposure/manager.go @@ -28,6 +28,8 @@ const ( dnsTakeoverEvery = 100 * time.Millisecond ) +const maxHTTPSCertificateCacheEntries = 128 + type Dialer func(ctx context.Context, sandboxID string, port int) (net.Conn, error) type RegisterRequest struct { @@ -56,14 +58,17 @@ type Manager struct { tlsDir string logger *log.Logger - mu sync.RWMutex - byOwner map[string][]*route - tcpRoutes map[int]*route - httpsRoutes map[string]*route - tcpServers map[int]*tcpServer + mu sync.RWMutex + byOwner map[string][]*route + tcpRoutes map[int]*route + httpsRoutes map[string]*route + httpsPatternRoutes []*route + tcpServers map[int]*tcpServer - httpsServer *http.Server - httpsLn net.Listener + httpsServer *http.Server + httpsLn net.Listener + certCache map[string]*tls.Certificate + certCacheLimit int dnsServers []*dns.Server closeOnce sync.Once @@ -78,6 +83,7 @@ type route struct { hostPort int name string hostname string + wildcard bool url string dialer Dialer @@ -109,18 +115,20 @@ func NewManager(cfg Config) *Manager { httpsListen = DefaultHTTPSListen } return &Manager{ - domain: domain, - tcpHost: tcpHost, - dnsListen: dnsListen, - httpsListen: httpsListen, - fixedHTTPS: fixedHTTPS, - tlsDir: strings.TrimSpace(cfg.TLSDir), - logger: cfg.Logger, - byOwner: map[string][]*route{}, - tcpRoutes: map[int]*route{}, - httpsRoutes: map[string]*route{}, - tcpServers: map[int]*tcpServer{}, - closed: make(chan struct{}), + domain: domain, + tcpHost: tcpHost, + dnsListen: dnsListen, + httpsListen: httpsListen, + fixedHTTPS: fixedHTTPS, + tlsDir: strings.TrimSpace(cfg.TLSDir), + logger: cfg.Logger, + byOwner: map[string][]*route{}, + tcpRoutes: map[int]*route{}, + httpsRoutes: map[string]*route{}, + certCache: map[string]*tls.Certificate{}, + certCacheLimit: maxHTTPSCertificateCacheEntries, + tcpServers: map[int]*tcpServer{}, + closed: make(chan struct{}), } } @@ -169,6 +177,8 @@ func (m *Manager) ReleaseOwner(ownerID string) { } case "https": delete(m.httpsRoutes, r.hostname) + m.removeHTTPSPatternRouteLocked(r) + m.removeHTTPSRouteCertificatesLocked(r) closeHTTPSRouteIdleConnections(r) } } @@ -201,6 +211,7 @@ func (m *Manager) startDNS(ctx context.Context) error { } handler := dns.NewServeMux() handler.HandleFunc(dns.Fqdn(m.domain), m.handleDNS) + handler.HandleFunc(dns.Fqdn("localhost"), m.handleDNS) udp := &dns.Server{Addr: m.dnsListen, Net: "udp", Handler: handler} tcp := &dns.Server{Addr: m.dnsListen, Net: "tcp", Handler: handler} @@ -282,6 +293,8 @@ func (m *Manager) Close() error { } m.tcpRoutes = map[int]*route{} m.httpsRoutes = map[string]*route{} + m.httpsPatternRoutes = nil + m.certCache = map[string]*tls.Certificate{} m.byOwner = map[string][]*route{} if m.httpsServer != nil { if err := m.httpsServer.Close(); err != nil && !errors.Is(err, http.ErrServerClosed) { @@ -369,10 +382,10 @@ func (m *Manager) registerHTTPS(ctx context.Context, ownerID, sandboxID string, if name == "" { name = sandboxID } - if err := validateDNSLabel(name); err != nil { + host, wildcard, err := m.normalizeHTTPSRouteHost(name) + if err != nil { return nil, err } - host := name + "." + m.domain r := &route{ ownerID: ownerID, sandboxID: sandboxID, @@ -380,6 +393,7 @@ func (m *Manager) registerHTTPS(ctx context.Context, ownerID, sandboxID string, guestPort: guestPort, name: name, hostname: host, + wildcard: wildcard, dialer: dialer, } r.httpsProxy, r.httpsTransport = m.newHTTPSProxy(r) @@ -390,6 +404,9 @@ func (m *Manager) registerHTTPS(ctx context.Context, ownerID, sandboxID string, return nil, fmt.Errorf("https route %s is already exposed", host) } m.httpsRoutes[host] = r + if wildcard { + m.httpsPatternRoutes = append(m.httpsPatternRoutes, r) + } m.byOwner[ownerID] = append(m.byOwner[ownerID], r) m.mu.Unlock() @@ -421,6 +438,8 @@ func (m *Manager) releaseRoute(target *route) { case "https": if m.httpsRoutes[target.hostname] == target { delete(m.httpsRoutes, target.hostname) + m.removeHTTPSPatternRouteLocked(target) + m.removeHTTPSRouteCertificatesLocked(target) closeHTTPSRouteIdleConnections(target) } } @@ -485,10 +504,12 @@ func (m *Manager) startHTTPS(ctx context.Context) error { return nil } - cert, err := GenerateServerCertificate(m.domain, m.tlsDir) + defaultCertName := m.defaultHTTPSCertificateName() + cert, err := GenerateServerCertificate(defaultCertName, m.tlsDir) if err != nil { return err } + m.certCache[defaultCertName] = &cert var listenConfig net.ListenConfig ln, err := listenConfig.Listen(ctx, "tcp", m.httpsListen) if err != nil { @@ -503,8 +524,11 @@ func (m *Manager) startHTTPS(ctx context.Context) error { m.httpsListen = ln.Addr().String() m.httpsLn = ln m.httpsServer = &http.Server{ - Handler: http.HandlerFunc(m.handleHTTPS), - TLSConfig: &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12}, + Handler: http.HandlerFunc(m.handleHTTPS), + TLSConfig: &tls.Config{ + GetCertificate: m.getHTTPSCertificate, + MinVersion: tls.VersionTLS12, + }, } go func() { err := m.httpsServer.ServeTLS(ln, "", "") @@ -553,7 +577,7 @@ func (m *Manager) handleHTTPS(w http.ResponseWriter, req *http.Request) { host = normalizeDomain(req.TLS.ServerName) } m.mu.RLock() - r := m.httpsRoutes[host] + r := m.matchHTTPSRouteLocked(host) m.mu.RUnlock() if r == nil { http.NotFound(w, req) @@ -602,9 +626,273 @@ func (m *Manager) hasKnownDNSQuestion(msg *dns.Msg) bool { } func (m *Manager) handlesDNSName(name string) bool { + name = normalizeDomain(name) + if handlesLocalhostName(name) { + return true + } + if m.handlesManagedDomainName(name) { + return true + } + m.mu.RLock() + defer m.mu.RUnlock() + return m.matchHTTPSRouteLocked(name) != nil +} + +func (m *Manager) handlesManagedDomainName(name string) bool { return name == m.domain || strings.HasSuffix(name, "."+m.domain) } +func handlesLocalhostName(name string) bool { + return name == "localhost" || strings.HasSuffix(name, ".localhost") +} + +func (m *Manager) normalizeHTTPSRouteHost(name string) (string, bool, error) { + return normalizeHTTPSRouteHostForDomain(name, m.domain) +} + +// ValidateHTTPSRouteName reports whether name is valid for an HTTPS exposure route. +func ValidateHTTPSRouteName(name string) error { + _, _, err := normalizeHTTPSRouteHostForDomain(name, Domain) + return err +} + +func normalizeHTTPSRouteHostForDomain(name, domain string) (string, bool, error) { + name = normalizeDomain(name) + if name == "" { + return "", false, errors.New("missing https route name") + } + if !strings.Contains(name, ".") && !strings.Contains(name, "*") { + if err := validateDNSLabel(name); err != nil { + return "", false, fmt.Errorf("invalid https route name: %w", err) + } + return name + "." + domain, false, nil + } + host, wildcard, err := normalizeLocalhostRoutePattern(name) + if err != nil { + return "", false, fmt.Errorf("invalid https route host %q: %w", name, err) + } + return host, wildcard, nil +} + +func normalizeLocalhostRoutePattern(host string) (string, bool, error) { + host = normalizeDomain(host) + if host == "" { + return "", false, errors.New("missing host") + } + labels := strings.Split(host, ".") + if len(labels) < 2 || labels[len(labels)-1] != "localhost" { + return "", false, errors.New("host must be a subdomain of localhost") + } + if len(labels) == 2 && labels[0] == "*" { + return "", false, errors.New("wildcard host must include a concrete localhost subdomain") + } + + wildcard := false + seenConcrete := false + for i, label := range labels { + if label == "*" { + if seenConcrete { + return "", false, errors.New("wildcard labels must be leading labels") + } + wildcard = true + continue + } + seenConcrete = true + if err := validateDNSLabel(label); err != nil { + return "", false, fmt.Errorf("label %d: %w", i, err) + } + } + if !wildcard && labels[0] == "localhost" { + return "", false, errors.New("host must be a subdomain of localhost") + } + return host, wildcard, nil +} + +func (m *Manager) matchHTTPSRouteLocked(host string) *route { + host = normalizeDomain(host) + if host == "" { + return nil + } + if r := m.httpsRoutes[host]; r != nil && !r.wildcard { + return r + } + var best *route + bestLabels := -1 + bestConcreteLabels := -1 + for _, candidate := range m.httpsPatternRoutes { + if candidate == nil || !routePatternMatchesHost(candidate.hostname, host) { + continue + } + labelCount, concreteLabels := routePatternSpecificity(candidate.hostname) + if labelCount > bestLabels || (labelCount == bestLabels && concreteLabels > bestConcreteLabels) { + best = candidate + bestLabels = labelCount + bestConcreteLabels = concreteLabels + } + } + return best +} + +func routePatternSpecificity(pattern string) (int, int) { + pattern = normalizeDomain(pattern) + if pattern == "" { + return 0, 0 + } + labels := strings.Split(pattern, ".") + concreteLabels := 0 + for _, label := range labels { + if label != "*" { + concreteLabels++ + } + } + return len(labels), concreteLabels +} + +func routePatternMatchesHost(pattern, host string) bool { + pattern = normalizeDomain(pattern) + host = normalizeDomain(host) + if pattern == "" || host == "" { + return false + } + patternLabels := strings.Split(pattern, ".") + hostLabels := strings.Split(host, ".") + if len(patternLabels) != len(hostLabels) { + return false + } + for i, label := range patternLabels { + if label == "*" { + continue + } + if label != hostLabels[i] { + return false + } + } + return true +} + +func (m *Manager) removeHTTPSPatternRouteLocked(target *route) { + if target == nil || !target.wildcard { + return + } + for i, r := range m.httpsPatternRoutes { + if r == target { + m.httpsPatternRoutes = append(m.httpsPatternRoutes[:i], m.httpsPatternRoutes[i+1:]...) + return + } + } +} + +func (m *Manager) removeHTTPSRouteCertificatesLocked(target *route) { + if target == nil { + return + } + if !target.wildcard { + delete(m.certCache, target.hostname) + return + } + if certName, ok := routeWildcardCertificateName(target); ok { + delete(m.certCache, certName) + return + } + for name := range m.certCache { + if routePatternMatchesHost(target.hostname, name) { + delete(m.certCache, name) + } + } +} + +func (m *Manager) getHTTPSCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + name := "" + if hello != nil { + name = normalizeDomain(hello.ServerName) + } + if name == "" { + name = m.defaultHTTPSCertificateName() + } + certName, wildcardRoute, allowed := m.certificateNameFor(name) + if !allowed { + return nil, fmt.Errorf("unhandled https exposure hostname %q", name) + } + + m.mu.RLock() + if cert := m.certCache[certName]; cert != nil { + m.mu.RUnlock() + return cert, nil + } + cacheFull := wildcardRoute && len(m.certCache) >= m.certCacheLimit + m.mu.RUnlock() + if cacheFull { + return nil, fmt.Errorf("https exposure certificate cache is full") + } + + cert, err := GenerateServerCertificate(certName, m.tlsDir) + if err != nil { + return nil, err + } + m.mu.Lock() + defer m.mu.Unlock() + if existing := m.certCache[certName]; existing != nil { + return existing, nil + } + if wildcardRoute && len(m.certCache) >= m.certCacheLimit { + return nil, fmt.Errorf("https exposure certificate cache is full") + } + m.certCache[certName] = &cert + return &cert, nil +} + +func (m *Manager) defaultHTTPSCertificateName() string { + return "*." + m.domain +} + +func (m *Manager) certificateNameFor(name string) (string, bool, bool) { + name = normalizeDomain(name) + if name == "" { + return "", false, false + } + if name == m.domain { + return name, false, true + } + m.mu.RLock() + r := m.matchHTTPSRouteLocked(name) + m.mu.RUnlock() + if r != nil { + if certName, ok := routeWildcardCertificateName(r); ok { + return certName, false, true + } + return name, r.wildcard, true + } + if m.handlesManagedWildcardCertificateName(name) { + return "*." + m.domain, false, true + } + return "", false, false +} + +func routeWildcardCertificateName(r *route) (string, bool) { + if r == nil || !r.wildcard { + return "", false + } + labels := strings.Split(r.hostname, ".") + if len(labels) < 2 || labels[0] != "*" { + return "", false + } + for _, label := range labels[1:] { + if label == "*" { + return "", false + } + } + return r.hostname, true +} + +func (m *Manager) handlesManagedWildcardCertificateName(name string) bool { + suffix := "." + m.domain + if !strings.HasSuffix(name, suffix) { + return false + } + prefix := strings.TrimSuffix(name, suffix) + return prefix != "" && !strings.Contains(prefix, ".") +} + func (m *Manager) httpsURL(host string) string { _, port, err := net.SplitHostPort(m.httpsListen) if err != nil || port == "" || port == "443" { diff --git a/internal/exposure/manager_test.go b/internal/exposure/manager_test.go index 71ff523b..05e7b099 100644 --- a/internal/exposure/manager_test.go +++ b/internal/exposure/manager_test.go @@ -2,6 +2,8 @@ package exposure import ( "context" + "crypto/tls" + "crypto/x509" "fmt" "io" "net" @@ -410,6 +412,392 @@ func TestRegisterHTTPSReusesProxyTransport(t *testing.T) { } } +func TestRegisterHTTPSRoutesWildcardLocalhostPattern(t *testing.T) { + t.Parallel() + + seen := make(chan string, 1) + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + seen <- req.Host + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(backend.Close) + backendURL, err := url.Parse(backend.URL) + if err != nil { + t.Fatalf("parse backend URL: %v", err) + } + + manager := NewManager(Config{ + HTTPSListen: net.JoinHostPort("127.0.0.1", strconv.Itoa(freeTCPPort(t))), + TLSDir: t.TempDir(), + }) + t.Cleanup(func() { + if err := manager.Close(); err != nil { + t.Fatalf("Close returned error: %v", err) + } + }) + + exposed, err := manager.Register(context.Background(), RegisterRequest{ + OwnerID: "owner-1", + SandboxID: "sandbox-1", + Exposure: &cleanroomv1.PortExposure{ + Protocol: "https", + Name: "*.sandbox-1.localhost", + GuestPort: 3000, + }, + Dialer: func(ctx context.Context, _ string, _ int) (net.Conn, error) { + var dialer net.Dialer + return dialer.DialContext(ctx, "tcp", backendURL.Host) + }, + }) + if err != nil { + t.Fatalf("Register returned error: %v", err) + } + if got, want := exposed.GetHostname(), "*.sandbox-1.localhost"; got != want { + t.Fatalf("unexpected hostname: got %q want %q", got, want) + } + + req := httptest.NewRequest(http.MethodGet, "https://api.sandbox-1.localhost/", nil) + req.Host = "api.sandbox-1.localhost" + rr := httptest.NewRecorder() + manager.handleHTTPS(rr, req) + if got, want := rr.Code, http.StatusNoContent; got != want { + t.Fatalf("unexpected status: got %d want %d", got, want) + } + select { + case got := <-seen: + if got != "api.sandbox-1.localhost" { + t.Fatalf("unexpected backend host: %q", got) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for backend request") + } + + req = httptest.NewRequest(http.MethodGet, "https://deep.api.sandbox-1.localhost/", nil) + req.Host = "deep.api.sandbox-1.localhost" + rr = httptest.NewRecorder() + manager.handleHTTPS(rr, req) + if got, want := rr.Code, http.StatusNotFound; got != want { + t.Fatalf("unexpected deep wildcard status: got %d want %d", got, want) + } +} + +func TestHTTPSWildcardRouteSelectionPrefersMostSpecificPattern(t *testing.T) { + t.Parallel() + + manager := NewManager(Config{ + HTTPSListen: net.JoinHostPort("127.0.0.1", strconv.Itoa(freeTCPPort(t))), + TLSDir: t.TempDir(), + }) + t.Cleanup(func() { + if err := manager.Close(); err != nil { + t.Fatalf("Close returned error: %v", err) + } + }) + + for _, tc := range []struct { + ownerID string + sandboxID string + name string + guestPort int32 + }{ + {ownerID: "owner-1", sandboxID: "sandbox-1", name: "*.*.sandbox-1.localhost", guestPort: 3000}, + {ownerID: "owner-2", sandboxID: "sandbox-2", name: "*.api.sandbox-1.localhost", guestPort: 4000}, + } { + if _, err := manager.Register(context.Background(), RegisterRequest{ + OwnerID: tc.ownerID, + SandboxID: tc.sandboxID, + Exposure: &cleanroomv1.PortExposure{ + Protocol: "https", + Name: tc.name, + GuestPort: tc.guestPort, + }, + Dialer: testDialer, + }); err != nil { + t.Fatalf("Register(%q) returned error: %v", tc.name, err) + } + } + + manager.mu.RLock() + route := manager.matchHTTPSRouteLocked("foo.api.sandbox-1.localhost") + manager.mu.RUnlock() + if route == nil { + t.Fatal("expected wildcard route match") + } + if got, want := route.hostname, "*.api.sandbox-1.localhost"; got != want { + t.Fatalf("unexpected route host: got %q want %q", got, want) + } + if got, want := route.guestPort, 4000; got != want { + t.Fatalf("unexpected route guest port: got %d want %d", got, want) + } +} + +func TestHTTPSCertificateGenerationUsesBoundedManagedWildcardFallback(t *testing.T) { + t.Parallel() + + manager := NewManager(Config{ + HTTPSListen: net.JoinHostPort("127.0.0.1", strconv.Itoa(freeTCPPort(t))), + TLSDir: t.TempDir(), + }) + t.Cleanup(func() { + if err := manager.Close(); err != nil { + t.Fatalf("Close returned error: %v", err) + } + }) + + _, err := manager.Register(context.Background(), RegisterRequest{ + OwnerID: "owner-1", + SandboxID: "sandbox-1", + Exposure: &cleanroomv1.PortExposure{ + Protocol: "https", + Name: "buildkite", + GuestPort: 3000, + }, + Dialer: testDialer, + }) + if err != nil { + t.Fatalf("Register returned error: %v", err) + } + manager.mu.RLock() + cacheEntries := len(manager.certCache) + manager.mu.RUnlock() + + cert, err := manager.getHTTPSCertificate(&tls.ClientHelloInfo{ServerName: "missing.cleanroom.localhost"}) + if err != nil { + t.Fatalf("expected one-label managed hostname to use wildcard fallback certificate: %v", err) + } + if cert.Leaf == nil { + t.Fatal("expected fallback certificate to include parsed leaf") + } + if err := cert.Leaf.VerifyHostname("missing.cleanroom.localhost"); err != nil { + t.Fatalf("expected fallback certificate to verify missing hostname: %v", err) + } + manager.mu.RLock() + if got, want := len(manager.certCache), cacheEntries; got != want { + manager.mu.RUnlock() + t.Fatalf("expected one wildcard fallback certificate cache entry: got %d want %d", got, want) + } + if manager.certCache["*.cleanroom.localhost"] == nil { + manager.mu.RUnlock() + t.Fatal("expected wildcard fallback certificate to be cached by pattern") + } + manager.mu.RUnlock() + + _, err = manager.getHTTPSCertificate(&tls.ClientHelloInfo{ServerName: "deep.missing.cleanroom.localhost"}) + if err == nil { + t.Fatal("expected unregistered deep managed hostname to be rejected") + } + if !strings.Contains(err.Error(), "unhandled") { + t.Fatalf("unexpected certificate error: %v", err) + } + manager.mu.RLock() + defer manager.mu.RUnlock() + if got, want := len(manager.certCache), cacheEntries; got != want { + t.Fatalf("expected rejected deep hostname not to populate certificate cache: got %d want %d", got, want) + } +} + +func TestHTTPSDefaultCertificateCoversManagedWildcardWithoutSNI(t *testing.T) { + t.Parallel() + + tlsDir := t.TempDir() + manager := NewManager(Config{ + HTTPSListen: net.JoinHostPort("127.0.0.1", strconv.Itoa(freeTCPPort(t))), + TLSDir: tlsDir, + }) + t.Cleanup(func() { + if err := manager.Close(); err != nil { + t.Fatalf("Close returned error: %v", err) + } + }) + _, err := manager.Register(context.Background(), RegisterRequest{ + OwnerID: "owner-1", + SandboxID: "sandbox-1", + Exposure: &cleanroomv1.PortExposure{ + Protocol: "https", + Name: "buildkite", + GuestPort: 3000, + }, + Dialer: testDialer, + }) + if err != nil { + t.Fatalf("Register returned error: %v", err) + } + ca, err := EnsureRuntimeCertificateAuthority(Domain, tlsDir) + if err != nil { + t.Fatalf("load local certificate authority: %v", err) + } + roots := x509.NewCertPool() + roots.AddCert(ca.Cert) + + conn, err := tls.Dial("tcp", manager.httpsListen, &tls.Config{ + MinVersion: tls.VersionTLS12, + InsecureSkipVerify: true, + VerifyConnection: func(state tls.ConnectionState) error { + if len(state.PeerCertificates) == 0 { + return fmt.Errorf("missing peer certificate") + } + _, err := state.PeerCertificates[0].Verify(x509.VerifyOptions{ + DNSName: "buildkite.cleanroom.localhost", + Roots: roots, + }) + return err + }, + }) + if err != nil { + t.Fatalf("expected no-SNI connection to receive managed wildcard certificate: %v", err) + } + _ = conn.Close() +} + +func TestHTTPSCertificateCacheLimitBoundsWildcardRouteCertificates(t *testing.T) { + t.Parallel() + + manager := NewManager(Config{ + HTTPSListen: net.JoinHostPort("127.0.0.1", strconv.Itoa(freeTCPPort(t))), + TLSDir: t.TempDir(), + }) + manager.certCacheLimit = 3 + t.Cleanup(func() { + if err := manager.Close(); err != nil { + t.Fatalf("Close returned error: %v", err) + } + }) + + _, err := manager.Register(context.Background(), RegisterRequest{ + OwnerID: "owner-1", + SandboxID: "sandbox-1", + Exposure: &cleanroomv1.PortExposure{ + Protocol: "https", + Name: "*.*.sandbox-1.localhost", + GuestPort: 3000, + }, + Dialer: testDialer, + }) + if err != nil { + t.Fatalf("Register returned error: %v", err) + } + for _, name := range []string{"a.b.sandbox-1.localhost", "c.d.sandbox-1.localhost"} { + if _, err := manager.getHTTPSCertificate(&tls.ClientHelloInfo{ServerName: name}); err != nil { + t.Fatalf("getHTTPSCertificate(%q) returned error: %v", name, err) + } + } + _, err = manager.getHTTPSCertificate(&tls.ClientHelloInfo{ServerName: "e.f.sandbox-1.localhost"}) + if err == nil { + t.Fatal("expected certificate cache limit error") + } + if !strings.Contains(err.Error(), "cache is full") { + t.Fatalf("unexpected certificate error: %v", err) + } + + _, err = manager.Register(context.Background(), RegisterRequest{ + OwnerID: "owner-2", + SandboxID: "sandbox-2", + Exposure: &cleanroomv1.PortExposure{ + Protocol: "https", + Name: "buildkite", + GuestPort: 3000, + }, + Dialer: testDialer, + }) + if err != nil { + t.Fatalf("Register exact route returned error: %v", err) + } + if _, err := manager.getHTTPSCertificate(&tls.ClientHelloInfo{ServerName: "buildkite.cleanroom.localhost"}); err != nil { + t.Fatalf("expected exact route certificate to bypass wildcard cache limit, got %v", err) + } +} + +func TestReleaseOwnerEvictsWildcardRouteCertificates(t *testing.T) { + t.Parallel() + + manager := NewManager(Config{ + HTTPSListen: net.JoinHostPort("127.0.0.1", strconv.Itoa(freeTCPPort(t))), + TLSDir: t.TempDir(), + }) + manager.certCacheLimit = 3 + t.Cleanup(func() { + if err := manager.Close(); err != nil { + t.Fatalf("Close returned error: %v", err) + } + }) + + _, err := manager.Register(context.Background(), RegisterRequest{ + OwnerID: "owner-1", + SandboxID: "sandbox-1", + Exposure: &cleanroomv1.PortExposure{ + Protocol: "https", + Name: "*.*.sandbox-1.localhost", + GuestPort: 3000, + }, + Dialer: testDialer, + }) + if err != nil { + t.Fatalf("Register returned error: %v", err) + } + for _, name := range []string{"a.b.sandbox-1.localhost", "c.d.sandbox-1.localhost"} { + if _, err := manager.getHTTPSCertificate(&tls.ClientHelloInfo{ServerName: name}); err != nil { + t.Fatalf("getHTTPSCertificate(%q) returned error: %v", name, err) + } + } + + manager.ReleaseOwner("owner-1") + manager.mu.RLock() + for _, name := range []string{"a.b.sandbox-1.localhost", "c.d.sandbox-1.localhost"} { + if manager.certCache[name] != nil { + manager.mu.RUnlock() + t.Fatalf("expected released wildcard certificate %q to be evicted", name) + } + } + manager.mu.RUnlock() + + _, err = manager.Register(context.Background(), RegisterRequest{ + OwnerID: "owner-2", + SandboxID: "sandbox-2", + Exposure: &cleanroomv1.PortExposure{ + Protocol: "https", + Name: "*.*.sandbox-2.localhost", + GuestPort: 3000, + }, + Dialer: testDialer, + }) + if err != nil { + t.Fatalf("Register second wildcard route returned error: %v", err) + } + for _, name := range []string{"a.b.sandbox-2.localhost", "c.d.sandbox-2.localhost"} { + if _, err := manager.getHTTPSCertificate(&tls.ClientHelloInfo{ServerName: name}); err != nil { + t.Fatalf("expected released certificates not to consume cache capacity, got %v", err) + } + } +} + +func TestRegisterHTTPSRejectsExternalRouteHost(t *testing.T) { + t.Parallel() + + manager := NewManager(Config{TLSDir: t.TempDir()}) + t.Cleanup(func() { + if err := manager.Close(); err != nil { + t.Fatalf("Close returned error: %v", err) + } + }) + + _, err := manager.Register(context.Background(), RegisterRequest{ + OwnerID: "owner-1", + SandboxID: "sandbox-1", + Exposure: &cleanroomv1.PortExposure{ + Protocol: "https", + Name: "google.com", + GuestPort: 3000, + }, + Dialer: testDialer, + }) + if err == nil { + t.Fatal("expected external host to be rejected") + } + if !strings.Contains(err.Error(), "localhost") { + t.Fatalf("unexpected error: %v", err) + } +} + func TestHTTPSProxyForwardsTrustedHeadersAndPreservesHost(t *testing.T) { t.Parallel() @@ -562,6 +950,75 @@ func TestDNSReturnsLoopbackForWildcardNames(t *testing.T) { } } +func TestDNSReturnsLoopbackForLocalhostNamesOutsideRoutes(t *testing.T) { + t.Parallel() + + manager := NewManager(Config{}) + for _, name := range []string{"localhost.", "tool.localhost."} { + t.Run(name, func(t *testing.T) { + msg := new(dns.Msg) + msg.SetQuestion(name, dns.TypeA) + w := &captureDNSResponseWriter{} + manager.handleDNS(w, msg) + + if w.msg == nil { + t.Fatal("expected DNS response") + } + if got, want := w.msg.Rcode, dns.RcodeSuccess; got != want { + t.Fatalf("unexpected DNS rcode: got %d want %d", got, want) + } + if len(w.msg.Answer) != 1 { + t.Fatalf("expected one A answer, got %v", w.msg.Answer) + } + a, ok := w.msg.Answer[0].(*dns.A) + if !ok { + t.Fatalf("expected A answer, got %T", w.msg.Answer[0]) + } + if !a.A.Equal(net.ParseIP("127.0.0.1")) { + t.Fatalf("unexpected A answer: got %v", a.A) + } + }) + } +} + +func TestDNSReturnsLoopbackForConfiguredLocalhostRoute(t *testing.T) { + t.Parallel() + + manager := NewManager(Config{TLSDir: t.TempDir()}) + t.Cleanup(func() { + if err := manager.Close(); err != nil { + t.Fatalf("Close returned error: %v", err) + } + }) + if _, err := manager.Register(context.Background(), RegisterRequest{ + OwnerID: "owner-1", + SandboxID: "sandbox-1", + Exposure: &cleanroomv1.PortExposure{ + Protocol: "https", + Name: "*.sandbox-1.localhost", + GuestPort: 3000, + }, + Dialer: testDialer, + }); err != nil { + t.Fatalf("Register returned error: %v", err) + } + + msg := new(dns.Msg) + msg.SetQuestion("api.sandbox-1.localhost.", dns.TypeA) + w := &captureDNSResponseWriter{} + manager.handleDNS(w, msg) + + if w.msg == nil { + t.Fatal("expected DNS response") + } + if got, want := w.msg.Rcode, dns.RcodeSuccess; got != want { + t.Fatalf("unexpected DNS rcode: got %d want %d", got, want) + } + if len(w.msg.Answer) != 1 { + t.Fatalf("expected one A answer, got %v", w.msg.Answer) + } +} + func TestStartDNSReusesExistingWildcardDNS(t *testing.T) { t.Parallel() diff --git a/internal/exposure/tls.go b/internal/exposure/tls.go index bcabac12..4686ccde 100644 --- a/internal/exposure/tls.go +++ b/internal/exposure/tls.go @@ -23,6 +23,8 @@ const ( LocalCertificateKeyFilename = "exposure-cert.key" ) +var ErrLocalCertificateRequiresInstall = errors.New("exposure certificate must be refreshed with cleanroom dns install") + type LocalCertificate struct { Cert *x509.Certificate Key *rsa.PrivateKey @@ -38,7 +40,7 @@ func DefaultTLSDir() (string, error) { func EnsureLocalCertificate(domain, dir string) (*LocalCertificate, error) { domain = normalizeCertificateDomain(domain) if domain == "" { - return nil, errors.New("missing exposure certificate domain") + return nil, errors.New("missing exposure certificate authority domain") } dir = strings.TrimSpace(dir) if dir == "" { @@ -59,7 +61,7 @@ func EnsureLocalCertificate(domain, dir string) (*LocalCertificate, error) { if err != nil { return nil, err } - if localCertificateMatchesDomain(cert.Cert, domain) { + if localCertificateIsUsableCA(cert.Cert) && localCertificateKeyMatches(cert.Cert, cert.Key) { cert.CertPath = certPath cert.KeyPath = keyPath return cert, nil @@ -72,7 +74,7 @@ func EnsureLocalCertificate(domain, dir string) (*LocalCertificate, error) { if err := os.MkdirAll(dir, 0o700); err != nil { return nil, fmt.Errorf("create exposure TLS directory %s: %w", dir, err) } - cert, err := generateLocalCertificate(domain) + cert, err := generateLocalCertificateAuthority(domain) if err != nil { return nil, err } @@ -108,19 +110,94 @@ func RemoveLocalCertificateFiles(dir string) error { } func GenerateServerCertificate(domain, dir string) (tls.Certificate, error) { - cert, err := EnsureLocalCertificate(domain, dir) + domain = normalizeCertificateDomain(domain) + if domain == "" { + return tls.Certificate{}, errors.New("missing exposure certificate domain") + } + ca, err := EnsureRuntimeCertificateAuthority(domain, dir) if err != nil { return tls.Certificate{}, err } - keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(cert.Key)}) - pair, err := tls.X509KeyPair(cert.CertPEM, keyPEM) + key, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { - return tls.Certificate{}, fmt.Errorf("load exposure certificate: %w", err) + return tls.Certificate{}, fmt.Errorf("generate exposure server certificate key: %w", err) } - pair.Leaf = cert.Cert + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return tls.Certificate{}, fmt.Errorf("generate exposure server certificate serial: %w", err) + } + now := time.Now() + tpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + CommonName: domain, + }, + NotBefore: now.Add(-time.Hour), + NotAfter: now.AddDate(0, 1, 0), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + DNSNames: []string{domain}, + } + certDER, err := x509.CreateCertificate(rand.Reader, tpl, ca.Cert, &key.PublicKey, ca.Key) + if err != nil { + return tls.Certificate{}, fmt.Errorf("create exposure server certificate: %w", err) + } + leaf, err := x509.ParseCertificate(certDER) + if err != nil { + return tls.Certificate{}, fmt.Errorf("parse generated exposure server certificate: %w", err) + } + certPEM := append( + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}), + ca.CertPEM..., + ) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) + pair, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return tls.Certificate{}, fmt.Errorf("load exposure server certificate: %w", err) + } + pair.Leaf = leaf return pair, nil } +// EnsureRuntimeCertificateAuthority loads the local CA used to mint HTTPS exposure leaves. +func EnsureRuntimeCertificateAuthority(domain, dir string) (*LocalCertificate, error) { + domain = normalizeCertificateDomain(domain) + if domain == "" { + return nil, errors.New("missing exposure certificate authority domain") + } + dir = strings.TrimSpace(dir) + if dir == "" { + var err error + dir, err = DefaultTLSDir() + if err != nil { + return nil, err + } + } + certPath := filepath.Join(dir, LocalCertificateFilename) + keyPath := filepath.Join(dir, LocalCertificateKeyFilename) + + certPEM, certErr := os.ReadFile(certPath) + keyPEM, keyErr := os.ReadFile(keyPath) + switch { + case certErr == nil && keyErr == nil: + cert, err := parseLocalCertificate(certPEM, keyPEM) + if err != nil { + return nil, err + } + if localCertificateIsUsableCA(cert.Cert) && localCertificateKeyMatches(cert.Cert, cert.Key) { + cert.CertPath = certPath + cert.KeyPath = keyPath + return cert, nil + } + return nil, fmt.Errorf("%w: run sudo cleanroom dns install to refresh %s", ErrLocalCertificateRequiresInstall, certPath) + case errors.Is(certErr, os.ErrNotExist) && errors.Is(keyErr, os.ErrNotExist): + return EnsureLocalCertificate(domain, dir) + default: + return nil, fmt.Errorf("load exposure certificate from %s: cert error=%v key error=%v", dir, certErr, keyErr) + } +} + func parseLocalCertificate(certPEM, keyPEM []byte) (*LocalCertificate, error) { certBlock, _ := pem.Decode(certPEM) if certBlock == nil || certBlock.Type != "CERTIFICATE" { @@ -141,35 +218,35 @@ func parseLocalCertificate(certPEM, keyPEM []byte) (*LocalCertificate, error) { return &LocalCertificate{Cert: cert, Key: key, CertPEM: certPEM}, nil } -func generateLocalCertificate(domain string) (*LocalCertificate, error) { +func generateLocalCertificateAuthority(domain string) (*LocalCertificate, error) { key, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { - return nil, fmt.Errorf("generate exposure certificate key: %w", err) + return nil, fmt.Errorf("generate exposure certificate authority key: %w", err) } serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) if err != nil { - return nil, fmt.Errorf("generate exposure certificate serial: %w", err) + return nil, fmt.Errorf("generate exposure certificate authority serial: %w", err) } now := time.Now() tpl := &x509.Certificate{ SerialNumber: serial, Subject: pkix.Name{ - CommonName: domain, + CommonName: "Cleanroom Local Exposure CA (" + domain + ")", }, NotBefore: now.Add(-time.Hour), - NotAfter: now.AddDate(1, 0, 0), - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + NotAfter: now.AddDate(5, 0, 0), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, BasicConstraintsValid: true, - DNSNames: []string{domain, "*." + domain}, + IsCA: true, + MaxPathLenZero: true, } certDER, err := x509.CreateCertificate(rand.Reader, tpl, tpl, &key.PublicKey, key) if err != nil { - return nil, fmt.Errorf("create exposure certificate: %w", err) + return nil, fmt.Errorf("create exposure certificate authority: %w", err) } cert, err := x509.ParseCertificate(certDER) if err != nil { - return nil, fmt.Errorf("parse generated exposure certificate: %w", err) + return nil, fmt.Errorf("parse generated exposure certificate authority: %w", err) } return &LocalCertificate{ Cert: cert, @@ -182,12 +259,23 @@ func normalizeCertificateDomain(domain string) string { return strings.TrimSuffix(strings.ToLower(strings.TrimSpace(domain)), ".") } -func localCertificateMatchesDomain(cert *x509.Certificate, domain string) bool { - if cert == nil || cert.IsCA { +func localCertificateIsUsableCA(cert *x509.Certificate) bool { + if cert == nil || !cert.IsCA { return false } if time.Until(cert.NotAfter) < 24*time.Hour { return false } - return cert.VerifyHostname(domain) == nil && cert.VerifyHostname("buildkite."+domain) == nil + return cert.KeyUsage&x509.KeyUsageCertSign != 0 +} + +func localCertificateKeyMatches(cert *x509.Certificate, key *rsa.PrivateKey) bool { + if cert == nil || key == nil { + return false + } + pub, ok := cert.PublicKey.(*rsa.PublicKey) + if !ok { + return false + } + return pub.E == key.PublicKey.E && pub.N.Cmp(key.PublicKey.N) == 0 } diff --git a/internal/exposure/tls_test.go b/internal/exposure/tls_test.go index 67f08325..3e3e48e6 100644 --- a/internal/exposure/tls_test.go +++ b/internal/exposure/tls_test.go @@ -1,14 +1,21 @@ package exposure import ( + "bytes" + "crypto/rand" + "crypto/rsa" "crypto/x509" + "crypto/x509/pkix" "encoding/pem" + "errors" + "math/big" "os" "path/filepath" "testing" + "time" ) -func TestEnsureLocalCertificateCreatesReusableLeafCertificate(t *testing.T) { +func TestEnsureLocalCertificateCreatesReusableCertificateAuthority(t *testing.T) { t.Parallel() dir := t.TempDir() @@ -23,22 +30,15 @@ func TestEnsureLocalCertificateCreatesReusableLeafCertificate(t *testing.T) { if !first.Cert.Equal(second.Cert) { t.Fatal("expected second call to reuse generated certificate") } - if first.Cert.IsCA { - t.Fatal("expected exposure certificate to be a leaf certificate, not a CA") - } - if err := first.Cert.VerifyHostname("buildkite." + Domain); err != nil { - t.Fatalf("expected certificate to verify buildkite hostname: %v", err) - } - if err := first.Cert.VerifyHostname("example.com"); err == nil { - t.Fatal("expected certificate not to verify arbitrary hostnames") + if !first.Cert.IsCA { + t.Fatal("expected exposure certificate to be a CA") } roots := x509.NewCertPool() roots.AddCert(first.Cert) if _, err := first.Cert.Verify(x509.VerifyOptions{ - DNSName: "buildkite." + Domain, - Roots: roots, + Roots: roots, }); err != nil { - t.Fatalf("expected certificate to verify as a directly trusted leaf: %v", err) + t.Fatalf("expected certificate authority to verify as a trusted root: %v", err) } if info, err := os.Stat(filepath.Join(dir, LocalCertificateKeyFilename)); err != nil { t.Fatalf("stat certificate key: %v", err) @@ -47,7 +47,7 @@ func TestEnsureLocalCertificateCreatesReusableLeafCertificate(t *testing.T) { } } -func TestGenerateServerCertificateUsesTrustedLeafCertificate(t *testing.T) { +func TestGenerateServerCertificateUsesLocalCertificateAuthority(t *testing.T) { t.Parallel() dir := t.TempDir() @@ -55,7 +55,7 @@ func TestGenerateServerCertificateUsesTrustedLeafCertificate(t *testing.T) { if err != nil { t.Fatalf("EnsureLocalCertificate returned error: %v", err) } - cert, err := GenerateServerCertificate(Domain, dir) + cert, err := GenerateServerCertificate("buildkite."+Domain, dir) if err != nil { t.Fatalf("GenerateServerCertificate returned error: %v", err) } @@ -63,16 +63,110 @@ func TestGenerateServerCertificateUsesTrustedLeafCertificate(t *testing.T) { if err != nil { t.Fatalf("parse leaf certificate: %v", err) } - if !leaf.Equal(local.Cert) { - t.Fatal("expected server certificate to use the locally trusted leaf certificate") + if leaf.Equal(local.Cert) { + t.Fatal("expected server certificate to use a dynamically generated leaf") } if leaf.IsCA { t.Fatal("expected server certificate not to be a CA") } + if len(cert.Certificate) < 2 { + t.Fatalf("expected server certificate chain to include the local CA, got %d certificates", len(cert.Certificate)) + } + issuer, err := x509.ParseCertificate(cert.Certificate[1]) + if err != nil { + t.Fatalf("parse issuer certificate: %v", err) + } + if !issuer.Equal(local.Cert) { + t.Fatal("expected server certificate chain to include the local CA") + } if block, _ := pem.Decode(local.CertPEM); block == nil || block.Type != "CERTIFICATE" { t.Fatal("expected local certificate PEM to contain a certificate") } if err := leaf.VerifyHostname("buildkite." + Domain); err != nil { t.Fatalf("expected leaf to verify buildkite hostname: %v", err) } + roots := x509.NewCertPool() + roots.AddCert(local.Cert) + if _, err := leaf.Verify(x509.VerifyOptions{ + DNSName: "buildkite." + Domain, + Roots: roots, + }); err != nil { + t.Fatalf("expected server certificate to verify against local CA: %v", err) + } +} + +func TestGenerateServerCertificateDoesNotReplaceLegacyLeafCertificate(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + writeLegacyLeafCertificate(t, dir, Domain) + certPath := filepath.Join(dir, LocalCertificateFilename) + before, err := os.ReadFile(certPath) + if err != nil { + t.Fatalf("read legacy certificate: %v", err) + } + + _, err = GenerateServerCertificate("buildkite."+Domain, dir) + if !errors.Is(err, ErrLocalCertificateRequiresInstall) { + t.Fatalf("expected install-required error, got %v", err) + } + after, err := os.ReadFile(certPath) + if err != nil { + t.Fatalf("read certificate after GenerateServerCertificate: %v", err) + } + if !bytes.Equal(after, before) { + t.Fatal("expected runtime server certificate generation not to replace the trusted legacy certificate") + } + + migrated, err := EnsureLocalCertificate(Domain, dir) + if err != nil { + t.Fatalf("EnsureLocalCertificate returned error: %v", err) + } + if !migrated.Cert.IsCA { + t.Fatal("expected dns install path to replace the legacy leaf with a CA") + } + if bytes.Equal(migrated.CertPEM, before) { + t.Fatal("expected dns install path to write a new certificate authority") + } +} + +func writeLegacyLeafCertificate(t *testing.T, dir, domain string) { + t.Helper() + + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("generate legacy key: %v", err) + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + t.Fatalf("generate legacy serial: %v", err) + } + now := time.Now() + tpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + CommonName: domain, + }, + NotBefore: now.Add(-time.Hour), + NotAfter: now.AddDate(1, 0, 0), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + DNSNames: []string{domain, "*." + domain}, + } + certDER, err := x509.CreateCertificate(rand.Reader, tpl, tpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("create legacy certificate: %v", err) + } + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatalf("create legacy certificate dir: %v", err) + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + if err := os.WriteFile(filepath.Join(dir, LocalCertificateFilename), certPEM, 0o644); err != nil { + t.Fatalf("write legacy certificate: %v", err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) + if err := os.WriteFile(filepath.Join(dir, LocalCertificateKeyFilename), keyPEM, 0o600); err != nil { + t.Fatalf("write legacy key: %v", err) + } } diff --git a/internal/policy/policy.go b/internal/policy/policy.go index 54abaf5f..7cbfc11a 100644 --- a/internal/policy/policy.go +++ b/internal/policy/policy.go @@ -35,6 +35,7 @@ type Loader struct{} type rawPolicy struct { Version int `yaml:"version"` Repository *rawRepository `yaml:"repository"` + Expose rawExpose `yaml:"expose"` Sandbox struct { Image struct { Ref string `yaml:"ref"` @@ -57,6 +58,20 @@ type rawRepository struct { Network *rawStageNetworkConfig `yaml:"network"` } +type rawExpose struct { + HTTPS rawExposeHTTPS `yaml:"https"` +} + +type rawExposeHTTPS struct { + Base string `yaml:"base"` + Routes []rawExposeHTTPSRoute `yaml:"routes"` +} + +type rawExposeHTTPSRoute struct { + Port int `yaml:"port"` + Hosts []string `yaml:"hosts"` +} + type rawDependencyCommandSpec []string type rawRunConfig struct { @@ -142,6 +157,28 @@ type RepositoryConfig struct { Submodules bool `json:"submodules"` } +type ExposeConfig struct { + HTTPS ExposeHTTPSConfig `json:"https,omitempty"` +} + +func (c ExposeConfig) IsZero() bool { + return c.HTTPS.IsZero() +} + +type ExposeHTTPSConfig struct { + Base string `json:"base,omitempty"` + Routes []ExposeHTTPSRoute `json:"routes,omitempty"` +} + +func (c ExposeHTTPSConfig) IsZero() bool { + return strings.TrimSpace(c.Base) == "" && len(c.Routes) == 0 +} + +type ExposeHTTPSRoute struct { + Port int `json:"port"` + Hosts []string `json:"hosts"` +} + type Services struct { Blocks []StageBlock `json:"blocks,omitempty"` Command []string `json:"-"` @@ -293,6 +330,49 @@ func normalizeRawNetworkStages(raw rawPolicy) (*NetworkStagePolicies, error) { return &out, nil } +func normalizeExposeConfig(raw rawExpose) (ExposeConfig, error) { + https, err := normalizeExposeHTTPSConfig(raw.HTTPS) + if err != nil { + return ExposeConfig{}, err + } + return ExposeConfig{HTTPS: https}, nil +} + +func normalizeExposeHTTPSConfig(raw rawExposeHTTPS) (ExposeHTTPSConfig, error) { + base := strings.TrimSpace(strings.ToLower(raw.Base)) + if base == "" && len(raw.Routes) == 0 { + return ExposeHTTPSConfig{}, nil + } + if len(raw.Routes) == 0 { + return ExposeHTTPSConfig{}, errors.New("expose.https.routes must include at least one route") + } + routes := make([]ExposeHTTPSRoute, 0, len(raw.Routes)) + for i, route := range raw.Routes { + field := fmt.Sprintf("expose.https.routes[%d]", i) + if route.Port < 1 || route.Port > 65535 { + return ExposeHTTPSConfig{}, fmt.Errorf("%s.port must be in range 1-65535", field) + } + if len(route.Hosts) == 0 { + return ExposeHTTPSConfig{}, fmt.Errorf("%s.hosts must include at least one host", field) + } + hosts := make([]string, 0, len(route.Hosts)) + seen := map[string]struct{}{} + for j, host := range route.Hosts { + host = strings.TrimSpace(strings.ToLower(host)) + if host == "" { + return ExposeHTTPSConfig{}, fmt.Errorf("%s.hosts[%d] cannot be empty", field, j) + } + if _, ok := seen[host]; ok { + continue + } + seen[host] = struct{}{} + hosts = append(hosts, host) + } + routes = append(routes, ExposeHTTPSRoute{Port: route.Port, Hosts: hosts}) + } + return ExposeHTTPSConfig{Base: base, Routes: routes}, nil +} + func (l Loader) LoadAndCompile(root string) (*CompiledPolicy, string, error) { raw, source, err := l.Load(root) if err != nil { @@ -320,6 +400,19 @@ func (l Loader) LoadRepository(root string) (RepositoryConfig, string, error) { return cfg, source, nil } +func (l Loader) LoadExpose(root string) (ExposeConfig, string, error) { + raw, source, err := l.Load(root) + if err != nil { + return ExposeConfig{}, "", err + } + + cfg, err := normalizeExposeConfig(raw.Expose) + if err != nil { + return ExposeConfig{}, source, err + } + return cfg, source, nil +} + func (l Loader) Load(root string) (rawPolicy, string, error) { primary := filepath.Join(root, PrimaryPolicyPath) fallback := filepath.Join(root, FallbackPolicyPath) @@ -386,6 +479,9 @@ func Compile(raw rawPolicy) (*CompiledPolicy, error) { if err != nil { return nil, err } + if _, err := normalizeExposeConfig(raw.Expose); err != nil { + return nil, err + } if err := validateRepositoryScopedBlocks(raw, repository); err != nil { return nil, err } diff --git a/internal/policy/policy_test.go b/internal/policy/policy_test.go index b3de9a74..61c3e95b 100644 --- a/internal/policy/policy_test.go +++ b/internal/policy/policy_test.go @@ -1446,6 +1446,81 @@ sandbox: } } +func TestLoadExposeNormalizesConfiguredHTTPS(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, PrimaryPolicyPath), []byte(` +version: 1 +expose: + https: + base: "{sandbox_id}.LOCALHOST" + routes: + - port: 3000 + hosts: + - "{base}" + - "*.{base}" + - "*.{base}" +sandbox: + image: + ref: ghcr.io/buildkite/cleanroom-base/alpine@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + network: + default: deny +`), 0o644); err != nil { + t.Fatalf("write policy: %v", err) + } + + cfg, source, err := Loader{}.LoadExpose(dir) + if err != nil { + t.Fatalf("LoadExpose returned error: %v", err) + } + if got, want := source, filepath.Join(dir, PrimaryPolicyPath); got != want { + t.Fatalf("unexpected source: got %q want %q", got, want) + } + if got, want := cfg.HTTPS.Base, "{sandbox_id}.localhost"; got != want { + t.Fatalf("unexpected https base: got %q want %q", got, want) + } + if got, want := len(cfg.HTTPS.Routes), 1; got != want { + t.Fatalf("unexpected route count: got %d want %d", got, want) + } + if got, want := cfg.HTTPS.Routes[0].Port, 3000; got != want { + t.Fatalf("unexpected route port: got %d want %d", got, want) + } + if got, want := cfg.HTTPS.Routes[0].Hosts, []string{"{base}", "*.{base}"}; strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("unexpected route hosts: got %v want %v", got, want) + } +} + +func TestCompileValidatesExposeWithoutChangingHash(t *testing.T) { + t.Parallel() + + raw := baseRawPolicy() + baseline, err := Compile(raw) + if err != nil { + t.Fatalf("Compile baseline returned error: %v", err) + } + + raw.Expose.HTTPS = rawExposeHTTPS{ + Base: "{sandbox_id}.localhost", + Routes: []rawExposeHTTPSRoute{{ + Port: 3000, + Hosts: []string{"{base}", "*.{base}", "*.*.{base}"}, + }}, + } + withExpose, err := Compile(raw) + if err != nil { + t.Fatalf("Compile with expose returned error: %v", err) + } + if got, want := withExpose.Hash, baseline.Hash; got != want { + t.Fatalf("expected expose config to stay out of policy hash: got %q want %q", got, want) + } + + raw.Expose.HTTPS.Routes[0].Port = 0 + if _, err := Compile(raw); err == nil { + t.Fatal("expected Compile to validate expose config") + } +} + func TestFromProtoRejectsMismatchedImageDigest(t *testing.T) { t.Parallel()