diff --git a/agent/agent_configuration.go b/agent/agent_configuration.go index 143e139676..ec196a1a5a 100644 --- a/agent/agent_configuration.go +++ b/agent/agent_configuration.go @@ -24,6 +24,7 @@ type AgentConfiguration struct { GitCloneMirrorFlags string GitCleanFlags string GitFetchFlags string + GitSparseCheckoutPaths []string GitSubmodules bool GitSubmoduleCloneConfig []string SkipCheckout bool diff --git a/agent/job_runner.go b/agent/job_runner.go index 9f717d11ac..f95aa09c5a 100644 --- a/agent/job_runner.go +++ b/agent/job_runner.go @@ -597,6 +597,10 @@ BUILDKITE_AGENT_JWKS_KEY_ID` } setEnv("BUILDKITE_LOCAL_HOOKS_ENABLED", fmt.Sprint(r.conf.AgentConfiguration.LocalHooksEnabled)) + if len(r.conf.AgentConfiguration.GitSparseCheckoutPaths) > 0 { + setCheckoutEnv("BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS", strings.Join(r.conf.AgentConfiguration.GitSparseCheckoutPaths, ",")) + } + setEnv("BUILDKITE_GIT_CLONE_MIRROR_FLAGS", r.conf.AgentConfiguration.GitCloneMirrorFlags) setCheckoutEnv("BUILDKITE_GIT_CHECKOUT_FLAGS", r.conf.AgentConfiguration.GitCheckoutFlags) setCheckoutEnv("BUILDKITE_GIT_CLONE_FLAGS", r.conf.AgentConfiguration.GitCloneFlags) setCheckoutEnv("BUILDKITE_GIT_FETCH_FLAGS", r.conf.AgentConfiguration.GitFetchFlags) diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go index 515c357edb..cc80ad7c87 100644 --- a/clicommand/agent_start.go +++ b/clicommand/agent_start.go @@ -162,6 +162,7 @@ type AgentStartConfig struct { GitCloneMirrorFlags string `cli:"git-clone-mirror-flags"` GitCleanFlags string `cli:"git-clean-flags"` GitFetchFlags string `cli:"git-fetch-flags"` + GitSparseCheckoutPaths []string `cli:"git-sparse-checkout-paths" normalize:"list"` GitMirrorsPath string `cli:"git-mirrors-path" normalize:"filepath"` GitMirrorCheckoutMode string `cli:"git-mirror-checkout-mode"` GitMirrorsLockTimeout int `cli:"git-mirrors-lock-timeout"` @@ -541,6 +542,7 @@ var AgentStartCommand = cli.Command{ GitCloneFlagsFlag, GitCleanFlagsFlag, GitFetchFlagsFlag, + GitSparseCheckoutPathsFlag, GitCloneMirrorFlagsFlag, GitMirrorsPathFlag, GitMirrorCheckoutModeFlag, @@ -1076,6 +1078,7 @@ var AgentStartCommand = cli.Command{ GitCloneMirrorFlags: cfg.GitCloneMirrorFlags, GitCleanFlags: cfg.GitCleanFlags, GitFetchFlags: cfg.GitFetchFlags, + GitSparseCheckoutPaths: cfg.GitSparseCheckoutPaths, GitSubmodules: !cfg.NoGitSubmodules, GitSubmoduleCloneConfig: cfg.GitSubmoduleCloneConfig, SkipCheckout: cfg.SkipCheckout, diff --git a/clicommand/bootstrap.go b/clicommand/bootstrap.go index 0ca91f6a55..e9064b6022 100644 --- a/clicommand/bootstrap.go +++ b/clicommand/bootstrap.go @@ -74,6 +74,7 @@ type BootstrapConfig struct { GitCheckoutFlags string `cli:"git-checkout-flags"` GitCloneFlags string `cli:"git-clone-flags"` GitFetchFlags string `cli:"git-fetch-flags"` + GitSparseCheckoutPaths []string `cli:"git-sparse-checkout-paths" normalize:"list"` GitCloneMirrorFlags string `cli:"git-clone-mirror-flags"` GitCleanFlags string `cli:"git-clean-flags"` GitMirrorsPath string `cli:"git-mirrors-path" normalize:"filepath"` @@ -249,6 +250,7 @@ var BootstrapCommand = cli.Command{ GitCloneMirrorFlagsFlag, GitCleanFlagsFlag, GitFetchFlagsFlag, + GitSparseCheckoutPathsFlag, GitMirrorsPathFlag, GitMirrorCheckoutModeFlag, GitMirrorsLockTimeoutFlag, @@ -451,6 +453,7 @@ var BootstrapCommand = cli.Command{ GitCloneFlags: cfg.GitCloneFlags, GitCloneMirrorFlags: cfg.GitCloneMirrorFlags, GitFetchFlags: cfg.GitFetchFlags, + GitSparseCheckoutPaths: cfg.GitSparseCheckoutPaths, GitMirrorsLockTimeout: cfg.GitMirrorsLockTimeout, GitMirrorsPath: cfg.GitMirrorsPath, GitMirrorCheckoutMode: cfg.GitMirrorCheckoutMode, diff --git a/clicommand/global.go b/clicommand/global.go index 99f6d1b556..b25c16353c 100644 --- a/clicommand/global.go +++ b/clicommand/global.go @@ -242,6 +242,13 @@ var ( EnvVar: "BUILDKITE_GIT_FETCH_FLAGS", } + GitSparseCheckoutPathsFlag = cli.StringSliceFlag{ + Name: "git-sparse-checkout-paths", + Value: &cli.StringSlice{}, + Usage: "Comma-separated list of paths for git sparse checkout (cone mode). When set, only the listed paths are materialized in the working tree.", + EnvVar: "BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS", + } + GitMirrorsPathFlag = cli.StringFlag{ Name: "git-mirrors-path", Value: "", diff --git a/internal/job/checkout.go b/internal/job/checkout.go index af93dbf8ce..31b3040f08 100644 --- a/internal/job/checkout.go +++ b/internal/job/checkout.go @@ -1,6 +1,7 @@ package job import ( + "bytes" "context" "errors" "fmt" @@ -343,6 +344,117 @@ func hasGitSubmodules(sh *shell.Shell) bool { return osutil.FileExists(filepath.Join(sh.Getwd(), ".gitmodules")) } +func cleanGitSparseCheckoutPaths(paths []string) []string { + cleaned := make([]string, 0, len(paths)) + for _, path := range paths { + path = strings.TrimSpace(path) + if path != "" { + cleaned = append(cleaned, path) + } + } + return cleaned +} + +func parseGitVersion(output string) (major, minor int, ok bool) { + if _, err := fmt.Sscanf(output, "git version %d.%d", &major, &minor); err != nil { + return 0, 0, false + } + return major, minor, true +} + +func gitVersionAtLeast(ctx context.Context, sh *shell.Shell, major, minor int) (bool, error) { + output, err := sh.Command("git", "--version").RunAndCaptureStdout(ctx) + if err != nil { + return false, err + } + + gitMajor, gitMinor, ok := parseGitVersion(strings.TrimSpace(output)) + if !ok { + return false, fmt.Errorf("parsing git version from %q", strings.TrimSpace(output)) + } + + if gitMajor != major { + return gitMajor > major, nil + } + return gitMinor >= minor, nil +} + +// sparseCheckoutMayBeConfigured does a cheap filesystem check for marker files +// that indicate sparse checkout (or the worktree-config extension that +// `sparse-checkout` enables) might already be in effect, so we can avoid +// shelling out to `git config` on every checkout. It resolves the .git dir +// directly to handle the worktree/submodule case where .git is a file +// containing `gitdir: `. +func sparseCheckoutMayBeConfigured(sh *shell.Shell) bool { + gitDir := filepath.Join(sh.Getwd(), ".git") + if data, err := os.ReadFile(gitDir); err == nil && bytes.HasPrefix(data, []byte("gitdir:")) { + gitDirValue := strings.TrimSpace(string(bytes.TrimPrefix(data, []byte("gitdir:")))) + if !filepath.IsAbs(gitDirValue) { + gitDirValue = filepath.Join(sh.Getwd(), gitDirValue) + } + gitDir = gitDirValue + } + + return osutil.FileExists(filepath.Join(gitDir, "info", "sparse-checkout")) || + osutil.FileExists(filepath.Join(gitDir, "config.worktree")) +} + +func (e *Executor) disableSparseCheckoutIfConfigured(ctx context.Context) { + if !sparseCheckoutMayBeConfigured(e.shell) { + return + } + + sparseOutput, err := e.shell.Command("git", "config", "--get", "core.sparseCheckout").RunAndCaptureStdout(ctx, shell.ShowStderr(false)) + if err != nil || strings.TrimSpace(sparseOutput) != "true" { + return + } + + e.shell.Commentf("Disabling sparse checkout from previous build") + if err := e.shell.Command("git", "sparse-checkout", "disable").Run(ctx); err != nil { + e.shell.Warningf("Failed to disable sparse checkout: %v", err) + } + + // `sparse-checkout disable` leaves extensions.worktreeConfig set, which + // can cause problems for subsequent git operations. Only unset it if no + // other worktree-scoped config remains, to avoid clobbering user config. + worktreeConfig, err := e.shell.Command("git", "config", "--worktree", "--list").RunAndCaptureStdout(ctx, shell.ShowStderr(false)) + if err == nil && strings.TrimSpace(worktreeConfig) == "" { + _ = e.shell.Command("git", "config", "--unset", "extensions.worktreeConfig").Run(ctx) + } +} + +// setupSparseCheckout configures (or disables) git sparse checkout for the +// current working tree. It returns true if sparse checkout was successfully +// applied for this build, so callers can adjust later behaviour (e.g. skip +// submodule init, which requires the full tree). +func (e *Executor) setupSparseCheckout(ctx context.Context) (bool, error) { + paths := cleanGitSparseCheckoutPaths(e.GitSparseCheckoutPaths) + if len(paths) == 0 { + e.disableSparseCheckoutIfConfigured(ctx) + return false, nil + } + + ok, err := gitVersionAtLeast(ctx, e.shell, 2, 26) + if err != nil { + e.shell.Warningf("Sparse checkout requires git >= 2.26; falling back to full checkout (%v)", err) + e.disableSparseCheckoutIfConfigured(ctx) + return false, nil + } + if !ok { + e.shell.Warningf("Sparse checkout requires git >= 2.26; falling back to full checkout") + e.disableSparseCheckoutIfConfigured(ctx) + return false, nil + } + + e.shell.Commentf("Setting up sparse checkout for paths: %s", strings.Join(paths, ",")) + args := append([]string{"sparse-checkout", "set", "--cone"}, paths...) + if err := e.shell.Command("git", args...).Run(ctx); err != nil { + return false, fmt.Errorf("setting sparse checkout paths: %w", err) + } + + return true, nil +} + func hasGitCommit(ctx context.Context, sh *shell.Shell, gitDir, commit string) bool { // Resolve commit to an actual commit object output, err := sh.Command("git", "--git-dir", gitDir, "rev-parse", commit+"^{commit}").RunAndCaptureStdout(ctx, shell.ShowStderr(false)) @@ -918,6 +1030,11 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) error { return err } + sparseCheckoutActive, err := e.setupSparseCheckout(ctx) + if err != nil { + return err + } + gitCheckoutFlags := e.GitCheckoutFlags if e.Commit == "HEAD" { @@ -932,10 +1049,13 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) error { gitSubmodules := false if hasGitSubmodules(e.shell) { - if e.GitSubmodules { + switch { + case sparseCheckoutActive: + e.shell.Commentf("Submodule initialization skipped during sparse checkout") + case e.GitSubmodules: e.shell.Commentf("Git submodules detected") gitSubmodules = true - } else { + default: e.shell.OptionalWarningf("submodules-disabled", "This repository has submodules, but submodules are disabled") } } diff --git a/internal/job/checkout_test.go b/internal/job/checkout_test.go index a23167d737..16108b4396 100644 --- a/internal/job/checkout_test.go +++ b/internal/job/checkout_test.go @@ -1,14 +1,18 @@ package job import ( + "bytes" "context" "os" + "path/filepath" + "strings" "testing" "time" "github.com/buildkite/agent/v3/internal/job/githttptest" "github.com/buildkite/agent/v3/internal/race" "github.com/buildkite/agent/v3/internal/shell" + "github.com/buildkite/bintest/v3" ) func TestDefaultCheckoutPhase(t *testing.T) { @@ -177,6 +181,204 @@ func TestSkipCheckout(t *testing.T) { } } +func TestCleanGitSparseCheckoutPaths(t *testing.T) { + t.Parallel() + + got := cleanGitSparseCheckoutPaths([]string{" .buildkite/ ", "src/", "", " docs "}) + want := []string{".buildkite/", "src/", "docs"} + if len(got) != len(want) { + t.Fatalf("cleanGitSparseCheckoutPaths() = %#v, want %#v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("cleanGitSparseCheckoutPaths() = %#v, want %#v", got, want) + } + } +} + +func TestParseGitVersion(t *testing.T) { + t.Parallel() + + tests := []struct { + output string + wantMajor int + wantMinor int + wantOK bool + }{ + {output: "git version 2.39.5", wantMajor: 2, wantMinor: 39, wantOK: true}, + {output: "git version 2.26.0.windows.1", wantMajor: 2, wantMinor: 26, wantOK: true}, + {output: "not git", wantOK: false}, + } + + for _, tt := range tests { + t.Run(tt.output, func(t *testing.T) { + t.Parallel() + gotMajor, gotMinor, gotOK := parseGitVersion(tt.output) + if gotMajor != tt.wantMajor || gotMinor != tt.wantMinor || gotOK != tt.wantOK { + t.Fatalf("parseGitVersion(%q) = (%d, %d, %t), want (%d, %d, %t)", tt.output, gotMajor, gotMinor, gotOK, tt.wantMajor, tt.wantMinor, tt.wantOK) + } + }) + } +} + +func TestSetupSparseCheckout_Enable(t *testing.T) { + executor, git, out := newSparseCheckoutTestExecutor(t) + defer git.Close() //nolint:errcheck // Best-effort cleanup. + executor.GitSparseCheckoutPaths = []string{".buildkite/", "src/"} + + git.Expect("--version").AndWriteToStdout("git version 2.39.0").AndExitWith(0) + git.Expect("sparse-checkout", "set", "--cone", ".buildkite/", "src/").AndExitWith(0) + + active, err := executor.setupSparseCheckout(t.Context()) + if err != nil { + t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err) + } + if !active { + t.Fatalf("executor.setupSparseCheckout(ctx) active = false, want true") + } + if got, want := out.String(), "Setting up sparse checkout for paths: .buildkite/,src/"; !strings.Contains(got, want) { + t.Fatalf("shell output = %q, want to contain %q", got, want) + } + + git.Check(t) +} + +func TestSetupSparseCheckout_DisableWithPriorSparseConfig(t *testing.T) { + executor, git, out := newSparseCheckoutTestExecutor(t) + defer git.Close() //nolint:errcheck // Best-effort cleanup. + createSparseCheckoutFile(t, executor.shell.Getwd()) + + git.Expect("config", "--get", "core.sparseCheckout").AndWriteToStdout("true\n").AndExitWith(0) + git.Expect("sparse-checkout", "disable").AndExitWith(0) + git.Expect("config", "--worktree", "--list").AndWriteToStdout("").AndExitWith(0) + git.Expect("config", "--unset", "extensions.worktreeConfig").AndExitWith(0) + + active, err := executor.setupSparseCheckout(t.Context()) + if err != nil { + t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err) + } + if active { + t.Fatalf("executor.setupSparseCheckout(ctx) active = true, want false") + } + if got, want := out.String(), "Disabling sparse checkout from previous build"; !strings.Contains(got, want) { + t.Fatalf("shell output = %q, want to contain %q", got, want) + } + + git.Check(t) +} + +func TestSetupSparseCheckout_DisablePreservesOtherWorktreeConfig(t *testing.T) { + executor, git, _ := newSparseCheckoutTestExecutor(t) + defer git.Close() //nolint:errcheck // Best-effort cleanup. + createSparseCheckoutFile(t, executor.shell.Getwd()) + + git.Expect("config", "--get", "core.sparseCheckout").AndWriteToStdout("true\n").AndExitWith(0) + git.Expect("sparse-checkout", "disable").AndExitWith(0) + git.Expect("config", "--worktree", "--list").AndWriteToStdout("user.something=value\n").AndExitWith(0) + git.Expect("config", "--unset", "extensions.worktreeConfig").NotCalled() + + if _, err := executor.setupSparseCheckout(t.Context()); err != nil { + t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err) + } + + git.Check(t) +} + +func TestSetupSparseCheckout_DisableWithoutPriorSparseConfig(t *testing.T) { + executor, git, _ := newSparseCheckoutTestExecutor(t) + defer git.Close() //nolint:errcheck // Best-effort cleanup. + + git.Expect("config").WithAnyArguments().NotCalled() + git.Expect("sparse-checkout").WithAnyArguments().NotCalled() + + if _, err := executor.setupSparseCheckout(t.Context()); err != nil { + t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err) + } + + git.Check(t) +} + +func TestSetupSparseCheckout_VersionFallback(t *testing.T) { + executor, git, out := newSparseCheckoutTestExecutor(t) + defer git.Close() //nolint:errcheck // Best-effort cleanup. + executor.GitSparseCheckoutPaths = []string{"src/"} + + git.Expect("--version").AndWriteToStdout("git version 2.25.4").AndExitWith(0) + git.Expect("sparse-checkout").WithAnyArguments().NotCalled() + + active, err := executor.setupSparseCheckout(t.Context()) + if err != nil { + t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err) + } + if active { + t.Fatalf("executor.setupSparseCheckout(ctx) active = true, want false") + } + if got, want := out.String(), "Sparse checkout requires git >= 2.26; falling back to full checkout"; !strings.Contains(got, want) { + t.Fatalf("shell output = %q, want to contain %q", got, want) + } + + git.Check(t) +} + +func TestSetupSparseCheckout_VersionFallbackDisablesPriorSparseConfig(t *testing.T) { + executor, git, out := newSparseCheckoutTestExecutor(t) + defer git.Close() //nolint:errcheck // Best-effort cleanup. + executor.GitSparseCheckoutPaths = []string{"src/"} + createSparseCheckoutFile(t, executor.shell.Getwd()) + + git.Expect("--version").AndWriteToStdout("git version 2.25.4").AndExitWith(0) + git.Expect("config", "--get", "core.sparseCheckout").AndWriteToStdout("true\n").AndExitWith(0) + git.Expect("sparse-checkout", "disable").AndExitWith(0) + git.Expect("config", "--worktree", "--list").AndWriteToStdout("").AndExitWith(0) + git.Expect("config", "--unset", "extensions.worktreeConfig").AndExitWith(0) + + if _, err := executor.setupSparseCheckout(t.Context()); err != nil { + t.Fatalf("executor.setupSparseCheckout(ctx) error = %v, want nil", err) + } + if got, want := out.String(), "Sparse checkout requires git >= 2.26; falling back to full checkout"; !strings.Contains(got, want) { + t.Fatalf("shell output = %q, want to contain %q", got, want) + } + if got, want := out.String(), "Disabling sparse checkout from previous build"; !strings.Contains(got, want) { + t.Fatalf("shell output = %q, want to contain %q", got, want) + } + + git.Check(t) +} + +func newSparseCheckoutTestExecutor(t *testing.T) (*Executor, *bintest.Mock, *bytes.Buffer) { + t.Helper() + + pathDir := t.TempDir() + git, err := bintest.NewMock(filepath.Join(pathDir, "git")) + if err != nil { + t.Fatalf("bintest.NewMock(git) error = %v, want nil", err) + } + + t.Setenv("PATH", pathDir) + + out := new(bytes.Buffer) + sh := shell.NewTestShell(t, + shell.WithLogger(shell.NewWriterLogger(out, false, nil)), + shell.WithStdout(out), + shell.WithWD(t.TempDir()), + ) + sh.Env.Set("PATH", pathDir) + + return &Executor{shell: sh}, git, out +} + +func createSparseCheckoutFile(t *testing.T, checkoutDir string) { + t.Helper() + + sparseCheckoutPath := filepath.Join(checkoutDir, ".git", "info", "sparse-checkout") + if err := os.MkdirAll(filepath.Dir(sparseCheckoutPath), 0o755); err != nil { + t.Fatalf("os.MkdirAll(%q) error = %v, want nil", filepath.Dir(sparseCheckoutPath), err) + } + if err := os.WriteFile(sparseCheckoutPath, []byte("/*\n"), 0o600); err != nil { + t.Fatalf("os.WriteFile(%q) error = %v, want nil", sparseCheckoutPath, err) + } +} + func TestDefaultCheckoutPhase_DelayedRefCreation(t *testing.T) { if race.IsRaceTest { t.Skip("this test simulates the agent recovering from a race condition, and needs to create one to test it.") diff --git a/internal/job/config.go b/internal/job/config.go index f1073fd91f..c65667994c 100644 --- a/internal/job/config.go +++ b/internal/job/config.go @@ -80,6 +80,9 @@ type ExecutorConfig struct { // Skip the checkout phase entirely SkipCheckout bool `env:"BUILDKITE_SKIP_CHECKOUT"` + // Comma-separated list of paths for git sparse checkout (cone mode). + GitSparseCheckoutPaths []string `env:"BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS"` + // Skip git fetch if the commit already exists locally GitSkipFetchExistingCommits bool `env:"BUILDKITE_GIT_SKIP_FETCH_EXISTING_COMMITS"` diff --git a/internal/job/config_test.go b/internal/job/config_test.go index 4a148ce249..46cb4f5ea4 100644 --- a/internal/job/config_test.go +++ b/internal/job/config_test.go @@ -1,6 +1,7 @@ package job import ( + "strings" "testing" "github.com/buildkite/agent/v3/env" @@ -14,6 +15,7 @@ func TestEnvVarsAreMappedToConfig(t *testing.T) { Repository: "https://original.host/repo.git", AutomaticArtifactUploadPaths: "llamas/", GitCloneFlags: "--prune", + GitSparseCheckoutPaths: []string{"old-path/"}, GitCleanFlags: "-v", AgentName: "myAgent", CleanCheckout: false, @@ -24,6 +26,7 @@ func TestEnvVarsAreMappedToConfig(t *testing.T) { environ := env.FromSlice([]string{ "BUILDKITE_ARTIFACT_PATHS=newpath", "BUILDKITE_GIT_CLONE_FLAGS=-f", + "BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS=.buildkite/,src/", "BUILDKITE_SOMETHING_ELSE=1", "BUILDKITE_REPO=https://my.mirror/repo.git", "BUILDKITE_CLEAN_CHECKOUT=true", @@ -35,6 +38,7 @@ func TestEnvVarsAreMappedToConfig(t *testing.T) { wantChanges := map[string]string{ "BUILDKITE_ARTIFACT_PATHS": "newpath", "BUILDKITE_GIT_CLONE_FLAGS": "-f", + "BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS": ".buildkite/,src/", "BUILDKITE_REPO": "https://my.mirror/repo.git", "BUILDKITE_CLEAN_CHECKOUT": "true", "BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH": "true", @@ -64,6 +68,10 @@ func TestEnvVarsAreMappedToConfig(t *testing.T) { if got, want := config.GitSubmodules, true; got != want { t.Errorf("config.GitSubmodules = %t, want %t", got, want) } + + if got, want := strings.Join(config.GitSparseCheckoutPaths, ","), ".buildkite/,src/"; got != want { + t.Errorf("config.GitSparseCheckoutPaths = %q, want %q", got, want) + } } func TestReadFromEnvironmentIgnoresMalformedBooleans(t *testing.T) { diff --git a/internal/job/integration/checkout_git_mirrors_integration_test.go b/internal/job/integration/checkout_git_mirrors_integration_test.go index cf992be131..e1c2d80c8e 100644 --- a/internal/job/integration/checkout_git_mirrors_integration_test.go +++ b/internal/job/integration/checkout_git_mirrors_integration_test.go @@ -143,6 +143,55 @@ func TestCheckingOutLocalGitProject_WithGitMirrors(t *testing.T) { tester.RunAndCheck(t, env...) } +func TestCheckingOutLocalGitProjectWithSparseCheckout_WithGitMirrors(t *testing.T) { + t.Parallel() + skipIfGitSparseCheckoutUnsupported(t) + + tester, err := NewExecutorTester(mainCtx) + if err != nil { + t.Fatalf("NewExecutorTester() error = %v", err) + } + defer tester.Close() + addSparseCheckoutFixture(t, tester.Repo) + + if err := tester.EnableGitMirrors(); err != nil { + t.Fatalf("EnableGitMirrors() error = %v", err) + } + + env := []string{ + "BUILDKITE_GIT_CLONE_FLAGS=-v --filter=blob:none --sparse", + "BUILDKITE_GIT_CLONE_MIRROR_FLAGS=--bare", + "BUILDKITE_GIT_CLEAN_FLAGS=-fdq", + "BUILDKITE_GIT_FETCH_FLAGS=-v --filter=blob:none", + "BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS=.buildkite/,src/", + } + + git := tester. + MustMock(t, "git"). + PassthroughToLocalCommand() + + git.ExpectAll([][]any{ + {"clone", "--mirror", "--bare", "--", tester.Repo.Path, matchSubDir(tester.GitMirrorsDir)}, + {"clone", "-v", "--filter=blob:none", "--sparse", "--reference", matchSubDir(tester.GitMirrorsDir), "--", tester.Repo.Path, "."}, + {"clean", "-fdq"}, + {"fetch", "-v", "--filter=blob:none", "--", "origin", "main"}, + {"--version"}, + {"sparse-checkout", "set", "--cone", ".buildkite/", "src/"}, + {"-c", "advice.detachedHead=false", "checkout", "-f", "FETCH_HEAD"}, + {"clean", "-fdq"}, + {"--no-pager", "log", "-1", "HEAD", "-s", "--no-color", gitShowFormatArg}, + }) + + agent := tester.MockAgent(t) + agent.Expect("meta-data", "exists", job.CommitMetadataKey).AndExitWith(1) + agent.Expect("meta-data", "set", job.CommitMetadataKey).WithStdin(commitPattern) + + tester.RunAndCheck(t, env...) + requireCheckoutPath(t, tester.CheckoutDir(), ".buildkite/pipeline.yml", true) + requireCheckoutPath(t, tester.CheckoutDir(), "src/main.txt", true) + requireCheckoutPath(t, tester.CheckoutDir(), "docs/readme.md", false) +} + func TestCheckingOutLocalGitProjectWithSubmodules_WithGitMirrors(t *testing.T) { t.Parallel() diff --git a/internal/job/integration/checkout_integration_test.go b/internal/job/integration/checkout_integration_test.go index d12cd844ab..070cfe05e9 100644 --- a/internal/job/integration/checkout_integration_test.go +++ b/internal/job/integration/checkout_integration_test.go @@ -29,6 +29,60 @@ var commitPattern = bintest.MatchPattern(`(?ms)\Acommit [0-9a-f]+\nabbrev-commit // We expect this arg multiple times, just define it once. const gitShowFormatArg = "--format=commit %H%nabbrev-commit %h%nAuthor: %an <%ae>%n%n%w(0,4,4)%B" +func skipIfGitSparseCheckoutUnsupported(t *testing.T) { + t.Helper() + + out, err := exec.Command("git", "--version").Output() + if err != nil { + t.Skipf("git --version failed: %v", err) + } + var major, minor int + if _, err := fmt.Sscanf(string(out), "git version %d.%d", &major, &minor); err != nil { + t.Skipf("couldn't parse git version from %q: %v", strings.TrimSpace(string(out)), err) + } + if major < 2 || (major == 2 && minor < 26) { + t.Skipf("git sparse-checkout --cone requires git >= 2.26, got %s", strings.TrimSpace(string(out))) + } +} + +func addSparseCheckoutFixture(t *testing.T, repo *gitRepository) { + t.Helper() + + files := map[string]string{ + ".buildkite/pipeline.yml": "steps:\n - command: true\n", + "src/main.txt": "hello from src\n", + "docs/readme.md": "hello from docs\n", + } + for name, contents := range files { + path := filepath.Join(repo.Path, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("os.MkdirAll(%q) error = %v, want nil", filepath.Dir(path), err) + } + if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatalf("os.WriteFile(%q) error = %v, want nil", path, err) + } + if err := repo.Add(name); err != nil { + t.Fatalf("repo.Add(%q) error = %v, want nil", name, err) + } + } + if err := repo.Commit("Add sparse checkout fixture"); err != nil { + t.Fatalf("repo.Commit(Add sparse checkout fixture) error = %v, want nil", err) + } +} + +func requireCheckoutPath(t *testing.T, checkoutDir, name string, exists bool) { + t.Helper() + + path := filepath.Join(checkoutDir, filepath.FromSlash(name)) + _, err := os.Stat(path) + switch { + case exists && err != nil: + t.Fatalf("os.Stat(%q) error = %v, want nil", path, err) + case !exists && !errors.Is(err, os.ErrNotExist): + t.Fatalf("os.Stat(%q) error = %v, want not exist", path, err) + } +} + func TestWithResolvingCommitExperiment(t *testing.T) { t.Parallel() @@ -103,6 +157,109 @@ func TestCheckingOutLocalGitProject(t *testing.T) { tester.RunAndCheck(t, env...) } +func TestCheckingOutLocalGitProjectWithSparseCheckout(t *testing.T) { + t.Parallel() + skipIfGitSparseCheckoutUnsupported(t) + + tester, err := NewExecutorTester(mainCtx) + if err != nil { + t.Fatalf("NewExecutorTester() error = %v", err) + } + defer tester.Close() + addSparseCheckoutFixture(t, tester.Repo) + + env := []string{ + "BUILDKITE_GIT_CLONE_FLAGS=-v --filter=blob:none --sparse", + "BUILDKITE_GIT_CLEAN_FLAGS=-fdq", + "BUILDKITE_GIT_FETCH_FLAGS=-v --filter=blob:none", + "BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS=.buildkite/,src/", + } + + git := tester. + MustMock(t, "git"). + PassthroughToLocalCommand() + + git.ExpectAll([][]any{ + {"clone", "-v", "--filter=blob:none", "--sparse", "--", tester.Repo.Path, "."}, + {"clean", "-fdq"}, + {"fetch", "-v", "--filter=blob:none", "--", "origin", "main"}, + {"--version"}, + {"sparse-checkout", "set", "--cone", ".buildkite/", "src/"}, + {"-c", "advice.detachedHead=false", "checkout", "-f", "FETCH_HEAD"}, + {"clean", "-fdq"}, + {"--no-pager", "log", "-1", "HEAD", "-s", "--no-color", gitShowFormatArg}, + }) + + agent := tester.MockAgent(t) + agent.Expect("meta-data", "exists", job.CommitMetadataKey).AndExitWith(1) + agent.Expect("meta-data", "set", job.CommitMetadataKey).WithStdin(commitPattern) + + tester.RunAndCheck(t, env...) + + requireCheckoutPath(t, tester.CheckoutDir(), ".buildkite/pipeline.yml", true) + requireCheckoutPath(t, tester.CheckoutDir(), "src/main.txt", true) + requireCheckoutPath(t, tester.CheckoutDir(), "docs/readme.md", false) +} + +func TestCheckingOutLocalGitProjectWithSparseCheckoutExistingGitDir(t *testing.T) { + t.Parallel() + skipIfGitSparseCheckoutUnsupported(t) + + tester, err := NewExecutorTester(mainCtx) + if err != nil { + t.Fatalf("NewExecutorTester() error = %v", err) + } + defer tester.Close() + addSparseCheckoutFixture(t, tester.Repo) + + env := []string{ + "BUILDKITE_GIT_CLONE_FLAGS=-v --filter=blob:none --sparse", + "BUILDKITE_GIT_CLEAN_FLAGS=-fdq", + "BUILDKITE_GIT_FETCH_FLAGS=-v --filter=blob:none", + "BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS=src/", + } + + git := tester. + MustMock(t, "git"). + PassthroughToLocalCommand() + agent := tester.MockAgent(t) + + agent.Expect("meta-data", "exists", job.CommitMetadataKey).AndExitWith(1) + agent.Expect("meta-data", "set", job.CommitMetadataKey).WithStdin(commitPattern) + git.ExpectAll([][]any{ + {"clone", "-v", "--filter=blob:none", "--sparse", "--", tester.Repo.Path, "."}, + {"clean", "-fdq"}, + {"fetch", "-v", "--filter=blob:none", "--", "origin", "main"}, + {"--version"}, + {"sparse-checkout", "set", "--cone", "src/"}, + {"-c", "advice.detachedHead=false", "checkout", "-f", "FETCH_HEAD"}, + {"clean", "-fdq"}, + {"--no-pager", "log", "-1", "HEAD", "-s", "--no-color", gitShowFormatArg}, + }) + + tester.RunAndCheck(t, env...) + requireCheckoutPath(t, tester.CheckoutDir(), "src/main.txt", true) + requireCheckoutPath(t, tester.CheckoutDir(), ".buildkite/pipeline.yml", false) + requireCheckoutPath(t, tester.CheckoutDir(), "docs/readme.md", false) + + env[len(env)-1] = "BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS=.buildkite/" + agent.Expect("meta-data", "exists", job.CommitMetadataKey).AndExitWith(0) + git.ExpectAll([][]any{ + {"config", "--get-all", "remote.origin.url"}, + {"clean", "-fdq"}, + {"fetch", "-v", "--filter=blob:none", "--", "origin", "main"}, + {"--version"}, + {"sparse-checkout", "set", "--cone", ".buildkite/"}, + {"-c", "advice.detachedHead=false", "checkout", "-f", "FETCH_HEAD"}, + {"clean", "-fdq"}, + }) + + tester.RunAndCheck(t, env...) + requireCheckoutPath(t, tester.CheckoutDir(), ".buildkite/pipeline.yml", true) + requireCheckoutPath(t, tester.CheckoutDir(), "src/main.txt", false) + requireCheckoutPath(t, tester.CheckoutDir(), "docs/readme.md", false) +} + func TestCheckingOutLocalGitProjectWithSubmodules(t *testing.T) { t.Parallel() @@ -173,6 +330,71 @@ func TestCheckingOutLocalGitProjectWithSubmodules(t *testing.T) { tester.RunAndCheck(t, env...) } +func TestCheckingOutLocalGitProjectWithSparseCheckoutSkipsSubmodules(t *testing.T) { + t.Parallel() + skipIfGitSparseCheckoutUnsupported(t) + + // Git for windows seems to struggle with local submodules in the temp dir + if runtime.GOOS == "windows" { + t.Skip() + } + + tester, err := NewExecutorTester(mainCtx) + if err != nil { + t.Fatalf("NewExecutorTester() error = %v", err) + } + defer tester.Close() + + submoduleRepo, err := createTestGitRespository() + if err != nil { + t.Fatalf("createTestGitRespository() error = %v", err) + } + defer submoduleRepo.Close() + + out, err := tester.Repo.Execute("-c", "protocol.file.allow=always", "submodule", "add", submoduleRepo.Path) + if err != nil { + t.Fatalf("tester.Repo.Execute(submodule, add, %q) error = %v\nout = %s", submoduleRepo.Path, err, out) + } + + out, err = tester.Repo.Execute("commit", "-am", "Add example submodule") + if err != nil { + t.Fatalf(`tester.Repo.Execute(commit, -am, "Add example submodule") error = %v\nout = %s`, err, out) + } + + env := []string{ + "BUILDKITE_GIT_CLONE_FLAGS=-v --sparse", + "BUILDKITE_GIT_CLEAN_FLAGS=-fdq", + "BUILDKITE_GIT_FETCH_FLAGS=-v", + "BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS=src/", + } + + git := tester. + MustMock(t, "git"). + PassthroughToLocalCommand() + git.Expect("submodule", "sync", "--recursive").NotCalled() + git.Expect("submodule", "update").WithAnyArguments().NotCalled() + git.ExpectAll([][]any{ + {"clone", "-v", "--sparse", "--", tester.Repo.Path, "."}, + {"clean", "-fdq"}, + {"submodule", "foreach", "--recursive", "git clean -fdq"}, + {"fetch", "-v", "--", "origin", "main"}, + {"--version"}, + {"sparse-checkout", "set", "--cone", "src/"}, + {"-c", "advice.detachedHead=false", "checkout", "-f", "FETCH_HEAD"}, + {"clean", "-fdq"}, + {"--no-pager", "log", "-1", "HEAD", "-s", "--no-color", gitShowFormatArg}, + }) + + agent := tester.MockAgent(t) + agent.Expect("meta-data", "exists", job.CommitMetadataKey).AndExitWith(1) + agent.Expect("meta-data", "set", job.CommitMetadataKey).WithStdin(commitPattern) + + tester.RunAndCheck(t, env...) + if got, want := tester.Output, "Submodule initialization skipped during sparse checkout"; !strings.Contains(got, want) { + t.Fatalf("bootstrap output does not contain %q:\n%s", want, got) + } +} + func TestCheckingOutLocalGitProjectWithSubmodulesDisabled(t *testing.T) { t.Parallel() diff --git a/internal/job/integration/hooks_integration_test.go b/internal/job/integration/hooks_integration_test.go index d518eae315..283b485a85 100644 --- a/internal/job/integration/hooks_integration_test.go +++ b/internal/job/integration/hooks_integration_test.go @@ -169,7 +169,7 @@ func TestEnvironmentHookNoCheckoutOverride(t *testing.T) { tester.ExpectGlobalHook("command").Once().AndExitWith(0).AndCallFunc(func(c *bintest.Call) { if got, want := c.GetEnv("BUILDKITE_SKIP_CHECKOUT"), tc.wantSkipCheckoutEnv; got != want { - fmt.Fprintf(c.Stderr, "Expected BUILDKITE_SKIP_CHECKOUT=%q, got %q\n", want, got) + _, _ = fmt.Fprintf(c.Stderr, "Expected BUILDKITE_SKIP_CHECKOUT=%q, got %q\n", want, got) c.Exit(1) return } diff --git a/internal/job/integration/secrets_integration_test.go b/internal/job/integration/secrets_integration_test.go index 6409884e17..8599df8a7d 100644 --- a/internal/job/integration/secrets_integration_test.go +++ b/internal/job/integration/secrets_integration_test.go @@ -600,7 +600,7 @@ func TestSecretsIntegration_NoCheckoutOverride(t *testing.T) { if !tc.wantErr { tester.ExpectGlobalHook("command").AndCallFunc(func(c *bintest.Call) { if got, want := c.GetEnv("BUILDKITE_GIT_CLONE_FLAGS"), "--mirror"; got != want { - fmt.Fprintf(c.Stderr, "Expected BUILDKITE_GIT_CLONE_FLAGS=%q, got %q\n", want, got) + _, _ = fmt.Fprintf(c.Stderr, "Expected BUILDKITE_GIT_CLONE_FLAGS=%q, got %q\n", want, got) c.Exit(1) return }