feat: add sparse checkout support

agent/agent_configuration.go

@@ -24,6 +24,7 @@ type AgentConfiguration struct {
 	GitCloneMirrorFlags             string
 	GitCleanFlags                   string
 	GitFetchFlags                   string
+	GitSparseCheckoutPaths          []string
 	GitSubmodules                   bool
 	GitSubmoduleCloneConfig         []string
 	SkipCheckout                    bool

agent/job_runner.go

@@ -597,6 +597,10 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
 	}
 	setEnv("BUILDKITE_LOCAL_HOOKS_ENABLED", fmt.Sprint(r.conf.AgentConfiguration.LocalHooksEnabled))
 
+	if len(r.conf.AgentConfiguration.GitSparseCheckoutPaths) > 0 {
+		setCheckoutEnv("BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS", strings.Join(r.conf.AgentConfiguration.GitSparseCheckoutPaths, ","))
+	}
+	setEnv("BUILDKITE_GIT_CLONE_MIRROR_FLAGS", r.conf.AgentConfiguration.GitCloneMirrorFlags)
 	setCheckoutEnv("BUILDKITE_GIT_CHECKOUT_FLAGS", r.conf.AgentConfiguration.GitCheckoutFlags)
 	setCheckoutEnv("BUILDKITE_GIT_CLONE_FLAGS", r.conf.AgentConfiguration.GitCloneFlags)
 	setCheckoutEnv("BUILDKITE_GIT_FETCH_FLAGS", r.conf.AgentConfiguration.GitFetchFlags)

clicommand/agent_start.go

@@ -162,6 +162,7 @@ type AgentStartConfig struct {
 	GitCloneMirrorFlags         string   `cli:"git-clone-mirror-flags"`
 	GitCleanFlags               string   `cli:"git-clean-flags"`
 	GitFetchFlags               string   `cli:"git-fetch-flags"`
+	GitSparseCheckoutPaths      []string `cli:"git-sparse-checkout-paths" normalize:"list"`
 	GitMirrorsPath              string   `cli:"git-mirrors-path" normalize:"filepath"`
 	GitMirrorCheckoutMode       string   `cli:"git-mirror-checkout-mode"`
 	GitMirrorsLockTimeout       int      `cli:"git-mirrors-lock-timeout"`
@@ -541,6 +542,7 @@ var AgentStartCommand = cli.Command{
 		GitCloneFlagsFlag,
 		GitCleanFlagsFlag,
 		GitFetchFlagsFlag,
+		GitSparseCheckoutPathsFlag,
 		GitCloneMirrorFlagsFlag,
 		GitMirrorsPathFlag,
 		GitMirrorCheckoutModeFlag,
@@ -1076,6 +1078,7 @@ var AgentStartCommand = cli.Command{
 			GitCloneMirrorFlags:             cfg.GitCloneMirrorFlags,
 			GitCleanFlags:                   cfg.GitCleanFlags,
 			GitFetchFlags:                   cfg.GitFetchFlags,
+			GitSparseCheckoutPaths:          cfg.GitSparseCheckoutPaths,
 			GitSubmodules:                   !cfg.NoGitSubmodules,
 			GitSubmoduleCloneConfig:         cfg.GitSubmoduleCloneConfig,
 			SkipCheckout:                    cfg.SkipCheckout,

clicommand/bootstrap.go

@@ -74,6 +74,7 @@ type BootstrapConfig struct {
 	GitCheckoutFlags             string   `cli:"git-checkout-flags"`
 	GitCloneFlags                string   `cli:"git-clone-flags"`
 	GitFetchFlags                string   `cli:"git-fetch-flags"`
+	GitSparseCheckoutPaths       []string `cli:"git-sparse-checkout-paths" normalize:"list"`
 	GitCloneMirrorFlags          string   `cli:"git-clone-mirror-flags"`
 	GitCleanFlags                string   `cli:"git-clean-flags"`
 	GitMirrorsPath               string   `cli:"git-mirrors-path" normalize:"filepath"`
@@ -249,6 +250,7 @@ var BootstrapCommand = cli.Command{
 		GitCloneMirrorFlagsFlag,
 		GitCleanFlagsFlag,
 		GitFetchFlagsFlag,
+		GitSparseCheckoutPathsFlag,
 		GitMirrorsPathFlag,
 		GitMirrorCheckoutModeFlag,
 		GitMirrorsLockTimeoutFlag,
@@ -451,6 +453,7 @@ var BootstrapCommand = cli.Command{
 			GitCloneFlags:                cfg.GitCloneFlags,
 			GitCloneMirrorFlags:          cfg.GitCloneMirrorFlags,
 			GitFetchFlags:                cfg.GitFetchFlags,
+			GitSparseCheckoutPaths:       cfg.GitSparseCheckoutPaths,
 			GitMirrorsLockTimeout:        cfg.GitMirrorsLockTimeout,
 			GitMirrorsPath:               cfg.GitMirrorsPath,
 			GitMirrorCheckoutMode:        cfg.GitMirrorCheckoutMode,

clicommand/global.go

@@ -242,6 +242,13 @@ var (
 		EnvVar: "BUILDKITE_GIT_FETCH_FLAGS",
 	}
 
+	GitSparseCheckoutPathsFlag = cli.StringSliceFlag{
+		Name:   "git-sparse-checkout-paths",
+		Value:  &cli.StringSlice{},
+		Usage:  "Comma-separated list of paths for git sparse checkout (cone mode). When set, only the listed paths are materialized in the working tree.",
+		EnvVar: "BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS",
+	}
+
 	GitMirrorsPathFlag = cli.StringFlag{
 		Name:   "git-mirrors-path",
 		Value:  "",

internal/job/checkout.go

@@ -1,6 +1,7 @@
 package job
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -343,6 +344,117 @@ func hasGitSubmodules(sh *shell.Shell) bool {
 	return osutil.FileExists(filepath.Join(sh.Getwd(), ".gitmodules"))
 }
 
+func cleanGitSparseCheckoutPaths(paths []string) []string {
+	cleaned := make([]string, 0, len(paths))
+	for _, path := range paths {
+		path = strings.TrimSpace(path)
+		if path != "" {
+			cleaned = append(cleaned, path)
+		}
+	}
+	return cleaned
+}
+
+func parseGitVersion(output string) (major, minor int, ok bool) {
+	if _, err := fmt.Sscanf(output, "git version %d.%d", &major, &minor); err != nil {
+		return 0, 0, false
+	}
+	return major, minor, true
+}
+
+func gitVersionAtLeast(ctx context.Context, sh *shell.Shell, major, minor int) (bool, error) {
+	output, err := sh.Command("git", "--version").RunAndCaptureStdout(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	gitMajor, gitMinor, ok := parseGitVersion(strings.TrimSpace(output))
+	if !ok {
+		return false, fmt.Errorf("parsing git version from %q", strings.TrimSpace(output))
+	}
+
+	if gitMajor != major {
+		return gitMajor > major, nil
+	}
+	return gitMinor >= minor, nil
+}
+
+// sparseCheckoutMayBeConfigured does a cheap filesystem check for marker files
+// that indicate sparse checkout (or the worktree-config extension that
+// `sparse-checkout` enables) might already be in effect, so we can avoid
+// shelling out to `git config` on every checkout. It resolves the .git dir
+// directly to handle the worktree/submodule case where .git is a file
+// containing `gitdir: <path>`.
+func sparseCheckoutMayBeConfigured(sh *shell.Shell) bool {
+	gitDir := filepath.Join(sh.Getwd(), ".git")
+	if data, err := os.ReadFile(gitDir); err == nil && bytes.HasPrefix(data, []byte("gitdir:")) {
+		gitDirValue := strings.TrimSpace(string(bytes.TrimPrefix(data, []byte("gitdir:"))))
+		if !filepath.IsAbs(gitDirValue) {
+			gitDirValue = filepath.Join(sh.Getwd(), gitDirValue)
+		}
+		gitDir = gitDirValue
+	}
+
+	return osutil.FileExists(filepath.Join(gitDir, "info", "sparse-checkout")) ||
+		osutil.FileExists(filepath.Join(gitDir, "config.worktree"))
+}
+
+func (e *Executor) disableSparseCheckoutIfConfigured(ctx context.Context) {
+	if !sparseCheckoutMayBeConfigured(e.shell) {
+		return
+	}
+
+	sparseOutput, err := e.shell.Command("git", "config", "--get", "core.sparseCheckout").RunAndCaptureStdout(ctx, shell.ShowStderr(false))
+	if err != nil || strings.TrimSpace(sparseOutput) != "true" {
+		return
+	}
+
+	e.shell.Commentf("Disabling sparse checkout from previous build")
+	if err := e.shell.Command("git", "sparse-checkout", "disable").Run(ctx); err != nil {
+		e.shell.Warningf("Failed to disable sparse checkout: %v", err)
+	}
+
+	// `sparse-checkout disable` leaves extensions.worktreeConfig set, which
+	// can cause problems for subsequent git operations. Only unset it if no
+	// other worktree-scoped config remains, to avoid clobbering user config.
+	worktreeConfig, err := e.shell.Command("git", "config", "--worktree", "--list").RunAndCaptureStdout(ctx, shell.ShowStderr(false))
+	if err == nil && strings.TrimSpace(worktreeConfig) == "" {
+		_ = e.shell.Command("git", "config", "--unset", "extensions.worktreeConfig").Run(ctx)
+	}
+}
+
+// setupSparseCheckout configures (or disables) git sparse checkout for the
+// current working tree. It returns true if sparse checkout was successfully
+// applied for this build, so callers can adjust later behaviour (e.g. skip
+// submodule init, which requires the full tree).
+func (e *Executor) setupSparseCheckout(ctx context.Context) (bool, error) {
+	paths := cleanGitSparseCheckoutPaths(e.GitSparseCheckoutPaths)
+	if len(paths) == 0 {
+		e.disableSparseCheckoutIfConfigured(ctx)
+		return false, nil
+	}
+
+	ok, err := gitVersionAtLeast(ctx, e.shell, 2, 26)
+	if err != nil {
+		e.shell.Warningf("Sparse checkout requires git >= 2.26; falling back to full checkout (%v)", err)
+		e.disableSparseCheckoutIfConfigured(ctx)
+		return false, nil
+	}
+	if !ok {
+		e.shell.Warningf("Sparse checkout requires git >= 2.26; falling back to full checkout")
+		e.disableSparseCheckoutIfConfigured(ctx)
+		return false, nil
+	}
+
+	e.shell.Commentf("Setting up sparse checkout for paths: %s", strings.Join(paths, ","))
+	args := append([]string{"sparse-checkout", "set", "--cone"}, paths...)
+	if err := e.shell.Command("git", args...).Run(ctx); err != nil {
+		return false, fmt.Errorf("setting sparse checkout paths: %w", err)
+	}
+
+	return true, nil
+}
+
 func hasGitCommit(ctx context.Context, sh *shell.Shell, gitDir, commit string) bool {
 	// Resolve commit to an actual commit object
 	output, err := sh.Command("git", "--git-dir", gitDir, "rev-parse", commit+"^{commit}").RunAndCaptureStdout(ctx, shell.ShowStderr(false))
@@ -918,6 +1030,11 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) error {
 		return err
 	}
 
+	sparseCheckoutActive, err := e.setupSparseCheckout(ctx)
+	if err != nil {
+		return err
+	}
+
 	gitCheckoutFlags := e.GitCheckoutFlags
 
 	if e.Commit == "HEAD" {
@@ -932,10 +1049,13 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) error {
 
 	gitSubmodules := false
 	if hasGitSubmodules(e.shell) {
-		if e.GitSubmodules {
+		switch {
+		case sparseCheckoutActive:
+			e.shell.Commentf("Submodule initialization skipped during sparse checkout")
+		case e.GitSubmodules:
 			e.shell.Commentf("Git submodules detected")
 			gitSubmodules = true
-		} else {
+		default:
 			e.shell.OptionalWarningf("submodules-disabled", "This repository has submodules, but submodules are disabled")
 		}
 	}