Add support for GIT LFS checkout

clicommand/bootstrap.go

@@ -60,6 +60,7 @@ type BootstrapConfig struct {
 	PullRequest                  string   `cli:"pullrequest"`
 	PullRequestUsingMergeRefspec bool     `cli:"pull-request-using-merge-refspec"`
 	GitSubmodules                bool     `cli:"git-submodules"`
+	GitLFSEnabled                bool     `cli:"git-lfs-enabled"`
 	SSHKeyscan                   bool     `cli:"ssh-keyscan"`
 	AgentName                    string   `cli:"agent" validate:"required"`
 	Queue                        string   `cli:"queue"`
@@ -298,6 +299,11 @@ var BootstrapCommand = cli.Command{
 			Usage:  "Enable git submodules (default: true)",
 			EnvVar: "BUILDKITE_GIT_SUBMODULES",
 		},
+		cli.BoolFlag{
+			Name:   "git-lfs-enabled",
+			Usage:  "Enable Git LFS object download during checkout (default: false)",
+			EnvVar: "BUILDKITE_GIT_LFS_ENABLED",
+		},
 		cli.BoolTFlag{
 			Name:   "pty",
 			Usage:  "Run jobs within a pseudo terminal (default: true)",
@@ -440,6 +446,7 @@ var BootstrapCommand = cli.Command{
 			GitCloneFlags:                cfg.GitCloneFlags,
 			GitCloneMirrorFlags:          cfg.GitCloneMirrorFlags,
 			GitFetchFlags:                cfg.GitFetchFlags,
+			GitLFSEnabled:                cfg.GitLFSEnabled,
 			GitMirrorsLockTimeout:        cfg.GitMirrorsLockTimeout,
 			GitMirrorsPath:               cfg.GitMirrorsPath,
 			GitMirrorCheckoutMode:        cfg.GitMirrorCheckoutMode,

internal/job/checkout.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"slices"
@@ -902,6 +903,13 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) error {
 		}
 	}
 
+	// Fail fast before any git work if git-lfs is required but missing.
+	if e.GitLFSEnabled {
+		if _, err := exec.LookPath("git-lfs"); err != nil {
+			return fmt.Errorf("BUILDKITE_GIT_LFS_ENABLED=true but git-lfs binary is not found on PATH: %w", err)
+		}
+	}
+
 	// Git clean prior to checkout, we do this even if submodules have been
 	// disabled to ensure previous submodules are cleaned up
 	if hasGitSubmodules(e.shell) {
@@ -914,6 +922,20 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) error {
 		return fmt.Errorf("cleaning git repository: %w", err)
 	}
 
+	// Install LFS filter before fetch so the filter is registered before any
+	// network operation, following the conventional git-lfs setup order.
+	if e.GitLFSEnabled {
+		e.shell.Commentf("Installing Git LFS filter")
+		if err := e.shell.Command("git", "lfs", "install", "--local").Run(ctx); err != nil {
+			return fmt.Errorf("installing git lfs filter: %w", err)
+		}
+		// Force-set GIT_LFS_SKIP_SMUDGE=1 so checkout writes pointer files to
+		// disk rather than downloading objects inline. Intentionally not
+		// restored — git lfs checkout materialises files from cache without
+		// triggering the smudge filter.
+		e.shell.Env.Set("GIT_LFS_SKIP_SMUDGE", "1")
+	}
+
 	if err := e.fetchSource(ctx); err != nil {
 		return err
 	}
@@ -1019,6 +1041,12 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) error {
 		}
 	}
 
+	if e.GitLFSEnabled {
+		if err := gitLFSFetchCheckout(ctx, e.shell); err != nil {
+			return err
+		}
+	}
+
 	// Git clean after checkout. We need to do this because submodules could have
 	// changed in between the last checkout and this one. A double clean is the only
 	// good solution to this problem that we've found

internal/job/checkout_test.go

@@ -2,7 +2,12 @@ package job
 
 import (
 	"context"
+	"fmt"
 	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
 	"testing"
 	"time"
 
@@ -276,3 +281,166 @@ func TestDefaultCheckoutPhase_DelayedRefCreation(t *testing.T) {
 		t.Fatalf("tt.executor.defaultCheckoutPhase(ctx) error = %v, want nil", err)
 	}
 }
+
+func TestDefaultCheckoutPhase_GitLFS(t *testing.T) {
+	// Not parallel: subtests manipulate PATH via t.Setenv, which modifies
+	// process-global state.
+	ctx := context.Background()
+
+	t.Setenv("GIT_AUTHOR_NAME", "Buildkite Agent")
+	t.Setenv("GIT_AUTHOR_EMAIL", "agent@example.com")
+	t.Setenv("GIT_COMMITTER_NAME", "Buildkite Agent")
+	t.Setenv("GIT_COMMITTER_EMAIL", "agent@example.com")
+
+	// gitOnlyBinDir returns a temp dir containing git (via a symlink on Unix or
+	// a .bat wrapper on Windows) but no git-lfs, so exec.LookPath("git-lfs")
+	// will fail while git commands still work.
+	gitOnlyBinDir := func(t *testing.T) string {
+		t.Helper()
+		gitBin, err := exec.LookPath("git")
+		if err != nil {
+			t.Fatalf("exec.LookPath(\"git\") error = %v", err)
+		}
+		binDir := t.TempDir()
+		if runtime.GOOS == "windows" {
+			// Use a .bat wrapper to avoid copying the multi-MB binary and to
+			// sidestep the symlink-privilege requirement on Windows.
+			wrapper := fmt.Sprintf("@echo off\r\n\"%s\" %%*\r\n", gitBin)
+			if err := os.WriteFile(filepath.Join(binDir, "git.bat"), []byte(wrapper), 0o755); err != nil {
+				t.Fatalf("os.WriteFile() error = %v", err)
+			}
+		} else {
+			if err := os.Symlink(gitBin, filepath.Join(binDir, "git")); err != nil {
+				t.Fatalf("os.Symlink() error = %v", err)
+			}
+		}
+		return binDir
+	}
+
+	// fakeLFSBinDir returns a temp dir that has git (via gitOnlyBinDir) plus a
+	// fake git-lfs whose behaviour is defined by the provided scripts.
+	// unixScript is a #!/bin/sh script; winBatch is a .bat file body.
+	fakeLFSBinDir := func(t *testing.T, unixScript, winBatch string) string {
+		t.Helper()
+		binDir := gitOnlyBinDir(t)
+		if runtime.GOOS == "windows" {
+			if err := os.WriteFile(filepath.Join(binDir, "git-lfs.bat"), []byte(winBatch), 0o755); err != nil {
+				t.Fatalf("os.WriteFile() error = %v", err)
+			}
+		} else {
+			if err := os.WriteFile(filepath.Join(binDir, "git-lfs"), []byte(unixScript), 0o755); err != nil {
+				t.Fatalf("os.WriteFile() error = %v", err)
+			}
+		}
+		return binDir
+	}
+
+	tests := []struct {
+		name       string
+		lfsEnabled bool
+		setupPath  func(t *testing.T)
+		wantErr    string
+	}{
+		{
+			name:       "LFS disabled",
+			lfsEnabled: false,
+		},
+		{
+			name:       "LFS enabled binary present",
+			lfsEnabled: true,
+			setupPath: func(t *testing.T) {
+				if _, err := exec.LookPath("git-lfs"); err != nil {
+					t.Skip("git-lfs not installed")
+				}
+			},
+		},
+		{
+			name:       "LFS enabled binary missing",
+			lfsEnabled: true,
+			setupPath: func(t *testing.T) {
+				t.Setenv("PATH", gitOnlyBinDir(t))
+			},
+			wantErr: "git-lfs binary is not found on PATH",
+		},
+		{
+			name:       "LFS enabled git lfs command fails",
+			lfsEnabled: true,
+			setupPath: func(t *testing.T) {
+				t.Setenv("PATH", fakeLFSBinDir(t,
+					"#!/bin/sh\nexit 1\n",
+					"@echo off\r\nexit /b 1\r\n",
+				))
+			},
+			wantErr: "installing git lfs filter",
+		},
+		{
+			name:       "LFS enabled git lfs fetch fails",
+			lfsEnabled: true,
+			setupPath: func(t *testing.T) {
+				t.Setenv("PATH", fakeLFSBinDir(t,
+					"#!/bin/sh\ncase \"$1\" in\n  install) exit 0 ;;\n  *) exit 1 ;;\nesac\n",
+					"@echo off\r\nif \"%1\"==\"install\" exit /b 0\r\nexit /b 1\r\n",
+				))
+			},
+			wantErr: "git lfs fetch",
+		},
+	}
+
+	s := githttptest.NewServer()
+	t.Cleanup(s.Close)
+
+	for i, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Set up the remote repository BEFORE restricting PATH so that
+			// githttptest's git operations use the real git binary.
+			projectName := fmt.Sprintf("lfs-test-%d", i)
+			if err := s.CreateRepository(projectName); err != nil {
+				t.Fatalf("s.CreateRepository(%q) error = %v", projectName, err)
+			}
+			out, err := s.InitRepository(projectName)
+			if err != nil {
+				t.Fatalf("s.InitRepository(%q) error = %v, output: %s", projectName, err, out)
+			}
+
+			// Restrict PATH after the repo is initialised.
+			if tt.setupPath != nil {
+				tt.setupPath(t)
+			}
+
+			sh, err := shell.New()
+			if err != nil {
+				t.Fatalf("shell.New() error = %v", err)
+			}
+
+			checkoutDir := t.TempDir()
+			sh.Env.Set("BUILDKITE_BUILD_CHECKOUT_PATH", checkoutDir)
+
+			executor := &Executor{
+				shell: sh,
+				ExecutorConfig: ExecutorConfig{
+					Commit:        "HEAD",
+					Branch:        "main",
+					GitCleanFlags: "-f -d -x",
+					BuildPath:     t.TempDir(),
+					Repository:    s.RepoURL(projectName),
+					GitLFSEnabled: tt.lfsEnabled,
+				},
+			}
+
+			err = executor.defaultCheckoutPhase(ctx)
+			if tt.wantErr == "" {
+				if err != nil {
+					t.Errorf("defaultCheckoutPhase() error = %v, want nil", err)
+				}
+				return
+			}
+			if err == nil {
+				t.Errorf("defaultCheckoutPhase() error = nil, want error containing %q", tt.wantErr)
+				return
+			}
+			if !strings.Contains(err.Error(), tt.wantErr) {
+				t.Errorf("defaultCheckoutPhase() error = %q, want it to contain %q", err.Error(), tt.wantErr)
+			}
+		})
+	}
+}

internal/job/config.go

@@ -53,6 +53,9 @@ type ExecutorConfig struct {
 	// Should git submodules be checked out
 	GitSubmodules bool `env:"BUILDKITE_GIT_SUBMODULES"`
 
+	// Whether to enable Git LFS operations during checkout
+	GitLFSEnabled bool `env:"BUILDKITE_GIT_LFS_ENABLED"`
+
 	// If the commit was part of a pull request, this will container the PR number
 	PullRequest string
 

internal/job/executor.go

@@ -907,6 +907,10 @@ func (e *Executor) setUp(ctx context.Context) error {
 	// Disable any interactive Git/SSH prompting
 	e.shell.Env.Set("GIT_TERMINAL_PROMPT", "0")
 
+	// Force-set GIT_LFS_SKIP_SMUDGE=1 before any git operations so LFS
+	// objects are not downloaded automatically during checkout.
+	e.shell.Env.Set("GIT_LFS_SKIP_SMUDGE", "1")
+
 	// Fetch and set secrets before environment hook execution
 	if e.Secrets != "" {
 		if err := e.fetchAndSetSecrets(ctx); err != nil {

internal/job/git.go

@@ -133,6 +133,19 @@ func gitCleanSubmodules(ctx context.Context, sh *shell.Shell, gitCleanFlags stri
 	return nil
 }
 
+// gitLFSFetchCheckout fetches LFS objects for the current HEAD then materialises
+// them. Fetch and checkout failures are wrapped with distinct messages so that a
+// caller can tell which step failed from the error string alone.
+func gitLFSFetchCheckout(ctx context.Context, sh *shell.Shell) error {
+	if err := sh.Command("git", "lfs", "fetch").Run(ctx); err != nil {
+		return fmt.Errorf("git lfs fetch: %w", err)
+	}
+	if err := sh.Command("git", "lfs", "checkout").Run(ctx); err != nil {
+		return fmt.Errorf("git lfs checkout: %w", err)
+	}
+	return nil
+}
+
 func gitRepack(ctx context.Context, sh *shell.Shell, args ...string) error {
 	commandArgs := []string{"repack"}
 	commandArgs = append(commandArgs, args...)