diff --git a/sandboxexec/sandbox/BUILD b/sandboxexec/sandbox/BUILD index 149f54925d..8703d71524 100644 --- a/sandboxexec/sandbox/BUILD +++ b/sandboxexec/sandbox/BUILD @@ -10,6 +10,7 @@ go_library( srcs = [ "oci.go", "sandbox.go", + "storage.go", ], visibility = ["//:__subpackages__"], deps = ["@com_github_opencontainers_runtime_spec//specs-go:go_default_library"], @@ -17,7 +18,9 @@ go_library( go_test( name = "sandbox_test", - srcs = ["oci_test.go"], + srcs = [ + "oci_test.go", + ], library = ":sandbox", deps = ["@com_github_opencontainers_runtime_spec//specs-go:go_default_library"], ) diff --git a/sandboxexec/sandbox/oci.go b/sandboxexec/sandbox/oci.go index 4bf4506d6e..932b821675 100644 --- a/sandboxexec/sandbox/oci.go +++ b/sandboxexec/sandbox/oci.go @@ -23,8 +23,8 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" ) -// NewBundle creates a temporary OCI bundle on the fly. -func NewBundle(sandboxID string, runscRuntimeDir string) (string, error) { +// NewBundle creates a temporary OCI bundle on the fly with optional custom annotations. +func NewBundle(sandboxID string, runscRuntimeDir string, annotations map[string]string) (string, error) { // Create a bundle directory for the sandbox. bundleDir := filepath.Join(runscRuntimeDir, sandboxID) rootfsDir := filepath.Join(bundleDir, "rootfs") @@ -35,7 +35,8 @@ func NewBundle(sandboxID string, runscRuntimeDir string) (string, error) { // Define the OCI Specification programmatically. spec := &specs.Spec{ - Version: "1.0.0", + Version: "1.0.0", + Annotations: annotations, Root: &specs.Root{ Path: "rootfs", // The root filesystem is read-only for now. We can add support for diff --git a/sandboxexec/sandbox/oci_test.go b/sandboxexec/sandbox/oci_test.go index 5ea4d09751..54699fe56a 100644 --- a/sandboxexec/sandbox/oci_test.go +++ b/sandboxexec/sandbox/oci_test.go @@ -27,7 +27,7 @@ func TestNewBundle(t *testing.T) { tempDir := t.TempDir() sandboxID := "test-sandbox" - bundleDir, err := NewBundle(sandboxID, tempDir) + bundleDir, err := NewBundle(sandboxID, tempDir, nil) if err != nil { t.Fatalf("NewBundle failed: %v", err) } @@ -79,3 +79,33 @@ func TestNewBundle(t *testing.T) { } } } + +func TestNewBundleWithAnnotations(t *testing.T) { + tempDir := t.TempDir() + sandboxID := "test-sandbox-annotations" + annotations := map[string]string{ + "dev.gvisor.tar.rootfs.upper": "/tmp/test.tar", + } + + bundleDir, err := NewBundle(sandboxID, tempDir, annotations) + if err != nil { + t.Fatalf("NewBundle failed: %v", err) + } + defer os.RemoveAll(bundleDir) + + configPath := filepath.Join(bundleDir, "config.json") + configFile, err := os.Open(configPath) + if err != nil { + t.Fatalf("failed to open config.json: %v", err) + } + defer configFile.Close() + + var spec specs.Spec + if err := json.NewDecoder(configFile).Decode(&spec); err != nil { + t.Fatalf("failed to decode config.json: %v", err) + } + + if val, ok := spec.Annotations["dev.gvisor.tar.rootfs.upper"]; !ok || val != "/tmp/test.tar" { + t.Errorf("expected annotation 'dev.gvisor.tar.rootfs.upper' with value '/tmp/test.tar', got spec.Annotations: %+v", spec.Annotations) + } +} diff --git a/sandboxexec/sandbox/sandbox.go b/sandboxexec/sandbox/sandbox.go index 1ab954c585..3f9ec5c755 100644 --- a/sandboxexec/sandbox/sandbox.go +++ b/sandboxexec/sandbox/sandbox.go @@ -19,18 +19,22 @@ package sandbox import ( "bytes" "context" + "encoding/json" "fmt" "io" "math/rand" "os" "os/exec" "path/filepath" + "time" ) // Options holds the configuration for a Sandbox. type Options struct { - runtimeDir string - id string + runtimeDir string + id string + restoreID string + restoreStore SnapshotStorage } // Option configures the Options struct. @@ -50,6 +54,16 @@ func WithID(id string) Option { } } +// WithRestore configures the sandbox to restore state from the given storage and snapshot ID. +// The sandbox automatically reads the snapshot metadata to determine if it is a +// full Checkpoint/Restore, Filesystem snapshot, or Rootfs Tar snapshot. +func WithRestore(snapshotID string, storage SnapshotStorage) Option { + return func(o *Options) { + o.restoreID = snapshotID + o.restoreStore = storage + } +} + // Sandbox represents a running gVisor sandbox where applications // run inside. type Sandbox struct { @@ -116,7 +130,56 @@ func New(ctx context.Context, opts ...Option) (*Sandbox, error) { return nil, fmt.Errorf("sandbox state directory has incorrect permissions: got %v, want %v", fi.Mode().Perm(), os.FileMode(0700)) } - bundleDir, err := NewBundle(options.id, runDir) + var annotations map[string]string + var extraArgs []string + var isCheckpointRestore bool + var checkpointRestoreDir string + + if options.restoreID != "" && options.restoreStore != nil { + // 1. Fetch metadata.json from store. + metaReader, err := options.restoreStore.GetReader(ctx, options.restoreID, "metadata.json") + if err != nil { + return nil, fmt.Errorf("failed to read snapshot metadata: %w", err) + } + defer metaReader.Close() + + var meta SnapshotMetadata + if err := json.NewDecoder(metaReader).Decode(&meta); err != nil { + return nil, fmt.Errorf("failed to parse snapshot metadata: %w", err) + } + + // 2. Perform restore based on Type. + switch meta.Type { + case RootfsTarSnapshot: + // Skeleton: Download rootfs.tar from store to a local temp file under stateDir. + tarPath := filepath.Join(stateDir, "rootfs.tar") + // skeleton: download asset "rootfs.tar" from restoreStore to tarPath... + annotations = map[string]string{ + "dev.gvisor.tar.rootfs.upper": tarPath, + } + extraArgs = append(extraArgs, "--allow-rootfs-tar-annotation") + + case FilesystemSnapshot: + // Skeleton: Download all assets to a local directory under stateDir. + fsRestoreDir := filepath.Join(stateDir, "fs-restore") + if err := os.MkdirAll(fsRestoreDir, 0755); err != nil { + return nil, err + } + // skeleton: download all assets from restoreStore to fsRestoreDir... + extraArgs = append(extraArgs, fmt.Sprintf("--fs-restore-image-path=%s", fsRestoreDir)) + + case CheckpointRestore: + // Skeleton: Download all assets to a local directory under stateDir. + checkpointRestoreDir = filepath.Join(stateDir, "checkpoint-restore") + if err := os.MkdirAll(checkpointRestoreDir, 0755); err != nil { + return nil, err + } + // skeleton: download all assets from restoreStore to checkpointRestoreDir... + isCheckpointRestore = true + } + } + + bundleDir, err := NewBundle(options.id, runDir, annotations) if err != nil { return nil, fmt.Errorf("failed to create OCI bundle: %v", err) } @@ -128,10 +191,18 @@ func New(ctx context.Context, opts ...Option) (*Sandbox, error) { rootState: stateDir, } - // Launch the sandbox in detached mode via os/exec, we use `runsc run` here - // as a shortcut for `runsc create` and `runsc start`. - args := []string{"--root", sb.rootState, "run", "--bundle", sb.bundleDir, "--detach", sb.id} - cmd := exec.CommandContext(ctx, sb.runscPath, args...) + // Launch the sandbox in detached mode via os/exec. + var cmd *exec.Cmd + if isCheckpointRestore { + // Use runsc restore for CheckpointRestore. + args := []string{"--root", sb.rootState, "restore", "--image-path", checkpointRestoreDir, "--detach", sb.id} + cmd = exec.CommandContext(ctx, sb.runscPath, args...) + } else { + // Use runsc run with optional extra arguments (e.g. tar annotation or filesystem snapshot path). + args := append([]string{"--root", sb.rootState, "run", "--bundle", sb.bundleDir, "--detach"}, extraArgs...) + args = append(args, sb.id) + cmd = exec.CommandContext(ctx, sb.runscPath, args...) + } if err := cmd.Run(); err != nil { return nil, fmt.Errorf("failed to create sandbox via subprocess: %v", err) @@ -180,3 +251,66 @@ func (s *Sandbox) Close(ctx context.Context) error { func (s *Sandbox) Bundle() string { return s.bundleDir } + +// SaveOptions holds configuration for saving a snapshot. +type SaveOptions struct { + LeaveRunning bool +} + +// SaveOption configures SaveOptions. +type SaveOption func(*SaveOptions) + +// WithLeaveRunning keeps the sandbox running after taking the snapshot. +func WithLeaveRunning(leaveRunning bool) SaveOption { + return func(o *SaveOptions) { + o.LeaveRunning = leaveRunning + } +} + +// SaveSnapshot serializes and saves the sandbox state to storage. +// Depending on the snapshotType, it will perform a full Checkpoint, a Filesystem Snapshot, or a Rootfs Tar Snapshot. +// It also automatically generates and writes "metadata.json" into the storage. +func (s *Sandbox) SaveSnapshot(ctx context.Context, snapshotID string, snapshotType SnapshotType, storage SnapshotStorage, opts ...SaveOption) error { + options := SaveOptions{ + LeaveRunning: false, // Default is false. + } + for _, o := range opts { + o(&options) + } + + // 1. Write the metadata file to the storage. + meta := SnapshotMetadata{ + Type: snapshotType, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + } + + metaWriter, err := storage.PutWriter(ctx, snapshotID, "metadata.json") + if err != nil { + return fmt.Errorf("failed to create metadata.json in storage: %w", err) + } + defer metaWriter.Close() + + if err := json.NewEncoder(metaWriter).Encode(&meta); err != nil { + return fmt.Errorf("failed to write metadata.json to storage: %w", err) + } + + // 2. Perform the actual snapshot action. + switch snapshotType { + case RootfsTarSnapshot: + // Skeleton: + // Run `runsc tar rootfs-upper --file= `. + // Upload `` to storage with asset name "rootfs.tar". + + case FilesystemSnapshot: + // Skeleton: + // Run `runsc fscheckpoint --image-path= [--leave-running] `. + // Walk `` and upload each file to storage. + + case CheckpointRestore: + // Skeleton: + // Run `runsc checkpoint --image-path= [--leave-running] `. + // Walk `` and upload each file to storage. + } + + return nil +} diff --git a/sandboxexec/sandbox/storage.go b/sandboxexec/sandbox/storage.go new file mode 100644 index 0000000000..8bbbff5cda --- /dev/null +++ b/sandboxexec/sandbox/storage.go @@ -0,0 +1,128 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sandbox + +import ( + "context" + "io" + "os" + "path/filepath" +) + +// SnapshotType defines the type of snapshot. +type SnapshotType string + +const ( + // CheckpointRestore represents a full process state checkpoint and restore. + CheckpointRestore SnapshotType = "CheckpointRestore" + + // FilesystemSnapshot represents a snapshot of the container's filesystems. + FilesystemSnapshot SnapshotType = "FilesystemSnapshot" + + // RootfsTarSnapshot represents a tar file snapshot of rootfs changes. + RootfsTarSnapshot SnapshotType = "RootfsTarSnapshot" +) + +// SnapshotMetadata stores the metadata of a snapshot. +type SnapshotMetadata struct { + Type SnapshotType `json:"type"` + CreatedAt string `json:"created_at"` +} + +// SnapshotStorage defines a pluggable storage interface for snapshots. +type SnapshotStorage interface { + // PutWriter returns a WriteCloser to write a file asset of a snapshot. + PutWriter(ctx context.Context, snapshotID string, assetName string) (io.WriteCloser, error) + + // GetReader returns a ReadCloser to read a file asset of a snapshot. + GetReader(ctx context.Context, snapshotID string, assetName string) (io.ReadCloser, error) + + // Delete deletes all assets associated with a snapshot ID. + Delete(ctx context.Context, snapshotID string) error + + // List returns all snapshot IDs known to this storage. + List(ctx context.Context) ([]string, error) + + // ListAssets returns all asset names associated with a snapshot ID. + ListAssets(ctx context.Context, snapshotID string) ([]string, error) +} + +// FilesystemStorage implements SnapshotStorage using a local directory. +type FilesystemStorage struct { + rootDir string +} + +// NewFilesystemStorage creates a new FilesystemStorage at the given root directory. +func NewFilesystemStorage(rootDir string) (*FilesystemStorage, error) { + if err := os.MkdirAll(rootDir, 0755); err != nil { + return nil, err + } + return &FilesystemStorage{rootDir: rootDir}, nil +} + +// PutWriter returns a WriteCloser to write a file asset of a snapshot. +func (f *FilesystemStorage) PutWriter(ctx context.Context, snapshotID string, assetName string) (io.WriteCloser, error) { + path := filepath.Join(f.rootDir, snapshotID, assetName) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return nil, err + } + return os.Create(path) +} + +// GetReader returns a ReadCloser to read a file asset of a snapshot. +func (f *FilesystemStorage) GetReader(ctx context.Context, snapshotID string, assetName string) (io.ReadCloser, error) { + path := filepath.Join(f.rootDir, snapshotID, assetName) + return os.Open(path) +} + +// Delete deletes all assets associated with a snapshot ID. +func (f *FilesystemStorage) Delete(ctx context.Context, snapshotID string) error { + path := filepath.Join(f.rootDir, snapshotID) + return os.RemoveAll(path) +} + +// List returns all snapshot IDs known to this storage. +func (f *FilesystemStorage) List(ctx context.Context) ([]string, error) { + entries, err := os.ReadDir(f.rootDir) + if err != nil { + return nil, err + } + var ids []string + for _, entry := range entries { + if entry.IsDir() { + ids = append(ids, entry.Name()) + } + } + return ids, nil +} + +// ListAssets returns all asset names associated with a snapshot ID. +func (f *FilesystemStorage) ListAssets(ctx context.Context, snapshotID string) ([]string, error) { + dir := filepath.Join(f.rootDir, snapshotID) + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + var assets []string + for _, entry := range entries { + if !entry.IsDir() { + assets = append(assets, entry.Name()) + } + } + return assets, nil +}