Preview: A unified gVisor checkpoint API.

sandboxexec/sandbox/BUILD

@@ -10,14 +10,17 @@ go_library(
     srcs = [
         "oci.go",
         "sandbox.go",
+        "storage.go",
     ],
     visibility = ["//:__subpackages__"],
     deps = ["@com_github_opencontainers_runtime_spec//specs-go:go_default_library"],
 )
 
 go_test(
     name = "sandbox_test",
-    srcs = ["oci_test.go"],
+    srcs = [
+        "oci_test.go",
+    ],
     library = ":sandbox",
     deps = ["@com_github_opencontainers_runtime_spec//specs-go:go_default_library"],
 )

sandboxexec/sandbox/oci.go

@@ -23,8 +23,8 @@ import (
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
 
-// NewBundle creates a temporary OCI bundle on the fly.
-func NewBundle(sandboxID string, runscRuntimeDir string) (string, error) {
+// NewBundle creates a temporary OCI bundle on the fly with optional custom annotations.
+func NewBundle(sandboxID string, runscRuntimeDir string, annotations map[string]string) (string, error) {
 	// Create a bundle directory for the sandbox.
 	bundleDir := filepath.Join(runscRuntimeDir, sandboxID)
 	rootfsDir := filepath.Join(bundleDir, "rootfs")
@@ -35,7 +35,8 @@ func NewBundle(sandboxID string, runscRuntimeDir string) (string, error) {
 
 	// Define the OCI Specification programmatically.
 	spec := &specs.Spec{
-		Version: "1.0.0",
+		Version:     "1.0.0",
+		Annotations: annotations,
 		Root: &specs.Root{
 			Path: "rootfs",
 			// The root filesystem is read-only for now. We can add support for

sandboxexec/sandbox/oci_test.go

@@ -27,7 +27,7 @@ func TestNewBundle(t *testing.T) {
 	tempDir := t.TempDir()
 	sandboxID := "test-sandbox"
 
-	bundleDir, err := NewBundle(sandboxID, tempDir)
+	bundleDir, err := NewBundle(sandboxID, tempDir, nil)
 	if err != nil {
 		t.Fatalf("NewBundle failed: %v", err)
 	}
@@ -79,3 +79,33 @@ func TestNewBundle(t *testing.T) {
 		}
 	}
 }
+
+func TestNewBundleWithAnnotations(t *testing.T) {
+	tempDir := t.TempDir()
+	sandboxID := "test-sandbox-annotations"
+	annotations := map[string]string{
+		"dev.gvisor.tar.rootfs.upper": "/tmp/test.tar",
+	}
+
+	bundleDir, err := NewBundle(sandboxID, tempDir, annotations)
+	if err != nil {
+		t.Fatalf("NewBundle failed: %v", err)
+	}
+	defer os.RemoveAll(bundleDir)
+
+	configPath := filepath.Join(bundleDir, "config.json")
+	configFile, err := os.Open(configPath)
+	if err != nil {
+		t.Fatalf("failed to open config.json: %v", err)
+	}
+	defer configFile.Close()
+
+	var spec specs.Spec
+	if err := json.NewDecoder(configFile).Decode(&spec); err != nil {
+		t.Fatalf("failed to decode config.json: %v", err)
+	}
+
+	if val, ok := spec.Annotations["dev.gvisor.tar.rootfs.upper"]; !ok || val != "/tmp/test.tar" {
+		t.Errorf("expected annotation 'dev.gvisor.tar.rootfs.upper' with value '/tmp/test.tar', got spec.Annotations: %+v", spec.Annotations)
+	}
+}

sandboxexec/sandbox/sandbox.go

@@ -19,18 +19,22 @@ package sandbox
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"math/rand"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"time"
 )
 
 // Options holds the configuration for a Sandbox.
 type Options struct {
-	runtimeDir string
-	id         string
+	runtimeDir   string
+	id           string
+	restoreID    string
+	restoreStore SnapshotStorage
 }
 
 // Option configures the Options struct.
@@ -50,6 +54,16 @@ func WithID(id string) Option {
 	}
 }
 
+// WithRestore configures the sandbox to restore state from the given storage and snapshot ID.
+// The sandbox automatically reads the snapshot metadata to determine if it is a
+// full Checkpoint/Restore, Filesystem snapshot, or Rootfs Tar snapshot.
+func WithRestore(snapshotID string, storage SnapshotStorage) Option {
+	return func(o *Options) {
+		o.restoreID = snapshotID
+		o.restoreStore = storage
+	}
+}
+
 // Sandbox represents a running gVisor sandbox where applications
 // run inside.
 type Sandbox struct {
@@ -116,7 +130,56 @@ func New(ctx context.Context, opts ...Option) (*Sandbox, error) {
 		return nil, fmt.Errorf("sandbox state directory has incorrect permissions: got %v, want %v", fi.Mode().Perm(), os.FileMode(0700))
 	}
 
-	bundleDir, err := NewBundle(options.id, runDir)
+	var annotations map[string]string
+	var extraArgs []string
+	var isCheckpointRestore bool
+	var checkpointRestoreDir string
+
+	if options.restoreID != "" && options.restoreStore != nil {
+		// 1. Fetch metadata.json from store.
+		metaReader, err := options.restoreStore.GetReader(ctx, options.restoreID, "metadata.json")
+		if err != nil {
+			return nil, fmt.Errorf("failed to read snapshot metadata: %w", err)
+		}
+		defer metaReader.Close()
+
+		var meta SnapshotMetadata
+		if err := json.NewDecoder(metaReader).Decode(&meta); err != nil {
+			return nil, fmt.Errorf("failed to parse snapshot metadata: %w", err)
+		}
+
+		// 2. Perform restore based on Type.
+		switch meta.Type {
+		case RootfsTarSnapshot:
+			// Skeleton: Download rootfs.tar from store to a local temp file under stateDir.
+			tarPath := filepath.Join(stateDir, "rootfs.tar")
+			// skeleton: download asset "rootfs.tar" from restoreStore to tarPath...
+			annotations = map[string]string{
+				"dev.gvisor.tar.rootfs.upper": tarPath,
+			}
+			extraArgs = append(extraArgs, "--allow-rootfs-tar-annotation")
+
+		case FilesystemSnapshot:
+			// Skeleton: Download all assets to a local directory under stateDir.
+			fsRestoreDir := filepath.Join(stateDir, "fs-restore")
+			if err := os.MkdirAll(fsRestoreDir, 0755); err != nil {
+				return nil, err
+			}
+			// skeleton: download all assets from restoreStore to fsRestoreDir...
+			extraArgs = append(extraArgs, fmt.Sprintf("--fs-restore-image-path=%s", fsRestoreDir))
+
+		case CheckpointRestore:
+			// Skeleton: Download all assets to a local directory under stateDir.
+			checkpointRestoreDir = filepath.Join(stateDir, "checkpoint-restore")
+			if err := os.MkdirAll(checkpointRestoreDir, 0755); err != nil {
+				return nil, err
+			}
+			// skeleton: download all assets from restoreStore to checkpointRestoreDir...
+			isCheckpointRestore = true
+		}
+	}
+
+	bundleDir, err := NewBundle(options.id, runDir, annotations)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create OCI bundle: %v", err)
 	}
@@ -128,10 +191,18 @@ func New(ctx context.Context, opts ...Option) (*Sandbox, error) {
 		rootState: stateDir,
 	}
 
-	// Launch the sandbox in detached mode via os/exec, we use `runsc run` here
-	// as a shortcut for `runsc create` and `runsc start`.
-	args := []string{"--root", sb.rootState, "run", "--bundle", sb.bundleDir, "--detach", sb.id}
-	cmd := exec.CommandContext(ctx, sb.runscPath, args...)
+	// Launch the sandbox in detached mode via os/exec.
+	var cmd *exec.Cmd
+	if isCheckpointRestore {
+		// Use runsc restore for CheckpointRestore.
+		args := []string{"--root", sb.rootState, "restore", "--image-path", checkpointRestoreDir, "--detach", sb.id}
+		cmd = exec.CommandContext(ctx, sb.runscPath, args...)
+	} else {
+		// Use runsc run with optional extra arguments (e.g. tar annotation or filesystem snapshot path).
+		args := append([]string{"--root", sb.rootState, "run", "--bundle", sb.bundleDir, "--detach"}, extraArgs...)
+		args = append(args, sb.id)
+		cmd = exec.CommandContext(ctx, sb.runscPath, args...)
+	}
 
 	if err := cmd.Run(); err != nil {
 		return nil, fmt.Errorf("failed to create sandbox via subprocess: %v", err)
@@ -180,3 +251,66 @@ func (s *Sandbox) Close(ctx context.Context) error {
 func (s *Sandbox) Bundle() string {
 	return s.bundleDir
 }
+
+// SaveOptions holds configuration for saving a snapshot.
+type SaveOptions struct {
+	LeaveRunning bool
+}
+
+// SaveOption configures SaveOptions.
+type SaveOption func(*SaveOptions)
+
+// WithLeaveRunning keeps the sandbox running after taking the snapshot.
+func WithLeaveRunning(leaveRunning bool) SaveOption {
+	return func(o *SaveOptions) {
+		o.LeaveRunning = leaveRunning
+	}
+}
+
+// SaveSnapshot serializes and saves the sandbox state to storage.
+// Depending on the snapshotType, it will perform a full Checkpoint, a Filesystem Snapshot, or a Rootfs Tar Snapshot.
+// It also automatically generates and writes "metadata.json" into the storage.
+func (s *Sandbox) SaveSnapshot(ctx context.Context, snapshotID string, snapshotType SnapshotType, storage SnapshotStorage, opts ...SaveOption) error {
+	options := SaveOptions{
+		LeaveRunning: false, // Default is false.
+	}
+	for _, o := range opts {
+		o(&options)
+	}
+
+	// 1. Write the metadata file to the storage.
+	meta := SnapshotMetadata{
+		Type:      snapshotType,
+		CreatedAt: time.Now().UTC().Format(time.RFC3339),
+	}
+
+	metaWriter, err := storage.PutWriter(ctx, snapshotID, "metadata.json")
+	if err != nil {
+		return fmt.Errorf("failed to create metadata.json in storage: %w", err)
+	}
+	defer metaWriter.Close()
+
+	if err := json.NewEncoder(metaWriter).Encode(&meta); err != nil {
+		return fmt.Errorf("failed to write metadata.json to storage: %w", err)
+	}
+
+	// 2. Perform the actual snapshot action.
+	switch snapshotType {
+	case RootfsTarSnapshot:
+		// Skeleton:
+		// Run `runsc tar rootfs-upper --file=<localTempTar> <sandboxID>`.
+		// Upload `<localTempTar>` to storage with asset name "rootfs.tar".
+
+	case FilesystemSnapshot:
+		// Skeleton:
+		// Run `runsc fscheckpoint --image-path=<localTempDir> [--leave-running] <sandboxID>`.
+		// Walk `<localTempDir>` and upload each file to storage.
+
+	case CheckpointRestore:
+		// Skeleton:
+		// Run `runsc checkpoint --image-path=<localTempDir> [--leave-running] <sandboxID>`.
+		// Walk `<localTempDir>` and upload each file to storage.
+	}
+
+	return nil
+}