From 8c4dc0bd2c4ae7cf3e071b91fe3537a3901356a4 Mon Sep 17 00:00:00 2001 From: Shayon Mukherjee Date: Tue, 19 May 2026 22:35:18 -0700 Subject: [PATCH] fsgofer: add extension interface for custom backends Building a custom gofer (e.g. for network-backed storage, encrypted filesystems, or tiered caches) currently requires forking the runsc binary and copying/maintaining unexported setup and seccomp code. This adds an `Extension` interface that lets custom filesystem backends register with the stock gofer and serve LISAFS connections for specific mounts without forking. This builds on #13180, which moved LISAFS implementation selection and connection options to the connection. With that in place, custom backends can plug into the stock gofer without creating separate `lisafs.Server` instances. All mounts continue to share one `lisafs.Server`, preserving the server-side filesystem tree and synchronization across stock and extension-backed mounts. Registered extensions are queried in order for each mount. `NewConnection` returns a nil `lisafs.ConnectionImpl` to decline a mount, and the first extension that returns a non-nil implementation handles it. `NewConnection` receives the sandbox's OCI runtime spec, the specific `*specs.Mount` being served, the resolved mount path, and readonly state, so extensions can read sandbox-wide configuration from `spec.Annotations` and per-mount configuration from the mount itself without a side channel. Stock `fsgofer` remains the default when no extension claims a mount. Extensions only choose the per-connection `lisafs.ConnectionImpl` and `lisafs.ConnectionOpts` now supported by `lisafs.Server.CreateConnection`. `SeccompRules` lets extensions declare additional syscalls, merged with the stock gofer allowlist before installation. There are no behavior changes when no extensions are registered. Also adds documentation in `g3doc/user_guide/filesystem.md` and `pkg/lisafs/README.md` describing how to use the extension interface. FUTURE_COPYBARA_INTEGRATE_REVIEW=https://github.com/google/gvisor/pull/12950 from shayonj:s/gofer-backend-v2 7139233f005d63db701b34c19501353c1af226d2 PiperOrigin-RevId: 918225708 --- g3doc/user_guide/filesystem.md | 28 +++++++++++ pkg/lisafs/README.md | 4 ++ runsc/cmd/BUILD | 1 + runsc/cmd/gofer.go | 35 +++++++++++-- runsc/fsgofer/extension/BUILD | 29 +++++++++++ runsc/fsgofer/extension/extension.go | 61 +++++++++++++++++++++++ runsc/fsgofer/extension/extension_test.go | 56 +++++++++++++++++++++ runsc/fsgofer/filter/BUILD | 1 + runsc/fsgofer/filter/filter.go | 11 ++-- runsc/fsgofer/filter/filter_test.go | 13 +++++ 10 files changed, 229 insertions(+), 10 deletions(-) create mode 100644 runsc/fsgofer/extension/BUILD create mode 100644 runsc/fsgofer/extension/extension.go create mode 100644 runsc/fsgofer/extension/extension_test.go diff --git a/g3doc/user_guide/filesystem.md b/g3doc/user_guide/filesystem.md index 6a592c08eb..ed6d02578d 100644 --- a/g3doc/user_guide/filesystem.md +++ b/g3doc/user_guide/filesystem.md @@ -273,3 +273,31 @@ runsc --root=/path/to/rootdir debug --mount erofs:{source}:{destination} ``` [Production guide]: production.md + +## Custom Gofer Extensions + +The stock gofer serves host filesystem mounts via LISAFS. For workloads that +need a different filesystem backend (a network-backed store, an encrypted +filesystem, a tiered cache), a custom runsc build can import +`runsc/fsgofer/extension` and register an extension that supplies a +`lisafs.ConnectionImpl` for selected mounts. + +An extension registers itself at startup and claims mounts by path. For example, +a binary might register one extension that handles mounts under `/storage` via +an HTTP blob service, and another that handles `/encrypted` via a local +encryption layer. Unclaimed mounts fall through to the stock fsgofer as usual. + +To build a custom gofer: + +1. Implement the `extension.Extension` interface: `Name`, `TryHandleMount`, and + `SeccompRules`. +2. Register your extension in `init()` or early `main()`. +3. Build a runsc binary that imports `runsc/cli` and your extension package. + +The extension's `TryHandleMount` returns a `lisafs.ConnectionImpl` and +`lisafs.ConnectionOpts` for mounts it handles, or a nil implementation to +decline. All mounts share the same `lisafs.Server`, preserving server-side +filesystem tree synchronization across stock and extension-backed mounts. +Configuration may be read from OCI annotations and mount fields such as source, +type, and options. `SeccompRules` declares any additional syscalls the extension +needs beyond the stock gofer allowlist. diff --git a/pkg/lisafs/README.md b/pkg/lisafs/README.md index 0f76092f76..f74040c551 100644 --- a/pkg/lisafs/README.md +++ b/pkg/lisafs/README.md @@ -42,6 +42,10 @@ accessed/mutated via RPCs by LISAFS clients. The server is a trusted process. For security reasons, the server must assume that the client can be potentially compromised and act maliciously. +Mount behavior is selected per connection. A single server may serve multiple +connections with different implementations while sharing the same server-side +filesystem tree and synchronization. + #### Concurrency The server must execute file system operations under appropriate concurrency diff --git a/runsc/cmd/BUILD b/runsc/cmd/BUILD index 57b42f804f..c80cb7c6ad 100644 --- a/runsc/cmd/BUILD +++ b/runsc/cmd/BUILD @@ -121,6 +121,7 @@ go_library( "//runsc/container", "//runsc/flag", "//runsc/fsgofer", + "//runsc/fsgofer/extension", "//runsc/fsgofer/filter", "//runsc/metricserver/containermetrics", "//runsc/mitigate", diff --git a/runsc/cmd/gofer.go b/runsc/cmd/gofer.go index 09a61fea8f..24583a5bbd 100644 --- a/runsc/cmd/gofer.go +++ b/runsc/cmd/gofer.go @@ -36,6 +36,7 @@ import ( "gvisor.dev/gvisor/runsc/container" "gvisor.dev/gvisor/runsc/flag" "gvisor.dev/gvisor/runsc/fsgofer" + "gvisor.dev/gvisor/runsc/fsgofer/extension" "gvisor.dev/gvisor/runsc/fsgofer/filter" "gvisor.dev/gvisor/runsc/profile" "gvisor.dev/gvisor/runsc/specutils" @@ -310,6 +311,9 @@ func (g *Gofer) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomm LisafsNeeded: lisafsNeeded, CgoEnabled: config.CgoEnabled, } + for _, e := range extension.Registered() { + opts.ExtraRules = append(opts.ExtraRules, e.SeccompRules()) + } if err := filter.Install(opts); err != nil { util.Fatalf("installing seccomp filters: %v", err) } @@ -322,6 +326,7 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string, ruid i sock *unet.Socket mountPath string readonly bool + mount *specs.Mount } cfgs := make([]connectionConfig, 0, len(spec.Mounts)+1) @@ -339,8 +344,9 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string, ruid i } mountIdx := 1 // first one is the root - for _, m := range spec.Mounts { - if !specutils.HasMountConfig(m) { + for i := range spec.Mounts { + m := &spec.Mounts[i] + if !specutils.HasMountConfig(*m) { continue } mountConf := g.mountConfs[mountIdx] @@ -362,6 +368,7 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string, ruid i sock: sandboxsetup.NewSocket(ioFD), mountPath: m.Destination, readonly: readonly, + mount: m, }) log.Infof("Serving %q mapped on FD %d (ro: %t)", m.Destination, ioFD, readonly) } @@ -392,8 +399,28 @@ func (g *Gofer) serve(spec *specs.Spec, conf *config.Config, root string, ruid i // Create the server and start connections. server := lisafs.NewServer() for _, cfg := range cfgs { - connImpl := fsgofer.NewConnectionImpl(fsgoferConf) - conn, err := server.CreateConnection(cfg.sock, cfg.mountPath, fsgofer.ConnectionOpts(cfg.readonly), connImpl) + var connImpl lisafs.ConnectionImpl + var connOpts lisafs.ConnectionOpts + // /dev is always served by the stock fsgofer. + if cfg.mountPath != "/dev" { + for _, e := range extension.Registered() { + impl, opts, err := e.TryHandleMount(spec, cfg.mount, cfg.mountPath, cfg.readonly) + if err != nil { + util.Fatalf("extension %s for %q: %v", e.Name(), cfg.mountPath, err) + } + if impl != nil { + connImpl = impl + connOpts = opts + log.Infof("Serving %q via extension %s on FD %d", cfg.mountPath, e.Name(), cfg.sock.FD()) + break + } + } + } + if connImpl == nil { + connImpl = fsgofer.NewConnectionImpl(fsgoferConf) + connOpts = fsgofer.ConnectionOpts(cfg.readonly) + } + conn, err := server.CreateConnection(cfg.sock, cfg.mountPath, connOpts, connImpl) if err != nil { util.Fatalf("starting connection on FD %d for gofer mount failed: %v", cfg.sock.FD(), err) } diff --git a/runsc/fsgofer/extension/BUILD b/runsc/fsgofer/extension/BUILD new file mode 100644 index 0000000000..c57397accf --- /dev/null +++ b/runsc/fsgofer/extension/BUILD @@ -0,0 +1,29 @@ +load("//tools:defs.bzl", "go_library", "go_test") + +package( + default_applicable_licenses = ["//:license"], + licenses = ["notice"], +) + +go_library( + name = "extension", + srcs = ["extension.go"], + visibility = ["//runsc:__subpackages__"], + deps = [ + "//pkg/lisafs", + "//pkg/seccomp", + "@com_github_opencontainers_runtime_spec//specs-go:go_default_library", + ], +) + +go_test( + name = "extension_test", + size = "small", + srcs = ["extension_test.go"], + library = ":extension", + deps = [ + "//pkg/lisafs", + "//pkg/seccomp", + "@com_github_opencontainers_runtime_spec//specs-go:go_default_library", + ], +) diff --git a/runsc/fsgofer/extension/extension.go b/runsc/fsgofer/extension/extension.go new file mode 100644 index 0000000000..c463afaec9 --- /dev/null +++ b/runsc/fsgofer/extension/extension.go @@ -0,0 +1,61 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package extension defines registration hooks for custom filesystem gofer +// extensions. The stock fsgofer handles any mount no Extension claims. +package extension + +import ( + specs "github.com/opencontainers/runtime-spec/specs-go" + "gvisor.dev/gvisor/pkg/lisafs" + "gvisor.dev/gvisor/pkg/seccomp" +) + +// Extension is implemented by alternative LisaFS backends. The first +// registered Extension whose TryHandleMount returns a non-nil implementation +// handles the mount. +type Extension interface { + // Name identifies the extension in log messages. + Name() string + + // TryHandleMount returns a LisaFS connection implementation and options for + // the given mount, or (nil, lisafs.ConnectionOpts{}, nil) if this extension + // does not handle it. A non-nil error means the extension claims the mount + // but failed to initialize. + // + // mount is nil for the root filesystem (root is not present in + // spec.Mounts). Per-sandbox config may be read from spec.Annotations. + // + // All returned connections run on one shared lisafs.Server so server-side + // tree synchronization is preserved across stock and extension-backed + // mounts. + TryHandleMount(spec *specs.Spec, mount *specs.Mount, mountPath string, readonly bool) (lisafs.ConnectionImpl, lisafs.ConnectionOpts, error) + + // SeccompRules returns additional rules to merge into the stock + // gofer's seccomp allowlist. + SeccompRules() seccomp.SyscallRules +} + +var registered []Extension + +// Register adds e to the extension list. Must be called during init or +// early in main, before Registered is iterated. +func Register(e Extension) { + registered = append(registered, e) +} + +// Registered returns all registered extensions in registration order. +func Registered() []Extension { + return registered +} diff --git a/runsc/fsgofer/extension/extension_test.go b/runsc/fsgofer/extension/extension_test.go new file mode 100644 index 0000000000..a888e924eb --- /dev/null +++ b/runsc/fsgofer/extension/extension_test.go @@ -0,0 +1,56 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extension + +import ( + "testing" + + specs "github.com/opencontainers/runtime-spec/specs-go" + "gvisor.dev/gvisor/pkg/lisafs" + "gvisor.dev/gvisor/pkg/seccomp" +) + +type fakeExtension struct { + name string +} + +func (f fakeExtension) Name() string { + return f.name +} + +func (fakeExtension) TryHandleMount(*specs.Spec, *specs.Mount, string, bool) (lisafs.ConnectionImpl, lisafs.ConnectionOpts, error) { + return nil, lisafs.ConnectionOpts{}, nil +} + +func (fakeExtension) SeccompRules() seccomp.SyscallRules { + return seccomp.NewSyscallRules() +} + +func TestRegisterAndRegistered(t *testing.T) { + registered = nil + + e1 := fakeExtension{name: "first"} + e2 := fakeExtension{name: "second"} + Register(e1) + Register(e2) + + got := Registered() + if len(got) != 2 { + t.Fatalf("len(Registered()) = %d, want 2", len(got)) + } + if got[0] != e1 || got[1] != e2 { + t.Fatalf("Registered() = %v, want [%v %v]", got, e1, e2) + } +} diff --git a/runsc/fsgofer/filter/BUILD b/runsc/fsgofer/filter/BUILD index 16ad30e556..0d56547203 100644 --- a/runsc/fsgofer/filter/BUILD +++ b/runsc/fsgofer/filter/BUILD @@ -38,6 +38,7 @@ go_test( srcs = ["filter_test.go"], library = ":filter", deps = [ + "//pkg/seccomp", "@org_golang_x_sys//unix:go_default_library", ], ) diff --git a/runsc/fsgofer/filter/filter.go b/runsc/fsgofer/filter/filter.go index 617b00f670..02918b654b 100644 --- a/runsc/fsgofer/filter/filter.go +++ b/runsc/fsgofer/filter/filter.go @@ -30,13 +30,10 @@ type Options struct { DirectFS bool LisafsNeeded bool CgoEnabled bool + ExtraRules []seccomp.SyscallRules } -// Rules returns the seccomp rules for a gofer process without installing -// them. Callers can merge additional rules before building and installing -// the seccomp program. This is useful for custom gofer implementations -// that need the stock gofer's baseline syscall allowlist but also require -// additional syscalls (e.g. for networking or namespace switching). +// Rules returns the seccomp rules for a gofer process without installing them. func Rules(opt Options) seccomp.SyscallRules { s := allowedSyscalls.Copy() @@ -71,13 +68,15 @@ func Rules(opt Options) seccomp.SyscallRules { s.Merge(lisafsFilters) } + for _, rules := range opt.ExtraRules { + s.Merge(rules) + } return s } // Install installs seccomp filters. func Install(opt Options) error { s := Rules(opt) - program := &seccomp.Program{ RuleSets: []seccomp.RuleSet{ { diff --git a/runsc/fsgofer/filter/filter_test.go b/runsc/fsgofer/filter/filter_test.go index c80375dcf2..7fd56c52f9 100644 --- a/runsc/fsgofer/filter/filter_test.go +++ b/runsc/fsgofer/filter/filter_test.go @@ -18,6 +18,7 @@ import ( "testing" "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/seccomp" ) func TestRulesLisaFSFilters(t *testing.T) { @@ -56,3 +57,15 @@ func TestRulesLisaFSFilters(t *testing.T) { }) } } + +func TestRulesExtraRules(t *testing.T) { + const extraSyscall = uintptr(123456) + rules := Rules(Options{ + ExtraRules: []seccomp.SyscallRules{ + seccomp.NewSyscallRules().Add(extraSyscall, seccomp.MatchAll{}), + }, + }) + if !rules.Has(extraSyscall) { + t.Fatalf("Rules().Has(%d) = false, want true", extraSyscall) + } +}