From 6c756b30b69e2a5c9288be2b7f22e820b2145695 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Fri, 16 Oct 2026 21:23:55 +0000 Subject: [PATCH 1/5] Run gVisor sandboxes without a pause container Each actor sandbox had a root container that ran /pause from a pause image. atelet pulled that image and built its bundle, and every SandboxConfig had to name it. runsc can now boot a sandbox with no root container and checkpoint and restore it, so the pause container is not necessary. ateworker-gvisor still names the sandbox _pause, but now creates it with -no-root-container and a spec with no process, rootfs, or mounts. The spec keeps the network namespace, cgroup, resource limits, and durable-dir mount hints. Application containers do not change. Remove pauseImage from the SandboxConfig CRD, from atelet's sandbox record and snapshot manifest, from the sandbox bundle hash, and from runtime prewarm. Remove the pause_rootfs restore phase. The bundle hash changes, so snapshots taken with a pause container no longer match any worker's runtime. Signed-off-by: Jing Chen --- .golangci-kal.yaml | 2 +- benchmarking/analysis/phase_report.py | 2 +- cmd/ate-setup/differences.md | 6 +- cmd/ate-setup/internal/images/prebuilt.go | 14 +- .../internal/images/prebuilt_test.go | 8 +- .../workerpool_sandboxconfig_test.go | 5 - cmd/atelet/lifecycle_test.go | 99 +++++++------- cmd/atelet/main.go | 51 +++---- cmd/atelet/main_test.go | 80 +++-------- cmd/atelet/runtime_prewarm.go | 45 ++---- cmd/atelet/runtime_prewarm_test.go | 120 ++++------------ cmd/atelet/sandbox_assets.go | 22 +-- cmd/atelet/sandbox_runtime_index.go | 12 +- cmd/atelet/sandbox_runtime_index_test.go | 7 +- cmd/atelet/snapshot_transfer_test.go | 8 +- cmd/atelet/worker_sandbox_syncer.go | 5 +- cmd/atelet/worker_sandbox_syncer_test.go | 6 +- cmd/atelet/worker_sandbox_versions.go | 77 +++-------- cmd/atelet/worker_sandbox_versions_test.go | 125 +++-------------- cmd/ateworker-gvisor/cleanup_test.go | 2 +- cmd/ateworker-gvisor/main.go | 66 ++++----- cmd/ateworker-gvisor/phaselog.go | 10 +- cmd/ateworker-gvisor/phaselog_test.go | 2 - cmd/ateworker-gvisor/runsc.go | 52 +++---- cmd/ateworker-gvisor/runsc_test.go | 128 +++++++++++++----- cmd/ateworker-gvisor/stats.go | 17 ++- cmd/ateworker-gvisor/stats_test.go | 2 +- cmd/kubectl-ate/internal/cmd/actor_test.go | 6 +- docs/api-guide.md | 6 +- docs/architecture.md | 2 +- docs/glossary.md | 9 +- docs/observability.md | 4 +- docs/sandbox-asset-cache.md | 10 +- internal/hardware/hardware.go | 3 +- internal/imagecache/imagecache.go | 7 +- internal/ocispec/gvisor.go | 9 +- internal/ocispec/ocispec_test.go | 6 +- internal/ocispec/parity_test.go | 36 ++++- internal/proto/ateletpb/atelet.pb.go | 14 +- internal/proto/ateletpb/atelet.proto | 14 +- .../generated/ate.dev_sandboxconfigs.yaml | 29 +--- .../ate-install/sandboxconfig-gvisor.yaml | 6 +- pkg/api/v1alpha1/sandboxconfig_types.go | 27 +--- .../v1alpha1/sandboxconfig_validation_test.go | 68 ++-------- 44 files changed, 444 insertions(+), 785 deletions(-) diff --git a/.golangci-kal.yaml b/.golangci-kal.yaml index 8ae78cb43e..cc773f3336 100644 --- a/.golangci-kal.yaml +++ b/.golangci-kal.yaml @@ -83,7 +83,7 @@ linters: - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' text: '^(optionalfields|requiredfields): .*\bSandboxConfigSpec\.SandboxClass\b' - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' - text: '^(nomaps|optionalfields): .*\bSandboxVersionConfig\.(Assets|PauseImage)\b' + text: '^(nomaps|optionalfields): .*\bSandboxVersionConfig\.Assets\b' - path: 'pkg/api/v1alpha1/sandboxconfig_types\.go' text: '^(nonpointerstructs|requiredfields): .*\bSandboxConfig\.Spec\b' - path: 'pkg/api/v1alpha1/csidriverconfig_types\.go' diff --git a/benchmarking/analysis/phase_report.py b/benchmarking/analysis/phase_report.py index 2d7c154f40..cb7611880e 100755 --- a/benchmarking/analysis/phase_report.py +++ b/benchmarking/analysis/phase_report.py @@ -94,7 +94,7 @@ # ateworker restore: micro-VM (cmd/ateworker-microvm/phaselog.go) and gVisor # (cmd/ateworker-gvisor/phaselog.go), each in its own sequential order "prep", "egress_prepare", "net_setup", "bundles", "upper_join", "lowers", - "pause_rootfs", "pause_create", "pause_restore", + "pause_create", "pause_restore", "app_rootfs", "app_create", "app_restore", "tap", "vmm_launch", "vm_restore", "resume", "wakeup_probe", "activate", # atelet + ateworker checkpoint diff --git a/cmd/ate-setup/differences.md b/cmd/ate-setup/differences.md index d03d7b2385..e54eb99e31 100644 --- a/cmd/ate-setup/differences.md +++ b/cmd/ate-setup/differences.md @@ -121,9 +121,9 @@ The rewritten reference then keeps that tag and adds the digest it names, as tagged release. Resolving the digest is one HEAD request per image, cached for the install, so `--image-repo` needs read access to the registry from wherever `ate-setup` runs rather than only from the cluster. It is not optional: an -ActorTemplate's container image, an image volume's reference, and a -SandboxConfig's `pauseImage` each carry the CEL rule `self.contains('@')`, so an -unpinned reference is rejected by admission and every demo fails to deploy. +ActorTemplate's container image and an image volume's reference each carry the +CEL rule `self.contains('@')`, so an unpinned reference is rejected by admission +and every demo fails to deploy. Pinning also gives a pre-built install the property a `ko` install had for free, that a tag moving afterwards cannot change what is running. diff --git a/cmd/ate-setup/internal/images/prebuilt.go b/cmd/ate-setup/internal/images/prebuilt.go index 63656b29ad..873dcb01e8 100644 --- a/cmd/ate-setup/internal/images/prebuilt.go +++ b/cmd/ate-setup/internal/images/prebuilt.go @@ -79,13 +79,13 @@ func (p *Prebuilt) ResolveBytes(ctx context.Context, manifest []byte) ([]byte, e // it, such as envoy-dataplane: a release publishes that under the same // repository and tag as the rest, so it is found the same way. // -// The digest is not decoration. An ActorTemplate's container image, an image -// volume's reference, and a SandboxConfig's pauseImage each carry the CEL rule -// self.contains('@'), so a tag on its own is rejected by admission and every -// demo would fail to deploy. Pinning also restores what ko gave the control -// plane deployments for free: an install that cannot shift under a tag someone -// moves later. Keeping the tag alongside the digest is ko's own output shape, -// and leaves the reference legible in `kubectl get`. +// The digest is not decoration. An ActorTemplate's container image and an image +// volume's reference each carry the CEL rule self.contains('@'), so a tag on +// its own is rejected by admission and every demo would fail to deploy. Pinning +// also restores what ko gave the control plane deployments for free: an install +// that cannot shift under a tag someone moves later. Keeping the tag alongside +// the digest is ko's own output shape, and leaves the reference legible in +// `kubectl get`. func (p *Prebuilt) Pin(ctx context.Context, image string) (string, error) { tagged := p.src.Repo + "/" + image + ":" + p.src.Tag // A tag that already carries one is pinned as it stands. Looking it up diff --git a/cmd/ate-setup/internal/images/prebuilt_test.go b/cmd/ate-setup/internal/images/prebuilt_test.go index 1264caa09a..41a36bf181 100644 --- a/cmd/ate-setup/internal/images/prebuilt_test.go +++ b/cmd/ate-setup/internal/images/prebuilt_test.go @@ -370,10 +370,10 @@ func TestResolveEveryInstallManifest(t *testing.T) { if refs := koRefPattern.FindAllString(string(out), -1); len(refs) > 0 { t.Errorf("unresolved references survived: %v", refs) } - // Every reference has to name a digest. ActorTemplate images, - // image volume references, and a SandboxConfig's pauseImage all - // carry the CEL rule self.contains('@'), so an unpinned reference - // is not applied at all, it is rejected by admission. + // Every reference has to name a digest. ActorTemplate images and + // image volume references carry the CEL rule self.contains('@'), so + // an unpinned reference is not applied at all, it is rejected by + // admission. for _, ref := range resolvedRef.FindAllString(string(out), -1) { if !strings.Contains(ref, "@sha256:") { t.Errorf("%s is not pinned to a digest", ref) diff --git a/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig_test.go b/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig_test.go index f01ba39756..e5b1e7dbfc 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_sandboxconfig_test.go @@ -48,8 +48,6 @@ import ( // testGvisorDefault is the gvisor class default TestMain creates. const testGvisorDefault = "gvisor-default-test" -const testPauseImage = "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" - // testHoldFinalizer keeps a SandboxConfig on the API server after it is // deleted, so a test can observe how a config that is being deleted is treated. const testHoldFinalizer = "test.ate.dev/hold" @@ -2397,9 +2395,6 @@ func TestAddProtectionFinalizer(t *testing.T) { func makeSandboxConfig(name string, class atev1alpha1.SandboxClass, isDefault bool) *atev1alpha1.SandboxConfig { v := atev1alpha1.SandboxVersionConfig{Name: "v1"} - if class == atev1alpha1.SandboxClassGvisor { - v.PauseImage = testPauseImage - } sc := &atev1alpha1.SandboxConfig{ ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: atev1alpha1.SandboxConfigSpec{ diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index 322bf16051..b10a99f01a 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -16,6 +16,7 @@ package main import ( "context" + "crypto/sha256" "errors" "fmt" "net" @@ -246,7 +247,7 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity) { systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } - sandboxRec := testSandboxRecord(release, pinnedImage) + sandboxRec := testSandboxRecord(release) spec := &ateletpb.WorkloadSpec{ Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, } @@ -422,26 +423,27 @@ func TestRestoreUsesRequestSandboxRuntime(t *testing.T) { host := imageVolumeTestRegistry(t) image := host + "/actor:v1" pinnedImage := pushTestImage(t, image, singleFileLayer(t, "bin/app", "app")) - checkpointPause := pushTestImage(t, host+"/pause:v1", singleFileLayer(t, "pause", "pause-v1")) - restorePause := pushTestImage(t, host+"/pause:v2", singleFileLayer(t, "pause", "pause-v2")) - release := newGVisorTestRelease(t) + // The two releases differ in content, so they have distinct bundle hashes. + release := func(runsc string) gvisorTestRelease { + tarball := gvisorReleaseTarballWith(t, runsc) + return gvisorTestRelease{tarball: tarball, entry: assetEntry{URL: "gs://test-bucket/gvisor.tar", SHA256: fmt.Sprintf("%x", sha256.Sum256(tarball))}} + } + checkpointRelease, restoreRelease := release("runsc v1"), release("runsc v2") s := &Atelet{ workerDialer: newWorkerDialer(1), imageCache: newImageVolumeStore(t), sandboxAssetCache: newTestAssetCache(t), - anonGCSClient: fakeObjectStorage{data: release.tarball}, + anonGCSClient: fakeObjectStorage{data: checkpointRelease.tarball}, systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } - recWithPause := func(pause string) *sandboxAssetsRecord { - return testSandboxRecord(release, pause) - } + checkpointRec, restoreRec := testSandboxRecord(checkpointRelease), testSandboxRecord(restoreRelease) spec := &ateletpb.WorkloadSpec{ Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, } - advertiseRuntime(t, s, recWithPause(checkpointPause)) + advertiseRuntime(t, s, checkpointRec) if _, err := s.Run(ctx, &ateletpb.RunRequest{ Atespace: atespace, ActorName: actorName, @@ -449,7 +451,7 @@ func TestRestoreUsesRequestSandboxRuntime(t *testing.T) { ActorTemplateAtespace: "default", ActorTemplateName: "counter", WorkerPodUid: workerPodUID, - SandboxRuntime: runtimeFor(t, recWithPause(checkpointPause)), + SandboxRuntime: runtimeFor(t, checkpointRec), Spec: spec, }); err != nil { t.Fatalf("Run: %v", err) @@ -477,11 +479,12 @@ func TestRestoreUsesRequestSandboxRuntime(t *testing.T) { if err != nil { t.Fatalf("unmarshalling snapshot manifest: %v", err) } - if manifestRec.PauseImage != checkpointPause { - t.Fatalf("manifest pause image = %q, want %q", manifestRec.PauseImage, checkpointPause) + if got, want := manifestRec.Assets[gvisorAssetName].SHA256, checkpointRelease.entry.SHA256; got != want { + t.Fatalf("manifest gvisor sha256 = %q, want %q", got, want) } - advertiseRuntime(t, s, recWithPause(restorePause)) + s.anonGCSClient = fakeObjectStorage{data: restoreRelease.tarball} + advertiseRuntime(t, s, restoreRec) if _, err := s.Restore(ctx, &ateletpb.RestoreRequest{ Atespace: atespace, ActorName: actorName, @@ -489,7 +492,7 @@ func TestRestoreUsesRequestSandboxRuntime(t *testing.T) { ActorTemplateAtespace: "default", ActorTemplateName: "counter", WorkerPodUid: workerPodUID, - SandboxRuntime: runtimeFor(t, recWithPause(restorePause)), + SandboxRuntime: runtimeFor(t, restoreRec), Spec: spec, Snapshot: lifecycleSnapshot(actorUID, snapshotName, 1, ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY), }); err != nil { @@ -500,8 +503,8 @@ func TestRestoreUsesRequestSandboxRuntime(t *testing.T) { if err != nil { t.Fatalf("reading on-node sandbox record: %v", err) } - if got.PauseImage != restorePause { - t.Errorf("restored actor pause image = %q, want the request's %q", got.PauseImage, restorePause) + if sha, want := got.Assets[gvisorAssetName].SHA256, restoreRelease.entry.SHA256; sha != want { + t.Errorf("restored actor gvisor sha256 = %q, want the request's %q", sha, want) } } @@ -546,7 +549,7 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { release := newGVisorTestRelease(t) if tc.restore && tc.failure != "manifest" { writeLocalSnapshot(t, ateletpath.LocalSnapshotDir(actorUID, snapshotName), sandboxAssetsRecord{ - SandboxClass: "gvisor", PauseImage: testPauseImage, + SandboxClass: "gvisor", Assets: map[string]assetEntry{gvisorAssetName: release.entry}, SnapshotFiles: []string{"checkpoint.img"}, }, map[string]string{"checkpoint.img": "guest-memory"}) @@ -586,7 +589,7 @@ func TestActivationFailureBeforeRegistration(t *testing.T) { s := newPluginHerder(t, &recordingObjectStorage{}) s.anonGCSClient = storage s.systemInfoVolumes = refresher - rec := testSandboxRecord(release, testPauseImage) + rec := testSandboxRecord(release) advertiseRuntime(t, s, rec) if tc.restore { _, err = s.Restore(ctx, &ateletpb.RestoreRequest{ @@ -674,12 +677,13 @@ func TestRunFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } spec := &ateletpb.WorkloadSpec{ + Containers: []*ateletpb.Container{{Name: "app", Image: missingImage(t)}}, Volumes: []*ateletpb.Volume{{ Name: "trust", SystemInfo: trustVolumeSpec("ca.pem"), }}, } - rec := testSandboxRecord(release, missingPauseImage(t)) + rec := testSandboxRecord(release) advertiseRuntime(t, s, rec) _, err := s.Run(t.Context(), &ateletpb.RunRequest{ Atespace: "team-a", @@ -689,7 +693,7 @@ func TestRunFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { SandboxRuntime: runtimeFor(t, rec), Spec: spec, }) - if err == nil || !strings.Contains(err.Error(), "while creating pause OCI bundle") { + if err == nil || !strings.Contains(err.Error(), `while creating "app" OCI bundle`) { t.Fatalf("Run error = %v, want the post-registration OCI preparation failure", err) } if _, err := os.ReadFile(filepath.Join(ateletpath.SystemInfoVolumeRoot(actorUID, "trust"), "ca.pem")); err != nil { @@ -700,12 +704,12 @@ func TestRunFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { } } -// missingPauseImage returns a digest-pinned reference that passes request +// missingImage returns a digest-pinned reference that passes request // validation but names a manifest the local test registry does not serve, so // the pull fails only once OCI bundle preparation starts. -func missingPauseImage(t *testing.T) string { +func missingImage(t *testing.T) string { t.Helper() - return imageVolumeTestRegistry(t) + "/pause@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + return imageVolumeTestRegistry(t) + "/app@sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" } func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { @@ -714,22 +718,24 @@ func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { actorUID = "0f6c2b0e-1b7e-4b3f-9a6a-1d2c3e4f5a63" snapshotName = "pause-snap-after" ) - pauseImage := missingPauseImage(t) + image := missingImage(t) store := newCTBStore(t) store.set(t, string(testCertPEM(t))) refresher := newSystemInfoVolumeRefresher(trustbundle.NewSource(store.lister.Get, nil), nil) release := newGVisorTestRelease(t) writeLocalSnapshot(t, ateletpath.LocalSnapshotDir(actorUID, snapshotName), sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: pauseImage, Assets: map[string]assetEntry{gvisorAssetName: release.entry}, SnapshotFiles: []string{"checkpoint.img"}, }, map[string]string{"checkpoint.img": "guest-memory"}) - spec := &ateletpb.WorkloadSpec{Volumes: []*ateletpb.Volume{{ - Name: "trust", - SystemInfo: trustVolumeSpec("ca.pem"), - }}} + spec := &ateletpb.WorkloadSpec{ + Containers: []*ateletpb.Container{{Name: "app", Image: image}}, + Volumes: []*ateletpb.Volume{{ + Name: "trust", + SystemInfo: trustVolumeSpec("ca.pem"), + }}, + } s := &Atelet{ sandboxAssetCache: newTestAssetCache(t), anonGCSClient: fakeObjectStorage{data: release.tarball}, @@ -737,7 +743,7 @@ func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { systemInfoVolumes: refresher, snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } - rec := testSandboxRecord(release, pauseImage) + rec := testSandboxRecord(release) advertiseRuntime(t, s, rec) _, err := s.Restore(t.Context(), &ateletpb.RestoreRequest{ Atespace: "team-a", @@ -748,7 +754,7 @@ func TestRestoreFailureAfterRegistrationRemovesOwnRegistration(t *testing.T) { Spec: spec, Snapshot: lifecycleSnapshot(actorUID, snapshotName, 1, ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY), }) - if err == nil || !strings.Contains(err.Error(), "while creating pause OCI bundle") { + if err == nil || !strings.Contains(err.Error(), `while creating "app" OCI bundle`) { t.Fatalf("Restore error = %v, want the post-registration OCI preparation failure", err) } if _, err := os.ReadFile(filepath.Join(ateletpath.SystemInfoVolumeRoot(actorUID, "trust"), "ca.pem")); err != nil { @@ -780,13 +786,12 @@ func TestRestorePrefersLocalAndFallsBackToObjectStorage(t *testing.T) { pinnedImage := pushTestImage(t, host+"/actor:v1", singleFileLayer(t, "bin/app", "app")) release := newGVisorTestRelease(t) - sandboxRec := testSandboxRecord(release, pinnedImage) + sandboxRec := testSandboxRecord(release) spec := &ateletpb.WorkloadSpec{ Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, } manifestRec := sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: pinnedImage, Assets: map[string]assetEntry{gvisorAssetName: release.entry}, SnapshotFiles: []string{"checkpoint.img"}, } @@ -919,10 +924,9 @@ func TestRestoreFailureRemovesStagedSnapshotFiles(t *testing.T) { host := imageVolumeTestRegistry(t) pinnedImage := pushTestImage(t, host+"/actor:v1", singleFileLayer(t, "bin/app", "app")) release := newGVisorTestRelease(t) - sandboxRec := testSandboxRecord(release, pinnedImage) + sandboxRec := testSandboxRecord(release) manifestRec := sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: pinnedImage, Assets: map[string]assetEntry{gvisorAssetName: release.entry}, SnapshotFiles: []string{"checkpoint.img"}, } @@ -993,7 +997,7 @@ func TestRestoreWithoutUsableLocalCopy(t *testing.T) { snapshotName = "pause-snap-1" ) release := newGVisorTestRelease(t) - rec := testSandboxRecord(release, testPauseImage) + rec := testSandboxRecord(release) for _, tc := range []struct { name string // unusableLocal writes a manifest that lists a file that is not there. @@ -1011,7 +1015,7 @@ func TestRestoreWithoutUsableLocalCopy(t *testing.T) { useTempNodeDirs(t) if tc.unusableLocal { writeLocalSnapshot(t, ateletpath.LocalSnapshotDir(actorUID, snapshotName), sandboxAssetsRecord{ - SandboxClass: "gvisor", PauseImage: testPauseImage, SnapshotFiles: []string{"checkpoint.img"}, + SandboxClass: "gvisor", SnapshotFiles: []string{"checkpoint.img"}, }, nil) } // The object store is empty: nothing was ever uploaded. @@ -1138,7 +1142,7 @@ func runPluginActor(t *testing.T) *pluginActor { a.s.sandboxAssetCache = newTestAssetCache(t) a.s.anonGCSClient = fakeObjectStorage{data: release.tarball} a.s.systemInfoVolumes = newSystemInfoVolumeRefresher(nil, nil) - a.rec = testSandboxRecord(release, pinnedImage) + a.rec = testSandboxRecord(release) a.spec = &ateletpb.WorkloadSpec{ Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, } @@ -1512,7 +1516,7 @@ func TestConcurrentRunsShareOneAssetDownload(t *testing.T) { systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), snapshots: snapshottracker.New(t.Context(), nil, clock.RealClock{}), } - sandboxRec := testSandboxRecord(release, pinnedImage) + sandboxRec := testSandboxRecord(release) spec := &ateletpb.WorkloadSpec{ Containers: []*ateletpb.Container{{Name: "app", Image: pinnedImage, Command: []string{"/bin/app"}}}, } @@ -2004,10 +2008,9 @@ func TestPromoteSnapshotFoundAlreadyUploaded(t *testing.T) { // testSandboxRecord is the record of a gVisor runtime whose only asset is // release, as the syncer plans it from a SandboxConfig version. -func testSandboxRecord(release gvisorTestRelease, pauseImage string) *sandboxAssetsRecord { +func testSandboxRecord(release gvisorTestRelease) *sandboxAssetsRecord { return &sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: pauseImage, Assets: map[string]assetEntry{gvisorAssetName: release.entry}, } } @@ -2021,10 +2024,7 @@ func advertiseRuntime(t *testing.T, s *Atelet, rec *sandboxAssetsRecord) { if s.runtimes == nil { s.runtimes = newRuntimeIndex() } - hash, err := versionBundleHash(rec) - if err != nil { - t.Fatalf("versionBundleHash: %v", err) - } + hash := bundleHash(rec) s.runtimes.register("test-worker-pod", advertisedClass{ class: v1alpha1.SandboxClass(rec.SandboxClass), config: "test-config", @@ -2050,7 +2050,7 @@ func TestRunAndRestoreRejectUnadvertisedRuntime(t *testing.T) { anonGCSClient: fakeObjectStorage{data: release.tarball, gets: &gets}, systemInfoVolumes: newSystemInfoVolumeRefresher(nil, nil), } - rec := testSandboxRecord(release, testPauseImage) + rec := testSandboxRecord(release) for _, tc := range []struct { name string @@ -2106,10 +2106,7 @@ func TestRunAndRestoreRejectUnadvertisedRuntime(t *testing.T) { // registration, whose sandbox_bundle_hash is the hash of rec. func runtimeFor(t *testing.T, rec *sandboxAssetsRecord) *ateletpb.SandboxRuntime { t.Helper() - hash, err := versionBundleHash(rec) - if err != nil { - t.Fatalf("versionBundleHash: %v", err) - } + hash := bundleHash(rec) return &ateletpb.SandboxRuntime{ SandboxClass: rec.SandboxClass, Version: &ateletpb.VersionedSandboxCompat{ @@ -2447,7 +2444,6 @@ func TestPromoteSnapshotRejectsStaleGeneration(t *testing.T) { } { writeLocalSnapshot(t, ateletpath.LocalSnapshotDir(pluginActorUID, stale.GetSnapshotUuid()), sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: a.rec.PauseImage, SnapshotFiles: []string{"checkpoint.img"}, Fidelity: "MEMORY", }, map[string]string{"checkpoint.img": "stale-memory"}) @@ -2581,7 +2577,6 @@ func TestCheckpointSkipsStaleCommitAndKeepsNewerLocal(t *testing.T) { snap3Dir := ateletpath.LocalSnapshotDir(pluginActorUID, snap3.GetSnapshotUuid()) writeLocalSnapshot(t, snap3Dir, sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: a.rec.PauseImage, SnapshotFiles: []string{"checkpoint.img"}, Fidelity: "MEMORY", }, map[string]string{"checkpoint.img": "newer-memory"}) diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 1ce756b21b..916787d7c9 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -53,7 +53,6 @@ import ( "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/internal/installdefaults" "go.ate.dev/substrate/internal/nodepath" - "go.ate.dev/substrate/internal/ocispec" "go.ate.dev/substrate/internal/otlprelay" "go.ate.dev/substrate/internal/pluginsocket" "go.ate.dev/substrate/internal/preview" @@ -471,7 +470,7 @@ func main() { // the worker, so the first Run/Restore on this node hits the cache. // Best-effort: on failure the on-demand fetch in ensureSandboxAssets // still covers correctness. - prewarmer := newRuntimePrewarmer(runtimes, wmService, imageCache) + prewarmer := newRuntimePrewarmer(runtimes, wmService) go prewarmer.run(ctx) // Keep a sandbox runtimes file in each local worker's directory: the // runtimes of the enabled versions of its pool's SandboxConfig, @@ -781,10 +780,9 @@ func (s *Atelet) Run(ctx context.Context, req *ateletpb.RunRequest) (resp *atele actorUID := req.GetActorUid() actorRef := resources.ActorRef{Atespace: req.GetAtespace(), Name: req.GetActorName()} - // The sandbox (binaries + pause image) is the runtime the control plane - // assigned the actor to, resolved through the index. Resolved before any - // on-node work, BeginStart included, so an invalid request changes - // nothing. + // The sandbox binaries are the runtime the control plane assigned the + // actor to, resolved through the index. Resolved before any on-node work, + // BeginStart included, so an invalid request changes nothing. sandboxRec, err := s.advertisedRuntime(ctx, req.GetSandboxRuntime()) if err != nil { return nil, err @@ -854,9 +852,7 @@ func (s *Atelet) Run(ctx context.Context, req *ateletpb.RunRequest) (resp *atele if registration, err = s.systemInfoVolumes.Register(actorUID, actorRef, systemInfoVolumesFor(actorUID, req.GetSpec())); err != nil { return nil, err } - containerSpecs, err := s.prepareOCIBundles(ctx, actorUID, actorRef, - req.GetSpec(), sandboxRec.PauseImage, - ) + containerSpecs, err := s.prepareOCIBundles(ctx, actorUID, actorRef, req.GetSpec()) if err != nil { return nil, err } @@ -1428,11 +1424,11 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res actor := actorRef{atespace: req.GetAtespace(), name: req.GetActorName(), uid: actorUID} actorRef := resources.ActorRef{Atespace: req.GetAtespace(), Name: req.GetActorName()} - // The sandbox (binaries + pause image) that runs the restored workload is - // the runtime the control plane assigned the actor to, resolved through - // the index. The snapshot manifests only supply the files to restore and - // the actor identity. Resolved before any on-node work so an invalid - // request changes nothing. + // The sandbox binaries that run the restored workload are the runtime the + // control plane assigned the actor to, resolved through the index. The + // snapshot manifests only supply the files to restore and the actor + // identity. Resolved before any on-node work so an invalid request changes + // nothing. runtimeRec, err := s.advertisedRuntime(ctx, req.GetSandboxRuntime()) if err != nil { return nil, err @@ -1621,7 +1617,7 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res return err } t := time.Now() - containerSpecs, err = s.prepareOCIBundles(gctx, actorUID, actorRef, req.GetSpec(), runtimeRec.PauseImage) + containerSpecs, err = s.prepareOCIBundles(gctx, actorUID, actorRef, req.GetSpec()) dBundles = time.Since(t) if err != nil { prepFailedPhase = ateattr.SnapshotPhaseOCIUnpack @@ -1644,8 +1640,8 @@ func (s *Atelet) Restore(ctx context.Context, req *ateletpb.RestoreRequest) (res return nil, err } - // Tell worker to do runsc create + runsc restore for pause container and - // all application containers. + // Tell worker to do runsc create + runsc restore for the sandbox and all + // application containers. spec, err := buildWorkerWorkloadSpec(req.GetSpec(), containerSpecs) if err != nil { return nil, apierror.InvalidArgument("invalid workload spec: %v", err) @@ -1938,19 +1934,14 @@ func prepareDurableDirVolume(path string) error { return nil } -// prepareOCIBundles pulls images and assembles OCI bundles for the pause -// container and every application container in spec, in parallel. pauseImage -// comes from the sandbox record, not the workload spec: it is sandbox -// configuration, and on a restore it must be the image the snapshot was taken -// with. It is empty for sandboxes without a pause container, which get no -// pause bundle. It returns each application container's spec, indexed like -// spec.GetContainers(). +// prepareOCIBundles pulls images and assembles OCI bundles for every +// application container in spec, in parallel. It returns each application +// container's spec, indexed like spec.GetContainers(). func (s *Atelet) prepareOCIBundles( ctx context.Context, actorUID string, actorRef resources.ActorRef, spec *ateletpb.WorkloadSpec, - pauseImage string, ) ([]*ateworkerpb.ContainerSpec, error) { // Prepare host folders for volume types that need them. for _, vol := range spec.GetVolumes() { @@ -1963,16 +1954,6 @@ func (s *Atelet) prepareOCIBundles( g, gCtx := errgroup.WithContext(ctx) - if pauseImage != "" { - g.Go(func() error { - // pause is sandbox infra; it mounts no volumes. - if _, err := prepareOCIDirectory(gCtx, s.imageCache, actorUID, ocispec.PauseContainer, pauseImage, nil, nil); err != nil { - return wrapFileSystemErr("while creating pause OCI bundle", err) - } - return nil - }) - } - // Application containers. containerSpecs := make([]*ateworkerpb.ContainerSpec, len(spec.GetContainers())) for i, ctr := range spec.GetContainers() { diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index cae10e8d8b..3d804d9690 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -58,8 +58,6 @@ import ( "google.golang.org/protobuf/types/known/emptypb" ) -const testPauseImage = "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" - const ( snapshotOwnerUID = "3d7f1b62-8a04-4c19-b5e6-2f9c7a1d0e38" @@ -309,7 +307,7 @@ func TestSnapshotManifestActorMetadata(t *testing.T) { // Run/Restore, which has no fidelity, parses back with an empty fidelity and // does not serialize a fidelity key at all. func TestSnapshotManifestFidelityAbsent(t *testing.T) { - legacy := []byte(`{"sandboxClass":"gvisor","pauseImage":"` + testPauseImage + `","snapshotFiles":["checkpoint.img"]}`) + legacy := []byte(`{"sandboxClass":"gvisor","snapshotFiles":["checkpoint.img"]}`) rec, err := unmarshalSandboxRecord(legacy) if err != nil { t.Fatalf("unmarshalSandboxRecord(legacy manifest): %v", err) @@ -327,67 +325,26 @@ func TestSnapshotManifestFidelityAbsent(t *testing.T) { } } -// TestSnapshotManifestPauseImage pins that only a gVisor manifest needs a -// pause image: micro-VM sandboxes run no pause container. -func TestSnapshotManifestPauseImage(t *testing.T) { - for _, tc := range []struct { - name string - manifest string - wantErr bool - }{ - {"gvisor without pause image", `{"sandboxClass":"gvisor","snapshotFiles":["checkpoint.img"]}`, true}, - {"gvisor with pause image", `{"sandboxClass":"gvisor","pauseImage":"` + testPauseImage + `","snapshotFiles":["checkpoint.img"]}`, false}, - {"microvm without pause image", `{"sandboxClass":"microvm","snapshotFiles":["checkpoint.img"]}`, false}, - } { - t.Run(tc.name, func(t *testing.T) { - _, err := unmarshalSandboxRecord([]byte(tc.manifest)) - if tc.wantErr { - if err == nil { - t.Fatal("unmarshalSandboxRecord succeeded, want an error") - } - if !strings.Contains(err.Error(), "pauseImage") { - t.Errorf("error = %v, want it to name pauseImage", err) - } - } else if err != nil { - t.Fatalf("unmarshalSandboxRecord: %v", err) - } - }) - } -} - -// TestPrepareOCIBundlesPause pins that the pause bundle is built only when the -// sandbox has a pause image. -func TestPrepareOCIBundlesPause(t *testing.T) { +// TestPrepareOCIBundlesNoSandboxBundle pins that only application containers +// get a bundle. The worker writes the sandbox spec itself. +func TestPrepareOCIBundlesNoSandboxBundle(t *testing.T) { + useTempNodeDirs(t) host := imageVolumeTestRegistry(t) image := host + "/actor:v1" pushTestImage(t, image, singleFileLayer(t, "bin/app", "app")) spec := &ateletpb.WorkloadSpec{ Containers: []*ateletpb.Container{{Name: "app", Image: image, Command: []string{"/bin/app"}}}, } - - for _, tc := range []struct { - name string - pauseImage string - wantPause bool - }{ - {"with pause image", image, true}, - {"without pause image", "", false}, - } { - t.Run(tc.name, func(t *testing.T) { - useTempNodeDirs(t) - const actorUID = "actor-uid-1" - s := &Atelet{imageCache: newImageVolumeStore(t)} - if _, err := s.prepareOCIBundles(t.Context(), actorUID, resources.ActorRef{}, spec, tc.pauseImage); err != nil { - t.Fatalf("prepareOCIBundles: %v", err) - } - if _, err := os.Stat(filepath.Join(ateletpath.OCIBundlePath(actorUID, "app"), imagecache.OverlaySpecFileName)); err != nil { - t.Errorf("app bundle: %v", err) - } - _, err := os.Stat(ateletpath.OCIBundlePath(actorUID, ocispec.PauseContainer)) - if gotPause := err == nil; gotPause != tc.wantPause { - t.Errorf("pause bundle exists = %v, want %v (stat: %v)", gotPause, tc.wantPause, err) - } - }) + const actorUID = "actor-uid-1" + s := &Atelet{imageCache: newImageVolumeStore(t)} + if _, err := s.prepareOCIBundles(t.Context(), actorUID, resources.ActorRef{}, spec); err != nil { + t.Fatalf("prepareOCIBundles: %v", err) + } + if _, err := os.Stat(filepath.Join(ateletpath.OCIBundlePath(actorUID, "app"), imagecache.OverlaySpecFileName)); err != nil { + t.Errorf("app bundle: %v", err) + } + if _, err := os.Stat(ateletpath.OCIBundlePath(actorUID, ocispec.PauseContainer)); !os.IsNotExist(err) { + t.Errorf("sandbox bundle stat = %v, want it absent", err) } } @@ -413,7 +370,7 @@ func TestPrepareOCIBundlesFillsContainerSpec(t *testing.T) { } s := &Atelet{imageCache: newImageVolumeStore(t)} - got, err := s.prepareOCIBundles(t.Context(), "actor-uid-1", resources.ActorRef{}, spec, "") + got, err := s.prepareOCIBundles(t.Context(), "actor-uid-1", resources.ActorRef{}, spec) if err != nil { t.Fatalf("prepareOCIBundles: %v", err) } @@ -431,7 +388,6 @@ func TestPrepareOCIBundlesFillsContainerSpec(t *testing.T) { func TestSnapshotManifestRejectsNonLocalFile(t *testing.T) { manifest, err := json.Marshal(sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: testPauseImage, SnapshotFiles: []string{"../outside"}, }) if err != nil { @@ -445,7 +401,6 @@ func TestSnapshotManifestRejectsNonLocalFile(t *testing.T) { func TestSnapshotManifestRejectsDataFileNotInSnapshotFiles(t *testing.T) { manifest, err := json.Marshal(sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: testPauseImage, SnapshotFiles: []string{"checkpoint.img"}, DataSnapshotFiles: []string{"data.tar"}, }) @@ -1462,7 +1417,6 @@ func TestUploadLocalCheckpointDir(t *testing.T) { fullRec := func(class string) sandboxAssetsRecord { return sandboxAssetsRecord{ SandboxClass: class, - PauseImage: testPauseImage, SnapshotFiles: []string{"config.json", "memory-ranges", "data.tar"}, DataSnapshotFiles: []string{"data.tar"}, Fidelity: ateattr.SnapshotFidelityMemory, @@ -1513,7 +1467,6 @@ func TestUploadLocalCheckpointDir(t *testing.T) { dir := filepath.Join(t.TempDir(), "pause-snap-1") writeLocalSnapshot(t, dir, sandboxAssetsRecord{ SandboxClass: "microvm", - PauseImage: testPauseImage, SnapshotFiles: []string{"rootfs-upper-app.tar", "data.tar"}, DataSnapshotFiles: []string{"data.tar"}, Fidelity: ateattr.SnapshotFidelityRootfs, @@ -1577,7 +1530,6 @@ func TestUploadLocalCheckpointDir(t *testing.T) { dir := filepath.Join(t.TempDir(), "pause-snap-1") writeLocalSnapshot(t, dir, sandboxAssetsRecord{ SandboxClass: "microvm", - PauseImage: testPauseImage, SnapshotFiles: []string{"data.tar"}, }, map[string]string{"data.tar": "data"}) diff --git a/cmd/atelet/runtime_prewarm.go b/cmd/atelet/runtime_prewarm.go index 6801ed0706..ec648b138f 100644 --- a/cmd/atelet/runtime_prewarm.go +++ b/cmd/atelet/runtime_prewarm.go @@ -21,19 +21,14 @@ import ( "io/fs" "log/slog" "os" - "sync" "time" "k8s.io/client-go/util/workqueue" - - "go.ate.dev/substrate/internal/imagecache" ) // defaultPrewarmAttemptTimeout bounds one prewarm attempt. The queue is // drained by one worker, so an attempt that never returned would block -// every other runtime's prewarm. The asset downloads are bounded by the -// asset cache's own fetch timeout; this covers the pause image pull beside -// them. +// every other runtime's prewarm. const defaultPrewarmAttemptTimeout = 10 * time.Minute // prewarmMaxRetries bounds how often one failed prewarm is retried before @@ -60,7 +55,7 @@ type prewarmItem struct { } // runtimePrewarmer fetches the default runtime of each worker on this node -// into the asset and image caches before any actor asks for it, so the +// into the asset cache before any actor asks for it, so the // fetch inside the first Run/Restore on the node is a cache hit instead of // a download+extract on the critical path. The worker sandbox syncer hands // it each default runtime a worker advertises, once the worker's file says @@ -76,14 +71,12 @@ type prewarmItem struct { // without fetching, retrying, or recreating anything. // // Prewarming is purely a latency optimization: every failure is logged and -// left to the on-demand fetch in ensureSandboxAssets and the pull inside -// prepareOCIBundles, which remain the correctness path. Only default -// runtimes are prewarmed; the first restore of another version a worker -// advertises pays its download. +// left to the on-demand fetch in ensureSandboxAssets, which remains the +// correctness path. Only default runtimes are prewarmed; the first restore of +// another version a worker advertises pays its download. type runtimePrewarmer struct { index *runtimeIndex assets sandboxAssetFetcher - images *imagecache.Store // queue carries items. It dedupes while an item waits, and it // rate-limits retries after failures. A single worker drains it, which // also serializes downloads so concurrent prewarms never compete for @@ -100,11 +93,10 @@ type runtimePrewarmer struct { onFinished func(item prewarmItem, err error) } -func newRuntimePrewarmer(index *runtimeIndex, assets sandboxAssetFetcher, images *imagecache.Store) *runtimePrewarmer { +func newRuntimePrewarmer(index *runtimeIndex, assets sandboxAssetFetcher) *runtimePrewarmer { return &runtimePrewarmer{ index: index, assets: assets, - images: images, attemptTimeout: defaultPrewarmAttemptTimeout, // Downloads fail on the scale of network timeouts, not API conflicts, // so back off in seconds and cap in minutes rather than the @@ -183,11 +175,10 @@ func (p *runtimePrewarmer) process(ctx context.Context, item prewarmItem) { } // prewarm fetches the runtime's assets into the asset cache, linked into -// the item's directory, and its pause image into the image cache, the two -// side by side. Racing an on-demand ensureSandboxAssets for the same assets -// is safe: the asset cache runs one download per asset and serves every -// caller from it, and the image cache collapses concurrent pulls of one -// digest. A directory the syncer already filled costs a stat per asset. +// the item's directory. Racing an on-demand ensureSandboxAssets for the same +// assets is safe: the asset cache runs one download per asset and serves +// every caller from it. A directory the syncer already filled costs a stat +// per asset. // // Nothing here creates a directory: the staging goes into the item's // directory as it is, so a directory removed before or during the work, by @@ -204,20 +195,7 @@ func (p *runtimePrewarmer) prewarm(ctx context.Context, item prewarmItem, rt *in return err } - var assetErr, imageErr error - var wg sync.WaitGroup - wg.Go(func() { - _, assetErr = p.assets.stageSandboxAssets(ctx, rt.rec, item.dir) - }) - if rt.rec.PauseImage != "" { - wg.Go(func() { - if _, err := p.images.EnsureImage(ctx, rt.rec.PauseImage); err != nil { - imageErr = fmt.Errorf("while prewarming pause image %q: %w", rt.rec.PauseImage, err) - } - }) - } - wg.Wait() - if err := errors.Join(assetErr, imageErr); err != nil { + if _, err := p.assets.stageSandboxAssets(ctx, rt.rec, item.dir); err != nil { if gone, gerr := dirGone(item.dir); gerr == nil && gone { slog.DebugContext(ctx, "Sandbox runtime prewarm superseded while staging", slog.String("runtime_hash", item.hash), slog.String("dir", item.dir)) return nil @@ -229,7 +207,6 @@ func (p *runtimePrewarmer) prewarm(ctx context.Context, item prewarmItem, rt *in slog.String("config", rt.config), slog.String("version", rt.version), slog.Int("assets", len(rt.rec.Assets)), - slog.String("pause_image", rt.rec.PauseImage), slog.String("dir", item.dir), slog.Duration("duration", time.Since(t))) return nil diff --git a/cmd/atelet/runtime_prewarm_test.go b/cmd/atelet/runtime_prewarm_test.go index 4dc1d72482..19105e3cf9 100644 --- a/cmd/atelet/runtime_prewarm_test.go +++ b/cmd/atelet/runtime_prewarm_test.go @@ -30,11 +30,6 @@ import ( "testing" "time" - "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/empty" - "github.com/google/go-containerregistry/pkg/v1/mutate" - "github.com/google/go-containerregistry/pkg/v1/remote" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sruntime "k8s.io/apimachinery/pkg/runtime" kubeinformers "k8s.io/client-go/informers" @@ -42,34 +37,11 @@ import ( "go.ate.dev/substrate/cmd/atelet/internal/ateletpath" "go.ate.dev/substrate/cmd/atelet/internal/filecache" - "go.ate.dev/substrate/internal/imagecache" "go.ate.dev/substrate/pkg/api/v1alpha1" atefake "go.ate.dev/substrate/pkg/client/clientset/versioned/fake" "go.ate.dev/substrate/pkg/client/informers/externalversions" ) -// pushPauseImage pushes a tiny image to ref and returns its manifest digest, -// so tests can later assert a cache hit by digest with the registry gone. -func pushPauseImage(t *testing.T, ref string) v1.Hash { - t.Helper() - img, err := mutate.AppendLayers(empty.Image, singleFileLayer(t, "pause", "pause bytes")) - if err != nil { - t.Fatalf("mutate.AppendLayers: %v", err) - } - digest, err := img.Digest() - if err != nil { - t.Fatalf("img.Digest: %v", err) - } - tag, err := name.ParseReference(ref, name.Insecure) - if err != nil { - t.Fatalf("name.ParseReference(%q): %v", ref, err) - } - if err := remote.Write(tag, img); err != nil { - t.Fatalf("remote.Write(%q): %v", ref, err) - } - return digest -} - func gvisorConfig(name, url, sha string) *v1alpha1.SandboxConfig { return &v1alpha1.SandboxConfig{ ObjectMeta: metav1.ObjectMeta{Name: name}, @@ -77,8 +49,7 @@ func gvisorConfig(name, url, sha string) *v1alpha1.SandboxConfig { SandboxClass: v1alpha1.SandboxClassGvisor, DefaultVersion: "v1", Versions: []v1alpha1.SandboxVersionConfig{{ - Name: "v1", - PauseImage: "registry.k8s.io/pause@sha256:abc", + Name: "v1", Assets: map[string]map[string]v1alpha1.AssetFile{ runtime.GOARCH: { gvisorAssetName: {URL: url, SHA256: sha}, @@ -100,35 +71,21 @@ func (hangingObjectStorage) GetObject(ctx context.Context, _, _ string) (io.Read func (hangingObjectStorage) PutObject(_ context.Context, _, _ string, _ io.Reader) error { return nil } -// pauseImageInTestRegistry pushes a pause image to a test registry and -// returns its digest-pinned reference, the only form a runtime hash accepts. -func pauseImageInTestRegistry(t *testing.T) (ref string, host string) { - t.Helper() - host = imageVolumeTestRegistry(t) - digest := pushPauseImage(t, host+"/pause:3.10") - return host + "/pause@" + digest.String(), host -} - // indexedTestRuntime registers a runtime with one gvisor release asset, the -// tarball content, and the given pause image as a worker's default, and -// returns its hash. -func indexedTestRuntime(t *testing.T, index *runtimeIndex, content []byte, pauseImage string) string { +// tarball content, as a worker's default, and returns its hash. +func indexedTestRuntime(t *testing.T, index *runtimeIndex, content []byte) string { t.Helper() - return indexedTestRuntimeAt(t, index, "gs://bucket/gvisor.tar", content, pauseImage) + return indexedTestRuntimeAt(t, index, "gs://bucket/gvisor.tar", content) } // indexedTestRuntimeAt is indexedTestRuntime with the asset at url. -func indexedTestRuntimeAt(t *testing.T, index *runtimeIndex, url string, content []byte, pauseImage string) string { +func indexedTestRuntimeAt(t *testing.T, index *runtimeIndex, url string, content []byte) string { t.Helper() rec := &sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: pauseImage, Assets: map[string]assetEntry{gvisorAssetName: {URL: url, SHA256: fmt.Sprintf("%x", sha256.Sum256(content))}}, } - hash, err := versionBundleHash(rec) - if err != nil { - t.Fatal(err) - } + hash := bundleHash(rec) index.register("pod-a", advertisedClass{class: v1alpha1.SandboxClassGvisor, config: "gvisor-release", hashed: []hashedVersion{{ versionPlan: versionPlan{name: "v1", isDefault: true, rec: rec}, hash: hash, }}}) @@ -147,25 +104,16 @@ func defaultItem(t *testing.T, root, hash string) prewarmItem { return prewarmItem{hash: hash, dir: dir} } -func newTestImageStore(t *testing.T) *imagecache.Store { - t.Helper() - store, err := imagecache.New(t.TempDir()) - if err != nil { - t.Fatalf("imagecache.New: %v", err) - } - return store -} - // TestRuntimePrewarmProcess covers the worker's failure handling: a failed // prewarm is requeued with backoff until prewarmMaxRetries and then dropped, // and an item naming a hash that is not in the index is dropped at once. func TestRuntimePrewarmProcess(t *testing.T) { ctx := context.Background() index := newRuntimeIndex() - hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t), "") + hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t)) root := t.TempDir() herder := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}} - p := newRuntimePrewarmer(index, herder, newTestImageStore(t)) + p := newRuntimePrewarmer(index, herder) item := defaultItem(t, root, hash) for i := 1; i <= prewarmMaxRetries; i++ { @@ -194,7 +142,7 @@ func TestRuntimePrewarmSuperseded(t *testing.T) { ctx := context.Background() index := newRuntimeIndex() content := gvisorReleaseTarball(t) - hash := indexedTestRuntime(t, index, content, "") + hash := indexedTestRuntime(t, index, content) root := t.TempDir() herder := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: content}} @@ -203,7 +151,7 @@ func TestRuntimePrewarmSuperseded(t *testing.T) { if err := os.RemoveAll(item.dir); err != nil { t.Fatal(err) } - p := newRuntimePrewarmer(index, herder, newTestImageStore(t)) + p := newRuntimePrewarmer(index, herder) p.process(ctx, item) if got := p.queue.NumRequeues(item); got != 0 { t.Errorf("NumRequeues after a superseded prewarm = %d, want 0 (not retried)", got) @@ -219,7 +167,7 @@ func TestRuntimePrewarmSuperseded(t *testing.T) { t.Run("worker gone while staging", func(t *testing.T) { item := defaultItem(t, root, hash) removing := &removingFetcher{inner: herder, remove: filepath.Join(root, "workers", "pod-a")} - p := newRuntimePrewarmer(index, removing, newTestImageStore(t)) + p := newRuntimePrewarmer(index, removing) p.process(ctx, item) if got := p.queue.NumRequeues(item); got != 0 { t.Errorf("NumRequeues after the worker went mid-staging = %d, want 0 (not retried)", got) @@ -248,25 +196,23 @@ func (r *removingFetcher) stageSandboxAssets(ctx context.Context, rec *sandboxAs return r.inner.stageSandboxAssets(ctx, rec, dstDir) } -// TestRuntimePrewarmFillsBothCaches covers a successful prewarm: the asset +// TestRuntimePrewarmFillsTheAssetCache covers a successful prewarm: the asset // lands in the asset cache, linked into the runtime's directory so an -// eviction pass keeps it, and the pause image lands in the image cache. A -// second prewarm of the same directory fetches nothing. -func TestRuntimePrewarmFillsBothCaches(t *testing.T) { +// eviction pass keeps it. A second prewarm of the same directory fetches +// nothing. +func TestRuntimePrewarmFillsTheAssetCache(t *testing.T) { ctx := context.Background() - pauseRef, _ := pauseImageInTestRegistry(t) content := gvisorReleaseTarball(t) index := newRuntimeIndex() - hash := indexedTestRuntime(t, index, content, pauseRef) + hash := indexedTestRuntime(t, index, content) rt, _ := index.lookup(hash) root := t.TempDir() - images := newTestImageStore(t) var gets atomic.Int32 // No min age, so the only thing standing between the entry and a pass is // the directory's link. s := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root, filecache.WithMinAge(0)), anonGCSClient: fakeObjectStorage{data: content, gets: &gets}} - p := newRuntimePrewarmer(index, s, images) + p := newRuntimePrewarmer(index, s) item := defaultItem(t, root, hash) if err := p.prewarm(ctx, item, rt); err != nil { t.Fatalf("prewarm: %v", err) @@ -277,9 +223,6 @@ func TestRuntimePrewarmFillsBothCaches(t *testing.T) { if _, err := os.Stat(filepath.Join(item.dir, gvisorAssetName, "runsc")); err != nil { t.Errorf("prewarm did not link the runtime's asset into its directory: %v", err) } - if _, err := images.EnsureImage(ctx, pauseRef); err != nil { - t.Errorf("pause image not prewarmed into the image cache: %v", err) - } stats, err := s.sandboxAssetCache.EvictUnused(ctx, 1<<40, false) if err != nil { t.Fatalf("EvictUnused: %v", err) @@ -307,24 +250,19 @@ func TestRuntimePrewarmFillsBothCaches(t *testing.T) { } } -// TestRuntimePrewarmAssetFailureStillPullsImage pins that the two halves -// are independent: a failing asset fetch does not stop the pause image pull. -func TestRuntimePrewarmAssetFailureStillPullsImage(t *testing.T) { +// TestRuntimePrewarmAssetFailure pins that a failing asset fetch fails the +// prewarm. +func TestRuntimePrewarmAssetFailure(t *testing.T) { ctx := context.Background() - pauseRef, _ := pauseImageInTestRegistry(t) index := newRuntimeIndex() - hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t), pauseRef) + hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t)) rt, _ := index.lookup(hash) - images := newTestImageStore(t) root := t.TempDir() - p := newRuntimePrewarmer(index, &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}}, images) + p := newRuntimePrewarmer(index, &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{err: errors.New("bucket unavailable")}}) if err := p.prewarm(ctx, defaultItem(t, root, hash), rt); err == nil { t.Error("prewarm returned nil despite the asset fetch failing") } - if _, err := images.EnsureImage(ctx, pauseRef); err != nil { - t.Errorf("pause image not pulled when the asset fetch failed: %v", err) - } } // TestRuntimePrewarmAttemptTimeout verifies a single attempt is bounded by @@ -332,10 +270,10 @@ func TestRuntimePrewarmAssetFailureStillPullsImage(t *testing.T) { // that never returned would block every other runtime's prewarm. func TestRuntimePrewarmAttemptTimeout(t *testing.T) { index := newRuntimeIndex() - hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t), "") + hash := indexedTestRuntime(t, index, gvisorReleaseTarball(t)) rt, _ := index.lookup(hash) root := t.TempDir() - p := newRuntimePrewarmer(index, &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: hangingObjectStorage{}}, newTestImageStore(t)) + p := newRuntimePrewarmer(index, &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: hangingObjectStorage{}}) p.attemptTimeout = 50 * time.Millisecond item := defaultItem(t, root, hash) @@ -479,9 +417,9 @@ func (r *recordingPrewarm) count() int { // request. func TestDefaultRuntimePrewarmThroughSyncer(t *testing.T) { useTempAssetScratch(t) // the syncer's runsc probe stages into it - // The worker's own prewarm call returns only when the asset has landed - // and the pause-image pull has finished. Waiting for the call keeps the - // pull's last writes from racing the TempDir cleanup. + // The worker's own prewarm call returns only when the asset has landed. + // Waiting for the call keeps its last writes from racing the TempDir + // cleanup. prewarmed := make(chan error, 1) onFinished := func(_ prewarmItem, err error) { // Non-blocking: a failed attempt is retried, and the worker must not @@ -492,10 +430,8 @@ func TestDefaultRuntimePrewarmThroughSyncer(t *testing.T) { } } - pauseRef, _ := pauseImageInTestRegistry(t) content := gvisorReleaseTarball(t) cfg := gvisorConfig("gvisor-release", "gs://bucket/gvisor.tar", fmt.Sprintf("%x", sha256.Sum256(content))) - cfg.Spec.Versions[0].PauseImage = pauseRef kube := kubefake.NewClientset([]k8sruntime.Object{workerPod("pod-a", "pool")}...) ate := atefake.NewSimpleClientset(resolvedPool("pool", "gvisor-release"), cfg) @@ -507,7 +443,7 @@ func TestDefaultRuntimePrewarmThroughSyncer(t *testing.T) { workersDir := filepath.Join(root, "workers") // on the cache's filesystem index := newRuntimeIndex() herder := &Atelet{sandboxAssetCache: newTestAssetCacheAt(t, root), anonGCSClient: fakeObjectStorage{data: content}} - p := newRuntimePrewarmer(index, herder, newTestImageStore(t)) + p := newRuntimePrewarmer(index, herder) p.onFinished = onFinished go p.run(ctx) // Join the worker before the TempDir cleanups remove what it writes into. diff --git a/cmd/atelet/sandbox_assets.go b/cmd/atelet/sandbox_assets.go index c5043cbecc..bea59b8a62 100644 --- a/cmd/atelet/sandbox_assets.go +++ b/cmd/atelet/sandbox_assets.go @@ -42,7 +42,6 @@ import ( "go.ate.dev/substrate/cmd/atelet/internal/filecache" "go.ate.dev/substrate/internal/apierror" "go.ate.dev/substrate/internal/resources" - "go.ate.dev/substrate/pkg/api/v1alpha1" "go.ate.dev/substrate/pkg/objectstorage" objectstorev1 "go.ate.dev/substrate/pkg/proto/objectstorepb/v1" "google.golang.org/grpc/codes" @@ -77,19 +76,14 @@ func (e assetEntry) displayURL() string { } // sandboxAssetsRecord is the sandbox runtime an actor is running, projected onto -// the local node's architecture: the sandbox class and pause image plus the -// asset set keyed by asset name (gVisor uses a single "gvisor" release-tarball -// asset). +// the local node's architecture: the sandbox class plus the asset set keyed by +// asset name (gVisor uses a single "gvisor" release-tarball asset). // It is both the per-actor on-node record (written at Run/Restore, read at // Checkpoint) and the snapshot manifest (written at Checkpoint, read at // Restore). type sandboxAssetsRecord struct { SandboxClass string `json:"sandboxClass"` Assets map[string]assetEntry `json:"assets"` - // PauseImage is the root sandbox container's image. It is recorded here - // rather than taken from the request at Restore so a snapshot is rebuilt - // with the same sandbox it was captured from. Only gVisor has one. - PauseImage string `json:"pauseImage,omitempty"` // Actor identity makes a flat snapshot self-identifying if control-plane // persistence is unavailable. Atespace string `json:"atespace,omitempty"` @@ -114,15 +108,6 @@ type sandboxAssetsRecord struct { Fidelity string `json:"fidelity,omitempty"` } -// checkPauseImage requires a pause image for gVisor, the only sandbox class -// that runs a pause container. -func checkPauseImage(sandboxClass, pauseImage string) error { - if sandboxClass == string(v1alpha1.SandboxClassGvisor) && pauseImage == "" { - return fmt.Errorf("gvisor sandbox has no pauseImage") - } - return nil -} - // ensureSandboxAssets stages every asset in the record under dstDir, named // by asset name, and returns a map of asset name to the local path the worker // runs it from. gVisor has a single "gvisor" release-tarball asset (staged @@ -676,9 +661,6 @@ func unmarshalSandboxRecord(data []byte) (*sandboxAssetsRecord, error) { if err := json.Unmarshal(data, rec); err != nil { return nil, fmt.Errorf("while parsing sandbox record/manifest: %w", err) } - if err := checkPauseImage(rec.SandboxClass, rec.PauseImage); err != nil { - return nil, fmt.Errorf("invalid sandbox record/manifest: %w", err) - } if err := validateSnapshotFiles(rec.SnapshotFiles); err != nil { return nil, fmt.Errorf("sandbox record/manifest has invalid snapshotFiles: %w", err) } diff --git a/cmd/atelet/sandbox_runtime_index.go b/cmd/atelet/sandbox_runtime_index.go index 0dc8050808..335e06935d 100644 --- a/cmd/atelet/sandbox_runtime_index.go +++ b/cmd/atelet/sandbox_runtime_index.go @@ -158,11 +158,7 @@ func checkIndexedRuntime(hash string, e indexedRuntimeFile) error { if e.Record == nil { return errors.New("no record") } - got, err := versionBundleHash(e.Record) - if err != nil { - return fmt.Errorf("while hashing the record: %w", err) - } - if got != hash { + if got := bundleHash(e.Record); got != hash { return fmt.Errorf("record hashes to %s", got) } return nil @@ -237,9 +233,9 @@ func (x *runtimeIndex) registerRuntimes(config string, hashed []hashedVersion) e // addRuntime records v under its hash, unless the hash is already known, // persisting the index first when it has a path. The hash covers every -// asset's name and digest and the pause image, so two versions with one -// hash differ at most in where identical bytes are fetched from; keeping -// the first keeps the version the file was built from. Called with mu held. +// asset's name and digest, so two versions with one hash differ at most in +// where identical bytes are fetched from; keeping the first keeps the version +// the file was built from. Called with mu held. func (x *runtimeIndex) addRuntime(config string, v hashedVersion) error { if _, ok := x.runtimes[v.hash]; ok { return nil diff --git a/cmd/atelet/sandbox_runtime_index_test.go b/cmd/atelet/sandbox_runtime_index_test.go index 535939924c..d58195c3e9 100644 --- a/cmd/atelet/sandbox_runtime_index_test.go +++ b/cmd/atelet/sandbox_runtime_index_test.go @@ -35,14 +35,9 @@ func hashedTestVersion(t *testing.T, name string, isDefault bool, sha string) ha t.Helper() rec := &sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: testPauseImage, Assets: map[string]assetEntry{gvisorAssetName: {URL: "gs://bucket/" + name, SHA256: sha}}, } - h, err := versionBundleHash(rec) - if err != nil { - t.Fatal(err) - } - return hashedVersion{versionPlan: versionPlan{name: name, isDefault: isDefault, rec: rec}, hash: h} + return hashedVersion{versionPlan: versionPlan{name: name, isDefault: isDefault, rec: rec}, hash: bundleHash(rec)} } func TestRuntimeIndex(t *testing.T) { diff --git a/cmd/atelet/snapshot_transfer_test.go b/cmd/atelet/snapshot_transfer_test.go index 3847a66a50..466ac3c850 100644 --- a/cmd/atelet/snapshot_transfer_test.go +++ b/cmd/atelet/snapshot_transfer_test.go @@ -105,7 +105,7 @@ func TestUploadSnapshotManifestLast(t *testing.T) { t.Fatal(err) } dir := t.TempDir() - rec := sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: testPauseImage, SnapshotFiles: []string{"a", "b", "c"}} + rec := sandboxAssetsRecord{SandboxClass: "gvisor", SnapshotFiles: []string{"a", "b", "c"}} for _, f := range rec.SnapshotFiles { if err := os.WriteFile(dir+"/"+f, []byte(f), 0o600); err != nil { t.Fatal(err) @@ -181,7 +181,6 @@ func TestObjectStorePluginErrorCodeThroughAtelet(t *testing.T) { } rec := sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: testPauseImage, SnapshotFiles: []string{"a"}, Fidelity: ateattr.SnapshotFidelityMemory, } @@ -245,7 +244,7 @@ func TestMintErrorCodeThroughAtelet(t *testing.T) { if err != nil { t.Fatal(err) } - rec := sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: testPauseImage, SnapshotFiles: []string{"a"}, Fidelity: ateattr.SnapshotFidelityMemory} + rec := sandboxAssetsRecord{SandboxClass: "gvisor", SnapshotFiles: []string{"a"}, Fidelity: ateattr.SnapshotFidelityMemory} transfers := []struct { name string retryable bool @@ -304,7 +303,7 @@ func TestObjectStorePluginErrorKeepsCredentialsOut(t *testing.T) { if err != nil { t.Fatal(err) } - rec := sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: testPauseImage, SnapshotFiles: []string{"a"}, Fidelity: ateattr.SnapshotFidelityMemory} + rec := sandboxAssetsRecord{SandboxClass: "gvisor", SnapshotFiles: []string{"a"}, Fidelity: ateattr.SnapshotFidelityMemory} transfers := []struct { name string uri string // the URI the transfer names in its error, when it takes one @@ -462,7 +461,6 @@ func TestCheckpointUploadWaitsOutPluginOutage(t *testing.T) { } rec := sandboxAssetsRecord{ SandboxClass: "gvisor", - PauseImage: testPauseImage, SnapshotFiles: []string{"a"}, Fidelity: ateattr.SnapshotFidelityMemory, } diff --git a/cmd/atelet/worker_sandbox_syncer.go b/cmd/atelet/worker_sandbox_syncer.go index ac2b7582f8..81ff49866f 100644 --- a/cmd/atelet/worker_sandbox_syncer.go +++ b/cmd/atelet/worker_sandbox_syncer.go @@ -369,10 +369,7 @@ func (s *workerSandboxSyncer) registerAhead(pool *v1alpha1.WorkerPool) { if err != nil { return } - hashed, err := hashVersions(context.Background(), plan) - if err != nil { - return - } + hashed := hashVersions(plan) if err := s.index.registerRuntimes(plan.config, hashed); err != nil { // The syncs write it again, and report it if it keeps failing. slog.Warn("Could not index a changed pool's runtimes ahead of its sync", slog.String("pool", pool.Name), slog.Any("err", err)) diff --git a/cmd/atelet/worker_sandbox_syncer_test.go b/cmd/atelet/worker_sandbox_syncer_test.go index 38cb0a5b3b..c6585e5e00 100644 --- a/cmd/atelet/worker_sandbox_syncer_test.go +++ b/cmd/atelet/worker_sandbox_syncer_test.go @@ -394,11 +394,7 @@ func TestWorkerSandboxSyncerRegistersAheadOfJitter(t *testing.T) { if err != nil { t.Fatal(err) } - hashed, err := hashVersions(context.Background(), plan) - if err != nil { - t.Fatal(err) - } - for _, v := range hashed { + for _, v := range hashVersions(plan) { if _, ok := s.index.lookup(v.hash); !ok { t.Errorf("runtime %s (%s) does not resolve before the jittered sync", v.name, v.hash) } diff --git a/cmd/atelet/worker_sandbox_versions.go b/cmd/atelet/worker_sandbox_versions.go index d964c9c4de..f5bfcb0e54 100644 --- a/cmd/atelet/worker_sandbox_versions.go +++ b/cmd/atelet/worker_sandbox_versions.go @@ -18,10 +18,10 @@ package main // in the steps syncPod runs: // - resolve: find the SandboxConfig of the pod's WorkerPool // - plan: pick the config's versions this node can run -// - build: hash each version's assets and pause image digests into its -// sandbox_bundle_hash, and probe what a snapshot taken on the runtime -// must find again: the host CPU identity for microVM, the runsc -// binary's CPU features for gVisor +// - build: hash each version's asset digests into its sandbox_bundle_hash, +// and probe what a snapshot taken on the runtime must find again: the +// host CPU identity for microVM, the runsc binary's CPU features for +// gVisor // - write: replace the file in the worker directory if its content changed // // worker_sandbox_syncer.go decides when each pod is synced, and creates and @@ -42,7 +42,6 @@ import ( "strings" "time" - "github.com/google/go-containerregistry/pkg/name" corev1 "k8s.io/api/core/v1" "go.ate.dev/substrate/internal/hardware" @@ -55,8 +54,8 @@ import ( // errPermanent marks a failure that retrying cannot fix, so the syncer drops // the item instead of backing off until the next event for the pod: // - the pod has no worker pool label -// - the config's default version is missing, has no assets for this -// node's architecture, or has a pause image not pinned by digest +// - the config's default version is missing or has no assets for this +// node's architecture // - the default version's runsc runs but fails, prints no features, or // prints one this atelet does not know; the binary is content-addressed, // so running it again gives the same answer @@ -197,7 +196,6 @@ func recordForArch(cfg *v1alpha1.SandboxConfig, ver *v1alpha1.SandboxVersionConf } rec := &sandboxAssetsRecord{ SandboxClass: string(cfg.Spec.SandboxClass), - PauseImage: ver.PauseImage, Assets: make(map[string]assetEntry, len(files)), } for name, f := range files { @@ -213,24 +211,13 @@ type hashedVersion struct { } // hashVersions hashes each version of plan. The hashes come from the -// SandboxConfig spec alone, so it does no IO. A version that cannot be -// hashed is left out, unless it is the default, without which no worker of -// the pool could start actors on this node. -func hashVersions(ctx context.Context, plan *sandboxPlan) ([]hashedVersion, error) { +// SandboxConfig spec alone, so it does no IO. +func hashVersions(plan *sandboxPlan) []hashedVersion { hashed := make([]hashedVersion, 0, len(plan.versions)) for _, v := range plan.versions { - h, err := versionBundleHash(v.rec) - if err != nil { - err = fmt.Errorf("SandboxConfig %q, version %q: %w", plan.config, v.name, err) - if v.isDefault { - return nil, fmt.Errorf("%w: %w", errPermanent, err) - } - slog.WarnContext(ctx, "Leaving a sandbox version out of the worker sandbox runtimes file", slog.Any("err", err)) - continue - } - hashed = append(hashed, hashedVersion{versionPlan: v, hash: h}) + hashed = append(hashed, hashedVersion{versionPlan: v, hash: bundleHash(v.rec)}) } - return hashed, nil + return hashed } // buildFile returns the file content naming the runtimes of the hashed @@ -248,10 +235,10 @@ func hashVersions(ctx context.Context, plan *sandboxPlan) ([]hashedVersion, erro // fetched and probed, serially. // // Listing a runtime does not promise its assets are on the node: probing -// fetches a gVisor version's runsc the first time this atelet sees it, but -// not its pause image, and nothing of a microVM version. The syncer has -// the default runtime prewarmed once the file is written, and Run/Restore -// fetch any other version on demand. +// fetches a gVisor version's runsc the first time this atelet sees it, and +// nothing of a microVM version. The syncer has the default runtime prewarmed +// once the file is written, and Run/Restore fetch any other version on +// demand. func (s *workerSandboxSyncer) buildFile(ctx context.Context, plan *sandboxPlan, hashed []hashedVersion) (*sandboxruntimes.File, error) { out := &sandboxruntimes.File{} seen := make(map[string]bool, len(hashed)) @@ -424,34 +411,15 @@ func runRunscCPUFeatures(ctx context.Context, runscPath string) ([]string, error return strings.Split(printed, ","), nil } -// versionBundleHash returns the bundle hash of one version. The CRD requires -// every pause image to be pinned by digest, which is the digest the image -// cache resolves the ref to, so the hash needs no pull. An unpinned one is -// errPermanent: only a spec change fixes it. -func versionBundleHash(rec *sandboxAssetsRecord) (string, error) { - var pauseDigest string - if rec.PauseImage != "" { - ref, err := name.NewDigest(rec.PauseImage) - if err != nil { - return "", fmt.Errorf("%w: pause image %q is not pinned by digest: %w", errPermanent, rec.PauseImage, err) - } - pauseDigest = ref.DigestStr() - } - return bundleHash(rec, pauseDigest), nil -} - // bundleHash is the sandbox_bundle_hash of a version: a sha256 over one -// sorted "key=value" line per asset digest and pause image digest. Version -// names and URLs are left out, so the same binaries under another name or in -// another bucket are the same runtime. -func bundleHash(rec *sandboxAssetsRecord, pauseDigest string) string { - lines := make([]string, 0, len(rec.Assets)+1) +// sorted "asset/=" line per asset. Version names and URLs are +// left out, so the same binaries under another name or in another bucket are +// the same runtime. +func bundleHash(rec *sandboxAssetsRecord) string { + lines := make([]string, 0, len(rec.Assets)) for name, a := range rec.Assets { lines = append(lines, "asset/"+name+"="+a.SHA256) } - if pauseDigest != "" { - lines = append(lines, "pause_image="+pauseDigest) - } slices.Sort(lines) sum := sha256.Sum256([]byte(strings.Join(lines, "\n"))) return "sha256:" + hex.EncodeToString(sum[:]) @@ -475,10 +443,7 @@ func (s *workerSandboxSyncer) syncPod(ctx context.Context, pod *corev1.Pod) erro if err != nil { return err } - hashed, err := hashVersions(ctx, plan) - if err != nil { - return err - } + hashed := hashVersions(plan) // Resolvable, and on disk, before the file can let the worker register // any of them. if err := s.index.registerRuntimes(plan.config, hashed); err != nil { @@ -501,7 +466,7 @@ func (s *workerSandboxSyncer) syncPod(ctx context.Context, pod *corev1.Pod) erro // Only now, with the file naming its defaults: a sync that failed above // left the file, and so the directories, as they were. For a gVisor // default the build fetched the release for its probe, so the prewarm - // is links and the pause image; a micro-VM default downloads here. + // is links; a micro-VM default downloads here. if s.prewarm != nil { var defaults []string for _, v := range writtenVersions(hashed, want) { diff --git a/cmd/atelet/worker_sandbox_versions_test.go b/cmd/atelet/worker_sandbox_versions_test.go index 7941cc1915..ae66b36659 100644 --- a/cmd/atelet/worker_sandbox_versions_test.go +++ b/cmd/atelet/worker_sandbox_versions_test.go @@ -37,21 +37,14 @@ import ( "go.ate.dev/substrate/pkg/api/v1alpha1" ) -// pauseDigest is the digest of the pause image of multiVersionConfig's -// version with the given name. -func pauseDigest(name string) string { - return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte("pause-"+name))) -} - // multiVersionConfig is a gVisor config whose default version "v2" is listed // after an Enabled "v1", a Disabled "v0", and an Enabled "other-arch" with no // assets for this node. func multiVersionConfig() *v1alpha1.SandboxConfig { version := func(name, sha string, state *v1alpha1.SandboxVersionState) v1alpha1.SandboxVersionConfig { return v1alpha1.SandboxVersionConfig{ - Name: name, - State: state, - PauseImage: "registry.k8s.io/pause@" + pauseDigest(name), + Name: name, + State: state, Assets: map[string]map[string]v1alpha1.AssetFile{ runtime.GOARCH: {gvisorAssetName: {URL: "gs://bucket/" + name, SHA256: sha}}, }, @@ -85,7 +78,6 @@ func TestRecordForArch(t *testing.T) { assets: map[string]map[string]v1alpha1.AssetFile{runtime.GOARCH: {gvisorAssetName: {URL: "gs://bucket/gvisor", SHA256: sha}}}, want: &sandboxAssetsRecord{ SandboxClass: string(v1alpha1.SandboxClassGvisor), - PauseImage: "registry.k8s.io/pause@sha256:abc", Assets: map[string]assetEntry{gvisorAssetName: {URL: "gs://bucket/gvisor", SHA256: sha}}, }, }, { @@ -109,35 +101,26 @@ func TestRecordForArch(t *testing.T) { func TestBundleHash(t *testing.T) { gvisor := map[string]assetEntry{gvisorAssetName: {URL: "gs://bucket/gvisor", SHA256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f"}} - pause := "sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" // Snapshots record the hash, so a change to any of these values strands // every snapshot taken before it. tests := []struct { - name string - rec *sandboxAssetsRecord - pauseDigest string - want string + name string + rec *sandboxAssetsRecord + want string }{{ - name: "gvisor asset and pause image", - rec: &sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: "registry.k8s.io/pause:3.10.2@" + pause, Assets: gvisor}, - pauseDigest: pause, - want: "sha256:7197896bb429f0b824accbc635a38c9ed5666c160070b833bbdc6e036ccd4f76", + name: "gvisor asset", + rec: &sandboxAssetsRecord{SandboxClass: "gvisor", Assets: gvisor}, + want: "sha256:f86e6f2bd8e25ac31a22e51e6e57e0eb6741eb0b0df77082989cc1e40f906fe1", }, { // Only the digests count, not the version's other fields. - name: "another URL and pause image ref", - rec: &sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: "mirror.example.com/pause@" + pause, Assets: map[string]assetEntry{gvisorAssetName: {URL: "gs://other/gvisor", SHA256: gvisor[gvisorAssetName].SHA256}}}, - pauseDigest: pause, - want: "sha256:7197896bb429f0b824accbc635a38c9ed5666c160070b833bbdc6e036ccd4f76", - }, { - name: "no pause image", - rec: &sandboxAssetsRecord{SandboxClass: "microvm", Assets: gvisor}, + name: "another URL and class", + rec: &sandboxAssetsRecord{SandboxClass: "microvm", Assets: map[string]assetEntry{gvisorAssetName: {URL: "gs://other/gvisor", SHA256: gvisor[gvisorAssetName].SHA256}}}, want: "sha256:f86e6f2bd8e25ac31a22e51e6e57e0eb6741eb0b0df77082989cc1e40f906fe1", }, { - name: "the asset under another name", - rec: &sandboxAssetsRecord{SandboxClass: "gvisor", Assets: map[string]assetEntry{"renamed": gvisor[gvisorAssetName]}}, - pauseDigest: pause, - want: "sha256:04bf18376ca0c7227ae9fa65675127b1045d7de33f05ea5d2375eb11a1a8e62e", + name: "the asset under another name", + rec: &sandboxAssetsRecord{SandboxClass: "gvisor", Assets: map[string]assetEntry{"renamed": gvisor[gvisorAssetName]}}, + want: "sha256:510948cc43e084528a7c76c1627c5bfb2d140be0b620f33282c03fc2debd65af", }, { name: "several assets", rec: &sandboxAssetsRecord{SandboxClass: "microvm", Assets: map[string]assetEntry{ @@ -148,7 +131,7 @@ func TestBundleHash(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := bundleHash(tt.rec, tt.pauseDigest); got != tt.want { + if got := bundleHash(tt.rec); got != tt.want { t.Errorf("bundleHash() = %q, want %q", got, tt.want) } }) @@ -245,48 +228,6 @@ func TestWorkerSandboxResolverResolve(t *testing.T) { } } -func TestVersionBundleHash(t *testing.T) { - gvisor := map[string]assetEntry{gvisorAssetName: {URL: "gs://bucket/gvisor", SHA256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f"}} - pause := "sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" - - tests := []struct { - name string - pauseImage string - want string - wantErr bool - }{{ - name: "pause image pinned by digest", - pauseImage: "registry.k8s.io/pause:3.10.2@" + pause, - want: bundleHash(&sandboxAssetsRecord{Assets: gvisor}, pause), - }, { - name: "pause image pinned by digest only", - pauseImage: "gcr.io/gke-release/pause@" + pause, - want: bundleHash(&sandboxAssetsRecord{Assets: gvisor}, pause), - }, { - name: "no pause image", - want: bundleHash(&sandboxAssetsRecord{Assets: gvisor}, ""), - }, { - // The CRD rejects this shape. - name: "pause image by tag", - pauseImage: "registry.k8s.io/pause:3.10.2", - wantErr: true, - }} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := versionBundleHash(&sandboxAssetsRecord{SandboxClass: "gvisor", PauseImage: tt.pauseImage, Assets: gvisor}) - if (err != nil) != tt.wantErr { - t.Fatalf("versionBundleHash() error = %v, want error %v", err, tt.wantErr) - } - if err != nil && !errors.Is(err, errPermanent) { - t.Errorf("versionBundleHash() error = %v, want errPermanent", err) - } - if got != tt.want { - t.Errorf("versionBundleHash() = %q, want %q", got, tt.want) - } - }) - } -} - // wantRuntime is the runtime buildFile builds from multiVersionConfig's // version with the given name when runsc reports testCPUFeatures() ("3"). func wantRuntime(name string) sandboxruntimes.Attributes { @@ -298,7 +239,7 @@ func wantRuntimeWithFeatures(name, cpuFeatures string) sandboxruntimes.Attribute return sandboxruntimes.Attributes{ hardware.AttrArchitecture: runtime.GOARCH, hardware.AttrCPUFeatures: cpuFeatures, - hardware.AttrSandboxBundleHash: bundleHash(rec, pauseDigest(name)), + hardware.AttrSandboxBundleHash: bundleHash(rec), } } @@ -316,11 +257,7 @@ func planAndBuild(s *workerSandboxSyncer, pod *corev1.Pod) (*sandboxruntimes.Fil if err != nil { return nil, err } - hashed, err := hashVersions(context.Background(), plan) - if err != nil { - return nil, err - } - return s.buildFile(context.Background(), plan, hashed) + return s.buildFile(context.Background(), plan, hashVersions(plan)) } func TestPlanAndBuild(t *testing.T) { @@ -331,7 +268,6 @@ func TestPlanAndBuild(t *testing.T) { // sameAsDefault has v1 run the default version's binaries. sameAsDefault := multiVersionConfig() - sameAsDefault.Spec.Versions[1].PauseImage = sameAsDefault.Spec.Versions[3].PauseImage sameAsDefault.Spec.Versions[1].Assets = sameAsDefault.Spec.Versions[3].Assets // v1Twice adds an Enabled "v3" running v1's binaries. @@ -340,13 +276,6 @@ func TestPlanAndBuild(t *testing.T) { v3.Name = "v3" v1Twice.Spec.Versions = append(v1Twice.Spec.Versions, v3) - // The CRD rejects a pause image not pinned by digest. - tagged := func(i int) *v1alpha1.SandboxConfig { - cfg := multiVersionConfig() - cfg.Spec.Versions[i].PauseImage = "registry.k8s.io/pause:3.10.2" - return cfg - } - unsupportedClass := multiVersionConfig() unsupportedClass.Spec.SandboxClass = "unknown" @@ -381,16 +310,6 @@ func TestPlanAndBuild(t *testing.T) { cfg: defaultNoArch, wantErr: true, wantPermanent: true, - }, { - // No worker of the pool could start actors on this node. - name: "default version pause image not pinned by digest", - cfg: tagged(3), - wantErr: true, - wantPermanent: true, - }, { - name: "non-default version pause image not pinned by digest is left out", - cfg: tagged(1), - want: onlyDefault, }, { name: "unsupported sandbox class", cfg: unsupportedClass, @@ -454,7 +373,7 @@ func TestPlanAndBuildMicroVM(t *testing.T) { hardware.AttrArchitecture: runtime.GOARCH, hardware.AttrCPUVendor: testCPUVendor, hardware.AttrCPUModel: testCPUModel, - hardware.AttrSandboxBundleHash: bundleHash(rec, ""), + hardware.AttrSandboxBundleHash: bundleHash(rec), } } @@ -534,11 +453,11 @@ func TestGVisorCPUFeaturesMemoizationAndErrors(t *testing.T) { t.Run("memoizes runsc cpu-features by asset digest across versions and syncs", func(t *testing.T) { cfg := multiVersionConfig() // Add a third version "v3" that shares v1's gvisor asset digest ("sha-v1") - // but has its own pause image digest, so it has a distinct bundle hash - // while reusing the same runsc binary. + // but has one more asset, so it has a distinct bundle hash while it + // uses the same runsc binary. v3 := *cfg.Spec.Versions[1].DeepCopy() v3.Name = "v3" - v3.PauseImage = "registry.k8s.io/pause@" + pauseDigest("v3") + v3.Assets[runtime.GOARCH]["extra"] = v1alpha1.AssetFile{URL: "gs://bucket/extra", SHA256: "sha-extra"} cfg.Spec.Versions = append(cfg.Spec.Versions, v3) s := newTestSyncer(t, resolvedPool("pool", "gvisor-release"), cfg) @@ -560,7 +479,7 @@ func TestGVisorCPUFeaturesMemoizationAndErrors(t *testing.T) { if err != nil { t.Fatalf("first planAndBuild() error = %v", err) } - v3Rec := &sandboxAssetsRecord{Assets: map[string]assetEntry{gvisorAssetName: {SHA256: "sha-v1"}}} + v3Rec := &sandboxAssetsRecord{Assets: map[string]assetEntry{gvisorAssetName: {SHA256: "sha-v1"}, "extra": {SHA256: "sha-extra"}}} want := &sandboxruntimes.File{ DefaultRuntime: wantRuntimeWithFeatures("v2", "3"), RestorableRuntimes: []sandboxruntimes.Attributes{ @@ -568,7 +487,7 @@ func TestGVisorCPUFeaturesMemoizationAndErrors(t *testing.T) { { hardware.AttrArchitecture: runtime.GOARCH, hardware.AttrCPUFeatures: "1", - hardware.AttrSandboxBundleHash: bundleHash(v3Rec, pauseDigest("v3")), + hardware.AttrSandboxBundleHash: bundleHash(v3Rec), }, }, } diff --git a/cmd/ateworker-gvisor/cleanup_test.go b/cmd/ateworker-gvisor/cleanup_test.go index 655f765872..3b3092958e 100644 --- a/cmd/ateworker-gvisor/cleanup_test.go +++ b/cmd/ateworker-gvisor/cleanup_test.go @@ -119,7 +119,7 @@ func TestCleanupContainers_SucceedsWhenEverythingIsAlreadyGone(t *testing.T) { assertCalls(t, f, "state app", "list", "state _pause", "list") } -// The pause container is the sandbox: it must outlive the deletes of the +// The sandbox must outlive the deletes of the // containers inside it, so stopping the actor leaves it to cleanup. func TestStopContainersLeavesTheSandboxToCleanup(t *testing.T) { f := newFakeRunsc() diff --git a/cmd/ateworker-gvisor/main.go b/cmd/ateworker-gvisor/main.go index d876d92e7f..0b14f86408 100644 --- a/cmd/ateworker-gvisor/main.go +++ b/cmd/ateworker-gvisor/main.go @@ -629,7 +629,7 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run // Contract with atelet: // // * Correct runsc version is downloaded and placed on disk. - // * All OCI bundles are set up, including for the pause container. + // * The OCI bundles of the application containers are set up. egress, err := s.tunnel.PrepareEgress(ctx, workerstats.ActorAttributionFromRequest(req), req.GetEgressGateway()) if err != nil { @@ -671,31 +671,27 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run } } }() - // Create and start pause container. The bundle rootfs is composed here — - // an overlay of the node's cached image layers plus the bundle's private - // upper — because mounting is ateworker's job (atelet runs with no - // capabilities); runsc's gofer resolves the mount in this pod's mount - // namespace. - if err := imagecache.SetupBundleRootfs(ociBundlePath(req.GetActorDirs(), ocispec.PauseContainer)); err != nil { - return nil, fmt.Errorf("while composing pause rootfs: %w", err) - } containersToDelete = append(containersToDelete, ocispec.PauseContainer) - // Attribute runsc CLI output and the pause container's stdio to the actor. + // Attribute the sandbox's runsc CLI output to the actor. pw, err := s.actorLogger.StartJSONLogPipe(attribution, ocispec.PauseContainer) if err != nil { - return nil, fmt.Errorf("while starting pause log pipe: %w", err) + return nil, fmt.Errorf("while starting sandbox log pipe: %w", err) } defer pw.Close() if err := rcmd.cmdCreate(ctx, pw, ocispec.PauseContainer, nil); err != nil { - return nil, fmt.Errorf("while creating pause container: %w", err) + return nil, fmt.Errorf("while creating the sandbox: %w", err) } s.pinSandbox(ctx, hosted, req.GetActorDirs()) if err := rcmd.cmdStart(ctx, pw, ocispec.PauseContainer); err != nil { - return nil, fmt.Errorf("while starting pause container: %w", err) + return nil, fmt.Errorf("while starting the sandbox: %w", err) } // Create and start each application container, each with its own log pipe so // every line is tagged with the originating container (ate.actor.container.name). + // The bundle rootfs is composed here — an overlay of the node's cached + // image layers plus the bundle's private upper — because mounting is + // ateworker's job (atelet runs with no capabilities); runsc's gofer + // resolves the mount in this pod's mount namespace. for _, ac := range req.GetSpec().GetContainers() { pw, err := s.actorLogger.StartJSONLogPipe(attribution, ac.GetName()) if err != nil { @@ -844,9 +840,9 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke return nil } case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: - // Checkpoint pause container (root of the sandbox), splitting container - // rootfs overlays into the rootfs bundle and durable-dir volumes into - // /fs when present. + // Checkpoint the sandbox, splitting container rootfs overlays into the + // rootfs bundle and durable-dir volumes into /fs when + // present. checkpointFn = func(ctx context.Context) error { var sandbox *sandboxProcess if hosted != nil { @@ -871,7 +867,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke hosted.stopping.Store(false) } timing.checkpoint = lap(&tLast) - return fmt.Errorf("while checkpointing pause: %w", err) + return fmt.Errorf("while checkpointing the sandbox: %w", err) } // the checkpoint stops the sandbox, runsc delete fails until it is reaped if sandbox != nil { @@ -932,10 +928,9 @@ func listSnapshotFiles(dir string) ([]string, error) { return files, nil } -// stopContainers stops the actor's application containers. The pause container -// is left running: it is the sandbox, and cleanupContainers deletes the others -// through it before deleting it last. Killing it first leaves the sentry a -// zombie until reaped, which runsc delete mistakes for a live sandbox. +// stopContainers stops the actor's application containers. The sandbox is left +// running: cleanupContainers deletes the others through it before deleting it +// last. runsc kill does not apply to a sandbox with no root container. func stopContainers(ctx context.Context, rcmd containerRuntime, containers []*ateworkerpb.Container) { for _, ctr := range containers { _ = rcmd.cmdKill(ctx, ctr.GetName(), "SIGKILL") @@ -944,7 +939,7 @@ func stopContainers(ctx context.Context, rcmd containerRuntime, containers []*at } func cleanupContainers(ctx context.Context, rcmd containerRuntime, containers []*ateworkerpb.Container) error { - // Application containers first, the pause (root) container last. + // Application containers first, the sandbox last. names := make([]string, 0, len(containers)+1) for _, ctr := range containers { names = append(names, ctr.GetName()) @@ -1044,7 +1039,7 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb // Contract with atelet: // // * Correct runsc version is downloaded and placed on disk. - // * All OCI bundles are set up, including for the pause container. + // * The OCI bundles of the application containers are set up. // * Checkpoint downloaded and placed on disk egress, err := s.tunnel.PrepareEgress(ctx, attribution, req.GetEgressGateway()) @@ -1091,49 +1086,40 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb return nil, err } - // Compose the pause rootfs before create (see RunWorkload). runsc restore - // only needs the rootfs to hold the correct content; whether it came from - // an untar or an overlay of cached layers is transparent to it. - err = imagecache.SetupBundleRootfs(ociBundlePath(req.GetActorDirs(), ocispec.PauseContainer)) - timing.pauseRootfs = lap(&tLast) - if err != nil { - return nil, fmt.Errorf("while composing pause rootfs: %w", err) - } - - // Attribute runsc CLI output and the pause container's stdio to the actor. + // Attribute the sandbox's runsc CLI output to the actor. pw, err := s.actorLogger.StartJSONLogPipe(attribution, ocispec.PauseContainer) if err != nil { - return nil, fmt.Errorf("while starting pause log pipe: %w", err) + return nil, fmt.Errorf("while starting sandbox log pipe: %w", err) } defer pw.Close() switch fidelity { case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: - // Create and start pause container (cold boot with filesystem checkpoint bundles restored) + // Create and start the sandbox (cold boot with filesystem checkpoint bundles restored) containersToDelete = append(containersToDelete, ocispec.PauseContainer) err := rcmd.cmdCreate(ctx, pw, ocispec.PauseContainer, restoreArgs) timing.pauseCreate = lap(&tLast) if err != nil { - return nil, fmt.Errorf("while creating pause container: %w", err) + return nil, fmt.Errorf("while creating the sandbox: %w", err) } s.pinSandbox(ctx, hosted, req.GetActorDirs()) err = rcmd.cmdStart(ctx, pw, ocispec.PauseContainer) timing.pauseRestore = lap(&tLast) if err != nil { - return nil, fmt.Errorf("while starting pause container: %w", err) + return nil, fmt.Errorf("while starting the sandbox: %w", err) } case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: - // Create and restore pause container (restores process state and filesystem checkpoint bundles) + // Create and restore the sandbox (restores process state and filesystem checkpoint bundles) containersToDelete = append(containersToDelete, ocispec.PauseContainer) err := rcmd.cmdCreate(ctx, pw, ocispec.PauseContainer, restoreArgs) timing.pauseCreate = lap(&tLast) if err != nil { - return nil, fmt.Errorf("while creating pause container: %w", err) + return nil, fmt.Errorf("while creating the sandbox: %w", err) } s.pinSandbox(ctx, hosted, req.GetActorDirs()) err = rcmd.cmdRestore(ctx, pw, ocispec.PauseContainer, checkpointDir) timing.pauseRestore = lap(&tLast) if err != nil { - return nil, fmt.Errorf("while restoring pause container: %w", err) + return nil, fmt.Errorf("while restoring the sandbox: %w", err) } } diff --git a/cmd/ateworker-gvisor/phaselog.go b/cmd/ateworker-gvisor/phaselog.go index 78ab0f2740..78fe1752cc 100644 --- a/cmd/ateworker-gvisor/phaselog.go +++ b/cmd/ateworker-gvisor/phaselog.go @@ -36,7 +36,6 @@ const ( phasePrep = "prep" phaseEgressPrepare = "egress_prepare" phaseNetSetup = "net_setup" - phasePauseRootfs = "pause_rootfs" phasePauseCreate = "pause_create" phasePauseRestore = "pause_restore" phaseAppRootfs = "app_rootfs" @@ -60,10 +59,10 @@ const containerCountKey = "ate.actor.container.count" // left at zero never ran. Under VOLUMES and ROOTFS fidelity, pauseRestore and // appRestore time the cold start that stands in for the restore. type restoreTiming struct { - prep, egressPrepare, netSetup time.Duration - pauseRootfs, pauseCreate, pauseRestore time.Duration - appRootfs, appCreate, appRestore time.Duration - wakeupProbe, activate, total time.Duration + prep, egressPrepare, netSetup time.Duration + pauseCreate, pauseRestore time.Duration + appRootfs, appCreate, appRestore time.Duration + wakeupProbe, activate, total time.Duration } func (t restoreTiming) phases() []phaselog.Phase { @@ -71,7 +70,6 @@ func (t restoreTiming) phases() []phaselog.Phase { {Name: phasePrep, D: t.prep}, {Name: phaseEgressPrepare, D: t.egressPrepare}, {Name: phaseNetSetup, D: t.netSetup}, - {Name: phasePauseRootfs, D: t.pauseRootfs}, {Name: phasePauseCreate, D: t.pauseCreate}, {Name: phasePauseRestore, D: t.pauseRestore}, {Name: phaseAppRootfs, D: t.appRootfs}, diff --git a/cmd/ateworker-gvisor/phaselog_test.go b/cmd/ateworker-gvisor/phaselog_test.go index 11efbe3973..7f80d39afa 100644 --- a/cmd/ateworker-gvisor/phaselog_test.go +++ b/cmd/ateworker-gvisor/phaselog_test.go @@ -51,7 +51,6 @@ func TestRestoreTimingPhases(t *testing.T) { prep: 10 * time.Millisecond, egressPrepare: 120 * time.Millisecond, netSetup: 30 * time.Millisecond, - pauseRootfs: 5 * time.Millisecond, pauseCreate: 200 * time.Millisecond, pauseRestore: 900 * time.Millisecond, appRootfs: 15 * time.Millisecond, @@ -78,7 +77,6 @@ func TestRestoreTimingPhases(t *testing.T) { "ateworker.actor.restore.duration.prep": 0.01, "ateworker.actor.restore.duration.egress_prepare": 0.12, "ateworker.actor.restore.duration.net_setup": 0.03, - "ateworker.actor.restore.duration.pause_rootfs": 0.005, "ateworker.actor.restore.duration.pause_create": 0.2, "ateworker.actor.restore.duration.pause_restore": 0.9, "ateworker.actor.restore.duration.app_rootfs": 0.015, diff --git a/cmd/ateworker-gvisor/runsc.go b/cmd/ateworker-gvisor/runsc.go index f41e6e7daf..1b3f718209 100644 --- a/cmd/ateworker-gvisor/runsc.go +++ b/cmd/ateworker-gvisor/runsc.go @@ -82,19 +82,19 @@ func durableVolumeNames(spec *ateworkerpb.WorkloadSpec) []string { return slices.Compact(names) } -// pauseContainer is the sandbox root container. It only reaps, so it gets no -// capabilities and mounts nothing. -func pauseContainer() *ateworkerpb.Container { - return &ateworkerpb.Container{ - Name: ocispec.PauseContainer, - ContainerSpec: &ateworkerpb.ContainerSpec{Args: []string{"/pause"}}, +// noRootContainerArgs returns the flag that tells runsc the sandbox has no +// root container. runsc refuses the flag for any other container. +func noRootContainerArgs(containerName string) []string { + if containerName == ocispec.PauseContainer { + return []string{"-no-root-container"} } + return nil } // writeSpec builds the container's OCI spec, shapes it for gVisor, and saves it // into the container's bundle. func (r *runsc) writeSpec(containerName string) error { - container := pauseContainer() + container := &ateworkerpb.Container{Name: ocispec.PauseContainer} if containerName != ocispec.PauseContainer { i := slices.IndexFunc(r.containers, func(c *ateworkerpb.Container) bool { return c.GetName() == containerName }) if i < 0 { @@ -102,6 +102,11 @@ func (r *runsc) writeSpec(containerName string) error { } container = r.containers[i] } + // atelet makes the bundles of the application containers only. + bundlePath := ociBundlePath(r.actorDirs, containerName) + if err := os.MkdirAll(bundlePath, 0o700); err != nil { + return fmt.Errorf("while creating bundle %q: %w", bundlePath, err) + } spec := ocispec.Build(ocispec.Options{ActorUID: r.actorUID, ActorDirs: r.actorDirs, Container: container}) ocispec.ShapeGVisor(spec, ocispec.GVisorOptions{ ActorUID: r.actorUID, @@ -110,7 +115,18 @@ func (r *runsc) writeSpec(containerName string) error { DurableVolumes: r.durableVolumes, Size: r.size, }) - return ocispec.Save(ociBundlePath(r.actorDirs, containerName), spec) + return ocispec.Save(bundlePath, spec) +} + +// createArgs builds the argv for `runsc create `. Factored out so +// the argument construction can be unit-tested without executing runsc. +func (r *runsc) createArgs(containerName string, additionalArgs []string) []string { + args := append(noRootContainerArgs(containerName), + "-bundle", ociBundlePath(r.actorDirs, containerName), + "-pid-file", pidFilePath(r.actorDirs, containerName), + ) + args = append(args, additionalArgs...) + return r.buildArgs("create", containerName, args...) } func (r *runsc) cmdCreate(ctx context.Context, out io.Writer, containerName string, additionalArgs []string) error { @@ -120,18 +136,7 @@ func (r *runsc) cmdCreate(ctx context.Context, out io.Writer, containerName stri return fmt.Errorf("while writing the OCI spec for %q: %w", containerName, err) } - args := make([]string, 0, 4+len(additionalArgs)) - args = append(args, - "-bundle", ociBundlePath(r.actorDirs, containerName), - "-pid-file", pidFilePath(r.actorDirs, containerName), - ) - args = append(args, additionalArgs...) - - cmd := exec.CommandContext( - ctx, - r.path, - r.buildArgs("create", containerName, args...)..., - ) + cmd := exec.CommandContext(ctx, r.path, r.createArgs(containerName, additionalArgs)...) cmd.Stdout = out cmd.Stderr = out @@ -210,15 +215,16 @@ func (r *runsc) cmdFsCheckpoint(ctx context.Context, containerName, checkpointPa // restoreArgs builds the argv for `runsc restore `. Factored out so // the argument construction can be unit-tested without executing runsc. func (r *runsc) restoreArgs(containerName, checkpointPath string) []string { - return r.buildArgs("restore", containerName, + args := append(noRootContainerArgs(containerName), "-bundle", ociBundlePath(r.actorDirs, containerName), "-image-path", checkpointPath, "-pid-file", pidFilePath(r.actorDirs, containerName), ) + return r.buildArgs("restore", containerName, args...) } -// We take a checkpoint only of the root container of the sandbox, but we need -// to call restore on each container, using the same checkpoint. +// We take a checkpoint only of the sandbox, but we need to call restore on each +// container, using the same checkpoint. func (r *runsc) cmdRestore(ctx context.Context, out io.Writer, containerName, checkpointPath string) error { slog.InfoContext(ctx, "About to run runsc restore", slog.String("container", containerName)) diff --git a/cmd/ateworker-gvisor/runsc_test.go b/cmd/ateworker-gvisor/runsc_test.go index badafe71be..fc8d8073ef 100644 --- a/cmd/ateworker-gvisor/runsc_test.go +++ b/cmd/ateworker-gvisor/runsc_test.go @@ -40,8 +40,8 @@ var testActorDirs = &ateworkerpb.ActorDirs{ OciBundleDir: "/node/actors/test-actor-123/bundle", } -// writeSpec builds a workload container's spec from the request and the pause -// container's from ateworker's own description. +// writeSpec builds a workload container's spec from the request, and a +// sandbox spec with no process and no rootfs. It makes the sandbox bundle. func TestWriteSpec(t *testing.T) { root := t.TempDir() actorDirs := &ateworkerpb.ActorDirs{RootDir: root, OciBundleDir: filepath.Join(root, "bundles")} @@ -50,18 +50,11 @@ func TestWriteSpec(t *testing.T) { actorDirs: actorDirs, containers: []*ateworkerpb.Container{{Name: "app", ContainerSpec: &ateworkerpb.ContainerSpec{Args: []string{"/app", "serve"}}}}, } - for name, wantArgs := range map[string][]string{ - "app": {"/app", "serve"}, - ocispec.PauseContainer: {"/pause"}, - } { - bundle := ociBundlePath(actorDirs, name) - if err := os.MkdirAll(bundle, 0o700); err != nil { - t.Fatal(err) - } + for _, name := range []string{"app", ocispec.PauseContainer} { if err := r.writeSpec(name); err != nil { t.Fatalf("writeSpec(%q) = %v", name, err) } - b, err := os.ReadFile(filepath.Join(bundle, "config.json")) + b, err := os.ReadFile(filepath.Join(ociBundlePath(actorDirs, name), "config.json")) if err != nil { t.Fatal(err) } @@ -69,14 +62,17 @@ func TestWriteSpec(t *testing.T) { if err := json.Unmarshal(b, &spec); err != nil { t.Fatal(err) } - if !slices.Equal(spec.Process.Args, wantArgs) { - t.Errorf("%s args = %v, want %v", name, spec.Process.Args, wantArgs) - } if got := spec.Annotations["io.kubernetes.cri.container-name"]; got != name { t.Errorf("%s container-name annotation = %q, want the spec shaped for gVisor", name, got) } - if name == ocispec.PauseContainer && len(spec.Process.Capabilities.Bounding) != 0 { - t.Errorf("pause capabilities = %v, want none", spec.Process.Capabilities.Bounding) + if name == ocispec.PauseContainer { + if spec.Process != nil || spec.Root != nil || spec.Mounts != nil { + t.Errorf("sandbox spec has process %v, root %v, mounts %v; want none", spec.Process, spec.Root, spec.Mounts) + } + continue + } + if want := []string{"/app", "serve"}; spec.Process == nil || !slices.Equal(spec.Process.Args, want) { + t.Errorf("%s process = %+v, want args %v", name, spec.Process, want) } } if err := r.writeSpec("missing"); err == nil { @@ -258,32 +254,98 @@ func TestFsCheckpointArgs(t *testing.T) { } } -func TestRestoreArgs(t *testing.T) { +// Only the sandbox gets -no-root-container: runsc refuses it for any other +// container. +func TestCreateArgs(t *testing.T) { r := &runsc{ path: "/usr/bin/runsc", actorUID: "test-actor-123", actorDirs: testActorDirs, flags: mustFlagStore(t), } - checkpointDir := "/node/actors/test-actor-123/checkpoints/snap-1" + for _, tc := range []struct { + container string + extra []string + want []string + }{{ + container: ocispec.PauseContainer, + extra: []string{"-fs-restore-image-path", "/snap/fs"}, + want: []string{ + "-global1", + "-global2", + "-root", "/node/actors/test-actor-123/runsc-state", + "create", + "-no-root-container", + "-bundle", "/node/actors/test-actor-123/bundle/" + ocispec.PauseContainer, + "-pid-file", "/node/actors/test-actor-123/pidfiles/" + ocispec.PauseContainer + ".pid", + "-fs-restore-image-path", "/snap/fs", + ocispec.PauseContainer, + }, + }, { + container: "app", + want: []string{ + "-global1", + "-global2", + "-root", "/node/actors/test-actor-123/runsc-state", + "create", + "-bundle", "/node/actors/test-actor-123/bundle/app", + "-pid-file", "/node/actors/test-actor-123/pidfiles/app.pid", + "app", + }, + }} { + if got := r.createArgs(tc.container, tc.extra); !reflect.DeepEqual(got, tc.want) { + t.Errorf("createArgs(%q) = %v, want %v", tc.container, got, tc.want) + } + } +} - got := r.restoreArgs(ocispec.PauseContainer, checkpointDir) - want := []string{ - "-global1", - "-global2", - "-restore-global", - "-root", "/node/actors/test-actor-123/runsc-state", - "restore", - "-restore-sub1", - "-restore-sub2", - "-bundle", "/node/actors/test-actor-123/bundle/" + ocispec.PauseContainer, - "-image-path", checkpointDir, - "-pid-file", "/node/actors/test-actor-123/pidfiles/" + ocispec.PauseContainer + ".pid", - ocispec.PauseContainer, +func TestRestoreArgs(t *testing.T) { + r := &runsc{ + path: "/usr/bin/runsc", + actorUID: "test-actor-123", + actorDirs: testActorDirs, + flags: mustFlagStore(t), } + checkpointDir := "/node/actors/test-actor-123/checkpoints/snap-1" - if !reflect.DeepEqual(got, want) { - t.Errorf("restoreArgs() = %v, want %v", got, want) + for _, tc := range []struct { + container string + want []string + }{{ + container: ocispec.PauseContainer, + want: []string{ + "-global1", + "-global2", + "-restore-global", + "-root", "/node/actors/test-actor-123/runsc-state", + "restore", + "-restore-sub1", + "-restore-sub2", + "-no-root-container", + "-bundle", "/node/actors/test-actor-123/bundle/" + ocispec.PauseContainer, + "-image-path", checkpointDir, + "-pid-file", "/node/actors/test-actor-123/pidfiles/" + ocispec.PauseContainer + ".pid", + ocispec.PauseContainer, + }, + }, { + container: "app", + want: []string{ + "-global1", + "-global2", + "-restore-global", + "-root", "/node/actors/test-actor-123/runsc-state", + "restore", + "-restore-sub1", + "-restore-sub2", + "-bundle", "/node/actors/test-actor-123/bundle/app", + "-image-path", checkpointDir, + "-pid-file", "/node/actors/test-actor-123/pidfiles/app.pid", + "app", + }, + }} { + if got := r.restoreArgs(tc.container, checkpointDir); !reflect.DeepEqual(got, tc.want) { + t.Errorf("restoreArgs(%q) = %v, want %v", tc.container, got, tc.want) + } } } diff --git a/cmd/ateworker-gvisor/stats.go b/cmd/ateworker-gvisor/stats.go index dce914cd1d..8b0c4766d4 100644 --- a/cmd/ateworker-gvisor/stats.go +++ b/cmd/ateworker-gvisor/stats.go @@ -44,10 +44,10 @@ const defaultCgroupRoot = "/sys/fs/cgroup" // sandbox as a whole. // // gVisor runs every container of a sandbox as threads inside one host process, -// the sentry. runsc starts that process from the root container's create and -// from inside that container's cgroup — container.createRoot wraps the sandbox -// and gofer spawn in cgroup.RunInCgroup — so the sentry lands in the leaf of -// the pause container, the first one RunWorkload and RestoreWorkload create. +// the sentry. runsc starts that process from the sandbox's create and from +// inside the sandbox's cgroup — container.createRoot wraps the sandbox spawn in +// cgroup.RunInCgroup — so the sentry lands in the sandbox's leaf. RunWorkload +// and RestoreWorkload create the sandbox first. // // The leaf is a direct child of the delegated scope rather than of ateworker's own // cgroup, because runsc resolves cgroupsPath against the parent of the cgroup @@ -72,11 +72,10 @@ const defaultCgroupRoot = "/sys/fs/cgroup" // // What the leaf holds besides the actor's own work: the sentry's own overhead // (its Go heap, page tables, netstack) and the gofers. Process listings taken -// on a live node in #161 put runsc-sandbox and both gofers — the pause -// container's and the actor container's — in the pause cgroup. Those runs -// predate #496, so they establish the leaf name and the fact that everything -// lands in one leaf, not the absolute path, which #496's delegation moved under -// the pod scope. +// on a live node in #161 put runsc-sandbox and the gofers in the _pause +// cgroup. Those runs predate #496, so they establish the leaf name and the +// fact that everything lands in one leaf, not the absolute path, which #496's +// delegation moved under the pod scope. // // So this measures the sandbox, not the actor's processes in isolation, which // is what the proto means by "the unit of measurement is the SANDBOX". Sizing diff --git a/cmd/ateworker-gvisor/stats_test.go b/cmd/ateworker-gvisor/stats_test.go index 7a211acc7c..dfeef7a253 100644 --- a/cmd/ateworker-gvisor/stats_test.go +++ b/cmd/ateworker-gvisor/stats_test.go @@ -673,7 +673,7 @@ func TestDeadSandboxCheckDuringDrain(t *testing.T) { // TestDrainNeverKillsPauseContainer: gracefulShutdown kills the names // containerNames takes from the spec. atelet requires container names to be -// DNS-1123 labels (k8s-short-name) and the pause container's name is not one, +// DNS-1123 labels (k8s-short-name) and the sandbox ID is not one, // so no spec can name it and the drain cannot stop the sandbox itself. func TestDrainNeverKillsPauseContainer(t *testing.T) { if resources.IsValidResourceName(ocispec.PauseContainer) { diff --git a/cmd/kubectl-ate/internal/cmd/actor_test.go b/cmd/kubectl-ate/internal/cmd/actor_test.go index 4c5db51c3e..573c9a42f3 100644 --- a/cmd/kubectl-ate/internal/cmd/actor_test.go +++ b/cmd/kubectl-ate/internal/cmd/actor_test.go @@ -523,13 +523,13 @@ func TestFilterAndDisplayLogLine(t *testing.T) { wantOutput: `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"hi"}`, }, { - name: "no filter excludes pause container diagnostics", + name: "no filter excludes sandbox diagnostics", line: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _pause]","labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"_pause"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: false, }, { - name: "explicit pause filter shows runsc diagnostics", + name: "explicit sandbox filter shows runsc diagnostics", line: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _pause]","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"_pause"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, container: "_pause", @@ -537,7 +537,7 @@ func TestFilterAndDisplayLogLine(t *testing.T) { wantOutput: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _pause]"}`, }, { - name: "application filter excludes pause container diagnostics", + name: "application filter excludes sandbox diagnostics", line: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _pause]","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"_pause"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, container: "counter", diff --git a/docs/api-guide.md b/docs/api-guide.md index 1e465a5baf..cab8815a76 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -104,7 +104,7 @@ The `ActorTemplate` defines the code, environment, and state-management policies | `volumes` | `[]Volume` | Optional. Volumes the containers may mount, each a `durableDir`, an `external` CSI volume source (see [CSI Volumes Guide](csi-volumes.md)), or a `systemInfo` volume (see [SystemInfo Volumes](#systeminfo-volumes)). Every declared volume must be mounted by at least one container. A `durableDir` is created `0777`, as Kubernetes creates an emptyDir, so a container running as any user can write it. A snapshot restores it with the mode it had, so the container's own chmod survives suspend and resume. | | `resources` | `*ResourceRequirements` | Required. Declares each actor's compute size via `limits`, which must set both `cpuMillicores` and `memoryBytes` — see [Sandbox Right-Sizing](#sandbox-right-sizing-resources). Immutable, like the rest of the template. | -The sandbox runtime and its binaries (e.g. the gVisor `runsc` binary and `pauseImage`, or a micro-VM kernel/firmware/config) are configured on the [`WorkerPool`](#1-workerpool-the-compute-foundation) via its `sandboxClasses` and the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) it resolves, rather than on the `ActorTemplate`. An `ActorTemplate` selects its `WorkerPool`s via `workerSelector` (AND'd with the actor's `worker_selector`). When an actor runs on a worker, the worker's active `SandboxRuntime` is stamped onto the actor's assignment and snapshots so subsequent resumes prefer runtime-compatible workers. +The sandbox runtime and its binaries (e.g. the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) are configured on the [`WorkerPool`](#1-workerpool-the-compute-foundation) via its `sandboxClasses` and the cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) it resolves, rather than on the `ActorTemplate`. An `ActorTemplate` selects its `WorkerPool`s via `workerSelector` (AND'd with the actor's `worker_selector`). When an actor runs on a worker, the worker's active `SandboxRuntime` is stamped onto the actor's assignment and snapshots so subsequent resumes prefer runtime-compatible workers. ### Sandbox Right-Sizing (`resources`) @@ -433,7 +433,7 @@ One consequence worth planning for: **a published snapshot is read from the ates ## 3. SandboxConfig: The Sandbox Itself -`SandboxConfig` is a **cluster-scoped** resource that decouples the sandbox — its binaries (the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) and the `pauseImage` that holds the sandbox's namespaces — from the workload definition in the `ActorTemplate`. A `WorkerPool`'s `sandboxClasses` entry resolves its `SandboxConfig` (either by explicit `configRef` or via the class default), and `atelet` provisions each worker pod with the enabled versions of that config, using `defaultVersion` for cold boots and keeping other enabled versions in `restorableRuntimes` so existing snapshots can still restore across upgrades. +`SandboxConfig` is a **cluster-scoped** resource that decouples the sandbox binaries (the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) from the workload definition in the `ActorTemplate`. A `WorkerPool`'s `sandboxClasses` entry resolves its `SandboxConfig` (either by explicit `configRef` or via the class default), and `atelet` provisions each worker pod with the enabled versions of that config, using `defaultVersion` for cold boots and keeping other enabled versions in `restorableRuntimes` so existing snapshots can still restore across upgrades. ### Specification (`SandboxConfigSpec`) @@ -449,7 +449,6 @@ Each entry of `versions` (`SandboxVersionConfig`): | :--- | :--- | :--- | | `name` | `string` | **Required.** Identifies the version within the config. A DNS label (lower-case alphanumerics and `-`, at most 63 characters). | | `state` | `string` | Optional. `Enabled` (default) or `Disabled`. The version named by `defaultVersion` cannot be `Disabled`. | -| `pauseImage` | `string` | **Required for `gvisor`; not allowed for `microvm`**, which runs no pause container. The image for the sandbox's root container (e.g. `registry.k8s.io/pause`, or `gcr.io/gke-release/pause` on GKE). Must include a digest (`...@sha256:...`) — it is recorded in each snapshot's manifest so a restore rebuilds the sandbox from the same image. | | `assets` | `map[arch]map[name]AssetFile` | Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.zstd`), which atelet auto-extracts. A micro-VM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair; `url` is at most 2048 bytes. A `ValidatingAdmissionPolicy` enforces each class's required assets on every version. atelet holds no storage credentials: it fetches every URL through the node's object-store plugin (`AssetProvider`), with the plugin's credentials. Which schemes work depends on the plugin; the in-tree one serves only its storage backend's scheme, `gs://` on GCS and `s3://` on S3, as `/`, and rejects other schemes without reading them. A `gs://` URL the plugin cannot serve (it has no such object, rejects the URL, does not implement `AssetProvider`, is unreachable, or fails with an error it does not recognize, such as GCS refusing its credentials for a public bucket) is then read anonymously, which serves public buckets such as `gs://gvisor`; any other refusal by the plugin is final. The plugin returns an object only if its content matches `sha256`, and refuses any URL under a snapshot location (an `ActorTemplate`'s `snapshotConfig.location` followed by `/atespaces/`), so assets may share the snapshot bucket but must not be stored there. atelet verifies the sha256 again before using an asset. Its logs and errors show an asset URL as `scheme://host/path`. | A cluster-wide gVisor `SandboxConfig` (`gvisor-default`, annotated as the class default) is installed with the platform, so gVisor `WorkerPool`s use it automatically without any extra setup. @@ -493,7 +492,6 @@ spec: versions: - name: v4 state: Enabled - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" assets: amd64: gvisor: diff --git a/docs/architecture.md b/docs/architecture.md index 7b40dbe937..8271f3dc30 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -335,7 +335,7 @@ The node-level subsystem manages the physical execution of sandboxes and the mov ### Sandbox Classes -A `WorkerPool` selects a **sandbox class** (`spec.sandboxClasses[].name`), and each class has a matching `ateworker` herder image. The sandbox binaries themselves are not baked into the worker image — they, and the pause image holding the sandbox's namespaces, come at runtime from the cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) the `WorkerPool` resolves for its class (an explicit `configRef`, else the class default) and are pinned into each snapshot's manifest so restores stay reproducible across runtime upgrades. +A `WorkerPool` selects a **sandbox class** (`spec.sandboxClasses[].name`), and each class has a matching `ateworker` herder image. The sandbox binaries themselves are not baked into the worker image — they come at runtime from the cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) the `WorkerPool` resolves for its class (an explicit `configRef`, else the class default) and are pinned into each snapshot's manifest so restores stay reproducible across runtime upgrades. * **gVisor** (`ateworker-gvisor`, the default): Runs the workload under `runsc` for kernel-level sandboxing. Suspend and resume leverage gVisor's native checkpoint/restore of the sandboxed process tree. `DurableDir` volumes and each container's rootfs overlay are saved as separate filesystem checkpoint bundles (`fs/` and `rootfs/`). A `MEMORY` snapshot splits them out of the process checkpoint; `VOLUMES` and `ROOTFS` snapshots save only the bundles they need with `runsc fscheckpoint`, without memory, and restore them on a cold boot. diff --git a/docs/glossary.md b/docs/glossary.md index 69e800fb48..156bf42d7d 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -24,11 +24,10 @@ For how the pieces fit together, see the [Architecture](architecture.md) and - **SandboxConfig** (Kubernetes CRD): a cluster-scoped resource holding the sandbox binaries for one runtime family (the gVisor `runsc` binary, or a - micro-VM kernel/firmware/config), plus the pause image for the sandbox's - root container. An actor resolves its sandbox at first cold boot from the - config its `ActorTemplate` names (naming one is currently required; - per-class cluster defaults are planned), so one config pins the runtime - version for many templates. + micro-VM kernel/firmware/config). An actor resolves its sandbox at first cold + boot from the config its `ActorTemplate` names (naming one is currently + required; per-class cluster defaults are planned), so one config pins the + runtime version for many templates. ## Records (dynamic state, in the control-plane store) diff --git a/docs/observability.md b/docs/observability.md index 60fa15aae8..0cf3204561 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -12,7 +12,7 @@ To make underlying infrastructure transitions transparent, Agent Substrate estab * `ate.actor.uid`: Server-assigned UID of the actor, unique to the lifetime of an actor. * `ate.template.name`: The name of the actor's ActorTemplate (e.g., `counter`). * `ate.template.atespace`: The atespace of the actor's ActorTemplate (e.g., `ate-demo-counter`). -* `ate.actor.container.name`: The name of the container within the actor that produced the log line (e.g., `counter`), so a multi-container actor's logs can be demultiplexed by container. Absent on the synthetic lifecycle records (`Actor starting`, `Actor restored`, …): those are about the actor, so no container produced them. The `runsc create`, `start`, and `restore` CLI output and the pause container's stdio are wrapped under `_pause`. This does not capture sentry or gofer output, which runsc directs to `/dev/null` when debug logging is disabled. +* `ate.actor.container.name`: The name of the container within the actor that produced the log line (e.g., `counter`), so a multi-container actor's logs can be demultiplexed by container. Absent on the synthetic lifecycle records (`Actor starting`, `Actor restored`, …): those are about the actor, so no container produced them. The `runsc create`, `start`, and `restore` CLI output for the sandbox is wrapped under `_pause`. This does not capture sentry or gofer output, which runsc directs to `/dev/null` when debug logging is disabled. Currently, Agent Substrate automatically wraps container output and injects these metadata labels into **container logs**. For metrics and distributed tracing, Agent Substrate provides foundational system telemetry and head-sampled request tracing (10% by default, 1% at the router), with roadmap plans to fully integrate actor-level correlation. @@ -74,7 +74,7 @@ An actor can run several containers. By default application container logs and s kubectl ate logs actor -a -c ``` -To view the pause container's runsc diagnostics explicitly: +To view the sandbox's runsc diagnostics explicitly: ```bash kubectl ate logs actor -a -c _pause diff --git a/docs/sandbox-asset-cache.md b/docs/sandbox-asset-cache.md index f1e5131aad..13df476b5a 100644 --- a/docs/sandbox-asset-cache.md +++ b/docs/sandbox-asset-cache.md @@ -38,7 +38,7 @@ invalidates: changed content arrives under a new key. 1. ate-api-server places the actor on a worker and names, in `Run` or `Restore`, the **sandbox runtime** that worker registered. Its `sandbox_bundle_hash` attribute is the hash of one `SandboxConfig` - version's assets and pause image. + version's assets. 2. `atelet` resolves that hash in its **runtime index** to the asset record the hash was computed from. A hash the index does not hold fails the request with `FailedPrecondition`: the actor reached a node whose workers @@ -74,8 +74,8 @@ pool runs, hashes every enabled version, and: says; - makes the worker's `default-runtime/` directory match the file's **default** runtimes, and hands each to the **prewarmer**, which fetches - its assets into the cache, links them into the runtime's directory, and - pulls its pause image, so the first actor on the node finds a warm cache. + its assets into the cache and links them into the runtime's directory, so + the first actor on the node finds a warm cache. A `SandboxConfig` or `WorkerPool` change registers the affected pools' runtimes at once and rewrites their files after a jitter of up to thirty @@ -104,8 +104,8 @@ which directories exist. Nothing counts references and nothing prunes. permanent `runsc` probe failure does, leaves the file and so the directories as they were, with the previous default still linked and still advertised. The prewarm starts late by only a little: a gVisor default's - release is fetched by the build's probe, so the prewarm is links and the - pause image pull, and a micro-VM build takes milliseconds. + release is fetched by the build's probe, so the prewarm is links, and a + micro-VM build takes milliseconds. - The prewarm fills what exists and creates nothing. It checks that its directory is there, stages into it, and on an error checks again: a directory gone, after a later sync or the pod's removal, means the item diff --git a/internal/hardware/hardware.go b/internal/hardware/hardware.go index ff858cf033..af796bcabf 100644 --- a/internal/hardware/hardware.go +++ b/internal/hardware/hardware.go @@ -31,8 +31,7 @@ const ( AttrArchitecture = "architecture" // AttrSandboxBundleHash identifies the sandbox runtime bundle: a - // sha256 over a SandboxConfig version's asset digests and pause image - // digest, computed by atelet. + // sha256 over a SandboxConfig version's asset digests, computed by atelet. AttrSandboxBundleHash = "sandbox_bundle_hash" // AttrCPUFeatures is the set of CPU features `runsc cpu-features` reports diff --git a/internal/imagecache/imagecache.go b/internal/imagecache/imagecache.go index 9afaa78ab9..630dc81f0f 100644 --- a/internal/imagecache/imagecache.go +++ b/internal/imagecache/imagecache.go @@ -148,10 +148,9 @@ var ( // sharedRegistries are the well-known communal registries whose rate limits // are shared across all anonymous clients. Everything not listed here is -// assumed dedicated: fleet-scale pulls realistically hit either the pause -// image (registry.k8s.io) or the customer's own registry, and enumerating -// the small stable set of communal hosts beats guessing at every private -// registry vendor. +// assumed dedicated: fleet-scale pulls realistically hit the customer's own +// registry, and enumerating the small stable set of communal hosts beats +// guessing at every private registry vendor. var sharedRegistries = map[string]bool{ "registry.k8s.io": true, // Pulls only ever present index.docker.io here: name.ParseReference diff --git a/internal/ocispec/gvisor.go b/internal/ocispec/gvisor.go index c532a0c67f..d0ffdf22c2 100644 --- a/internal/ocispec/gvisor.go +++ b/internal/ocispec/gvisor.go @@ -22,7 +22,8 @@ import ( "go.ate.dev/substrate/internal/sizing" ) -// PauseContainer is the name of the sandbox root container. The underscore +// PauseContainer is the runsc ID of the sandbox. runsc creates it with +// -no-root-container, so it has no process and no rootfs. The underscore // keeps it outside the k8s-short-name an ActorTemplate container // name is drawn from, so no actor container can collide with it. const PauseContainer = "_pause" @@ -45,13 +46,17 @@ func GVisorCgroupLeaf(actorUID, containerName string) string { } // ShapeGVisor adds runsc CRI annotations, durable-dir mount hints, and -// per-container cgroups to the spec. It is idempotent. +// per-container cgroups to the spec. It removes the process, rootfs, and +// mounts from the sandbox spec. It is idempotent. func ShapeGVisor(spec *specs.Spec, o GVisorOptions) { if spec.Annotations == nil { spec.Annotations = make(map[string]string) } spec.Annotations["io.kubernetes.cri.container-name"] = o.ContainerName if o.ContainerName == PauseContainer { + spec.Process = nil + spec.Root = nil + spec.Mounts = nil spec.Annotations["io.kubernetes.cri.container-type"] = "sandbox" for _, v := range o.DurableVolumes { spec.Annotations["dev.gvisor.spec.mount."+v+".type"] = "tmpfs" diff --git a/internal/ocispec/ocispec_test.go b/internal/ocispec/ocispec_test.go index 1d5438f2ca..4b4e7fb101 100644 --- a/internal/ocispec/ocispec_test.go +++ b/internal/ocispec/ocispec_test.go @@ -137,9 +137,9 @@ func TestBuild_Capabilities(t *testing.T) { } } -// The pause container gets no capabilities. -func TestBuild_NoCapabilitiesForPause(t *testing.T) { - spec := Build(Options{Container: &ateworkerpb.Container{ContainerSpec: &ateworkerpb.ContainerSpec{Args: []string{"/pause"}}}}) +// A container that declares no capabilities gets none. +func TestBuild_NoCapabilities(t *testing.T) { + spec := Build(Options{Container: &ateworkerpb.Container{ContainerSpec: &ateworkerpb.ContainerSpec{Args: []string{"/app"}}}}) caps := spec.Process.Capabilities if caps == nil { diff --git a/internal/ocispec/parity_test.go b/internal/ocispec/parity_test.go index e341285761..8bc028a772 100644 --- a/internal/ocispec/parity_test.go +++ b/internal/ocispec/parity_test.go @@ -16,6 +16,7 @@ package ocispec import ( "encoding/json" + "slices" "strings" "testing" @@ -140,15 +141,25 @@ func TestShapeMicroVM_UnstagedSourceIsAnError(t *testing.T) { // Shaping a spec twice does not accumulate mounts. func TestShapeGVisor_Idempotent(t *testing.T) { spec := Build(parityOptions) - o := GVisorOptions{ActorUID: testActorUID, ActorDirs: parityActorDirs, ContainerName: PauseContainer, DurableVolumes: []string{"data"}, Size: paritySize} + o := GVisorOptions{ActorUID: testActorUID, ActorDirs: parityActorDirs, ContainerName: "app", Size: paritySize} ShapeGVisor(spec, o) first := len(spec.Mounts) ShapeGVisor(spec, o) if len(spec.Mounts) != first { t.Errorf("mount count = %d after a second shaping, want %d", len(spec.Mounts), first) } +} + +// The sandbox spec has no process, rootfs, or mounts, and declares the +// durable-dir mount hints. +func TestShapeGVisor_Sandbox(t *testing.T) { + spec := Build(parityOptions) + ShapeGVisor(spec, GVisorOptions{ActorUID: testActorUID, ActorDirs: parityActorDirs, ContainerName: PauseContainer, DurableVolumes: []string{"data"}, Size: paritySize}) + if spec.Process != nil || spec.Root != nil || spec.Mounts != nil { + t.Errorf("sandbox spec has process %v, root %v, mounts %v; want none", spec.Process, spec.Root, spec.Mounts) + } if got := spec.Annotations["io.kubernetes.cri.container-type"]; got != "sandbox" { - t.Errorf("pause container-type = %q, want sandbox", got) + t.Errorf("sandbox container-type = %q, want sandbox", got) } if got := spec.Annotations["dev.gvisor.spec.mount.data.type"]; got != "tmpfs" { t.Errorf("durable-dir mount type annotation = %q, want tmpfs", got) @@ -163,11 +174,22 @@ func TestShapeGVisor_Idempotent(t *testing.T) { // gVisor sizes the container's cgroup leaf from the actor-level limits: one // sentry backs every container, so the sandbox cgroup is the only one that -// binds. +// binds. The sandbox spec keeps its limits, cgroup, and namespaces. func TestShapeGVisor_AppliesTheActorSize(t *testing.T) { - spec := Build(parityOptions) - ShapeGVisor(spec, GVisorOptions{ActorUID: testActorUID, ContainerName: "app", Size: paritySize}) - if got := *spec.Linux.Resources.Memory.Limit; got != paritySize.MemoryBytes { - t.Errorf("memory limit = %d, want %d", got, paritySize.MemoryBytes) + for _, name := range []string{"app", PauseContainer} { + spec := Build(parityOptions) + ShapeGVisor(spec, GVisorOptions{ActorUID: testActorUID, ContainerName: name, Size: paritySize}) + if got := *spec.Linux.Resources.Memory.Limit; got != paritySize.MemoryBytes { + t.Errorf("%s memory limit = %d, want %d", name, got, paritySize.MemoryBytes) + } + if want := "/" + GVisorCgroupLeaf(testActorUID, name); spec.Linux.CgroupsPath != want { + t.Errorf("%s cgroupsPath = %q, want %q", name, spec.Linux.CgroupsPath, want) + } + if !slices.ContainsFunc(spec.Linux.Namespaces, func(ns specs.LinuxNamespace) bool { return ns.Type == "network" && ns.Path != "" }) { + t.Errorf("%s namespaces = %v, want the actor's network namespace", name, spec.Linux.Namespaces) + } + if spec.Hostname == "" { + t.Errorf("%s hostname is empty", name) + } } } diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index fc781298ee..d7d8ba3e49 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -1682,9 +1682,9 @@ type RunRequest struct { // +k8s:optional Resources *ResourceRequirements `protobuf:"bytes,10,opt,name=resources,proto3" json:"resources,omitempty"` // The runtime the assigned worker runs this actor under, from - // WorkerAssignment.sandbox_runtime. atelet resolves the pause image and - // sandbox binaries through its advertised-runtime index and records the - // runtime for the actor so a later Checkpoint pins the snapshot to it. + // WorkerAssignment.sandbox_runtime. atelet resolves the sandbox binaries + // through its advertised-runtime index and records the runtime for the + // actor so a later Checkpoint pins the snapshot to it. // // +k8s:required SandboxRuntime *SandboxRuntime `protobuf:"bytes,12,opt,name=sandbox_runtime,json=sandboxRuntime,proto3" json:"sandbox_runtime,omitempty"` @@ -3600,10 +3600,10 @@ type RestoreRequest struct { // +k8s:optional Resources *ResourceRequirements `protobuf:"bytes,10,opt,name=resources,proto3" json:"resources,omitempty"` // The runtime the assigned worker restores this actor under, from - // WorkerAssignment.sandbox_runtime. atelet resolves the pause image and - // sandbox binaries through its advertised-runtime index and records the - // runtime for the actor so the next Checkpoint pins the snapshot to what ran - // this activation, not what the restored snapshot was taken under. + // WorkerAssignment.sandbox_runtime. atelet resolves the sandbox binaries + // through its advertised-runtime index and records the runtime for the + // actor so the next Checkpoint pins the snapshot to what ran this + // activation, not what the restored snapshot was taken under. // // +k8s:required SandboxRuntime *SandboxRuntime `protobuf:"bytes,12,opt,name=sandbox_runtime,json=sandboxRuntime,proto3" json:"sandbox_runtime,omitempty"` diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index b1ab598586..24e17cef7e 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -485,9 +485,9 @@ message RunRequest { ResourceRequirements resources = 10; // The runtime the assigned worker runs this actor under, from - // WorkerAssignment.sandbox_runtime. atelet resolves the pause image and - // sandbox binaries through its advertised-runtime index and records the - // runtime for the actor so a later Checkpoint pins the snapshot to it. + // WorkerAssignment.sandbox_runtime. atelet resolves the sandbox binaries + // through its advertised-runtime index and records the runtime for the + // actor so a later Checkpoint pins the snapshot to it. // // +k8s:required SandboxRuntime sandbox_runtime = 12; @@ -1101,10 +1101,10 @@ message RestoreRequest { ResourceRequirements resources = 10; // The runtime the assigned worker restores this actor under, from - // WorkerAssignment.sandbox_runtime. atelet resolves the pause image and - // sandbox binaries through its advertised-runtime index and records the - // runtime for the actor so the next Checkpoint pins the snapshot to what ran - // this activation, not what the restored snapshot was taken under. + // WorkerAssignment.sandbox_runtime. atelet resolves the sandbox binaries + // through its advertised-runtime index and records the runtime for the + // actor so the next Checkpoint pins the snapshot to what ran this + // activation, not what the restored snapshot was taken under. // // +k8s:required SandboxRuntime sandbox_runtime = 12; diff --git a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml index cdb8a6512d..9e0c24cfc0 100644 --- a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml +++ b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml @@ -92,8 +92,8 @@ spec: keyed by name. items: description: |- - SandboxVersionConfig is one version of a SandboxConfig: the exact pause image - and asset set a sandbox boots with. + SandboxVersionConfig is one version of a SandboxConfig: the asset set a + sandbox boots with. properties: assets: additionalProperties: @@ -143,26 +143,6 @@ spec: minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string - pauseImage: - description: |- - pauseImage is the container image used as the root sandbox container. - It holds the sandbox's namespaces and runs no workload code, so it is an - implementation detail of the sandbox rather than something actor authors - choose. Required for gvisor; not allowed for microvm, which runs no pause - container. It is captured in the snapshot manifest alongside the sandbox - binaries, so a restore always re-creates the sandbox from the same image - the snapshot was taken with. - - Typically, set it to [1] for on-gcp, and [2] for off-gcp - - - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da - - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 - maxLength: 512 - minLength: 1 - type: string - x-kubernetes-validations: - - message: All images must include a digest - rule: self.contains('@') state: default: Enabled description: |- @@ -191,11 +171,6 @@ spec: - message: the version named by defaultVersion must not be Disabled rule: '!self.versions.exists(v, v.name == self.defaultVersion && has(v.state) && v.state == ''Disabled'')' - - message: pauseImage is required on every version for gvisor and not - allowed for other sandbox classes - rule: 'self.sandboxClass == ''gvisor'' ? self.versions.all(v, has(v.pauseImage) - && size(v.pauseImage) > 0) : self.versions.all(v, !has(v.pauseImage) - || size(v.pauseImage) == 0)' required: - spec type: object diff --git a/manifests/ate-install/sandboxconfig-gvisor.yaml b/manifests/ate-install/sandboxconfig-gvisor.yaml index ab8fd3742c..d6b41e5857 100644 --- a/manifests/ate-install/sandboxconfig-gvisor.yaml +++ b/manifests/ate-install/sandboxconfig-gvisor.yaml @@ -32,12 +32,10 @@ spec: # gVisor nightly 2026-10-17, the first to include google/gvisor@b79256080dec # (save and restore multiple filesystem checkpoint bundles). ateworker-gvisor # uses it to save container rootfs overlays (rootfs/) and durable-dir volumes - # (fs/) as separate bundles. + # (fs/) as separate bundles, and to run the sandbox with no root container + # (google/gvisor#15551). - name: v4 state: Enabled - # The root sandbox container's image. On GCP, prefer the in-project mirror - # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da. - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" assets: amd64: gvisor: diff --git a/pkg/api/v1alpha1/sandboxconfig_types.go b/pkg/api/v1alpha1/sandboxconfig_types.go index 3672affbff..3c5276c418 100644 --- a/pkg/api/v1alpha1/sandboxconfig_types.go +++ b/pkg/api/v1alpha1/sandboxconfig_types.go @@ -96,8 +96,8 @@ const ( SandboxVersionStateDisabled SandboxVersionState = "Disabled" ) -// SandboxVersionConfig is one version of a SandboxConfig: the exact pause image -// and asset set a sandbox boots with. +// SandboxVersionConfig is one version of a SandboxConfig: the asset set a +// sandbox boots with. type SandboxVersionConfig struct { // name identifies this version within its SandboxConfig. It is a DNS label // (lower-case alphanumerics and '-'). @@ -115,28 +115,6 @@ type SandboxVersionConfig struct { // +kubebuilder:default=Enabled State *SandboxVersionState `json:"state,omitempty"` - // TODO: drop PauseImage once gVisor can run without a pause container: - // https://github.com/google/gvisor/pull/13981 - - // pauseImage is the container image used as the root sandbox container. - // It holds the sandbox's namespaces and runs no workload code, so it is an - // implementation detail of the sandbox rather than something actor authors - // choose. Required for gvisor; not allowed for microvm, which runs no pause - // container. It is captured in the snapshot manifest alongside the sandbox - // binaries, so a restore always re-creates the sandbox from the same image - // the snapshot was taken with. - // - // Typically, set it to [1] for on-gcp, and [2] for off-gcp - // - // - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da - // - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 - // - // +optional - // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:MaxLength=512 - // +kubebuilder:validation:XValidation:rule="self.contains('@')",message="All images must include a digest" - PauseImage string `json:"pauseImage,omitempty"` - // assets is the set of files atelet fetches for this runtime, keyed first by // architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The // asset names are interpreted by the sandbox backend: gVisor expects a @@ -154,7 +132,6 @@ type SandboxVersionConfig struct { // // +kubebuilder:validation:XValidation:rule="self.versions.exists(v, v.name == self.defaultVersion)",message="defaultVersion must name an entry in versions" // +kubebuilder:validation:XValidation:rule="!self.versions.exists(v, v.name == self.defaultVersion && has(v.state) && v.state == 'Disabled')",message="the version named by defaultVersion must not be Disabled" -// +kubebuilder:validation:XValidation:rule="self.sandboxClass == 'gvisor' ? self.versions.all(v, has(v.pauseImage) && size(v.pauseImage) > 0) : self.versions.all(v, !has(v.pauseImage) || size(v.pauseImage) == 0)",message="pauseImage is required on every version for gvisor and not allowed for other sandbox classes" type SandboxConfigSpec struct { // sandboxClass is the sandbox runtime family this config applies to. A // WorkerPool only uses SandboxConfigs whose sandboxClass matches the entry diff --git a/pkg/api/v1alpha1/sandboxconfig_validation_test.go b/pkg/api/v1alpha1/sandboxconfig_validation_test.go index 5336a93296..6860130a6a 100644 --- a/pkg/api/v1alpha1/sandboxconfig_validation_test.go +++ b/pkg/api/v1alpha1/sandboxconfig_validation_test.go @@ -40,15 +40,9 @@ const gvisorAssetMsg = "'gvisor' (release tarball) asset" // the install — loaded here so the test guards the policy we actually ship. const vapManifestPath = "../../../manifests/ate-install/sandboxconfig-validation.yaml" -const validPauseImage = "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" - -// sandboxVersion returns a version with a pause image iff the class needs one. -func sandboxVersion(name string, class SandboxClass, assets map[string]map[string]AssetFile) SandboxVersionConfig { - v := SandboxVersionConfig{Name: name, Assets: assets} - if class == SandboxClassGvisor { - v.PauseImage = validPauseImage - } - return v +// sandboxVersion returns a version with the given name and assets. +func sandboxVersion(name string, assets map[string]map[string]AssetFile) SandboxVersionConfig { + return SandboxVersionConfig{Name: name, Assets: assets} } // sandboxConfig returns a config with a single default version "v1". @@ -58,18 +52,11 @@ func sandboxConfig(name string, class SandboxClass, assets map[string]map[string Spec: SandboxConfigSpec{ SandboxClass: class, DefaultVersion: "v1", - Versions: []SandboxVersionConfig{sandboxVersion("v1", class, assets)}, + Versions: []SandboxVersionConfig{sandboxVersion("v1", assets)}, }, } } -// withPauseImage overrides the pause image of the first version on an -// otherwise-valid config. -func withPauseImage(sc *SandboxConfig, image string) *SandboxConfig { - sc.Spec.Versions[0].PauseImage = image - return sc -} - // withVersions appends versions to a config. func withVersions(sc *SandboxConfig, versions ...SandboxVersionConfig) *SandboxConfig { sc.Spec.Versions = append(sc.Spec.Versions, versions...) @@ -223,29 +210,14 @@ func TestSandboxConfigValidation(t *testing.T) { sc: sandboxConfig("bad-sha", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": {URL: "gs://bucket/gvisor.tar.bz2", SHA256: "deadbeef"}}}), wantErr: true, errMsg: "sha256", - }, { - name: "gvisor missing pauseImage", - sc: withPauseImage(sandboxConfig("bad-no-pause", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}}), ""), - wantErr: true, - errMsg: "pauseImage is required on every version for gvisor", - }, { - name: "microvm with pauseImage", - sc: withPauseImage(sandboxConfig("bad-microvm-pause", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": microVMAssets()}), validPauseImage), - wantErr: true, - errMsg: "pauseImage is required on every version for gvisor", - }, { - name: "unpinned pauseImage", - sc: withPauseImage(sandboxConfig("bad-unpinned-pause", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}}), "registry.k8s.io/pause:3.10.2"), - wantErr: true, - errMsg: "All images must include a digest", }, { name: "valid gvisor with two versions", - sc: withVersions(sandboxConfig("ok-two-versions", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", SandboxClassGvisor, gvisorAssets())), + sc: withVersions(sandboxConfig("ok-two-versions", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", gvisorAssets())), wantErr: false, }, { name: "valid non-default disabled version", sc: withVersions(sandboxConfig("ok-disabled-old", SandboxClassGvisor, gvisorAssets()), func() SandboxVersionConfig { - v := sandboxVersion("v0", SandboxClassGvisor, gvisorAssets()) + v := sandboxVersion("v0", gvisorAssets()) v.State = ptr.To(SandboxVersionStateDisabled) return v }()), @@ -271,7 +243,7 @@ func TestSandboxConfigValidation(t *testing.T) { errMsg: "spec.versions", }, { name: "duplicate version names", - sc: withVersions(sandboxConfig("bad-dup-versions", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v1", SandboxClassGvisor, gvisorAssets())), + sc: withVersions(sandboxConfig("bad-dup-versions", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v1", gvisorAssets())), wantErr: true, errMsg: "Duplicate value", }, { @@ -279,7 +251,7 @@ func TestSandboxConfigValidation(t *testing.T) { sc: func() *SandboxConfig { sc := sandboxConfig("bad-many-versions", SandboxClassGvisor, gvisorAssets()) for i := range 16 { - sc.Spec.Versions = append(sc.Spec.Versions, sandboxVersion(fmt.Sprintf("extra-%d", i), SandboxClassGvisor, gvisorAssets())) + sc.Spec.Versions = append(sc.Spec.Versions, sandboxVersion(fmt.Sprintf("extra-%d", i), gvisorAssets())) } return sc }(), @@ -295,32 +267,14 @@ func TestSandboxConfigValidation(t *testing.T) { }(), wantErr: true, errMsg: "spec.versions[0].name", - }, { - name: "gvisor second version missing pauseImage", - sc: withVersions(sandboxConfig("bad-v2-no-pause", SandboxClassGvisor, gvisorAssets()), func() SandboxVersionConfig { - v := sandboxVersion("v2", SandboxClassGvisor, gvisorAssets()) - v.PauseImage = "" - return v - }()), - wantErr: true, - errMsg: "pauseImage is required on every version for gvisor", - }, { - name: "microvm second version with pauseImage", - sc: withVersions(sandboxConfig("bad-microvm-v2-pause", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": microVMAssets()}), func() SandboxVersionConfig { - v := sandboxVersion("v2", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": microVMAssets()}) - v.PauseImage = validPauseImage - return v - }()), - wantErr: true, - errMsg: "pauseImage is required on every version for gvisor", }, { name: "gvisor second version missing the gvisor asset", - sc: withVersions(sandboxConfig("bad-v2-no-gvisor", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"other": otherAsset()}})), + sc: withVersions(sandboxConfig("bad-v2-no-gvisor", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", map[string]map[string]AssetFile{"amd64": {"other": otherAsset()}})), wantErr: true, errMsg: gvisorAssetMsg, }, { name: "microvm second version missing an asset", - sc: withVersions(sandboxConfig("bad-microvm-v2", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": microVMAssets()}), sandboxVersion("v2", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": func() map[string]AssetFile { + sc: withVersions(sandboxConfig("bad-microvm-v2", SandboxClassMicroVM, map[string]map[string]AssetFile{"amd64": microVMAssets()}), sandboxVersion("v2", map[string]map[string]AssetFile{"amd64": func() map[string]AssetFile { m := microVMAssets() delete(m, "kata-kernel") return m @@ -367,7 +321,7 @@ func TestSandboxConfigSandboxClassDefault(t *testing.T) { func TestSandboxConfigUpdateRemovingDefaultVersion(t *testing.T) { ctx := t.Context() - sc := withVersions(sandboxConfig("update-remove-default", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", SandboxClassGvisor, gvisorAssets())) + sc := withVersions(sandboxConfig("update-remove-default", SandboxClassGvisor, gvisorAssets()), sandboxVersion("v2", gvisorAssets())) if err := k8sClient.Create(ctx, sc); err != nil { t.Fatalf("Create() unexpected error: %v", err) } From 586e8936e9e083d64b46197ccf712ea6aff96430 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sun, 18 Oct 2026 01:41:07 +0000 Subject: [PATCH 2/5] Rename the gVisor sandbox from _pause to _sandbox The sandbox has no pause container, so the name _pause is misleading. Rename the sandbox's runsc ID to _sandbox and ocispec.PauseContainer to ocispec.SandboxID. The sandbox's runsc output is now logged under _sandbox, its cgroup leaf is -_sandbox, and the restore phases are sandbox_create and sandbox_restore. Remove the claim in the architecture doc that snapshot manifests pin the sandbox binaries for restore. A restore gets its runtime from the control plane. Signed-off-by: Jing Chen --- benchmarking/analysis/README.md | 2 +- benchmarking/analysis/phase_report.py | 2 +- .../internal/apivalidation/validation_test.go | 6 +-- cmd/atelet/main_test.go | 2 +- cmd/ateworker-gvisor/ckptcgroup.go | 24 +++++----- cmd/ateworker-gvisor/ckptcgroup_test.go | 34 +++++++------- cmd/ateworker-gvisor/cleanup_test.go | 8 ++-- cmd/ateworker-gvisor/crash_test.go | 2 +- cmd/ateworker-gvisor/main.go | 40 ++++++++-------- cmd/ateworker-gvisor/main_test.go | 12 ++--- cmd/ateworker-gvisor/phaselog.go | 28 +++++------ cmd/ateworker-gvisor/phaselog_test.go | 46 +++++++++---------- cmd/ateworker-gvisor/runsc.go | 6 +-- cmd/ateworker-gvisor/runsc_test.go | 28 +++++------ cmd/ateworker-gvisor/sandboxexit.go | 6 +-- cmd/ateworker-gvisor/sandboxexit_test.go | 2 +- cmd/ateworker-gvisor/stats.go | 10 ++-- cmd/ateworker-gvisor/stats_test.go | 14 +++--- cmd/kubectl-ate/internal/cmd/actor.go | 4 +- cmd/kubectl-ate/internal/cmd/actor_test.go | 28 +++++------ docs/architecture.md | 2 +- docs/observability.md | 8 ++-- .../ateworkervalidation/validation_test.go | 2 +- internal/ocispec/gvisor.go | 10 ++-- internal/ocispec/gvisor_test.go | 10 ++-- internal/ocispec/parity_test.go | 4 +- internal/proto/ateworkerpb/ateworker.pb.go | 4 +- internal/proto/ateworkerpb/ateworker.proto | 4 +- 28 files changed, 174 insertions(+), 174 deletions(-) diff --git a/benchmarking/analysis/README.md b/benchmarking/analysis/README.md index 3109de03fb..b2a6735bec 100644 --- a/benchmarking/analysis/README.md +++ b/benchmarking/analysis/README.md @@ -85,7 +85,7 @@ restore between `volume_mount`, `manifest_fetch`, `sandbox_assets`, `prep` / `pause` / `snapshot` / `durable_dir` / `rootfs_upper` / `teardown` on a checkpoint and `prep` / … / `vm_restore` / `wakeup_probe` on a restore; for gVisor `pause` / `checkpoint` / `fs_checkpoint` / `resume` / `teardown` and -`prep` / `net_setup` / … / `pause_restore` / `app_restore` / +`prep` / `net_setup` / … / `sandbox_restore` / `app_restore` / `wakeup_probe` / `activate` (the full lists are in `cmd/ateworker-*/phaselog.go`). The ateworker records carry no sandbox class or kind of their own; a paired one takes both diff --git a/benchmarking/analysis/phase_report.py b/benchmarking/analysis/phase_report.py index cb7611880e..edbe814b8c 100755 --- a/benchmarking/analysis/phase_report.py +++ b/benchmarking/analysis/phase_report.py @@ -94,7 +94,7 @@ # ateworker restore: micro-VM (cmd/ateworker-microvm/phaselog.go) and gVisor # (cmd/ateworker-gvisor/phaselog.go), each in its own sequential order "prep", "egress_prepare", "net_setup", "bundles", "upper_join", "lowers", - "pause_create", "pause_restore", + "sandbox_create", "sandbox_restore", "app_rootfs", "app_create", "app_restore", "tap", "vmm_launch", "vm_restore", "resume", "wakeup_probe", "activate", # atelet + ateworker checkpoint diff --git a/cmd/atelet/internal/apivalidation/validation_test.go b/cmd/atelet/internal/apivalidation/validation_test.go index fff8d0b2b6..9fcfdf371b 100644 --- a/cmd/atelet/internal/apivalidation/validation_test.go +++ b/cmd/atelet/internal/apivalidation/validation_test.go @@ -477,7 +477,7 @@ func TestValidateCheckpointRequest(t *testing.T) { }, { name: "spec errors surface at the spec's path", obj: valid(func(r *ateletpb.CheckpointRequest) { - r.Spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "_pause", Image: testDigestImage}}} + r.Spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "_sandbox", Image: testDigestImage}}} }), want: field.ErrorList{field.Invalid(field.NewPath("spec", "containers").Index(0).Child("name"), nil, "").WithOrigin("format=k8s-short-name")}, }, { @@ -652,7 +652,7 @@ func TestValidateRestoreRequest(t *testing.T) { }, { name: "spec errors surface at the spec's path", obj: valid(func(r *ateletpb.RestoreRequest) { - r.Spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "_pause", Image: testDigestImage}}} + r.Spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "_sandbox", Image: testDigestImage}}} }), want: field.ErrorList{field.Invalid(field.NewPath("spec", "containers").Index(0).Child("name"), nil, "").WithOrigin("format=k8s-short-name")}, }, { @@ -939,7 +939,7 @@ func TestValidateTerminateRequest(t *testing.T) { }, { name: "spec errors surface at the spec's path", obj: valid(func(r *ateletpb.TerminateRequest) { - r.Spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "_pause", Image: testDigestImage}}} + r.Spec = &ateletpb.WorkloadSpec{Containers: []*ateletpb.Container{{Name: "_sandbox", Image: testDigestImage}}} }), want: field.ErrorList{field.Invalid(field.NewPath("spec", "containers").Index(0).Child("name"), nil, "").WithOrigin("format=k8s-short-name")}, }, { diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index 3d804d9690..33436ab010 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -343,7 +343,7 @@ func TestPrepareOCIBundlesNoSandboxBundle(t *testing.T) { if _, err := os.Stat(filepath.Join(ateletpath.OCIBundlePath(actorUID, "app"), imagecache.OverlaySpecFileName)); err != nil { t.Errorf("app bundle: %v", err) } - if _, err := os.Stat(ateletpath.OCIBundlePath(actorUID, ocispec.PauseContainer)); !os.IsNotExist(err) { + if _, err := os.Stat(ateletpath.OCIBundlePath(actorUID, ocispec.SandboxID)); !os.IsNotExist(err) { t.Errorf("sandbox bundle stat = %v, want it absent", err) } } diff --git a/cmd/ateworker-gvisor/ckptcgroup.go b/cmd/ateworker-gvisor/ckptcgroup.go index d3f332c985..e5e26eefa9 100644 --- a/cmd/ateworker-gvisor/ckptcgroup.go +++ b/cmd/ateworker-gvisor/ckptcgroup.go @@ -35,8 +35,8 @@ import ( "golang.org/x/sys/unix" ) -// During checkpoint, moving the sentry from -_pause into a sibling -// -_ckpt cgroup (memory.max = "max", cpu.max copied from -_pause) +// During checkpoint, moving the sentry from -_sandbox into a sibling +// -_ckpt cgroup (memory.max = "max", cpu.max copied from -_sandbox) // prevents newly written pages.img cache from evicting clean pages or // OOM-killing the sentry under the actor's memory.max. Removing -_ckpt // after sandbox teardown reparents the cache to the worker scope for fast @@ -45,7 +45,7 @@ import ( // POSIX_FADV_DONTNEED once background restore page loading completes. const ( - // checkpointCgroupSuffix uses a leading underscore like _pause so it cannot + // checkpointCgroupSuffix uses a leading underscore like _sandbox so it cannot // collide with a DNS-label application container leaf (-). checkpointCgroupSuffix = "-_ckpt" ) @@ -60,8 +60,8 @@ func checkpointCgroupDir(root, actorUID string) string { return filepath.Join(root, actorUID+checkpointCgroupSuffix) } -// isolateCheckpointCache moves the actor's sentry from its pause leaf into a -// sibling -_ckpt cgroup, preserving the pause leaf's cpu.max so checkpoint +// isolateCheckpointCache moves the actor's sentry from its sandbox leaf into a +// sibling -_ckpt cgroup, preserving the sandbox leaf's cpu.max so checkpoint // serialization remains capped to the actor's CPU limit. func isolateCheckpointCache(root string, actorDirs *ateworkerpb.ActorDirs, actorUID string) (*checkpointCgroup, error) { pid, err := readSandboxPID(actorDirs) @@ -69,7 +69,7 @@ func isolateCheckpointCache(root string, actorDirs *ateworkerpb.ActorDirs, actor return nil, err } - leaf := filepath.Join(root, ocispec.GVisorCgroupLeaf(actorUID, ocispec.PauseContainer)) + leaf := filepath.Join(root, ocispec.GVisorCgroupLeaf(actorUID, ocispec.SandboxID)) if !cgroupHasPID(leaf, pid) { return nil, fmt.Errorf("sandbox pid %d is not in cgroup %q", pid, leaf) } @@ -90,7 +90,7 @@ func isolateCheckpointCache(root string, actorDirs *ateworkerpb.ActorDirs, actor } func readSandboxPID(actorDirs *ateworkerpb.ActorDirs) (int, error) { - pidFile := pidFilePath(actorDirs, ocispec.PauseContainer) + pidFile := pidFilePath(actorDirs, ocispec.SandboxID) b, err := os.ReadFile(pidFile) if err != nil { return 0, fmt.Errorf("while reading sandbox pid file %q: %w", pidFile, err) @@ -120,7 +120,7 @@ func writeCgroupPID(dir string, pid int) error { return nil } -// restore moves the sentry back into its pause leaf on checkpoint failure. It +// restore moves the sentry back into its sandbox leaf on checkpoint failure. It // returns nil once the sentry has exited; any other failure leaves the sentry // alive in the checkpoint cgroup, so the caller must terminate it. func (c *checkpointCgroup) restore() error { @@ -151,7 +151,7 @@ func (c *checkpointCgroup) remove() error { // checkpointIsolated isolates the sentry into a sibling cgroup (when the // worker's cgroup is delegated), runs checkpoint, and tears down the workload // before removing the sibling cgroup so its page cache reparents cleanly. If -// checkpoint fails, it moves the sentry back into its pause leaf (or terminates +// checkpoint fails, it moves the sentry back into its sandbox leaf (or terminates // the workload if that rollback fails). func checkpointIsolated( ctx context.Context, @@ -177,9 +177,9 @@ func checkpointIsolated( } err := ckptCgroup.remove() if err != nil { - // If terminate left the sentry alive, move it back to its pause leaf. + // If terminate left the sentry alive, move it back to its sandbox leaf. if rerr := ckptCgroup.restore(); rerr != nil { - slog.WarnContext(ctx, "Failed to move the sentry back into its pause leaf", + slog.WarnContext(ctx, "Failed to move the sentry back into its sandbox leaf", slog.String("actorUID", actorUID), slog.Any("err", rerr)) } else { err = ckptCgroup.remove() @@ -193,7 +193,7 @@ func checkpointIsolated( if err := checkpoint(ctx); err != nil { if rerr := ckptCgroup.restore(); rerr != nil { - slog.WarnContext(ctx, "Failed to move the sentry back into its pause leaf; terminating workload", + slog.WarnContext(ctx, "Failed to move the sentry back into its sandbox leaf; terminating workload", slog.String("actorUID", actorUID), slog.Any("err", rerr)) } else { terminateOnExit = false diff --git a/cmd/ateworker-gvisor/ckptcgroup_test.go b/cmd/ateworker-gvisor/ckptcgroup_test.go index 7cb2c83ac9..dfca0796e9 100644 --- a/cmd/ateworker-gvisor/ckptcgroup_test.go +++ b/cmd/ateworker-gvisor/ckptcgroup_test.go @@ -50,14 +50,14 @@ func newCkptFixture(t *testing.T, sandboxPID string) ckptFixture { CheckpointDir: t.TempDir(), }, } - f.leaf = filepath.Join(f.root, ocispec.GVisorCgroupLeaf(testActorUID, ocispec.PauseContainer)) + f.leaf = filepath.Join(f.root, ocispec.GVisorCgroupLeaf(testActorUID, ocispec.SandboxID)) mustMkdir(t, f.leaf) mustWrite(t, filepath.Join(f.leaf, "cgroup.procs"), "100\n"+sandboxPID+"\n") mustWrite(t, filepath.Join(f.leaf, "memory.max"), "805306368\n") mustWrite(t, filepath.Join(f.leaf, "cpu.max"), "200000 100000\n") if sandboxPID != "" { - pidFile := pidFilePath(f.actorDirs, ocispec.PauseContainer) + pidFile := pidFilePath(f.actorDirs, ocispec.SandboxID) mustMkdir(t, filepath.Dir(pidFile)) mustWrite(t, pidFile, sandboxPID+"\n") } @@ -69,7 +69,7 @@ func newCkptFixture(t *testing.T, sandboxPID string) ckptFixture { return err } // Emulate cgroup2 migration out of the sibling checkpoint cgroup so rmdir - // succeeds once the sentry moves back into its pause leaf. + // succeeds once the sentry moves back into its sandbox leaf. ckptDir := checkpointCgroupDir(f.root, testActorUID) if dir != ckptDir { _ = os.Remove(filepath.Join(ckptDir, "cgroup.procs")) @@ -125,7 +125,7 @@ func TestIsolateCheckpointCacheMovesOnlyTheSentry(t *testing.T) { t.Fatalf("restore: %v", err) } if got := mustRead(t, filepath.Join(f.leaf, "cgroup.procs")); got != "101" { - t.Errorf("pause leaf cgroup.procs after restore = %q, want 101", got) + t.Errorf("sandbox leaf cgroup.procs after restore = %q, want 101", got) } if err := c.remove(); err != nil { t.Fatalf("remove: %v", err) @@ -136,25 +136,25 @@ func TestIsolateCheckpointCacheWithoutPIDFile(t *testing.T) { f := newCkptFixture(t, "") if _, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID); err == nil { - t.Fatal("isolateCheckpointCache succeeded with no pause pid file") + t.Fatal("isolateCheckpointCache succeeded with no sandbox pid file") } if _, err := os.Stat(checkpointCgroupDir(f.root, testActorUID)); !os.IsNotExist(err) { t.Errorf("checkpoint cgroup left behind: stat err = %v", err) } } -func TestIsolateCheckpointCacheRejectsPIDOutsidePauseLeaf(t *testing.T) { +func TestIsolateCheckpointCacheRejectsPIDOutsideSandboxLeaf(t *testing.T) { f := newCkptFixture(t, "101") mustWrite(t, filepath.Join(f.leaf, "cgroup.procs"), "100\n") if _, err := isolateCheckpointCache(f.root, f.actorDirs, testActorUID); err == nil { - t.Fatal("isolateCheckpointCache succeeded with the pid outside the pause leaf") + t.Fatal("isolateCheckpointCache succeeded with the pid outside the sandbox leaf") } if _, err := os.Stat(checkpointCgroupDir(f.root, testActorUID)); !os.IsNotExist(err) { t.Errorf("checkpoint cgroup created for a reused pid: stat err = %v", err) } if got := mustRead(t, filepath.Join(f.leaf, "cgroup.procs")); got != "100\n" { - t.Errorf("pause leaf cgroup.procs = %q, want unchanged \"100\\n\"", got) + t.Errorf("sandbox leaf cgroup.procs = %q, want unchanged \"100\\n\"", got) } } @@ -185,14 +185,14 @@ func TestCheckpointCgroupRestoreSkipsExitedSentry(t *testing.T) { } // If the sentry exited (no longer in ckpt cgroup.procs) and its PID was - // recycled elsewhere, restore must not move it into the pause leaf. + // recycled elsewhere, restore must not move it into the sandbox leaf. mustWrite(t, filepath.Join(c.dir, "cgroup.procs"), "") mustWrite(t, filepath.Join(f.leaf, "cgroup.procs"), "100\n") if err := c.restore(); err != nil { t.Fatalf("restore with exited sentry: %v", err) } if got := mustRead(t, filepath.Join(f.leaf, "cgroup.procs")); got != "100\n" { - t.Errorf("pause leaf cgroup.procs = %q, want unchanged \"100\\n\"", got) + t.Errorf("sandbox leaf cgroup.procs = %q, want unchanged \"100\\n\"", got) } // If the sentry exits between cgroupHasPID and movePID (ESRCH), restore returns nil. @@ -296,7 +296,7 @@ func TestCheckpointIsolated(t *testing.T) { t.Error("terminate called after successful cgroup rollback") } if got := mustRead(t, filepath.Join(f.leaf, "cgroup.procs")); got != "101" { - t.Errorf("pause leaf cgroup.procs = %q, want 101", got) + t.Errorf("sandbox leaf cgroup.procs = %q, want 101", got) } if _, err := os.Stat(ckptDir); !os.IsNotExist(err) { t.Errorf("checkpoint cgroup left behind: stat err = %v", err) @@ -332,7 +332,7 @@ func TestCheckpointIsolated(t *testing.T) { } }) - t.Run("terminates workload if checkpoint fails and the pause leaf is gone", func(t *testing.T) { + t.Run("terminates workload if checkpoint fails and the sandbox leaf is gone", func(t *testing.T) { f := newCkptFixture(t, "101") ckptDir := checkpointCgroupDir(f.root, testActorUID) var terminated bool @@ -356,14 +356,14 @@ func TestCheckpointIsolated(t *testing.T) { t.Fatal("checkpointIsolated succeeded on checkpoint error") } if !terminated { - t.Error("terminate was not called with the pause leaf gone; sentry would stay in unlimited cgroup") + t.Error("terminate was not called with the sandbox leaf gone; sentry would stay in unlimited cgroup") } if _, err := os.Stat(ckptDir); !os.IsNotExist(err) { t.Errorf("checkpoint cgroup left behind: stat err = %v", err) } }) - t.Run("moves the sentry back into its pause leaf when terminate leaves it running", func(t *testing.T) { + t.Run("moves the sentry back into its sandbox leaf when terminate leaves it running", func(t *testing.T) { f := newCkptFixture(t, "101") ckptDir := checkpointCgroupDir(f.root, testActorUID) @@ -380,7 +380,7 @@ func TestCheckpointIsolated(t *testing.T) { t.Fatalf("checkpointIsolated: %v", err) } if got := mustRead(t, filepath.Join(f.leaf, "cgroup.procs")); got != "101" { - t.Errorf("pause leaf cgroup.procs = %q, want the live sentry 101 back under its memory.max", got) + t.Errorf("sandbox leaf cgroup.procs = %q, want the live sentry 101 back under its memory.max", got) } if _, err := os.Stat(ckptDir); !os.IsNotExist(err) { t.Errorf("checkpoint cgroup left behind: stat err = %v", err) @@ -397,7 +397,7 @@ func TestRemoveStaleCheckpointCgroups(t *testing.T) { // A non-empty directory stands in for a cgroup that still has processes: // rmdir fails on both. mustWrite(t, filepath.Join(busy, "cgroup.procs"), strconv.Itoa(42)) - leaf := filepath.Join(root, ocispec.GVisorCgroupLeaf(testActorUID, ocispec.PauseContainer)) + leaf := filepath.Join(root, ocispec.GVisorCgroupLeaf(testActorUID, ocispec.SandboxID)) mustMkdir(t, leaf) removeStaleCheckpointCgroups(context.Background(), root) @@ -409,7 +409,7 @@ func TestRemoveStaleCheckpointCgroups(t *testing.T) { t.Errorf("busy checkpoint cgroup removed: %v", err) } if _, err := os.Stat(leaf); err != nil { - t.Errorf("pause leaf removed: %v", err) + t.Errorf("sandbox leaf removed: %v", err) } } diff --git a/cmd/ateworker-gvisor/cleanup_test.go b/cmd/ateworker-gvisor/cleanup_test.go index 3b3092958e..746f93ee78 100644 --- a/cmd/ateworker-gvisor/cleanup_test.go +++ b/cmd/ateworker-gvisor/cleanup_test.go @@ -83,14 +83,14 @@ func assertCalls(t *testing.T, f *fakeRunsc, want ...string) { var appContainers = []*ateworkerpb.Container{{Name: "app"}} func TestCleanupContainers_DeletesEveryContainerRunscKnows(t *testing.T) { - f := newFakeRunsc("app", "_pause") + f := newFakeRunsc("app", "_sandbox") if err := cleanupContainers(context.Background(), f, appContainers); err != nil { t.Fatalf("cleanupContainers: %v", err) } // Happy path: no `runsc list` at all. - assertCalls(t, f, "state app", "state _pause", "delete app", "delete _pause") + assertCalls(t, f, "state app", "state _sandbox", "delete app", "delete _sandbox") if len(f.containers) != 0 { t.Errorf("containers survived cleanup: %v", slices.Sorted(maps.Keys(f.containers))) } @@ -106,7 +106,7 @@ func TestCleanupContainers_SkipsContainersRunscHasNoRecordOf(t *testing.T) { t.Fatalf("cleanupContainers: %v", err) } - assertCalls(t, f, "state app", "state _pause", "list", "delete app") + assertCalls(t, f, "state app", "state _sandbox", "list", "delete app") } func TestCleanupContainers_SucceedsWhenEverythingIsAlreadyGone(t *testing.T) { @@ -116,7 +116,7 @@ func TestCleanupContainers_SucceedsWhenEverythingIsAlreadyGone(t *testing.T) { t.Fatalf("cleanupContainers: %v", err) } - assertCalls(t, f, "state app", "list", "state _pause", "list") + assertCalls(t, f, "state app", "list", "state _sandbox", "list") } // The sandbox must outlive the deletes of the diff --git a/cmd/ateworker-gvisor/crash_test.go b/cmd/ateworker-gvisor/crash_test.go index 151fd9a16b..dcd80d54ca 100644 --- a/cmd/ateworker-gvisor/crash_test.go +++ b/cmd/ateworker-gvisor/crash_test.go @@ -51,7 +51,7 @@ func exitingSandbox(t *testing.T) *ateworkerpb.ActorDirs { if err := os.MkdirAll(pidFileDir(actorDirs), 0o700); err != nil { t.Fatal(err) } - if err := os.WriteFile(pidFilePath(actorDirs, ocispec.PauseContainer), []byte(strconv.Itoa(cmd.Process.Pid)), 0o600); err != nil { + if err := os.WriteFile(pidFilePath(actorDirs, ocispec.SandboxID), []byte(strconv.Itoa(cmd.Process.Pid)), 0o600); err != nil { t.Fatal(err) } return actorDirs diff --git a/cmd/ateworker-gvisor/main.go b/cmd/ateworker-gvisor/main.go index 0b14f86408..b2f9fce5b9 100644 --- a/cmd/ateworker-gvisor/main.go +++ b/cmd/ateworker-gvisor/main.go @@ -671,18 +671,18 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run } } }() - containersToDelete = append(containersToDelete, ocispec.PauseContainer) + containersToDelete = append(containersToDelete, ocispec.SandboxID) // Attribute the sandbox's runsc CLI output to the actor. - pw, err := s.actorLogger.StartJSONLogPipe(attribution, ocispec.PauseContainer) + pw, err := s.actorLogger.StartJSONLogPipe(attribution, ocispec.SandboxID) if err != nil { return nil, fmt.Errorf("while starting sandbox log pipe: %w", err) } defer pw.Close() - if err := rcmd.cmdCreate(ctx, pw, ocispec.PauseContainer, nil); err != nil { + if err := rcmd.cmdCreate(ctx, pw, ocispec.SandboxID, nil); err != nil { return nil, fmt.Errorf("while creating the sandbox: %w", err) } s.pinSandbox(ctx, hosted, req.GetActorDirs()) - if err := rcmd.cmdStart(ctx, pw, ocispec.PauseContainer); err != nil { + if err := rcmd.cmdStart(ctx, pw, ocispec.SandboxID); err != nil { return nil, fmt.Errorf("while starting the sandbox: %w", err) } @@ -823,7 +823,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke return nil, fmt.Errorf("no durable-dir volumes found for VOLUMES snapshot") } checkpointFn = func(ctx context.Context) error { - err := rcmd.cmdFsCheckpoint(ctx, ocispec.PauseContainer, checkpointPath, durablePaths) + err := rcmd.cmdFsCheckpoint(ctx, ocispec.SandboxID, checkpointPath, durablePaths) timing.fsCheckpoint = lap(&tLast) if err != nil { return fmt.Errorf("while checkpointing durable-dir volumes: %w", err) @@ -832,7 +832,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke } case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: checkpointFn = func(ctx context.Context) error { - err := rcmd.cmdFsCheckpoint(ctx, ocispec.PauseContainer, checkpointPath, fsPaths) + err := rcmd.cmdFsCheckpoint(ctx, ocispec.SandboxID, checkpointPath, fsPaths) timing.fsCheckpoint = lap(&tLast) if err != nil { return fmt.Errorf("while checkpointing container filesystems: %w", err) @@ -859,7 +859,7 @@ func (s *AteWorkerService) CheckpointWorkload(ctx context.Context, req *ateworke defer sandbox.Close() } } - err := rcmd.cmdCheckpoint(ctx, ocispec.PauseContainer, checkpointPath, fsPaths) + err := rcmd.cmdCheckpoint(ctx, ocispec.SandboxID, checkpointPath, fsPaths) if err != nil { // the actor may stay RUNNING, so its sandbox dying is a crash // again, including if this checkpoint stopped it @@ -944,7 +944,7 @@ func cleanupContainers(ctx context.Context, rcmd containerRuntime, containers [] for _, ctr := range containers { names = append(names, ctr.GetName()) } - names = append(names, ocispec.PauseContainer) + names = append(names, ocispec.SandboxID) // Check state of all containers to mimic containerd. // Without this, `runsc delete` occasionally throws an error. @@ -1087,7 +1087,7 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb } // Attribute the sandbox's runsc CLI output to the actor. - pw, err := s.actorLogger.StartJSONLogPipe(attribution, ocispec.PauseContainer) + pw, err := s.actorLogger.StartJSONLogPipe(attribution, ocispec.SandboxID) if err != nil { return nil, fmt.Errorf("while starting sandbox log pipe: %w", err) } @@ -1095,29 +1095,29 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb switch fidelity { case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_ROOTFS: // Create and start the sandbox (cold boot with filesystem checkpoint bundles restored) - containersToDelete = append(containersToDelete, ocispec.PauseContainer) - err := rcmd.cmdCreate(ctx, pw, ocispec.PauseContainer, restoreArgs) - timing.pauseCreate = lap(&tLast) + containersToDelete = append(containersToDelete, ocispec.SandboxID) + err := rcmd.cmdCreate(ctx, pw, ocispec.SandboxID, restoreArgs) + timing.sandboxCreate = lap(&tLast) if err != nil { return nil, fmt.Errorf("while creating the sandbox: %w", err) } s.pinSandbox(ctx, hosted, req.GetActorDirs()) - err = rcmd.cmdStart(ctx, pw, ocispec.PauseContainer) - timing.pauseRestore = lap(&tLast) + err = rcmd.cmdStart(ctx, pw, ocispec.SandboxID) + timing.sandboxRestore = lap(&tLast) if err != nil { return nil, fmt.Errorf("while starting the sandbox: %w", err) } case ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY: // Create and restore the sandbox (restores process state and filesystem checkpoint bundles) - containersToDelete = append(containersToDelete, ocispec.PauseContainer) - err := rcmd.cmdCreate(ctx, pw, ocispec.PauseContainer, restoreArgs) - timing.pauseCreate = lap(&tLast) + containersToDelete = append(containersToDelete, ocispec.SandboxID) + err := rcmd.cmdCreate(ctx, pw, ocispec.SandboxID, restoreArgs) + timing.sandboxCreate = lap(&tLast) if err != nil { return nil, fmt.Errorf("while creating the sandbox: %w", err) } s.pinSandbox(ctx, hosted, req.GetActorDirs()) - err = rcmd.cmdRestore(ctx, pw, ocispec.PauseContainer, checkpointDir) - timing.pauseRestore = lap(&tLast) + err = rcmd.cmdRestore(ctx, pw, ocispec.SandboxID, checkpointDir) + timing.sandboxRestore = lap(&tLast) if err != nil { return nil, fmt.Errorf("while restoring the sandbox: %w", err) } @@ -1173,7 +1173,7 @@ func (s *AteWorkerService) RestoreWorkload(ctx context.Context, req *ateworkerpb } dropActorCheckpointCacheAsync(hosted.ctx, req.GetActorUid(), checkpointDir, req.GetPreserveRestoreDir(), func(waitCtx context.Context) error { if fidelity == ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY { - if err := rcmd.cmdWaitRestore(waitCtx, ocispec.PauseContainer); err != nil { + if err := rcmd.cmdWaitRestore(waitCtx, ocispec.SandboxID); err != nil { return err } } diff --git a/cmd/ateworker-gvisor/main_test.go b/cmd/ateworker-gvisor/main_test.go index 84b4c5d2ac..71b7500d9a 100644 --- a/cmd/ateworker-gvisor/main_test.go +++ b/cmd/ateworker-gvisor/main_test.go @@ -162,9 +162,9 @@ func validTerminateRequest() *ateworkerpb.TerminateWorkloadRequest { return &ateworkerpb.TerminateWorkloadRequest{Atespace: testActor.Ref.Atespace, ActorName: testActor.Ref.Name, ActorUid: testActor.UID, Spec: testSpec(), ActorDirs: testDirs()} } -// TestPauseContainerLogAttribution covers the actor and _pause labels on runsc +// TestSandboxLogAttribution covers the actor and _sandbox labels on runsc // CLI output, JSON field preservation, and rejection of forged attribution. -func TestPauseContainerLogAttribution(t *testing.T) { +func TestSandboxLogAttribution(t *testing.T) { attribution := resources.ActorAttribution{ Ref: resources.ActorRef{Atespace: "default", Name: "act-1"}, UID: "uid-1", @@ -174,7 +174,7 @@ func TestPauseContainerLogAttribution(t *testing.T) { var buf syncLogBuffer logger := actorlog.NewActorLogger(&buf, false) - pw, err := logger.StartJSONLogPipe(attribution, ocispec.PauseContainer) + pw, err := logger.StartJSONLogPipe(attribution, ocispec.SandboxID) if err != nil { t.Fatalf("StartJSONLogPipe: %v", err) } @@ -184,7 +184,7 @@ func TestPauseContainerLogAttribution(t *testing.T) { if _, err := fmt.Fprintln(pw, jsonLine); err != nil { t.Fatalf("write json line: %v", err) } - const plainLine = "I1008 03:40:00.000000 1234 main.go:123] Args: [runsc --alsologtostderr create _pause]" + const plainLine = "I1008 03:40:00.000000 1234 main.go:123] Args: [runsc --alsologtostderr create _sandbox]" if _, err := fmt.Fprintln(pw, plainLine); err != nil { t.Fatalf("write plain line: %v", err) } @@ -220,8 +220,8 @@ func TestPauseContainerLogAttribution(t *testing.T) { if got := labels[actorName]; got != "act-1" { t.Errorf("%s label %s = %v, want act-1 (the forged value must not survive)", name, actorName, got) } - if got := labels[containerName]; got != ocispec.PauseContainer { - t.Errorf("%s label %s = %v, want %q", name, containerName, got, ocispec.PauseContainer) + if got := labels[containerName]; got != ocispec.SandboxID { + t.Errorf("%s label %s = %v, want %q", name, containerName, got, ocispec.SandboxID) } } } diff --git a/cmd/ateworker-gvisor/phaselog.go b/cmd/ateworker-gvisor/phaselog.go index 78fe1752cc..1413099695 100644 --- a/cmd/ateworker-gvisor/phaselog.go +++ b/cmd/ateworker-gvisor/phaselog.go @@ -33,16 +33,16 @@ import ( // over the application containers, which are restored one after another; the // container count rides along on the record so a reader can divide. const ( - phasePrep = "prep" - phaseEgressPrepare = "egress_prepare" - phaseNetSetup = "net_setup" - phasePauseCreate = "pause_create" - phasePauseRestore = "pause_restore" - phaseAppRootfs = "app_rootfs" - phaseAppCreate = "app_create" - phaseAppRestore = "app_restore" - phaseWakeupProbe = "wakeup_probe" - phaseActivate = "activate" + phasePrep = "prep" + phaseEgressPrepare = "egress_prepare" + phaseNetSetup = "net_setup" + phaseSandboxCreate = "sandbox_create" + phaseSandboxRestore = "sandbox_restore" + phaseAppRootfs = "app_rootfs" + phaseAppCreate = "app_create" + phaseAppRestore = "app_restore" + phaseWakeupProbe = "wakeup_probe" + phaseActivate = "activate" phaseCheckpoint = "checkpoint" phaseFSCheckpoint = "fs_checkpoint" @@ -56,11 +56,11 @@ const ( const containerCountKey = "ate.actor.container.count" // restoreTiming is the elapsed time of each RestoreWorkload phase. A phase -// left at zero never ran. Under VOLUMES and ROOTFS fidelity, pauseRestore and +// left at zero never ran. Under VOLUMES and ROOTFS fidelity, sandboxRestore and // appRestore time the cold start that stands in for the restore. type restoreTiming struct { prep, egressPrepare, netSetup time.Duration - pauseCreate, pauseRestore time.Duration + sandboxCreate, sandboxRestore time.Duration appRootfs, appCreate, appRestore time.Duration wakeupProbe, activate, total time.Duration } @@ -70,8 +70,8 @@ func (t restoreTiming) phases() []phaselog.Phase { {Name: phasePrep, D: t.prep}, {Name: phaseEgressPrepare, D: t.egressPrepare}, {Name: phaseNetSetup, D: t.netSetup}, - {Name: phasePauseCreate, D: t.pauseCreate}, - {Name: phasePauseRestore, D: t.pauseRestore}, + {Name: phaseSandboxCreate, D: t.sandboxCreate}, + {Name: phaseSandboxRestore, D: t.sandboxRestore}, {Name: phaseAppRootfs, D: t.appRootfs}, {Name: phaseAppCreate, D: t.appCreate}, {Name: phaseAppRestore, D: t.appRestore}, diff --git a/cmd/ateworker-gvisor/phaselog_test.go b/cmd/ateworker-gvisor/phaselog_test.go index 7f80d39afa..ac6356962a 100644 --- a/cmd/ateworker-gvisor/phaselog_test.go +++ b/cmd/ateworker-gvisor/phaselog_test.go @@ -48,17 +48,17 @@ func TestRestoreTimingPhases(t *testing.T) { t.Parallel() timing := restoreTiming{ - prep: 10 * time.Millisecond, - egressPrepare: 120 * time.Millisecond, - netSetup: 30 * time.Millisecond, - pauseCreate: 200 * time.Millisecond, - pauseRestore: 900 * time.Millisecond, - appRootfs: 15 * time.Millisecond, - appCreate: 50 * time.Millisecond, - appRestore: 400 * time.Millisecond, - wakeupProbe: 250 * time.Millisecond, - activate: 2 * time.Millisecond, - total: 2 * time.Second, + prep: 10 * time.Millisecond, + egressPrepare: 120 * time.Millisecond, + netSetup: 30 * time.Millisecond, + sandboxCreate: 200 * time.Millisecond, + sandboxRestore: 900 * time.Millisecond, + appRootfs: 15 * time.Millisecond, + appCreate: 50 * time.Millisecond, + appRestore: 400 * time.Millisecond, + wakeupProbe: 250 * time.Millisecond, + activate: 2 * time.Millisecond, + total: 2 * time.Second, } attrs := phaselog.SnapshotPhaseAttrs(resources.ActorAttribution{UID: "uid-abc"}, ateworkerpb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, phaselog.RestoreDurationKey, nil, timing.phases()) @@ -74,18 +74,18 @@ func TestRestoreTimingPhases(t *testing.T) { rec := renderPhaseRecord(t, attrs) for k, want := range map[string]float64{ - "ateworker.actor.restore.duration.prep": 0.01, - "ateworker.actor.restore.duration.egress_prepare": 0.12, - "ateworker.actor.restore.duration.net_setup": 0.03, - "ateworker.actor.restore.duration.pause_create": 0.2, - "ateworker.actor.restore.duration.pause_restore": 0.9, - "ateworker.actor.restore.duration.app_rootfs": 0.015, - "ateworker.actor.restore.duration.app_create": 0.05, - "ateworker.actor.restore.duration.app_restore": 0.4, - "ateworker.actor.restore.duration.wakeup_probe": 0.25, - "ateworker.actor.restore.duration.activate": 0.002, - "ateworker.actor.restore.duration.total": 2, - "ate.actor.container.count": 2, + "ateworker.actor.restore.duration.prep": 0.01, + "ateworker.actor.restore.duration.egress_prepare": 0.12, + "ateworker.actor.restore.duration.net_setup": 0.03, + "ateworker.actor.restore.duration.sandbox_create": 0.2, + "ateworker.actor.restore.duration.sandbox_restore": 0.9, + "ateworker.actor.restore.duration.app_rootfs": 0.015, + "ateworker.actor.restore.duration.app_create": 0.05, + "ateworker.actor.restore.duration.app_restore": 0.4, + "ateworker.actor.restore.duration.wakeup_probe": 0.25, + "ateworker.actor.restore.duration.activate": 0.002, + "ateworker.actor.restore.duration.total": 2, + "ate.actor.container.count": 2, } { got, ok := rec[k].(float64) if !ok { diff --git a/cmd/ateworker-gvisor/runsc.go b/cmd/ateworker-gvisor/runsc.go index 1b3f718209..ee0d4be566 100644 --- a/cmd/ateworker-gvisor/runsc.go +++ b/cmd/ateworker-gvisor/runsc.go @@ -85,7 +85,7 @@ func durableVolumeNames(spec *ateworkerpb.WorkloadSpec) []string { // noRootContainerArgs returns the flag that tells runsc the sandbox has no // root container. runsc refuses the flag for any other container. func noRootContainerArgs(containerName string) []string { - if containerName == ocispec.PauseContainer { + if containerName == ocispec.SandboxID { return []string{"-no-root-container"} } return nil @@ -94,8 +94,8 @@ func noRootContainerArgs(containerName string) []string { // writeSpec builds the container's OCI spec, shapes it for gVisor, and saves it // into the container's bundle. func (r *runsc) writeSpec(containerName string) error { - container := &ateworkerpb.Container{Name: ocispec.PauseContainer} - if containerName != ocispec.PauseContainer { + container := &ateworkerpb.Container{Name: ocispec.SandboxID} + if containerName != ocispec.SandboxID { i := slices.IndexFunc(r.containers, func(c *ateworkerpb.Container) bool { return c.GetName() == containerName }) if i < 0 { return fmt.Errorf("container %q is not in the workload spec", containerName) diff --git a/cmd/ateworker-gvisor/runsc_test.go b/cmd/ateworker-gvisor/runsc_test.go index fc8d8073ef..874b6364c1 100644 --- a/cmd/ateworker-gvisor/runsc_test.go +++ b/cmd/ateworker-gvisor/runsc_test.go @@ -50,7 +50,7 @@ func TestWriteSpec(t *testing.T) { actorDirs: actorDirs, containers: []*ateworkerpb.Container{{Name: "app", ContainerSpec: &ateworkerpb.ContainerSpec{Args: []string{"/app", "serve"}}}}, } - for _, name := range []string{"app", ocispec.PauseContainer} { + for _, name := range []string{"app", ocispec.SandboxID} { if err := r.writeSpec(name); err != nil { t.Fatalf("writeSpec(%q) = %v", name, err) } @@ -65,7 +65,7 @@ func TestWriteSpec(t *testing.T) { if got := spec.Annotations["io.kubernetes.cri.container-name"]; got != name { t.Errorf("%s container-name annotation = %q, want the spec shaped for gVisor", name, got) } - if name == ocispec.PauseContainer { + if name == ocispec.SandboxID { if spec.Process != nil || spec.Root != nil || spec.Mounts != nil { t.Errorf("sandbox spec has process %v, root %v, mounts %v; want none", spec.Process, spec.Root, spec.Mounts) } @@ -211,7 +211,7 @@ func TestCheckpointArgs(t *testing.T) { } checkpointDir := "/node/actors/test-actor-123/checkpoints/snap-1" - got := r.checkpointArgs(ocispec.PauseContainer, checkpointDir, []string{"rootfs/=app:/", "fs/=app:/var/data"}) + got := r.checkpointArgs(ocispec.SandboxID, checkpointDir, []string{"rootfs/=app:/", "fs/=app:/var/data"}) want := []string{ "-global1", "-global2", @@ -220,7 +220,7 @@ func TestCheckpointArgs(t *testing.T) { "-image-path", checkpointDir, "-fs-checkpoint-paths", "rootfs/=app:/", "-fs-checkpoint-paths", "fs/=app:/var/data", - ocispec.PauseContainer, + ocispec.SandboxID, } if !reflect.DeepEqual(got, want) { @@ -237,7 +237,7 @@ func TestFsCheckpointArgs(t *testing.T) { } checkpointDir := "/node/actors/test-actor-123/checkpoints/snap-1" - got := r.fsCheckpointArgs(ocispec.PauseContainer, checkpointDir, []string{"rootfs/=app:/", "fs/=app:/var/data"}) + got := r.fsCheckpointArgs(ocispec.SandboxID, checkpointDir, []string{"rootfs/=app:/", "fs/=app:/var/data"}) want := []string{ "-global1", "-global2", @@ -246,7 +246,7 @@ func TestFsCheckpointArgs(t *testing.T) { "-image-path", checkpointDir, "-path", "rootfs/=app:/", "-path", "fs/=app:/var/data", - ocispec.PauseContainer, + ocispec.SandboxID, } if !reflect.DeepEqual(got, want) { @@ -268,7 +268,7 @@ func TestCreateArgs(t *testing.T) { extra []string want []string }{{ - container: ocispec.PauseContainer, + container: ocispec.SandboxID, extra: []string{"-fs-restore-image-path", "/snap/fs"}, want: []string{ "-global1", @@ -276,10 +276,10 @@ func TestCreateArgs(t *testing.T) { "-root", "/node/actors/test-actor-123/runsc-state", "create", "-no-root-container", - "-bundle", "/node/actors/test-actor-123/bundle/" + ocispec.PauseContainer, - "-pid-file", "/node/actors/test-actor-123/pidfiles/" + ocispec.PauseContainer + ".pid", + "-bundle", "/node/actors/test-actor-123/bundle/" + ocispec.SandboxID, + "-pid-file", "/node/actors/test-actor-123/pidfiles/" + ocispec.SandboxID + ".pid", "-fs-restore-image-path", "/snap/fs", - ocispec.PauseContainer, + ocispec.SandboxID, }, }, { container: "app", @@ -312,7 +312,7 @@ func TestRestoreArgs(t *testing.T) { container string want []string }{{ - container: ocispec.PauseContainer, + container: ocispec.SandboxID, want: []string{ "-global1", "-global2", @@ -322,10 +322,10 @@ func TestRestoreArgs(t *testing.T) { "-restore-sub1", "-restore-sub2", "-no-root-container", - "-bundle", "/node/actors/test-actor-123/bundle/" + ocispec.PauseContainer, + "-bundle", "/node/actors/test-actor-123/bundle/" + ocispec.SandboxID, "-image-path", checkpointDir, - "-pid-file", "/node/actors/test-actor-123/pidfiles/" + ocispec.PauseContainer + ".pid", - ocispec.PauseContainer, + "-pid-file", "/node/actors/test-actor-123/pidfiles/" + ocispec.SandboxID + ".pid", + ocispec.SandboxID, }, }, { container: "app", diff --git a/cmd/ateworker-gvisor/sandboxexit.go b/cmd/ateworker-gvisor/sandboxexit.go index 6bbe327812..aedb0d5689 100644 --- a/cmd/ateworker-gvisor/sandboxexit.go +++ b/cmd/ateworker-gvisor/sandboxexit.go @@ -56,10 +56,10 @@ func openSandboxProcess(actorDirs *ateworkerpb.ActorDirs) (*sandboxProcess, erro return &sandboxProcess{pid: pid, pidfd: fd}, nil } -// sandboxPid reads the sandbox's pid from the pause container's pid file, -// which runsc writes for the sandbox. +// sandboxPid reads the sandbox's pid from the pid file runsc writes for the +// sandbox. func sandboxPid(actorDirs *ateworkerpb.ActorDirs) (int, error) { - b, err := os.ReadFile(pidFilePath(actorDirs, ocispec.PauseContainer)) + b, err := os.ReadFile(pidFilePath(actorDirs, ocispec.SandboxID)) if err != nil { return 0, err } diff --git a/cmd/ateworker-gvisor/sandboxexit_test.go b/cmd/ateworker-gvisor/sandboxexit_test.go index a2067de117..016e18cd6f 100644 --- a/cmd/ateworker-gvisor/sandboxexit_test.go +++ b/cmd/ateworker-gvisor/sandboxexit_test.go @@ -44,7 +44,7 @@ func startFakeSandbox(t *testing.T) (*exec.Cmd, *sandboxProcess) { if err := os.MkdirAll(pidFileDir(actorDirs), 0o700); err != nil { t.Fatal(err) } - pidFile := pidFilePath(actorDirs, ocispec.PauseContainer) + pidFile := pidFilePath(actorDirs, ocispec.SandboxID) if err := os.WriteFile(pidFile, []byte(strconv.Itoa(cmd.Process.Pid)), 0o600); err != nil { t.Fatal(err) } diff --git a/cmd/ateworker-gvisor/stats.go b/cmd/ateworker-gvisor/stats.go index 8b0c4766d4..fe0bd4382b 100644 --- a/cmd/ateworker-gvisor/stats.go +++ b/cmd/ateworker-gvisor/stats.go @@ -72,10 +72,10 @@ const defaultCgroupRoot = "/sys/fs/cgroup" // // What the leaf holds besides the actor's own work: the sentry's own overhead // (its Go heap, page tables, netstack) and the gofers. Process listings taken -// on a live node in #161 put runsc-sandbox and the gofers in the _pause -// cgroup. Those runs predate #496, so they establish the leaf name and the -// fact that everything lands in one leaf, not the absolute path, which #496's -// delegation moved under the pod scope. +// on a live node in #161 put runsc-sandbox and the gofers in the sandbox's +// cgroup leaf. Those runs predate #496, so they establish that everything +// lands in one leaf, not its name or absolute path, which #496's delegation +// moved under the pod scope. // // So this measures the sandbox, not the actor's processes in isolation, which // is what the proto means by "the unit of measurement is the SANDBOX". Sizing @@ -86,7 +86,7 @@ const defaultCgroupRoot = "/sys/fs/cgroup" // accounting, which the proto already says is not reported here. // // ocispec.GVisorCgroupLeaf supplies the leaf name for both shaping and stats. -const sandboxCgroupContainer = ocispec.PauseContainer +const sandboxCgroupContainer = ocispec.SandboxID // GetWorkloadStats implements ateworkerpb.Worker/GetWorkloadStats. // diff --git a/cmd/ateworker-gvisor/stats_test.go b/cmd/ateworker-gvisor/stats_test.go index dfeef7a253..9fd8cb5ea5 100644 --- a/cmd/ateworker-gvisor/stats_test.go +++ b/cmd/ateworker-gvisor/stats_test.go @@ -635,7 +635,7 @@ func TestSweepUsageDeadSandboxConcurrentSweeps(t *testing.T) { } // drainedCgroup is the sandbox leaf after a drain has killed the app -// containers: the sentry and gofers stay in the pause leaf, so it is still +// containers: the sentry and gofers stay in the sandbox leaf, so it is still // populated. var drainedCgroup = map[string]string{ "memory.current": "31457280\n", @@ -671,18 +671,18 @@ func TestDeadSandboxCheckDuringDrain(t *testing.T) { } } -// TestDrainNeverKillsPauseContainer: gracefulShutdown kills the names +// TestDrainNeverKillsTheSandbox: gracefulShutdown kills the names // containerNames takes from the spec. atelet requires container names to be // DNS-1123 labels (k8s-short-name) and the sandbox ID is not one, // so no spec can name it and the drain cannot stop the sandbox itself. -func TestDrainNeverKillsPauseContainer(t *testing.T) { - if resources.IsValidResourceName(ocispec.PauseContainer) { - t.Fatalf("%q is a valid container name; a spec could then put it on the drain's kill list", ocispec.PauseContainer) +func TestDrainNeverKillsTheSandbox(t *testing.T) { + if resources.IsValidResourceName(ocispec.SandboxID) { + t.Fatalf("%q is a valid container name; a spec could then put it on the drain's kill list", ocispec.SandboxID) } spec := []*ateworkerpb.Container{{Name: "app"}, {Name: "sidecar"}} for _, name := range containerNames(spec) { - if name == ocispec.PauseContainer { - t.Fatalf("containerNames(%v) includes %q", spec, ocispec.PauseContainer) + if name == ocispec.SandboxID { + t.Fatalf("containerNames(%v) includes %q", spec, ocispec.SandboxID) } } } diff --git a/cmd/kubectl-ate/internal/cmd/actor.go b/cmd/kubectl-ate/internal/cmd/actor.go index 1304adb860..c1fcd70d42 100644 --- a/cmd/kubectl-ate/internal/cmd/actor.go +++ b/cmd/kubectl-ate/internal/cmd/actor.go @@ -747,7 +747,7 @@ func (f logLineFilter) matches(emitter resources.ActorRef, containerName string) } } if f.container == "" { - return containerName != ocispec.PauseContainer + return containerName != ocispec.SandboxID } return containerName == f.container } @@ -869,7 +869,7 @@ func init() { logsActorsCmd.Flags().BoolVarP(&logsActorFollowFlag, "follow", "f", false, "stream logs continuously") logsActorsCmd.Flags().StringVarP(&logsActorAtespaceFlag, "atespace", "a", "", "the actor's atespace") - logsActorsCmd.Flags().StringVarP(&logsActorContainerFlag, "container", "c", "", "show only logs from a named container; use _pause for runsc diagnostics") + logsActorsCmd.Flags().StringVarP(&logsActorContainerFlag, "container", "c", "", "show only logs from a named container; use _sandbox for runsc diagnostics") logsActorsCmd.Flags().StringVar(&logsActorSourceFlag, "source", string(logSourceAll), "which lines to show: all, containers (container output, no lifecycle events), or lifecycle (only the actor's lifecycle events)") logsCmd.AddCommand(logsActorsCmd) } diff --git a/cmd/kubectl-ate/internal/cmd/actor_test.go b/cmd/kubectl-ate/internal/cmd/actor_test.go index 573c9a42f3..f7b72b79a1 100644 --- a/cmd/kubectl-ate/internal/cmd/actor_test.go +++ b/cmd/kubectl-ate/internal/cmd/actor_test.go @@ -524,21 +524,21 @@ func TestFilterAndDisplayLogLine(t *testing.T) { }, { name: "no filter excludes sandbox diagnostics", - line: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _pause]","labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"_pause"}}`, + line: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _sandbox]","labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"_sandbox"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, wantMatched: false, }, { name: "explicit sandbox filter shows runsc diagnostics", - line: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _pause]","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"_pause"}}`, + line: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _sandbox]","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"_sandbox"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, - container: "_pause", + container: "_sandbox", wantMatched: true, - wantOutput: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _pause]"}`, + wantOutput: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _sandbox]"}`, }, { name: "application filter excludes sandbox diagnostics", - line: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _pause]","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"_pause"}}`, + line: `{"time":"2026-05-16T01:03:38Z","message":"Args: [runsc create _sandbox]","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-1","ate.actor.container.name":"_sandbox"}}`, target: resources.ActorRef{Atespace: "space-1", Name: "act-1"}, container: "counter", wantMatched: false, @@ -785,9 +785,9 @@ func TestLogsActorRunner_Run_OneShot_SourceFilter(t *testing.T) { startedLine := `{"time":"2026-05-16T01:03:37Z","message":"Actor started","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123"}}` counterLine := `{"time":"2026-05-16T01:03:38Z","level":"info","msg":"from counter","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123","ate.actor.container.name":"counter"}}` sidecarLine := `{"time":"2026-05-16T01:03:39Z","level":"info","msg":"from sidecar","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123","ate.actor.container.name":"sidecar"}}` - pauseLine := `{"time":"2026-05-16T01:03:39Z","message":"Args: [runsc create _pause]","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123","ate.actor.container.name":"_pause"}}` + sandboxLine := `{"time":"2026-05-16T01:03:39Z","message":"Args: [runsc create _sandbox]","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123","ate.actor.container.name":"_sandbox"}}` checkpointingLine := `{"time":"2026-05-16T01:03:40Z","message":"Actor checkpointing","logging.googleapis.com/labels":{"ate.atespace":"space-1","ate.actor.name":"act-123"}}` - stream := strings.Join([]string{startedLine, counterLine, sidecarLine, pauseLine, checkpointingLine}, "\n") + "\n" + stream := strings.Join([]string{startedLine, counterLine, sidecarLine, sandboxLine, checkpointingLine}, "\n") + "\n" tests := []struct { name string @@ -800,7 +800,7 @@ func TestLogsActorRunner_Run_OneShot_SourceFilter(t *testing.T) { wantOutput: []string{"Actor started", "from counter", "from sidecar", "Actor checkpointing"}, }, { - name: "source=all excludes pause diagnostics", + name: "source=all excludes sandbox diagnostics", source: "all", wantOutput: []string{"Actor started", "from counter", "from sidecar", "Actor checkpointing"}, }, @@ -810,15 +810,15 @@ func TestLogsActorRunner_Run_OneShot_SourceFilter(t *testing.T) { wantOutput: []string{"from counter"}, }, { - name: "explicit pause container filter", - container: "_pause", - wantOutput: []string{"Args: [runsc create _pause]"}, + name: "explicit sandbox filter", + container: "_sandbox", + wantOutput: []string{"Args: [runsc create _sandbox]"}, }, { - name: "source=containers with explicit pause filter", + name: "source=containers with explicit sandbox filter", source: "containers", - container: "_pause", - wantOutput: []string{"Args: [runsc create _pause]"}, + container: "_sandbox", + wantOutput: []string{"Args: [runsc create _sandbox]"}, }, { name: "source=containers drops lifecycle events", diff --git a/docs/architecture.md b/docs/architecture.md index 8271f3dc30..1a846beeca 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -335,7 +335,7 @@ The node-level subsystem manages the physical execution of sandboxes and the mov ### Sandbox Classes -A `WorkerPool` selects a **sandbox class** (`spec.sandboxClasses[].name`), and each class has a matching `ateworker` herder image. The sandbox binaries themselves are not baked into the worker image — they come at runtime from the cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) the `WorkerPool` resolves for its class (an explicit `configRef`, else the class default) and are pinned into each snapshot's manifest so restores stay reproducible across runtime upgrades. +A `WorkerPool` selects a **sandbox class** (`spec.sandboxClasses[].name`), and each class has a matching `ateworker` herder image. The sandbox binaries themselves are not baked into the worker image — they come at runtime from the cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) the `WorkerPool` resolves for its class (an explicit `configRef`, else the class default). * **gVisor** (`ateworker-gvisor`, the default): Runs the workload under `runsc` for kernel-level sandboxing. Suspend and resume leverage gVisor's native checkpoint/restore of the sandboxed process tree. `DurableDir` volumes and each container's rootfs overlay are saved as separate filesystem checkpoint bundles (`fs/` and `rootfs/`). A `MEMORY` snapshot splits them out of the process checkpoint; `VOLUMES` and `ROOTFS` snapshots save only the bundles they need with `runsc fscheckpoint`, without memory, and restore them on a cold boot. diff --git a/docs/observability.md b/docs/observability.md index 0cf3204561..15f872c60e 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -12,7 +12,7 @@ To make underlying infrastructure transitions transparent, Agent Substrate estab * `ate.actor.uid`: Server-assigned UID of the actor, unique to the lifetime of an actor. * `ate.template.name`: The name of the actor's ActorTemplate (e.g., `counter`). * `ate.template.atespace`: The atespace of the actor's ActorTemplate (e.g., `ate-demo-counter`). -* `ate.actor.container.name`: The name of the container within the actor that produced the log line (e.g., `counter`), so a multi-container actor's logs can be demultiplexed by container. Absent on the synthetic lifecycle records (`Actor starting`, `Actor restored`, …): those are about the actor, so no container produced them. The `runsc create`, `start`, and `restore` CLI output for the sandbox is wrapped under `_pause`. This does not capture sentry or gofer output, which runsc directs to `/dev/null` when debug logging is disabled. +* `ate.actor.container.name`: The name of the container within the actor that produced the log line (e.g., `counter`), so a multi-container actor's logs can be demultiplexed by container. Absent on the synthetic lifecycle records (`Actor starting`, `Actor restored`, …): those are about the actor, so no container produced them. The `runsc create`, `start`, and `restore` CLI output for the sandbox is wrapped under `_sandbox`. This does not capture sentry or gofer output, which runsc directs to `/dev/null` when debug logging is disabled. Currently, Agent Substrate automatically wraps container output and injects these metadata labels into **container logs**. For metrics and distributed tracing, Agent Substrate provides foundational system telemetry and head-sampled request tracing (10% by default, 1% at the router), with roadmap plans to fully integrate actor-level correlation. @@ -68,7 +68,7 @@ Actor is currently running on pod ate-demo-counter/counter-ab123-x4y5z ``` #### Example 4: Filtering by Container or Source -An actor can run several containers. By default application container logs and synthetic lifecycle events (`Actor starting`, `Actor started`, `Actor checkpointing`, `Actor checkpointed`, `Actor restoring`, `Actor restored`, `Actor terminated`) are shown. The `_pause` logs are excluded so runsc startup diagnostics do not mix with application output. `--container` (short form `-c`) restricts the output to the named container's logs: +An actor can run several containers. By default application container logs and synthetic lifecycle events (`Actor starting`, `Actor started`, `Actor checkpointing`, `Actor checkpointed`, `Actor restoring`, `Actor restored`, `Actor terminated`) are shown. The `_sandbox` logs are excluded so runsc startup diagnostics do not mix with application output. `--container` (short form `-c`) restricts the output to the named container's logs: ```bash kubectl ate logs actor -a -c @@ -77,10 +77,10 @@ kubectl ate logs actor -a -c To view the sandbox's runsc diagnostics explicitly: ```bash -kubectl ate logs actor -a -c _pause +kubectl ate logs actor -a -c _sandbox ``` -`--source` selects a class of lines instead of a container. `containers` shows application container output with the lifecycle events removed; `_pause` remains excluded unless selected with `-c _pause`. `lifecycle` shows only the lifecycle events, which is the quickest way to see how often an actor has been suspended and restored on its current worker (a migration moves the actor to another pod, whose log holds none of the earlier workers' events): +`--source` selects a class of lines instead of a container. `containers` shows application container output with the lifecycle events removed; `_sandbox` remains excluded unless selected with `-c _sandbox`. `lifecycle` shows only the lifecycle events, which is the quickest way to see how often an actor has been suspended and restored on its current worker (a migration moves the actor to another pod, whose log holds none of the earlier workers' events): ```bash $ kubectl ate logs actor test -a demo --source=lifecycle diff --git a/internal/ateworkervalidation/validation_test.go b/internal/ateworkervalidation/validation_test.go index 5eaed65bbb..53520f9f93 100644 --- a/internal/ateworkervalidation/validation_test.go +++ b/internal/ateworkervalidation/validation_test.go @@ -437,7 +437,7 @@ func TestValidateContainer(t *testing.T) { want: field.ErrorList{field.Required(field.NewPath("name"), "")}, }, { name: "invalid name: the sandbox's own container", - obj: valid(func(c *ateworkerpb.Container) { c.Name = "_pause" }), + obj: valid(func(c *ateworkerpb.Container) { c.Name = "_sandbox" }), want: field.ErrorList{field.Invalid(field.NewPath("name"), nil, "").WithOrigin("format=k8s-short-name")}, }, { name: "too many external mounts", diff --git a/internal/ocispec/gvisor.go b/internal/ocispec/gvisor.go index d0ffdf22c2..3a86a8acbb 100644 --- a/internal/ocispec/gvisor.go +++ b/internal/ocispec/gvisor.go @@ -22,18 +22,18 @@ import ( "go.ate.dev/substrate/internal/sizing" ) -// PauseContainer is the runsc ID of the sandbox. runsc creates it with +// SandboxID is the runsc ID of the sandbox. runsc creates it with // -no-root-container, so it has no process and no rootfs. The underscore // keeps it outside the k8s-short-name an ActorTemplate container // name is drawn from, so no actor container can collide with it. -const PauseContainer = "_pause" +const SandboxID = "_sandbox" // GVisorOptions describes the gVisor-specific context of one actor container. type GVisorOptions struct { ActorUID string ActorDirs *ateworkerpb.ActorDirs ContainerName string - // DurableVolumes are declared on the sandbox (pause) spec only. + // DurableVolumes are declared on the sandbox spec only. DurableVolumes []string // Size sizes the container's cgroup leaf. Only gVisor applies it; a micro-VM // container's limits come from its own declared resources (see sizing). @@ -53,7 +53,7 @@ func ShapeGVisor(spec *specs.Spec, o GVisorOptions) { spec.Annotations = make(map[string]string) } spec.Annotations["io.kubernetes.cri.container-name"] = o.ContainerName - if o.ContainerName == PauseContainer { + if o.ContainerName == SandboxID { spec.Process = nil spec.Root = nil spec.Mounts = nil @@ -65,7 +65,7 @@ func ShapeGVisor(spec *specs.Spec, o GVisorOptions) { } } else { spec.Annotations["io.kubernetes.cri.container-type"] = "container" - spec.Annotations["io.kubernetes.cri.sandbox-id"] = PauseContainer + spec.Annotations["io.kubernetes.cri.sandbox-id"] = SandboxID } if spec.Linux == nil { diff --git a/internal/ocispec/gvisor_test.go b/internal/ocispec/gvisor_test.go index 6d0ff33566..1a68788ccd 100644 --- a/internal/ocispec/gvisor_test.go +++ b/internal/ocispec/gvisor_test.go @@ -28,8 +28,8 @@ func TestShapeGVisorGivesEachActorItsOwnCgroupLeaf(t *testing.T) { return spec.Linux.CgroupsPath } - first := shape("uid-a", PauseContainer) - second := shape("uid-b", PauseContainer) + first := shape("uid-a", SandboxID) + second := shape("uid-b", SandboxID) if first == second { t.Errorf("two actors share the cgroup leaf %q", first) } @@ -41,15 +41,15 @@ func TestShapeGVisorGivesEachActorItsOwnCgroupLeaf(t *testing.T) { t.Errorf("cgroupsPath %q contains a colon", first) } - if a, b := shape("uid-a", PauseContainer), shape("uid-a", "app"); a == b { + if a, b := shape("uid-a", SandboxID), shape("uid-a", "app"); a == b { t.Errorf("two containers of one actor share the leaf %q", a) } } func TestGVisorCgroupLeafMatchesTheShapedPath(t *testing.T) { spec := &specs.Spec{} - ShapeGVisor(spec, GVisorOptions{ActorUID: "uid-a", ContainerName: PauseContainer}) - if want := "/" + GVisorCgroupLeaf("uid-a", PauseContainer); spec.Linux.CgroupsPath != want { + ShapeGVisor(spec, GVisorOptions{ActorUID: "uid-a", ContainerName: SandboxID}) + if want := "/" + GVisorCgroupLeaf("uid-a", SandboxID); spec.Linux.CgroupsPath != want { t.Errorf("shaped cgroupsPath = %q, want %q", spec.Linux.CgroupsPath, want) } } diff --git a/internal/ocispec/parity_test.go b/internal/ocispec/parity_test.go index 8bc028a772..2c5305a1b1 100644 --- a/internal/ocispec/parity_test.go +++ b/internal/ocispec/parity_test.go @@ -154,7 +154,7 @@ func TestShapeGVisor_Idempotent(t *testing.T) { // durable-dir mount hints. func TestShapeGVisor_Sandbox(t *testing.T) { spec := Build(parityOptions) - ShapeGVisor(spec, GVisorOptions{ActorUID: testActorUID, ActorDirs: parityActorDirs, ContainerName: PauseContainer, DurableVolumes: []string{"data"}, Size: paritySize}) + ShapeGVisor(spec, GVisorOptions{ActorUID: testActorUID, ActorDirs: parityActorDirs, ContainerName: SandboxID, DurableVolumes: []string{"data"}, Size: paritySize}) if spec.Process != nil || spec.Root != nil || spec.Mounts != nil { t.Errorf("sandbox spec has process %v, root %v, mounts %v; want none", spec.Process, spec.Root, spec.Mounts) } @@ -176,7 +176,7 @@ func TestShapeGVisor_Sandbox(t *testing.T) { // sentry backs every container, so the sandbox cgroup is the only one that // binds. The sandbox spec keeps its limits, cgroup, and namespaces. func TestShapeGVisor_AppliesTheActorSize(t *testing.T) { - for _, name := range []string{"app", PauseContainer} { + for _, name := range []string{"app", SandboxID} { spec := Build(parityOptions) ShapeGVisor(spec, GVisorOptions{ActorUID: testActorUID, ContainerName: name, Size: paritySize}) if got := *spec.Linux.Resources.Memory.Limit; got != paritySize.MemoryBytes { diff --git a/internal/proto/ateworkerpb/ateworker.pb.go b/internal/proto/ateworkerpb/ateworker.pb.go index 322b7d1648..01670638ff 100644 --- a/internal/proto/ateworkerpb/ateworker.pb.go +++ b/internal/proto/ateworkerpb/ateworker.pb.go @@ -721,8 +721,8 @@ func (x *WorkloadSpec) GetContainers() []*Container { // +k8s:customValidation # mount paths are unique and do not nest across the four lists type Container struct { state protoimpl.MessageState `protogen:"open.v1"` - // The sandbox's own container, ocispec.PauseContainer, is not a short - // name, so the format keeps a workload container from taking its bundle. + // The sandbox ID, ocispec.SandboxID, is not a short name, so the format + // keeps a workload container from taking its bundle. // // +k8s:required // +k8s:format=k8s-short-name diff --git a/internal/proto/ateworkerpb/ateworker.proto b/internal/proto/ateworkerpb/ateworker.proto index b31556c3d7..fa23e6725a 100644 --- a/internal/proto/ateworkerpb/ateworker.proto +++ b/internal/proto/ateworkerpb/ateworker.proto @@ -273,8 +273,8 @@ message WorkloadSpec { // // +k8s:customValidation # mount paths are unique and do not nest across the four lists message Container { - // The sandbox's own container, ocispec.PauseContainer, is not a short - // name, so the format keeps a workload container from taking its bundle. + // The sandbox ID, ocispec.SandboxID, is not a short name, so the format + // keeps a workload container from taking its bundle. // // +k8s:required // +k8s:format=k8s-short-name From 9b35ccd274684356083d060d56215451a14c010c Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Tue, 20 Oct 2026 00:16:51 +0000 Subject: [PATCH 3/5] Build the gVisor sandbox spec directly writeSpec built a full container spec for the sandbox and ShapeGVisor then removed the process, rootfs, and mounts. Add ocispec.BuildSandbox, which gives only the hostname and namespaces, and use it for the sandbox. ShapeGVisor now only adds to a spec. Signed-off-by: Jing Chen --- cmd/ateworker-gvisor/runsc.go | 10 ++++---- internal/ocispec/gvisor.go | 6 +---- internal/ocispec/ocispec.go | 42 ++++++++++++++++++--------------- internal/ocispec/parity_test.go | 5 +++- 4 files changed, 34 insertions(+), 29 deletions(-) diff --git a/cmd/ateworker-gvisor/runsc.go b/cmd/ateworker-gvisor/runsc.go index ee0d4be566..b9bfb38113 100644 --- a/cmd/ateworker-gvisor/runsc.go +++ b/cmd/ateworker-gvisor/runsc.go @@ -27,6 +27,7 @@ import ( "slices" "strings" + "github.com/opencontainers/runtime-spec/specs-go" "go.ate.dev/substrate/internal/ocispec" "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/internal/sizing" @@ -94,20 +95,21 @@ func noRootContainerArgs(containerName string) []string { // writeSpec builds the container's OCI spec, shapes it for gVisor, and saves it // into the container's bundle. func (r *runsc) writeSpec(containerName string) error { - container := &ateworkerpb.Container{Name: ocispec.SandboxID} - if containerName != ocispec.SandboxID { + var spec *specs.Spec + if containerName == ocispec.SandboxID { + spec = ocispec.BuildSandbox(r.actorUID) + } else { i := slices.IndexFunc(r.containers, func(c *ateworkerpb.Container) bool { return c.GetName() == containerName }) if i < 0 { return fmt.Errorf("container %q is not in the workload spec", containerName) } - container = r.containers[i] + spec = ocispec.Build(ocispec.Options{ActorUID: r.actorUID, ActorDirs: r.actorDirs, Container: r.containers[i]}) } // atelet makes the bundles of the application containers only. bundlePath := ociBundlePath(r.actorDirs, containerName) if err := os.MkdirAll(bundlePath, 0o700); err != nil { return fmt.Errorf("while creating bundle %q: %w", bundlePath, err) } - spec := ocispec.Build(ocispec.Options{ActorUID: r.actorUID, ActorDirs: r.actorDirs, Container: container}) ocispec.ShapeGVisor(spec, ocispec.GVisorOptions{ ActorUID: r.actorUID, ActorDirs: r.actorDirs, diff --git a/internal/ocispec/gvisor.go b/internal/ocispec/gvisor.go index 3a86a8acbb..7d80fdd92c 100644 --- a/internal/ocispec/gvisor.go +++ b/internal/ocispec/gvisor.go @@ -46,17 +46,13 @@ func GVisorCgroupLeaf(actorUID, containerName string) string { } // ShapeGVisor adds runsc CRI annotations, durable-dir mount hints, and -// per-container cgroups to the spec. It removes the process, rootfs, and -// mounts from the sandbox spec. It is idempotent. +// per-container cgroups to the spec. It is idempotent. func ShapeGVisor(spec *specs.Spec, o GVisorOptions) { if spec.Annotations == nil { spec.Annotations = make(map[string]string) } spec.Annotations["io.kubernetes.cri.container-name"] = o.ContainerName if o.ContainerName == SandboxID { - spec.Process = nil - spec.Root = nil - spec.Mounts = nil spec.Annotations["io.kubernetes.cri.container-type"] = "sandbox" for _, v := range o.DurableVolumes { spec.Annotations["dev.gvisor.spec.mount."+v+".type"] = "tmpfs" diff --git a/internal/ocispec/ocispec.go b/internal/ocispec/ocispec.go index 14e76e852f..7a59103e94 100644 --- a/internal/ocispec/ocispec.go +++ b/internal/ocispec/ocispec.go @@ -77,6 +77,27 @@ func ociResources(r *ateworkerpb.ResourceRequirements) *specs.LinuxResources { return out } +// BuildSandbox returns the OCI spec of a sandbox with no root container: the +// hostname and namespaces, and no process, rootfs, or mounts. +func BuildSandbox(actorUID string) *specs.Spec { + return &specs.Spec{ + Hostname: Hostname, + Linux: &specs.Linux{Namespaces: namespaces(actorUID)}, + } +} + +// namespaces returns the namespaces of an actor's sandbox and containers. The +// network namespace is the actor's. +func namespaces(actorUID string) []specs.LinuxNamespace { + return []specs.LinuxNamespace{ + {Type: "pid"}, + {Type: "network", Path: nodepath.ActorNetNSPath(actorUID)}, + {Type: "ipc"}, + {Type: "uts"}, + {Type: "mount"}, + } +} + // Build returns a runtime-neutral OCI spec for an actor container. func Build(o Options) *specs.Spec { container := o.Container @@ -138,25 +159,8 @@ func Build(o Options) *specs.Spec { }, }, Linux: &specs.Linux{ - Namespaces: []specs.LinuxNamespace{ - { - Type: "pid", - }, - { - Type: "network", - Path: nodepath.ActorNetNSPath(o.ActorUID), - }, - { - Type: "ipc", - }, - { - Type: "uts", - }, - { - Type: "mount", - }, - }, - Resources: ociResources(containerSpec.GetResources()), + Namespaces: namespaces(o.ActorUID), + Resources: ociResources(containerSpec.GetResources()), }, } diff --git a/internal/ocispec/parity_test.go b/internal/ocispec/parity_test.go index 2c5305a1b1..f886892087 100644 --- a/internal/ocispec/parity_test.go +++ b/internal/ocispec/parity_test.go @@ -153,7 +153,7 @@ func TestShapeGVisor_Idempotent(t *testing.T) { // The sandbox spec has no process, rootfs, or mounts, and declares the // durable-dir mount hints. func TestShapeGVisor_Sandbox(t *testing.T) { - spec := Build(parityOptions) + spec := BuildSandbox(testActorUID) ShapeGVisor(spec, GVisorOptions{ActorUID: testActorUID, ActorDirs: parityActorDirs, ContainerName: SandboxID, DurableVolumes: []string{"data"}, Size: paritySize}) if spec.Process != nil || spec.Root != nil || spec.Mounts != nil { t.Errorf("sandbox spec has process %v, root %v, mounts %v; want none", spec.Process, spec.Root, spec.Mounts) @@ -178,6 +178,9 @@ func TestShapeGVisor_Sandbox(t *testing.T) { func TestShapeGVisor_AppliesTheActorSize(t *testing.T) { for _, name := range []string{"app", SandboxID} { spec := Build(parityOptions) + if name == SandboxID { + spec = BuildSandbox(testActorUID) + } ShapeGVisor(spec, GVisorOptions{ActorUID: testActorUID, ContainerName: name, Size: paritySize}) if got := *spec.Linux.Resources.Memory.Limit; got != paritySize.MemoryBytes { t.Errorf("%s memory limit = %d, want %d", name, got, paritySize.MemoryBytes) From 6ddbac1dc514b73eca430a8906628757f6fc9f44 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Tue, 20 Oct 2026 18:31:28 +0000 Subject: [PATCH 4/5] Make only the sandbox bundle in writeSpec, and fix the restore docs atelet makes the application containers' bundles, so writeSpec only makes the sandbox's. Say in the architecture doc how a memory restore stays on the snapshot's sandbox runtime. Signed-off-by: Jing Chen --- cmd/ateworker-gvisor/runsc.go | 10 +++++----- cmd/ateworker-gvisor/runsc_test.go | 7 +++++++ docs/architecture.md | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/cmd/ateworker-gvisor/runsc.go b/cmd/ateworker-gvisor/runsc.go index b9bfb38113..87e6a795e7 100644 --- a/cmd/ateworker-gvisor/runsc.go +++ b/cmd/ateworker-gvisor/runsc.go @@ -95,8 +95,13 @@ func noRootContainerArgs(containerName string) []string { // writeSpec builds the container's OCI spec, shapes it for gVisor, and saves it // into the container's bundle. func (r *runsc) writeSpec(containerName string) error { + bundlePath := ociBundlePath(r.actorDirs, containerName) var spec *specs.Spec if containerName == ocispec.SandboxID { + // atelet makes the application containers' bundles, not the sandbox's. + if err := os.MkdirAll(bundlePath, 0o700); err != nil { + return fmt.Errorf("while creating bundle %q: %w", bundlePath, err) + } spec = ocispec.BuildSandbox(r.actorUID) } else { i := slices.IndexFunc(r.containers, func(c *ateworkerpb.Container) bool { return c.GetName() == containerName }) @@ -105,11 +110,6 @@ func (r *runsc) writeSpec(containerName string) error { } spec = ocispec.Build(ocispec.Options{ActorUID: r.actorUID, ActorDirs: r.actorDirs, Container: r.containers[i]}) } - // atelet makes the bundles of the application containers only. - bundlePath := ociBundlePath(r.actorDirs, containerName) - if err := os.MkdirAll(bundlePath, 0o700); err != nil { - return fmt.Errorf("while creating bundle %q: %w", bundlePath, err) - } ocispec.ShapeGVisor(spec, ocispec.GVisorOptions{ ActorUID: r.actorUID, ActorDirs: r.actorDirs, diff --git a/cmd/ateworker-gvisor/runsc_test.go b/cmd/ateworker-gvisor/runsc_test.go index 874b6364c1..16e50620ec 100644 --- a/cmd/ateworker-gvisor/runsc_test.go +++ b/cmd/ateworker-gvisor/runsc_test.go @@ -50,6 +50,13 @@ func TestWriteSpec(t *testing.T) { actorDirs: actorDirs, containers: []*ateworkerpb.Container{{Name: "app", ContainerSpec: &ateworkerpb.ContainerSpec{Args: []string{"/app", "serve"}}}}, } + // atelet makes the application bundle; writeSpec does not make it. + if err := r.writeSpec("app"); err == nil { + t.Error("writeSpec(app) = nil without the bundle atelet makes") + } + if err := os.MkdirAll(ociBundlePath(actorDirs, "app"), 0o700); err != nil { + t.Fatal(err) + } for _, name := range []string{"app", ocispec.SandboxID} { if err := r.writeSpec(name); err != nil { t.Fatalf("writeSpec(%q) = %v", name, err) diff --git a/docs/architecture.md b/docs/architecture.md index 1a846beeca..940cfec7cf 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -335,7 +335,7 @@ The node-level subsystem manages the physical execution of sandboxes and the mov ### Sandbox Classes -A `WorkerPool` selects a **sandbox class** (`spec.sandboxClasses[].name`), and each class has a matching `ateworker` herder image. The sandbox binaries themselves are not baked into the worker image — they come at runtime from the cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) the `WorkerPool` resolves for its class (an explicit `configRef`, else the class default). +A `WorkerPool` selects a **sandbox class** (`spec.sandboxClasses[].name`), and each class has a matching `ateworker` herder image. The sandbox binaries themselves are not baked into the worker image — they come at runtime from the cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) the `WorkerPool` resolves for its class (an explicit `configRef`, else the class default). Each snapshot records the sandbox runtime it was taken on, and a memory restore runs on a worker that still offers that runtime, so restores stay reproducible across runtime upgrades. * **gVisor** (`ateworker-gvisor`, the default): Runs the workload under `runsc` for kernel-level sandboxing. Suspend and resume leverage gVisor's native checkpoint/restore of the sandboxed process tree. `DurableDir` volumes and each container's rootfs overlay are saved as separate filesystem checkpoint bundles (`fs/` and `rootfs/`). A `MEMORY` snapshot splits them out of the process checkpoint; `VOLUMES` and `ROOTFS` snapshots save only the bundles they need with `runsc fscheckpoint`, without memory, and restore them on a cold boot. From 30e99e6ab41f5e4f4d722fd3bf7ab4cccdf67603 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Tue, 20 Oct 2026 20:10:54 +0000 Subject: [PATCH 5/5] Tidy comments and drop a test for removed behavior Reword comments that used contrast pairs or em-dashes. Drop TestPrepareOCIBundlesNoSandboxBundle, which only checked that atelet no longer makes a bundle for the sandbox. Signed-off-by: Jing Chen --- cmd/atelet/main_test.go | 25 ------------------------- cmd/ateworker-gvisor/main.go | 8 ++++---- cmd/ateworker-gvisor/runsc.go | 2 +- cmd/ateworker-gvisor/stats.go | 11 +++++------ docs/architecture.md | 2 +- 5 files changed, 11 insertions(+), 37 deletions(-) diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index 33436ab010..ebe036aa8d 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -42,8 +42,6 @@ import ( "go.ate.dev/substrate/internal/ateattr" "go.ate.dev/substrate/internal/ateinterceptors" "go.ate.dev/substrate/internal/atelet" - "go.ate.dev/substrate/internal/imagecache" - "go.ate.dev/substrate/internal/ocispec" "go.ate.dev/substrate/internal/proto/ateletpb" "go.ate.dev/substrate/internal/proto/ateworkerpb" "go.ate.dev/substrate/internal/resources" @@ -325,29 +323,6 @@ func TestSnapshotManifestFidelityAbsent(t *testing.T) { } } -// TestPrepareOCIBundlesNoSandboxBundle pins that only application containers -// get a bundle. The worker writes the sandbox spec itself. -func TestPrepareOCIBundlesNoSandboxBundle(t *testing.T) { - useTempNodeDirs(t) - host := imageVolumeTestRegistry(t) - image := host + "/actor:v1" - pushTestImage(t, image, singleFileLayer(t, "bin/app", "app")) - spec := &ateletpb.WorkloadSpec{ - Containers: []*ateletpb.Container{{Name: "app", Image: image, Command: []string{"/bin/app"}}}, - } - const actorUID = "actor-uid-1" - s := &Atelet{imageCache: newImageVolumeStore(t)} - if _, err := s.prepareOCIBundles(t.Context(), actorUID, resources.ActorRef{}, spec); err != nil { - t.Fatalf("prepareOCIBundles: %v", err) - } - if _, err := os.Stat(filepath.Join(ateletpath.OCIBundlePath(actorUID, "app"), imagecache.OverlaySpecFileName)); err != nil { - t.Errorf("app bundle: %v", err) - } - if _, err := os.Stat(ateletpath.OCIBundlePath(actorUID, ocispec.SandboxID)); !os.IsNotExist(err) { - t.Errorf("sandbox bundle stat = %v, want it absent", err) - } -} - // TestPrepareOCIBundlesFillsContainerSpec pins that each container's spec // carries its resolved args, env, capabilities and resource limits. func TestPrepareOCIBundlesFillsContainerSpec(t *testing.T) { diff --git a/cmd/ateworker-gvisor/main.go b/cmd/ateworker-gvisor/main.go index b2f9fce5b9..6bd06a35ff 100644 --- a/cmd/ateworker-gvisor/main.go +++ b/cmd/ateworker-gvisor/main.go @@ -688,10 +688,10 @@ func (s *AteWorkerService) RunWorkload(ctx context.Context, req *ateworkerpb.Run // Create and start each application container, each with its own log pipe so // every line is tagged with the originating container (ate.actor.container.name). - // The bundle rootfs is composed here — an overlay of the node's cached - // image layers plus the bundle's private upper — because mounting is - // ateworker's job (atelet runs with no capabilities); runsc's gofer - // resolves the mount in this pod's mount namespace. + // The bundle rootfs is composed here, as an overlay of the node's cached + // image layers plus the bundle's private upper. atelet runs with no + // capabilities, so ateworker mounts it, and runsc's gofer resolves the + // mount in this pod's mount namespace. for _, ac := range req.GetSpec().GetContainers() { pw, err := s.actorLogger.StartJSONLogPipe(attribution, ac.GetName()) if err != nil { diff --git a/cmd/ateworker-gvisor/runsc.go b/cmd/ateworker-gvisor/runsc.go index 87e6a795e7..18faacc6ec 100644 --- a/cmd/ateworker-gvisor/runsc.go +++ b/cmd/ateworker-gvisor/runsc.go @@ -98,7 +98,7 @@ func (r *runsc) writeSpec(containerName string) error { bundlePath := ociBundlePath(r.actorDirs, containerName) var spec *specs.Spec if containerName == ocispec.SandboxID { - // atelet makes the application containers' bundles, not the sandbox's. + // atelet makes only the application containers' bundles. if err := os.MkdirAll(bundlePath, 0o700); err != nil { return fmt.Errorf("while creating bundle %q: %w", bundlePath, err) } diff --git a/cmd/ateworker-gvisor/stats.go b/cmd/ateworker-gvisor/stats.go index fe0bd4382b..d312cdd622 100644 --- a/cmd/ateworker-gvisor/stats.go +++ b/cmd/ateworker-gvisor/stats.go @@ -44,9 +44,9 @@ const defaultCgroupRoot = "/sys/fs/cgroup" // sandbox as a whole. // // gVisor runs every container of a sandbox as threads inside one host process, -// the sentry. runsc starts that process from the sandbox's create and from -// inside the sandbox's cgroup — container.createRoot wraps the sandbox spawn in -// cgroup.RunInCgroup — so the sentry lands in the sandbox's leaf. RunWorkload +// the sentry. runsc starts that process from the sandbox's create, inside the +// sandbox's cgroup (container.createRoot wraps the sandbox spawn in +// cgroup.RunInCgroup), so the sentry lands in the sandbox's leaf. RunWorkload // and RestoreWorkload create the sandbox first. // // The leaf is a direct child of the delegated scope rather than of ateworker's own @@ -73,9 +73,8 @@ const defaultCgroupRoot = "/sys/fs/cgroup" // What the leaf holds besides the actor's own work: the sentry's own overhead // (its Go heap, page tables, netstack) and the gofers. Process listings taken // on a live node in #161 put runsc-sandbox and the gofers in the sandbox's -// cgroup leaf. Those runs predate #496, so they establish that everything -// lands in one leaf, not its name or absolute path, which #496's delegation -// moved under the pod scope. +// cgroup leaf. Those runs predate #496 and the rename to _sandbox, so they +// establish only that everything lands in one leaf. // // So this measures the sandbox, not the actor's processes in isolation, which // is what the proto means by "the unit of measurement is the SANDBOX". Sizing diff --git a/docs/architecture.md b/docs/architecture.md index 940cfec7cf..7b0d97be15 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -335,7 +335,7 @@ The node-level subsystem manages the physical execution of sandboxes and the mov ### Sandbox Classes -A `WorkerPool` selects a **sandbox class** (`spec.sandboxClasses[].name`), and each class has a matching `ateworker` herder image. The sandbox binaries themselves are not baked into the worker image — they come at runtime from the cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) the `WorkerPool` resolves for its class (an explicit `configRef`, else the class default). Each snapshot records the sandbox runtime it was taken on, and a memory restore runs on a worker that still offers that runtime, so restores stay reproducible across runtime upgrades. +A `WorkerPool` selects a **sandbox class** (`spec.sandboxClasses[].name`), and each class has a matching `ateworker` herder image. The sandbox binaries themselves are not baked into the worker image. They come at runtime from the cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) the `WorkerPool` resolves for its class (an explicit `configRef`, else the class default). Each snapshot records the sandbox runtime it was taken on, and a memory restore runs on a worker that still offers that runtime, so restores stay reproducible across runtime upgrades. * **gVisor** (`ateworker-gvisor`, the default): Runs the workload under `runsc` for kernel-level sandboxing. Suspend and resume leverage gVisor's native checkpoint/restore of the sandboxed process tree. `DurableDir` volumes and each container's rootfs overlay are saved as separate filesystem checkpoint bundles (`fs/` and `rootfs/`). A `MEMORY` snapshot splits them out of the process checkpoint; `VOLUMES` and `ROOTFS` snapshots save only the bundles they need with `runsc fscheckpoint`, without memory, and restore them on a cold boot.