Skip to content

fix(vm): bind cached registry images to verified manifest digests #4342

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
alangou wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(vm): bind cached registry images to verified manifest digests #4342

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions crates/openshell-driver-vm/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,13 @@ prepared inside the bootstrap VM. The driver checks that a prepared disk
contains the unpacked rootfs before caching it; on failure it caches nothing
and reports the image-prep console tail. Set `OPENSHELL_VM_IMAGE_PULL_CONCURRENCY` to
tune registry layer download parallelism (default `4`, maximum `16`).
Registry pulls in both caches use the resolved digest as an immutable reference.
The driver verifies the downloaded manifest against that digest before downloading
layers. For multi-platform images, it verifies the index and then the selected
platform manifest. Digest-pinned requests retain the caller's digest.
Cache keys hash source identities to keep cache and staging filenames bounded.
Bootstrap cache layout v6 and prepared-image layout v4 rebuild older entries on
first use; older cache files remain on disk but are not reused for registry images.
Both caches are scoped by source image identity and OpenShell version, so an
OpenShell upgrade builds a fresh guest rootfs instead of reusing one with an old
embedded supervisor. Host-side bootstrap layer assembly preserves relative
Expand Down
121 changes: 86 additions & 35 deletions crates/openshell-driver-vm/src/driver.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@
#[path = "preparation.rs"]
mod preparation;

#[cfg(test)]
#[path = "registry_cache_tests.rs"]
mod registry_cache_tests;

use crate::gpu::{GpuInventory, allocate_vsock_cid};

use crate::isolation::VmBoundarySpec;
Expand Down Expand Up @@ -200,8 +204,9 @@ const GUEST_IMAGE_CONFIG_DIR: &str = "openshell-image";
const GUEST_IMAGE_OCI_LAYOUT_DIR: &str = "oci";
const GUEST_IMAGE_OCI_REF: &str = "openshell";
const IMAGE_EXPORT_ROOTFS_ARCHIVE: &str = "source-rootfs.tar";
const BOOTSTRAP_IMAGE_CACHE_LAYOUT_VERSION: &str = "sandbox-bootstrap-rootfs-ext4-v5";
const PREPARED_IMAGE_CACHE_LAYOUT_VERSION: &str = "sandbox-prepared-rootfs-ext4-umoci-v3";
// Older registry cache entries were not bound to the downloaded manifest.
const BOOTSTRAP_IMAGE_CACHE_LAYOUT_VERSION: &str = "sandbox-bootstrap-rootfs-ext4-v6";
const PREPARED_IMAGE_CACHE_LAYOUT_VERSION: &str = "sandbox-prepared-rootfs-ext4-umoci-v4";
const IMAGE_IDENTITY_FILE: &str = "image-identity";
const IMAGE_REFERENCE_FILE: &str = "image-reference";
const IMAGE_PREP_INIT_MODE: &str = "image-prep";
Expand Down Expand Up @@ -3199,10 +3204,27 @@ impl VmDriver {
return span_status.finish(result);
}

span_status.finish(
self.ensure_cached_registry_rootfs_image(
sandbox_id,
image_ref,
&registry_client(),
&registry_auth(image_ref)?,
)
.await,
)
}

async fn ensure_cached_registry_rootfs_image(
&self,
sandbox_id: &str,
image_ref: &str,
client: &OciClient,
auth: &RegistryAuth,
) -> Result<String, Status> {
let span_status = openshell_otel::ErrorStatusGuard::current();
info!(image_ref = %image_ref, "vm driver: ensuring cached root disk image (registry)");
let reference = parse_registry_reference(image_ref)?;
let client = registry_client();
let auth = registry_auth(image_ref)?;
info!(image_ref = %image_ref, "vm driver: authenticating with registry");
self.publish_vm_progress(
sandbox_id,
Expand All @@ -3214,7 +3236,7 @@ impl VmDriver {
]),
);
retry_registry_request("authenticate with registry", || {
client.auth(&reference, &auth, RegistryOperation::Pull)
client.auth(&reference, auth, RegistryOperation::Pull)
})
.await
.map_err(|err| {
Expand All @@ -3232,15 +3254,13 @@ impl VmDriver {
("image_source".to_string(), "registry".to_string()),
]),
);
let source_image_identity = retry_registry_request("fetch manifest digest", || {
client.fetch_manifest_digest(&reference, &auth)
})
.await
.map_err(|err| {
Status::failed_precondition(format!(
"failed to resolve vm sandbox image '{image_ref}': {err}"
))
})?;
let (reference, source_image_identity) = pin_registry_reference(client, &reference, auth)
.await
.map_err(|err| {
Status::failed_precondition(format!(
"failed to resolve vm sandbox image '{image_ref}': {err}"
))
})?;
info!(
image_ref = %image_ref,
image_identity = %source_image_identity,
Expand Down Expand Up @@ -3335,9 +3355,9 @@ impl VmDriver {

self.build_cached_registry_image_rootfs_image(
sandbox_id,
&client,
client,
&reference,
&auth,
auth,
image_ref,
&image_identity,
)
Expand Down Expand Up @@ -3522,8 +3542,14 @@ impl VmDriver {
.await;
}

self.ensure_prepared_registry_image_disk(sandbox_id, image_ref, bootstrap_root_disk)
.await
self.ensure_prepared_registry_image_disk(
sandbox_id,
image_ref,
bootstrap_root_disk,
&registry_client(),
&registry_auth(image_ref)?,
)
.await
}

async fn ensure_prepared_local_image_disk(
Expand Down Expand Up @@ -3746,10 +3772,10 @@ impl VmDriver {
sandbox_id: &str,
image_ref: &str,
bootstrap_root_disk: &Path,
client: &OciClient,
auth: &RegistryAuth,
) -> Result<PreparedImageDisk, Status> {
let reference = parse_registry_reference(image_ref)?;
let client = registry_client();
let auth = registry_auth(image_ref)?;

self.publish_vm_progress(
sandbox_id,
Expand All @@ -3761,7 +3787,7 @@ impl VmDriver {
]),
);
retry_registry_request("authenticate with registry", || {
client.auth(&reference, &auth, RegistryOperation::Pull)
client.auth(&reference, auth, RegistryOperation::Pull)
})
.await
.map_err(|err| {
Expand All @@ -3779,15 +3805,13 @@ impl VmDriver {
("image_source".to_string(), "registry".to_string()),
]),
);
let source_image_identity = retry_registry_request("fetch manifest digest", || {
client.fetch_manifest_digest(&reference, &auth)
})
.await
.map_err(|err| {
Status::failed_precondition(format!(
"failed to resolve vm sandbox image '{image_ref}': {err}"
))
})?;
let (reference, source_image_identity) = pin_registry_reference(client, &reference, auth)
.await
.map_err(|err| {
Status::failed_precondition(format!(
"failed to resolve vm sandbox image '{image_ref}': {err}"
))
})?;
let cache_identity = prepared_image_cache_identity(&source_image_identity, &self.config);
let image_path = image_cache_rootfs_image(&self.config.state_dir, &cache_identity);

Expand All @@ -3814,7 +3838,7 @@ impl VmDriver {
let layout_dir = staging_dir.join(GUEST_IMAGE_OCI_LAYOUT_DIR);

let (manifest, _) = retry_registry_request("pull image manifest", || {
client.pull_image_manifest(&reference, &auth)
client.pull_image_manifest(&reference, auth)
})
.await
.map_err(|err| {
Expand All @@ -3827,7 +3851,7 @@ impl VmDriver {
.map_err(|err| Status::internal(format!("create guest OCI layout failed: {err}")))?;

download_registry_descriptor_blob_file(
&client,
client,
&reference,
image_ref,
&layout_dir,
Expand Down Expand Up @@ -5202,6 +5226,28 @@ fn registry_client() -> OciClient {
})
}

async fn pin_registry_reference(
client: &OciClient,
reference: &Reference,
auth: &RegistryAuth,
) -> Result<(Reference, String), OciDistributionError> {
// Preserve caller pins, including their digest algorithm. A HEAD response
// is only a resolution hint for tags, never proof of the image's contents.
let digest = match reference.digest() {
Some(digest) => digest.to_string(),
None => {
retry_registry_request("fetch manifest digest", || {
client.fetch_manifest_digest(reference, auth)
})
.await?
}
};
// oci-client verifies GET bytes against this pin. For an index it verifies
// the index first, then the selected platform manifest against its entry.
// All retries must use this same reference before publishing to the cache.
Ok((reference.clone_with_digest(digest.clone()), digest))
}

async fn retry_registry_request<T, F, Fut>(
operation: &str,
request: F,
Expand Down Expand Up @@ -6465,6 +6511,9 @@ fn write_oci_layout_for_manifest(
}

fn bootstrap_image_cache_identity(image_identity: &str) -> String {
// Bound new cache/staging filenames even for long source digests such as
// SHA-512. Persisted identities still use the unchanged path lookup.
let image_identity = compute_bytes_sha256_hex(image_identity.as_bytes());
format!(
"{BOOTSTRAP_IMAGE_CACHE_LAYOUT_VERSION}:openshell-{}:guest-{}:{image_identity}",
openshell_core::VERSION,
Expand All @@ -6480,6 +6529,7 @@ fn configured_sandbox_identity(config: &VmDriverConfig) -> Option<(u32, u32)> {
}

fn prepared_image_cache_identity(image_identity: &str, config: &VmDriverConfig) -> String {
let image_identity = compute_bytes_sha256_hex(image_identity.as_bytes());
let identity = configured_sandbox_identity(config).map_or_else(
|| "image-account".to_string(),
|(uid, gid)| format!("configured-{uid}-{gid}"),
Expand Down Expand Up @@ -11493,11 +11543,12 @@ mod tests {
#[test]
fn prepared_image_cache_identity_includes_rootfs_layout_and_openshell_version() {
let image = "sha256:local-image";
let image_hash = compute_bytes_sha256_hex(image.as_bytes());
let image_account = prepared_image_cache_identity(image, &VmDriverConfig::default());
assert_eq!(
image_account,
format!(
"sandbox-prepared-rootfs-ext4-umoci-v3:openshell-{}:image-account:{image}",
"sandbox-prepared-rootfs-ext4-umoci-v4:openshell-{}:image-account:{image_hash}",
openshell_core::VERSION
)
);
Expand Down Expand Up @@ -11539,10 +11590,10 @@ mod tests {
fn bootstrap_image_cache_identity_includes_rootfs_layout_version_and_guest_runtime() {
let identity = bootstrap_image_cache_identity("sha256:bootstrap-image");
assert!(identity.starts_with(&format!(
"sandbox-bootstrap-rootfs-ext4-v5:openshell-{}:guest-",
"sandbox-bootstrap-rootfs-ext4-v6:openshell-{}:guest-",
openshell_core::VERSION
)));
assert!(identity.ends_with(":sha256:bootstrap-image"));
assert!(identity.ends_with(&compute_bytes_sha256_hex(b"sha256:bootstrap-image")));
assert!(identity.contains(&sandbox_guest_runtime_identity()));
}

Expand Down
Loading
Loading