Skip to content

Create unpacker protocol + ext4 unpacker #151

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 0 additions & 87 deletions Sources/Containerization/Image/Image.swift
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,6 @@ import ContainerizationOCI
import ContainerizationOS
import Foundation

#if os(macOS)
import ContainerizationArchive
import ContainerizationEXT4
import SystemPackage
import ContainerizationExtras
#endif

/// Type representing an OCI container image.
public struct Image: Sendable {
private let contentStore: ContentStore
Expand Down Expand Up @@ -135,83 +128,3 @@ public struct Image: Sendable {
return content
}
}

#if os(macOS)

extension Image {
/// Unpack the image into a filesystem.
public func unpack(for platform: Platform, at path: URL, blockSizeInBytes: UInt64 = 512.gib(), progress: ProgressHandler? = nil) async throws -> Mount {
let blockPath = try prepareUnpackPath(path: path)
let manifest = try await loadManifest(platform: platform)
return try await unpackContents(
path: blockPath,
manifest: manifest,
blockSizeInBytes: blockSizeInBytes,
progress: progress
)
}

private func loadManifest(platform: Platform) async throws -> Manifest {
let manifest = try await descriptor(for: platform)
guard let m: Manifest = try await self.contentStore.get(digest: manifest.digest) else {
throw ContainerizationError(.notFound, message: "content not found \(manifest.digest)")
}
return m
}

private func prepareUnpackPath(path: URL) throws -> String {
let blockPath = path.absolutePath()
guard !FileManager.default.fileExists(atPath: blockPath) else {
throw ContainerizationError(.exists, message: "block device already exists at \(blockPath)")
}
return blockPath
}

private func unpackContents(path: String, manifest: Manifest, blockSizeInBytes: UInt64, progress: ProgressHandler?) async throws -> Mount {
let filesystem = try EXT4.Formatter(FilePath(path), minDiskSize: blockSizeInBytes)
defer { try? filesystem.close() }

for layer in manifest.layers {
try Task.checkCancellation()
guard let content = try await self.contentStore.get(digest: layer.digest) else {
throw ContainerizationError(.notFound, message: "Content with digest \(layer.digest)")
}

switch layer.mediaType {
case MediaTypes.imageLayer, MediaTypes.dockerImageLayer:
try filesystem.unpack(
source: content.path,
format: .paxRestricted,
compression: .none,
progress: progress
)
case MediaTypes.imageLayerGzip, MediaTypes.dockerImageLayerGzip:
try filesystem.unpack(
source: content.path,
format: .paxRestricted,
compression: .gzip,
progress: progress
)
default:
throw ContainerizationError(.unsupported, message: "Media type \(layer.mediaType) not supported.")
}
}

return .block(
format: "ext4",
source: path,
destination: "/",
options: []
)
}
}

#else

extension Image {
public func unpack(for platform: Platform, at path: URL, blockSizeInBytes: UInt64 = 512.gib()) async throws -> Mount {
throw ContainerizationError(.unsupported, message: "Image unpack unsupported on current platform")
}
}

#endif
3 changes: 2 additions & 1 deletion Sources/Containerization/Image/InitImage.swift
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ public struct InitImage: Sendable {
extension InitImage {
/// Unpack the initial filesystem for the desired platform at a given path.
public func initBlock(at: URL, for platform: SystemPlatform) async throws -> Mount {
var fs = try await image.unpack(for: platform.ociPlatform(), at: at, blockSizeInBytes: 512.mib())
let unpacker = EXT4Unpacker(blockSizeInBytes: 512.mib())
var fs = try await unpacker.unpack(self.image, for: platform.ociPlatform(), at: at)
fs.options = ["ro"]
return fs
}
Expand Down
84 changes: 84 additions & 0 deletions Sources/Containerization/Image/Unpacker/EXT4Unpacker.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
//===----------------------------------------------------------------------===//
// Copyright © 2025 Apple Inc. and the Containerization project authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//

import ContainerizationError
import ContainerizationExtras
import ContainerizationOCI
import Foundation

#if os(macOS)
import ContainerizationArchive
import ContainerizationEXT4
import SystemPackage
#endif

public struct EXT4Unpacker: Unpacker {
let blockSizeInBytes: UInt64

public init(blockSizeInBytes: UInt64) {
self.blockSizeInBytes = blockSizeInBytes
}

public func unpack(_ image: Image, for platform: Platform, at path: URL, progress: ProgressHandler? = nil) async throws -> Mount {
#if !os(macOS)
throw ContainerizationError(.unsupported, message: "Cannot unpack an image on current platform")
#else
let blockPath = try prepareUnpackPath(path: path)
let manifest = try await image.manifest(for: platform)
let filesystem = try EXT4.Formatter(FilePath(path), minDiskSize: blockSizeInBytes)
defer { try? filesystem.close() }

for layer in manifest.layers {
try Task.checkCancellation()
let content = try await image.getContent(digest: layer.digest)

switch layer.mediaType {
case MediaTypes.imageLayer, MediaTypes.dockerImageLayer:
try filesystem.unpack(
source: content.path,
format: .paxRestricted,
compression: .none,
progress: progress
)
case MediaTypes.imageLayerGzip, MediaTypes.dockerImageLayerGzip:
try filesystem.unpack(
source: content.path,
format: .paxRestricted,
compression: .gzip,
progress: progress
)
default:
throw ContainerizationError(.unsupported, message: "Media type \(layer.mediaType) not supported.")
}
}
Comment on lines +48 to +66
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: Given both the unpacks are the exact same besides whether they supply compression or not, can we just assign a compression variable in the switch cases and then do the unpack outside of it?


return .block(
format: "ext4",
source: blockPath,
destination: "/",
options: []
)
#endif
}

private func prepareUnpackPath(path: URL) throws -> String {
let blockPath = path.absolutePath()
guard !FileManager.default.fileExists(atPath: blockPath) else {
throw ContainerizationError(.exists, message: "block device already exists at \(blockPath)")
}
return blockPath
}
}
40 changes: 40 additions & 0 deletions Sources/Containerization/Image/Unpacker/Unpacker.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
//===----------------------------------------------------------------------===//
// Copyright © 2025 Apple Inc. and the Containerization project authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//

import ContainerizationExtras
import ContainerizationOCI
import Foundation

/// The `Unpacker` protocol defines a standardized interface that involves
/// decompressing, extracting image layers and preparing it for use.
///
/// The `Unpacker` is responsible for managing the lifecycle of the
/// unpacking process, including any temporary files or resources, until the
/// `Mount` object is produced.
protocol Unpacker {

/// Unpacks the provided image to a specified path for a given platform.
///
/// This asynchronous method should handle the entire unpacking process, from reading
/// the `Image` layers for the given `Platform` via its `Manifest`,
/// to making the extracted contents available as a `Mount`.
/// Implementations of this method may apply platform-specific optimizations
/// or transformations during the unpacking.
///
/// Progress updates can be observed via the optional `progress` handler.
func unpack(_ image: Image, for platform: Platform, at path: URL, progress: ProgressHandler?) async throws -> Mount

}
3 changes: 2 additions & 1 deletion Sources/Integration/Suite.swift
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,8 @@ struct IntegrationSuite: AsyncParsableCommand {
let fs: Containerization.Mount = try await {
let fsPath = Self.testDir.appending(component: "rootfs.ext4")
do {
return try await image.unpack(for: platform, at: fsPath)
let unpacker = EXT4Unpacker(blockSizeInBytes: 2.gib())
return try await unpacker.unpack(image, for: platform, at: fsPath)
} catch let err as ContainerizationError {
if err.code == .exists {
return .block(
Expand Down
7 changes: 2 additions & 5 deletions Sources/cctl/ContainerStore.swift
Original file line number Diff line number Diff line change
Expand Up @@ -96,11 +96,8 @@ struct ContainerStore: Sendable {
let imageBlock: Containerization.Mount = try await {
let source = self.root.appendingPathComponent(blockName)
do {
return try await image.unpack(
for: .current,
at: source,
blockSizeInBytes: fsSizeInBytes
)
let unpacker = EXT4Unpacker(blockSizeInBytes: fsSizeInBytes)
return try await unpacker.unpack(image, for: .current, at: source)
} catch let err as ContainerizationError {
if err.code == .exists {
return .block(
Expand Down
23 changes: 19 additions & 4 deletions Sources/cctl/ImageCommand.swift
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,13 @@ extension Application {

@Option(name: .customLong("platform"), help: "Platform string in the form 'os/arch/variant'. Example 'linux/arm64/v8', 'linux/amd64'") var platformString: String?

@Option(
name: .customLong("unpack-path"), help: "Path to unpack image into",
transform: { str in
URL(fileURLWithPath: str, relativeTo: .currentDirectory()).absoluteURL.path(percentEncoded: false)
})
var unpackPath: String?

@Flag(help: "Pull via plain text http") var http: Bool = false

func run() async throws {
Expand Down Expand Up @@ -126,11 +133,20 @@ extension Application {
}

print("image pulled")
guard let unpackPath else {
return
}
guard !FileManager.default.fileExists(atPath: unpackPath) else {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this also returns true if the path is a directory, do we still want to error?

throw ContainerizationError(.invalidArgument, message: "File exists at \(unpackPath)")
}
let unpackUrl = URL(filePath: unpackPath)
try FileManager.default.createDirectory(at: unpackUrl, withIntermediateDirectories: true)

let unpacker = EXT4Unpacker.init(blockSizeInBytes: 2.gib())

let tempDir = FileManager.default.uniqueTemporaryDirectory(create: true)
if let platform {
let name = platform.description.replacingOccurrences(of: "/", with: "-")
let _ = try await image.unpack(for: platform, at: tempDir.appending(component: name))
let _ = try await unpacker.unpack(image, for: platform, at: unpackUrl.appending(component: name))
} else {
for descriptor in try await image.index().manifests {
if let referenceType = descriptor.annotations?["vnd.docker.reference.type"], referenceType == "attestation-manifest" {
Expand All @@ -140,11 +156,10 @@ extension Application {
continue
}
let name = descPlatform.description.replacingOccurrences(of: "/", with: "-")
let _ = try await image.unpack(for: descPlatform, at: tempDir.appending(component: name))
let _ = try await unpacker.unpack(image, for: descPlatform, at: unpackUrl.appending(component: name))
print("created snapshot for platform \(descPlatform.description)")
}
}
try? FileManager.default.removeItem(at: tempDir)
}
}

Expand Down