Skip to content

Commit

Permalink
preliminary EROFS support for local turboOCI convertion
Browse files Browse the repository at this point in the history
This is a very basic support for EROFS local OCI convertion.

Unlike EXT4, EROFS doesn't need a `--import` and `export` pair to
generate modified tar headers first since erofs-utils just needs to
parse tar streams once.

Even for parallel convertion, erofs-utils will use EROFS layer blobs
directly.

Also, EROFS doesn't have an explicit `--mkfs` step since `mkfs.erofs`
can handle layer imports directly.

Signed-off-by: Hongzhen Luo <[email protected]>
  • Loading branch information
Hongzhen Luo committed Mar 27, 2024
1 parent ccbce82 commit 00fc151
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 14 deletions.
34 changes: 23 additions & 11 deletions pkg/snapshot/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -534,20 +534,32 @@ func (o *snapshotter) constructOverlayBDSpec(ctx context.Context, key string, wr
// 1. generate tar meta for oci layer blob
// 2. convert local layer.tarmeta to overlaybd
// 3. create layer's config
log.G(ctx).Infof("generate metadata of layer blob (sn: %s)", id)
if err := utils.GenerateTarMeta(ctx, o.overlaybdOCILayerPath(id), o.overlaybdOCILayerMeta(id)); err != nil {
log.G(ctx).Errorf("generate tar metadata failed. (sn: %s)", id)
return err
}
opt := &utils.ConvertOption{
TarMetaPath: o.overlaybdOCILayerMeta(id),
Workdir: o.convertTempdir(id),
Ext4FSMetaPath: o.magicFilePath(id), // overlaybd.commit
Config: configJSON,
var opt *utils.ConvertOption
rootfs_type := o.defaultFsType
if rootfs_type == "erofs" {
opt = &utils.ConvertOption{
TarMetaPath: o.overlaybdOCILayerPath(id),
Workdir: o.convertTempdir(id),
Ext4FSMetaPath: o.magicFilePath(id), // overlaybd.commit
Config: configJSON,
}
} else {
log.G(ctx).Infof("generate metadata of layer blob (sn: %s)", id)
if err := utils.GenerateTarMeta(ctx, o.overlaybdOCILayerPath(id), o.overlaybdOCILayerMeta(id)); err != nil {
log.G(ctx).Errorf("generate tar metadata failed. (sn: %s)", id)
return err
}

opt = &utils.ConvertOption{
TarMetaPath: o.overlaybdOCILayerMeta(id),
Workdir: o.convertTempdir(id),
Ext4FSMetaPath: o.magicFilePath(id), // overlaybd.commit
Config: configJSON,
}
}
log.G(ctx).Infof("convert layer to turboOCI (sn: %s)", id)

if err := utils.ConvertLayer(ctx, opt); err != nil {
if err := utils.ConvertLayer(ctx, opt, rootfs_type); err != nil {
log.G(ctx).Error(err.Error())
os.RemoveAll(opt.Workdir)
os.Remove(opt.Ext4FSMetaPath)
Expand Down
10 changes: 7 additions & 3 deletions pkg/utils/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ func GenerateTarMeta(ctx context.Context, srcTarFile string, dstTarMeta string)
}

// ConvertLayer produce a turbooci layer, target is path of ext4.fs.meta
func ConvertLayer(ctx context.Context, opt *ConvertOption) error {
func ConvertLayer(ctx context.Context, opt *ConvertOption, fs_type string) error {
if opt.Workdir == "" {
opt.Workdir = "tmp_conv"
}
Expand All @@ -199,7 +199,7 @@ func ConvertLayer(ctx context.Context, opt *ConvertOption) error {

// overlaybd-create
args := []string{pathWritableData, pathWritableIndex, "256", "-s", "--turboOCI"}
if len(opt.Config.Lowers) == 0 {
if fs_type != "erofs" && len(opt.Config.Lowers) == 0 {
args = append(args, "--mkfs")
}
if out, err := exec.CommandContext(ctx, obdBinCreate, args...).CombinedOutput(); err != nil {
Expand Down Expand Up @@ -231,9 +231,13 @@ func ConvertLayer(ctx context.Context, opt *ConvertOption) error {
}
args = []string{
opt.TarMetaPath, pathConfig,
"--import",
"--service_config_path", pathService,
"--fstype", fs_type,
}
if fs_type != "erofs" {
args = append(args, "--import")
}

log.G(ctx).Debugf("%s %s", obdBinTurboOCIApply, strings.Join(args, " "))
if out, err := exec.CommandContext(ctx, obdBinTurboOCIApply,
args...,
Expand Down

0 comments on commit 00fc151

Please sign in to comment.