Add support for bootc images in lifecycle-agent

controllers/prep_handlers.go

@@ -78,12 +78,7 @@ func GetSeedImage(c client.Client, ctx context.Context, ibu *ibuv1.ImageBasedUpg
 
 // validateSeedImageConfig retrieves the labels for the seed image without downloading the image itself, then validates
 // the config data from the labels
-func (r *ImageBasedUpgradeReconciler) validateSeedImageConfig(ctx context.Context, ibu *ibuv1.ImageBasedUpgrade) error {
-	labels, err := r.getLabelsForSeedImage(ctx, ibu)
-	if err != nil {
-		return fmt.Errorf("failed to get seed image labels: %w", err)
-	}
-
+func (r *ImageBasedUpgradeReconciler) validateSeedImageConfig(ctx context.Context, labels map[string]string) error {
 	r.Log.Info("Checking seed image version compatibility")
 	if err := checkSeedImageVersionCompatibility(labels); err != nil {
 		return fmt.Errorf("checking seed image compatibility: %w", err)
@@ -569,19 +564,32 @@ func (r *ImageBasedUpgradeReconciler) handlePrep(ctx context.Context, ibu *ibuv1
 				return prepFailDoNotRequeue(r.Log, fmt.Sprintf("failed to initialize IBU workspace: %s", err.Error()), ibu)
 			}
 
+			labels, err := r.getLabelsForSeedImage(ctx, ibu)
+			if err != nil {
+				return requeueWithError(fmt.Errorf("failed to get labels for seed image: %w", err))
+			}
+
 			// Validate config information from the seed image labels, prior to launching the job and downloading the image
 			r.Log.Info("Validating seed information")
-			if err := r.validateSeedImageConfig(ctx, ibu); err != nil {
+			if err := r.validateSeedImageConfig(ctx, labels); err != nil {
 				return prepFailDoNotRequeue(r.Log, fmt.Sprintf("failed to validate seed image info: %s", err.Error()), ibu)
 			}
 
+			// Check the OCI labels of the seed image to detect bootc seed
+			useBootc := false
+			if val, exists := labels[common.SeedUseBootcOCILabel]; exists {
+				if val == "true" {
+					useBootc = true
+				}
+			}
+
 			r.Log.Info("Checking container storage disk space")
 			if err := r.containerStorageCleanup(ibu); err != nil {
 				return requeueWithError(fmt.Errorf("failed container storage cleanup: %w", err))
 			}
 
 			r.Log.Info("Launching a new stateroot job")
-			if _, err := prep.LaunchStaterootSetupJob(ctx, r.Client, ibu, r.Scheme, r.Log); err != nil {
+			if _, err := prep.LaunchStaterootSetupJob(ctx, r.Client, ibu, r.Scheme, r.Log, useBootc); err != nil {
 				return requeueWithError(fmt.Errorf("failed launch stateroot job: %w", err))
 			}
 			// start prep stage stateroot phase timing

controllers/seedgen_controller.go

@@ -466,6 +466,12 @@ func (r *SeedGeneratorReconciler) launchImager(seedgen *seedgenv1.SeedGenerator)
 		r.Log.Info(fmt.Sprintf("Skipping recert validation because %s=%s", EnvSkipRecert, skipRecertEnvValue))
 	}
 
+	useBootc := false
+	if val, ok := seedgen.Annotations[common.UnsupportedExperimentalUseBootcAnnotation]; ok && val == common.UnsupportedExperimentalUseBootcValue {
+		useBootc = true
+		r.Log.Info("Using bootc for seed image generation")
+	}
+
 	imagerCmdArgs := []string{
 		"podman", "run", "--privileged", "--pid=host",
 		fmt.Sprintf("--name=%s", imagerContainerName),
@@ -488,6 +494,10 @@ func (r *SeedGeneratorReconciler) launchImager(seedgen *seedgenv1.SeedGenerator)
 		imagerCmdArgs = append(imagerCmdArgs, "--skip-recert-validation")
 	}
 
+	if useBootc {
+		imagerCmdArgs = append(imagerCmdArgs, "--use-bootc")
+	}
+
 	// In order to have the imager container both survive the LCA pod shutdown and have continued network access
 	// after all other pods are shutdown, we're using systemd-run to launch it as a transient service-unit
 	systemdRunOpts := []string{

internal/common/consts.go

@@ -94,6 +94,11 @@ const (
 	ImageCleanupOnPrepAnnotation = "image-cleanup.lca.openshift.io/on-prep"
 	// ImageCleanupDisabledValue value to disable image cleanup
 	ImageCleanupDisabledValue = "Disabled"
+	// UnsupportedExperimentalUseBootcAnnotation configures the use of bootc for image-based upgrade
+	// Only acceptable value is UnsupportedExperimentalUseBootcDisabledValue. Any other value is ignored.
+	UnsupportedExperimentalUseBootcAnnotation = "unsupported-experimental.lca.openshift.io/use-bootc"
+	// UnsupportedExperimentalUseBootcValue is the value to use bootc
+	UnsupportedExperimentalUseBootcValue = "Use"
 
 	LcaNamespace = "openshift-lifecycle-agent"
 	Host         = "/host"
@@ -118,6 +123,10 @@ const (
 
 	SeedClusterInfoOCILabel = "com.openshift.lifecycle-agent.seed_cluster_info"
 
+	// SeedUseBootcOCILabel is the name of the OCI label applied to seed images
+	// to indicate whether they're bootc seed images or normal ones
+	SeedUseBootcOCILabel = "com.openshift.lifecycle-agent.bootc_seed"
+
 	PullSecretName           = "pull-secret"
 	PullSecretEmptyData      = "{\"auths\":{\"registry.connect.redhat.com\":{\"username\":\"empty\",\"password\":\"empty\",\"auth\":\"ZW1wdHk6ZW1wdHk=\",\"email\":\"\"}}}" //nolint:gosec
 	OpenshiftConfigNamespace = "openshift-config"

internal/ostreeclient/ostreeclient.go

@@ -15,7 +15,7 @@ import (
 type IClient interface {
 	PullLocal(repoPath string) error
 	OSInit(osname string) error
-	Deploy(osname, refsepc string, kargs []string, rpmOstreeClient rpmostreeclient.IClient, ibi bool) error
+	Deploy(osname, refsepc string, kargs []string, rpmOstreeClient rpmostreeclient.IClient, ibi bool, useBootc bool) error
 	Undeploy(ostreeIndex int) error
 	SetDefaultDeployment(index int) error
 	IsOstreeAdminSetDefaultFeatureEnabled() bool
@@ -59,31 +59,64 @@ func (c *Client) OSInit(osname string) error {
 	return nil
 }
 
-func (c *Client) Deploy(osname, refsepc string, kargs []string, rpmOstreeClient rpmostreeclient.IClient, ibi bool) error {
-	args := []string{"admin", "deploy", "--os", osname, "--no-prune"}
-	if c.ibi {
-		args = append(args, "--sysroot", "/mnt")
-	}
-	args = append(args, kargs...)
-	args = append(args, refsepc)
-	if !c.ibi && c.IsOstreeAdminSetDefaultFeatureEnabled() {
-		args = append(args, "--not-as-default")
-	}
+func (c *Client) Deploy(osname, refsepc string, kargs []string, rpmOstreeClient rpmostreeclient.IClient, ibi bool, useBootc bool) error {
+	if !useBootc {
+		args := []string{"admin", "deploy", "--os", osname, "--no-prune"}
+		if c.ibi {
+			args = append(args, "--sysroot", "/mnt")
+		}
 
-	// Run the command in bash to preserve the quoted kargs
-	args = append([]string{"ostree"}, args...)
-	if _, err := c.executor.Execute("bash", "-c", strings.Join(args, " ")); err != nil {
-		return fmt.Errorf("failed to run OSInit with args %s: %w", args, err)
-	}
+		for _, karg := range kargs {
+			args = append(args, "--karg-append", karg)
+		}
+
+		args = append(args, refsepc)
+		if !c.ibi && c.IsOstreeAdminSetDefaultFeatureEnabled() {
+			args = append(args, "--not-as-default")
+		}
 
-	if !ibi {
-		// In an IBU where both releases have the same underlying rhcos image, the parent commit of the deployment has
-		// unique commit IDs (due to import from seed), but the same checksum. In order to avoid pruning the original parent
-		// commit and corrupting the ostree, the previous "ostree admin deploy" command was called with the "--no-prune" option.
-		// This must also be followed up with a call to "rpm-ostree cleanup -b" to update the base refs.
-		if err := rpmOstreeClient.RpmOstreeCleanup(); err != nil {
-			return fmt.Errorf("failed rpm-ostree cleanup -b: %w", err)
+		// Run the command in bash to preserve the quoted kargs
+		args = append([]string{"ostree"}, args...)
+		if _, err := c.executor.Execute("bash", "-c", strings.Join(args, " ")); err != nil {
+			return fmt.Errorf("failed to run OSInit with args %s: %w", args, err)
 		}
+
+		if !ibi {
+			// In an IBU where both releases have the same underlying rhcos image, the parent commit of the deployment has
+			// unique commit IDs (due to import from seed), but the same checksum. In order to avoid pruning the original parent
+			// commit and corrupting the ostree, the previous "ostree admin deploy" command was called with the "--no-prune" option.
+			// This must also be followed up with a call to "rpm-ostree cleanup -b" to update the base refs.
+			if err := rpmOstreeClient.RpmOstreeCleanup(); err != nil {
+				return fmt.Errorf("failed rpm-ostree cleanup -b: %w", err)
+			}
+		}
+	} else {
+		args := []string{
+			"run", "--privileged",
+			"--env", "RUST_LOG=trace",
+			// TODO: We can probably remove many of these mounts and it would still work, due to recent improvements in bootc
+			"-v", "/:/target",
+			"-v", "/boot:/target/sysroot/boot",
+			"-v", "/var/tmp:/var/tmp",
+			"-v", "/var/lib/containers/storage:/var/lib/containers/storage",
+			"--pid=host", "-it",
+			refsepc,
+			"bootc", "install", "to-existing-root",
+			"--acknowledge-destructive",
+			"--stateroot", osname,
+		}
+
+		for _, karg := range kargs {
+			args = append(args, "--karg", karg)
+		}
+
+		// Run the command in bash to preserve the quoted kargs
+		args = append([]string{"podman"}, args...)
+		if _, err := c.executor.Execute("bash", "-c", strings.Join(args, " ")); err != nil {
+			return fmt.Errorf("failed to run bootc with args %s: %w", args, err)
+		}
+
+		return nil
 	}
 
 	return nil

internal/prep/prep.go

@@ -58,14 +58,13 @@ func buildKernelArgumentsFromMCOFile(path string) ([]string, error) {
 		return nil, fmt.Errorf("failed to read and decode machine config json file: %w", err)
 	}
 
-	args := make([]string, len(mc.Spec.KernelArguments)*2)
-	for i, karg := range mc.Spec.KernelArguments {
+	args := make([]string, 0)
+	for _, karg := range mc.Spec.KernelArguments {
 		// if we don't marshal the karg, `"` won't appear in the kernel arguments after reboot
 		if val, err := json.Marshal(karg); err != nil {
 			return nil, fmt.Errorf("failed to marshal karg %s: %w", karg, err)
 		} else {
-			args[2*i] = "--karg-append"
-			args[2*i+1] = string(val)
+			args = append(args, string(val))
 		}
 	}
 
@@ -123,7 +122,7 @@ func getDeploymentFromDeploymentID(deploymentID string) (string, error) {
 }
 
 func SetupStateroot(log logr.Logger, ops ops.Ops, ostreeClient ostreeclient.IClient,
-	rpmOstreeClient rpmostreeclient.IClient, seedImage, expectedVersion string, ibi bool) error {
+	rpmOstreeClient rpmostreeclient.IClient, seedImage, expectedVersion string, ibi bool, useBootc bool) error {
 	log.Info("Start setupstateroot")
 
 	defer ops.UnmountAndRemoveImage(seedImage)
@@ -132,7 +131,6 @@ func SetupStateroot(log logr.Logger, ops ops.Ops, ostreeClient ostreeclient.ICli
 	if err != nil {
 		return fmt.Errorf("failed to create temp directory %w", err)
 	}
-
 	defer func() {
 		if err := os.RemoveAll(workspaceOutsideChroot); err != nil {
 			log.Error(err, "failed to cleanup workspace")
@@ -157,15 +155,23 @@ func SetupStateroot(log logr.Logger, ops ops.Ops, ostreeClient ostreeclient.ICli
 		return fmt.Errorf("failed to mount seed image: %w", err)
 	}
 
+	if useBootc {
+		// The seed data for bootc images is not at the root but at /usr/lib/openshift/seed instead
+		mountpoint = filepath.Join(mountpoint, "usr/lib/openshift/seed")
+	}
+
 	ostreeRepo := filepath.Join(workspace, "ostree")
 	if err = os.Mkdir(common.PathOutsideChroot(ostreeRepo), 0o700); err != nil {
 		return fmt.Errorf("failed to create ostree repo directory: %w", err)
 	}
 
-	if err := ops.ExtractTarWithSELinux(
-		fmt.Sprintf("%s/ostree.tgz", mountpoint), ostreeRepo,
-	); err != nil {
-		return fmt.Errorf("failed to extract ostree.tgz: %w", err)
+	// The bootc image does not contain ostree.tgz, so it does not need to be extracted
+	if !useBootc {
+		if err := ops.ExtractTarWithSELinux(
+			fmt.Sprintf("%s/ostree.tgz", mountpoint), ostreeRepo,
+		); err != nil {
+			return fmt.Errorf("failed to extract ostree.tgz: %w", err)
+		}
 	}
 
 	// example:
@@ -194,12 +200,15 @@ func SetupStateroot(log logr.Logger, ops ops.Ops, ostreeClient ostreeclient.ICli
 
 	osname := common.GetStaterootName(expectedVersion)
 
-	if err = ostreeClient.PullLocal(ostreeRepo); err != nil {
-		return fmt.Errorf("failed ostree pull-local: %w", err)
-	}
+	// These stateroot preparations are handled by bootc
+	if !useBootc {
+		if err = ostreeClient.PullLocal(ostreeRepo); err != nil {
+			return fmt.Errorf("failed ostree pull-local: %w", err)
+		}
 
-	if err = ostreeClient.OSInit(osname); err != nil {
-		return fmt.Errorf("failed ostree admin os-init: %w", err)
+		if err = ostreeClient.OSInit(osname); err != nil {
+			return fmt.Errorf("failed ostree admin os-init: %w", err)
+		}
 	}
 
 	kargs, err := buildKernelArgumentsFromMCOFile(filepath.Join(common.PathOutsideChroot(mountpoint), "mco-currentconfig.json"))
@@ -209,10 +218,16 @@ func SetupStateroot(log logr.Logger, ops ops.Ops, ostreeClient ostreeclient.ICli
 
 	if !ibi {
 		// Append a unique karg
-		kargs = append(kargs, "--karg", "ibu="+expectedVersion)
+		kargs = append(kargs, "ibu="+expectedVersion)
+	}
+
+	ref := seedBootedRef
+	if useBootc {
+		// With bootc we deploy from the seed image directly
+		ref = seedImage
 	}
 
-	if err = ostreeClient.Deploy(osname, seedBootedRef, kargs, rpmOstreeClient, ibi); err != nil {
+	if err = ostreeClient.Deploy(osname, ref, kargs, rpmOstreeClient, ibi, useBootc); err != nil {
 		return fmt.Errorf("failed ostree admin deploy: %w", err)
 	}
 

internal/prep/prep_stateroot_setup_job.go

@@ -40,8 +40,8 @@ func GetStaterootSetupJob(ctx context.Context, c client.Client, log logr.Logger)
 	return job, nil
 }
 
-func LaunchStaterootSetupJob(ctx context.Context, c client.Client, ibu *ibuv1.ImageBasedUpgrade, scheme *runtime.Scheme, log logr.Logger) (*batchv1.Job, error) {
-	job, err := constructJobForStaterootSetup(ctx, c, ibu, scheme, log)
+func LaunchStaterootSetupJob(ctx context.Context, c client.Client, ibu *ibuv1.ImageBasedUpgrade, scheme *runtime.Scheme, log logr.Logger, useBootc bool) (*batchv1.Job, error) {
+	job, err := constructJobForStaterootSetup(ctx, c, ibu, scheme, log, useBootc)
 	if err != nil {
 		return nil, fmt.Errorf("failed to render job: %w", err)
 	}
@@ -56,7 +56,7 @@ func LaunchStaterootSetupJob(ctx context.Context, c client.Client, ibu *ibuv1.Im
 	return job, nil
 }
 
-func constructJobForStaterootSetup(ctx context.Context, c client.Client, ibu *ibuv1.ImageBasedUpgrade, scheme *runtime.Scheme, log logr.Logger) (*batchv1.Job, error) {
+func constructJobForStaterootSetup(ctx context.Context, c client.Client, ibu *ibuv1.ImageBasedUpgrade, scheme *runtime.Scheme, log logr.Logger, useBootc bool) (*batchv1.Job, error) {
 	log.Info("Getting lca deployment to configure stateroot setup job")
 	lcaDeployment := appsv1.Deployment{}
 	if err := c.Get(ctx, types.NamespacedName{Namespace: common.LcaNamespace, Name: "lifecycle-agent-controller-manager"}, &lcaDeployment); err != nil {
@@ -69,6 +69,11 @@ func constructJobForStaterootSetup(ctx context.Context, c client.Client, ibu *ib
 		return nil, fmt.Errorf("no 'manager' container found in deployment")
 	}
 
+	command := []string{"lca-cli", "ibu-stateroot-setup"}
+	if useBootc {
+		command = append(command, "--use-bootc")
+	}
+
 	var backoffLimit int32 = 0
 	job := &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
@@ -92,7 +97,7 @@ func constructJobForStaterootSetup(ctx context.Context, c client.Client, ibu *ib
 							Name:            StaterootSetupJobName,
 							Image:           manager.Image,
 							ImagePullPolicy: manager.ImagePullPolicy,
-							Command:         []string{"lca-cli", "ibu-stateroot-setup"},
+							Command:         command,
 							Env:             manager.Env,
 							EnvFrom:         manager.EnvFrom,
 							SecurityContext: manager.SecurityContext, // this is needed for podman

lca-cli/cmd/create.go

@@ -52,6 +52,8 @@ var (
 	recertSkipValidation bool
 
 	skipCleanup bool
+
+	useBootc bool
 )
 
 func init() {
@@ -114,7 +116,7 @@ func create() error {
 	}
 
 	seedCreator := seedcreator.NewSeedCreator(client, log, op, rpmOstreeClient, common.BackupDir, common.KubeconfigFile,
-		containerRegistry, authFile, recertContainerImage, recertSkipValidation)
+		containerRegistry, authFile, recertContainerImage, recertSkipValidation, useBootc)
 	if err = seedCreator.CreateSeedImage(); err != nil {
 		err = fmt.Errorf("failed to create seed image: %w", err)
 		log.Error(err)

lca-cli/cmd/ibuStaterootSetup.go

@@ -45,6 +45,7 @@ var ibuStaterootSetupCmd = &cobra.Command{
 }
 
 func init() {
+	addUseBootcFlag(ibuStaterootSetupCmd)
 	rootCmd.AddCommand(ibuStaterootSetupCmd)
 }
 
@@ -86,7 +87,7 @@ func ibuStaterootSetupRun() error {
 	}
 
 	logger.Info("Setting up stateroot")
-	if err := prep.SetupStateroot(logger, opsClient, ostreeClient, rpmOstreeClient, ibu.Spec.SeedImageRef.Image, ibu.Spec.SeedImageRef.Version, false); err != nil {
+	if err := prep.SetupStateroot(logger, opsClient, ostreeClient, rpmOstreeClient, ibu.Spec.SeedImageRef.Image, ibu.Spec.SeedImageRef.Version, false, useBootc); err != nil {
 		return fmt.Errorf("failed to complete stateroot setup: %w", err)
 	}