cmd/enter, cmd/run: Add --arch flag for cross-arch containers support

src/cmd/enter.go

@@ -27,6 +27,7 @@ import (
 
 var (
 	enterFlags struct {
+		arch      string
 		container string
 		distro    string
 		release   string
@@ -43,6 +44,12 @@ var enterCmd = &cobra.Command{
 func init() {
 	flags := enterCmd.Flags()
 
+	flags.StringVarP(&enterFlags.arch,
+		"arch",
+		"a",
+		"",
+		"Enter a Toolbx container for a different architecture than the host")
+
 	flags.StringVarP(&enterFlags.container,
 		"container",
 		"c",
@@ -104,11 +111,17 @@ func enter(cmd *cobra.Command, args []string) error {
 		defaultContainer = false
 	}
 
+	archID, err := resolveArchitectureID(enterFlags.arch, "")
+	if err != nil {
+		return err
+	}
+
 	container, image, release, err := resolveContainerAndImageNames(container,
 		containerArg,
 		enterFlags.distro,
 		"",
-		enterFlags.release)
+		enterFlags.release,
+		archID)
 
 	if err != nil {
 		return err

src/cmd/initContainer.go

@@ -28,6 +28,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/containers/toolbox/pkg/architecture"
 	"github.com/containers/toolbox/pkg/shell"
 	"github.com/containers/toolbox/pkg/utils"
 	"github.com/fsnotify/fsnotify"
@@ -41,6 +42,8 @@ import (
 
 var (
 	initContainerFlags struct {
+		archID      int
+		archInterp  string
 		gid         int
 		home        string
 		homeLink    bool
@@ -85,6 +88,16 @@ var initContainerCmd = &cobra.Command{
 func init() {
 	flags := initContainerCmd.Flags()
 
+	flags.IntVar(&initContainerFlags.archID,
+		"arch",
+		architecture.HostArchID,
+		"Specify the Toolbx container's architecture ID.")
+
+	flags.StringVar(&initContainerFlags.archInterp,
+		"arch-emulator-path",
+		"",
+		"Register an emulator using binfmt_misc with PATH as the interpreter for a non-native architecture container.")
+
 	flags.IntVar(&initContainerFlags.gid,
 		"gid",
 		0,
@@ -257,6 +270,31 @@ func initContainer(cmd *cobra.Command, args []string) error {
 		}
 	}
 
+	if !architecture.HasContainerNativeArch(initContainerFlags.archID) {
+		archName := architecture.GetArchNameOCI(initContainerFlags.archID)
+		interpreterPath := "/run/host" + initContainerFlags.archInterp
+
+		if err := validateCrossArchEmulation(initContainerFlags.archID); err != nil {
+			return err
+		}
+
+		logrus.Debugf("Mounting binfmt_misc file system in container for architecture %s", archName)
+		if err := architecture.MountBinfmtMisc(); err != nil {
+			return err
+		}
+
+		resolvedInterpreterPath, err := architecture.IsArchSupportedOnInitialization(initContainerFlags.archID, interpreterPath)
+		if err != nil {
+			errNotSupported := fmt.Errorf("Cannot run container for architecture %s:\n%s", archName, err)
+			return errNotSupported
+		}
+
+		logrus.Debugf("Registering QEMU emulator for architecture %s in binfmt_misc", archName)
+		if err := architecture.RegisterBinfmtMisc(initContainerFlags.archID, resolvedInterpreterPath); err != nil {
+			return err
+		}
+	}
+
 	for _, mount := range initContainerMounts {
 		if err := mountBind(mount.containerPath, mount.source, mount.flags); err != nil {
 			return err
@@ -1223,6 +1261,28 @@ func updateTimeZoneFromLocalTime() error {
 	return nil
 }
 
+func validateCrossArchEmulation(archID int) error {
+	archName := architecture.GetArchNameOCI(archID)
+	logrus.Debugf("Testing QEMU emulation for architecture %s", archName)
+
+	_, err := shell.RunWithExitCode2("true", nil, nil, nil)
+
+	if err != nil {
+		if errors.Is(err, syscall.ENOEXEC) {
+			return fmt.Errorf(
+				"QEMU emulation for architecture %s is not working\n"+
+					"Please verify that:\n"+
+					"  1. QEMU user-mode emulation is installed on the host system: qemu-user-static package\n"+
+					"  2. binfmt_misc is properly configured on the host system",
+				archName)
+		}
+		return fmt.Errorf("failed to test QEMU emulation for architecture %s: %w", archName, err)
+	}
+
+	logrus.Debugf("Test of QEMU emulation for architecture %s has succeeded", archName)
+	return nil
+}
+
 func writeTimeZone(timeZone string) error {
 	const etcTimeZone = "/etc/timezone"
 

src/cmd/rootMigrationPath.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/containers/toolbox/pkg/architecture"
 	"github.com/containers/toolbox/pkg/utils"
 	"github.com/spf13/cobra"
 )
@@ -56,7 +57,7 @@ func rootRunImpl(cmd *cobra.Command, args []string) error {
 		return &exitError{exitCode, err}
 	}
 
-	container, image, release, err := resolveContainerAndImageNames("", "", "", "", "")
+	container, image, release, err := resolveContainerAndImageNames("", "", "", "", "", architecture.HostArchID)
 	if err != nil {
 		return err
 	}

src/cmd/run.go

@@ -30,6 +30,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/containers/toolbox/pkg/architecture"
 	"github.com/containers/toolbox/pkg/nvidia"
 	"github.com/containers/toolbox/pkg/podman"
 	"github.com/containers/toolbox/pkg/shell"
@@ -52,6 +53,7 @@ type entryPointError struct {
 
 var (
 	runFlags struct {
+		arch        string
 		container   string
 		distro      string
 		preserveFDs uint
@@ -73,6 +75,12 @@ func init() {
 	flags := runCmd.Flags()
 	flags.SetInterspersed(false)
 
+	flags.StringVarP(&runFlags.arch,
+		"arch",
+		"a",
+		"",
+		"Run command inside a Toolbx container for a different architecture than the host")
+
 	flags.StringVarP(&runFlags.container,
 		"container",
 		"c",
@@ -141,11 +149,17 @@ func run(cmd *cobra.Command, args []string) error {
 
 	command := args
 
+	archID, err := resolveArchitectureID(runFlags.arch, "")
+	if err != nil {
+		return err
+	}
+
 	container, image, release, err := resolveContainerAndImageNames(runFlags.container,
 		"--container",
 		runFlags.distro,
 		"",
-		runFlags.release)
+		runFlags.release,
+		archID)
 
 	if err != nil {
 		return err
@@ -225,7 +239,7 @@ func runCommand(container string,
 				return nil
 			}
 
-			if err := createContainer(container, image, release, "", false); err != nil {
+			if err := createContainer(container, image, release, "", architecture.GetArchConfigDefault(), false); err != nil {
 				return err
 			}
 		} else if containersCount == 1 && defaultContainer {
@@ -962,8 +976,15 @@ func showEntryPointLog(line string) error {
 	}
 
 	if !logLevelFound {
-		errMsg, _ := strings.CutPrefix(line, "Error: ")
-		return &entryPointError{errMsg}
+		// Messages sent to stderr with a 'Warning:' prefix in the entry point
+		// are propagated to stderr on the host
+		if strings.HasPrefix(line, "Warning:") {
+			fmt.Fprintf(os.Stderr, "%s\n", line)
+			return nil
+		} else {
+			errMsg, _ := strings.CutPrefix(line, "Error: ")
+			return &entryPointError{errMsg}
+		}
 	}
 
 	logger := logrus.StandardLogger()

src/cmd/utils.go

@@ -31,6 +31,7 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/containers/toolbox/pkg/architecture"
 	"github.com/containers/toolbox/pkg/shell"
 	"github.com/containers/toolbox/pkg/utils"
 	"github.com/sirupsen/logrus"
@@ -261,6 +262,18 @@ func discardInputAsync(ctx context.Context) (<-chan int, <-chan error) {
 	return retValCh, errCh
 }
 
+func createErrorConflictingArchSpecs(archCLI, archTag int) error {
+	var builder strings.Builder
+	fmt.Fprintf(&builder, "conflicting architecture specifications\n")
+	fmt.Fprintf(&builder, "--arch=%s but image tag specifies %s\n",
+		architecture.GetArchNameOCI(archCLI),
+		architecture.GetArchNameOCI(archTag))
+	fmt.Fprintf(&builder, "Run '%s --help' for usage.", executableBase)
+
+	errMsg := builder.String()
+	return errors.New(errMsg)
+}
+
 func createErrorContainerNotFound(container string) error {
 	var builder strings.Builder
 	fmt.Fprintf(&builder, "container %s not found\n", container)
@@ -281,6 +294,15 @@ func createErrorDistroWithoutRelease(distro string) error {
 	return errors.New(errMsg)
 }
 
+func createErrorImagePull(image, domain string) error {
+	var builder strings.Builder
+	fmt.Fprintf(&builder, "failed to pull image %s\n", image)
+	fmt.Fprintf(&builder, "If it was a private image, log in with: podman login %s\n", domain)
+	fmt.Fprintf(&builder, "Use '%s --verbose ...' for further details.", executableBase)
+
+	return errors.New(builder.String())
+}
+
 func createErrorInvalidContainer(containerArg string) error {
 	var builder strings.Builder
 	fmt.Fprintf(&builder, "invalid argument for '%s'\n", containerArg)
@@ -345,6 +367,14 @@ func createErrorProfileDNotFound() error {
 	return errors.New(errMsg)
 }
 
+func createErrorSkopeoNotFound(imageFull string, archID int) error {
+	archName := architecture.GetArchNameOCI(archID)
+	return fmt.Errorf(
+		"Cannot inspect image %s for architecture %s: skopeo is not installed.\n"+
+			"Skopeo is required for creating non-native architecture containers.",
+		imageFull, archName)
+}
+
 func createErrorSudoersDNotFound() error {
 	const sudoersD = "/etc/sudoers.d"
 
@@ -483,9 +513,40 @@ func poll(pollFn pollFunc, eventFD int32, fds ...int32) error {
 	}
 }
 
-func resolveContainerAndImageNames(container, containerArg, distroCLI, imageCLI, releaseCLI string) (
+func resolveArchitectureID(arch string, image string) (int, error) {
+	archID := architecture.NotSpecified
+	if arch != "" {
+		archIDParsed, err := architecture.ParseArgArchValue(arch)
+		if err != nil {
+			return architecture.NotSpecified, err
+		}
+		archID = archIDParsed
+	}
+
+	if image != "" && utils.IsSupportedDistroImage(image) {
+		archIDFromTag := architecture.ImageReferenceGetArchFromTag(image)
+
+		if archID == architecture.NotSpecified && archIDFromTag != architecture.NotSpecified {
+			logrus.Debug("non-native architecture was detected in the image tag -> cross-architecture approach is going to be used")
+
+			archID = archIDFromTag
+		} else if archID != archIDFromTag && archIDFromTag != architecture.NotSpecified {
+			return architecture.NotSpecified, createErrorConflictingArchSpecs(archID, archIDFromTag)
+		}
+	}
+
+	if archID == architecture.NotSpecified {
+		archID = architecture.HostArchID
+	}
+
+	return archID, nil
+}
+
+func resolveContainerAndImageNames(container, containerArg, distroCLI, imageCLI, releaseCLI string, archID int) (
 	string, string, string, error,
 ) {
+	containerWasEmpty := container == ""
+
 	container, image, release, err := utils.ResolveContainerAndImageNames(container,
 		distroCLI,
 		imageCLI,
@@ -540,9 +601,35 @@ func resolveContainerAndImageNames(container, containerArg, distroCLI, imageCLI,
 		}
 	}
 
+	if containerWasEmpty && !architecture.HasContainerNativeArch(archID) {
+		archIDFromTag := architecture.ImageReferenceGetArchFromTag(image)
+
+		if archIDFromTag == architecture.NotSpecified {
+			archName := architecture.GetArchNameOCI(archID)
+			if archName != "" {
+				container = container + "-" + archName
+			}
+		}
+	}
+
 	return container, image, release, nil
 }
 
+func resolveImageNameWithArchitectureSuffix(image string, archID int) string {
+	if architecture.HasContainerNativeArch(archID) {
+		return image
+	}
+
+	archIDFromTag := architecture.ImageReferenceGetArchFromTag(image)
+	isSupportedDistroImage := utils.IsSupportedDistroImage(image)
+
+	if isSupportedDistroImage && archIDFromTag == architecture.NotSpecified {
+		return image + "-" + architecture.GetArchNameOCI(archID)
+	}
+
+	return image
+}
+
 // showManual tries to open the specified manual page using man on stdout
 func showManual(manual string) error {
 	manBinary, err := exec.LookPath("man")