From 1970ac19e4262f8f10dc20eccd305ffd7bebebaa Mon Sep 17 00:00:00 2001 From: Adphi Date: Thu, 23 Feb 2023 11:37:40 +0100 Subject: [PATCH] run/qemu: remove usb and device flags refactor: move qemu to its own package tests: implements end to end tests for the convert command with the following images: alpine:3.17, ubuntu:20.04, debian:11, centos:8 Signed-off-by: Adphi --- .dockerignore | 1 + Makefile | 5 +- cmd/d2vm/run/qemu.go | 373 ++---------------------- cmd/d2vm/run/util.go | 13 +- docs/content/reference/d2vm_run_qemu.md | 3 - e2e/e2e_test.go | 144 +++++++++ go.mod | 2 +- pkg/docker/check_term_unix.go | 28 ++ pkg/docker/check_term_windows.go | 36 +++ pkg/docker/docker.go | 17 +- pkg/qemu/config.go | 141 +++++++++ pkg/qemu/qemu.go | 363 +++++++++++++++++++++++ 12 files changed, 761 insertions(+), 365 deletions(-) create mode 100644 e2e/e2e_test.go create mode 100644 pkg/docker/check_term_unix.go create mode 100644 pkg/docker/check_term_windows.go create mode 100644 pkg/qemu/config.go create mode 100644 pkg/qemu/qemu.go diff --git a/.dockerignore b/.dockerignore index 471022c..5b63c3b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,3 +7,4 @@ bin dist images examples/build +e2e diff --git a/Makefile b/Makefile index 09ba957..c429ae7 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,10 @@ docker-run: .PHONY: tests tests: @go generate ./... - @go list ./...| xargs go test -exec sudo -count=1 -timeout 20m -v + @go list .| xargs go test -exec sudo -count=1 -timeout 20m -v + +e2e: docker-build .build + @go test -v -exec sudo -count=1 -timeout 60m -ldflags "-X '$(MODULE).Version=$(VERSION)' -X '$(MODULE).BuildDate=$(shell date)'" ./e2e docs-up-to-date: @$(MAKE) cli-docs diff --git a/cmd/d2vm/run/qemu.go b/cmd/d2vm/run/qemu.go index a27d672..126fdc8 100644 --- a/cmd/d2vm/run/qemu.go +++ b/cmd/d2vm/run/qemu.go @@ -1,18 +1,13 @@ package run import ( - "crypto/rand" - "fmt" - "net" "os" - "os/exec" "runtime" - "strconv" - "strings" - "github.com/google/uuid" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + + "go.linka.cloud/d2vm/pkg/qemu" ) const ( @@ -37,8 +32,6 @@ var ( qemuDetached bool networking string publishFlags MultipleFlag - deviceFlags MultipleFlag - usbEnabled bool QemuCmd = &cobra.Command{ Use: "qemu [options] [image-path]", @@ -71,7 +64,6 @@ func init() { // Paths and settings for disks flags.Var(&disks, "disk", "Disk config, may be repeated. [file=]path[,size=1G][,format=qcow2]") - flags.StringVar(&data, "data", "", "String of metadata to pass to VM") // VM configuration flags.StringVar(&accel, "accel", defaultAccel, "Choose acceleration mode. Use 'tcg' to disable it.") @@ -87,362 +79,45 @@ func init() { flags.StringVar(&networking, "networking", qemuNetworkingDefault, "Networking mode. Valid options are 'default', 'user', 'bridge[,name]', tap[,name] and 'none'. 'user' uses QEMUs userspace networking. 'bridge' connects to a preexisting bridge. 'tap' uses a prexisting tap device. 'none' disables networking.`") flags.Var(&publishFlags, "publish", "Publish a vm's port(s) to the host (default [])") - - // USB devices - flags.BoolVar(&usbEnabled, "usb", false, "Enable USB controller") - - flags.Var(&deviceFlags, "device", "Add USB host device(s). Format driver[,prop=value][,...] -- add device, like --device on the qemu command line.") - } func Qemu(cmd *cobra.Command, args []string) { - - // Generate UUID, so that /sys/class/dmi/id/product_uuid is populated - vmUUID := uuid.New() - // These envvars override the corresponding command line - // options. So this must remain after the `flags.Parse` above. - accel = GetStringValue("LINUXKIT_QEMU_ACCEL", accel, "") - path := args[0] if _, err := os.Stat(path); err != nil { log.Fatal(err) } - - for i, d := range disks { - id := "" - if i != 0 { - id = strconv.Itoa(i) - } - if d.Size != 0 && d.Format == "" { - d.Format = "qcow2" - } - if d.Size != 0 && d.Path == "" { - d.Path = "disk" + id + ".img" - } - if d.Path == "" { - log.Fatalf("disk specified with no size or name") - } - disks[i] = d - } - - disks = append(Disks{DiskConfig{Path: path}}, disks...) - - if networking == "" || networking == "default" { - dflt := qemuNetworkingDefault - networking = dflt - } - netMode := strings.SplitN(networking, ",", 2) - - var netdevConfig string - switch netMode[0] { - case qemuNetworkingUser: - netdevConfig = "user,id=t0" - case qemuNetworkingTap: - if len(netMode) != 2 { - log.Fatalf("Not enough arguments for %q networking mode", qemuNetworkingTap) - } - if len(publishFlags) != 0 { - log.Fatalf("Port publishing requires %q networking mode", qemuNetworkingUser) - } - netdevConfig = fmt.Sprintf("tap,id=t0,ifname=%s,script=no,downscript=no", netMode[1]) - case qemuNetworkingBridge: - if len(netMode) != 2 { - log.Fatalf("Not enough arguments for %q networking mode", qemuNetworkingBridge) - } - if len(publishFlags) != 0 { - log.Fatalf("Port publishing requires %q networking mode", qemuNetworkingUser) - } - netdevConfig = fmt.Sprintf("bridge,id=t0,br=%s", netMode[1]) - case qemuNetworkingNone: - if len(publishFlags) != 0 { - log.Fatalf("Port publishing requires %q networking mode", qemuNetworkingUser) - } - netdevConfig = "" - default: - log.Fatalf("Invalid networking mode: %s", netMode[0]) - } - - config := QemuConfig{ - Path: path, - GUI: enableGUI, - Disks: disks, - Arch: arch, - CPUs: cpus, - Memory: mem, - Accel: accel, - Detached: qemuDetached, - QemuBinPath: qemuCmd, - PublishedPorts: publishFlags, - NetdevConfig: netdevConfig, - UUID: vmUUID, - USB: usbEnabled, - Devices: deviceFlags, - } - - config, err := discoverBinaries(config) - if err != nil { - log.Fatal(err) - } - - if err = runQemuLocal(config); err != nil { - log.Fatal(err.Error()) - } -} - -func runQemuLocal(config QemuConfig) error { - var args []string - config, args = buildQemuCmdline(config) - - for _, d := range config.Disks { - // If disk doesn't exist then create one - if _, err := os.Stat(d.Path); err != nil { - if os.IsNotExist(err) { - log.Debugf("Creating new qemu disk [%s] format %s", d.Path, d.Format) - qemuImgCmd := exec.Command(config.QemuImgPath, "create", "-f", d.Format, d.Path, fmt.Sprintf("%dM", d.Size)) - log.Debugf("%v", qemuImgCmd.Args) - if err := qemuImgCmd.Run(); err != nil { - return fmt.Errorf("Error creating disk [%s] format %s: %s", d.Path, d.Format, err.Error()) - } - } else { - return err - } - } else { - log.Infof("Using existing disk [%s] format %s", d.Path, d.Format) - } - } - - // Detached mode is only supported in a container. - if config.Detached == true { - return fmt.Errorf("Detached mode is only supported when running in a container, not locally") - } - - qemuCmd := exec.Command(config.QemuBinPath, args...) - // If verbosity is enabled print out the full path/arguments - log.Debugf("%v", qemuCmd.Args) - - // If we're not using a separate window then link the execution to stdin/out - if config.GUI != true { - qemuCmd.Stdin = os.Stdin - qemuCmd.Stdout = os.Stdout - qemuCmd.Stderr = os.Stderr - } - - return qemuCmd.Run() -} - -func buildQemuCmdline(config QemuConfig) (QemuConfig, []string) { - // Iterate through the flags and build arguments - var qemuArgs []string - qemuArgs = append(qemuArgs, "-smp", fmt.Sprintf("%d", config.CPUs)) - qemuArgs = append(qemuArgs, "-m", fmt.Sprintf("%d", config.Memory)) - qemuArgs = append(qemuArgs, "-uuid", config.UUID.String()) - - // Need to specify the vcpu type when running qemu on arm64 platform, for security reason, - // the vcpu should be "host" instead of other names such as "cortex-a53"... - if config.Arch == "aarch64" { - if runtime.GOARCH == "arm64" { - qemuArgs = append(qemuArgs, "-cpu", "host") - } else { - qemuArgs = append(qemuArgs, "-cpu", "cortex-a57") - } - } - - // goArch is the GOARCH equivalent of config.Arch - var goArch string - switch config.Arch { - case "s390x": - goArch = "s390x" - case "aarch64": - goArch = "arm64" - case "x86_64": - goArch = "amd64" - default: - log.Fatalf("%s is an unsupported architecture.", config.Arch) - } - - if goArch != runtime.GOARCH { - log.Infof("Disable acceleration as %s != %s", config.Arch, runtime.GOARCH) - config.Accel = "" - } - - if config.Accel != "" { - switch config.Arch { - case "s390x": - qemuArgs = append(qemuArgs, "-machine", fmt.Sprintf("s390-ccw-virtio,accel=%s", config.Accel)) - case "aarch64": - gic := "" - // VCPU supports less PA bits (36) than requested by the memory map (40) - highmem := "highmem=off," - if runtime.GOOS == "linux" { - // gic-version=host requires KVM, which implies Linux - gic = "gic_version=host," - highmem = "" - } - qemuArgs = append(qemuArgs, "-machine", fmt.Sprintf("virt,%s%saccel=%s", gic, highmem, config.Accel)) - default: - qemuArgs = append(qemuArgs, "-machine", fmt.Sprintf("q35,accel=%s", config.Accel)) - } - } else { - switch config.Arch { - case "s390x": - qemuArgs = append(qemuArgs, "-machine", "s390-ccw-virtio") - case "aarch64": - qemuArgs = append(qemuArgs, "-machine", "virt") - default: - qemuArgs = append(qemuArgs, "-machine", "q35") - } - } - - // rng-random does not work on macOS - // Temporarily disable it until fixed upstream. - if runtime.GOOS != "darwin" { - rng := "rng-random,id=rng0" - if runtime.GOOS == "linux" { - rng = rng + ",filename=/dev/urandom" - } - if config.Arch == "s390x" { - qemuArgs = append(qemuArgs, "-object", rng, "-device", "virtio-rng-ccw,rng=rng0") - } else { - qemuArgs = append(qemuArgs, "-object", rng, "-device", "virtio-rng-pci,rng=rng0") - } - } - - var lastDisk int - for i, d := range config.Disks { - index := i - if d.Format != "" { - qemuArgs = append(qemuArgs, "-drive", "file="+d.Path+",format="+d.Format+",index="+strconv.Itoa(index)+",media=disk") - } else { - qemuArgs = append(qemuArgs, "-drive", "file="+d.Path+",index="+strconv.Itoa(index)+",media=disk") - } - lastDisk = index - } - - // Ensure CDROMs start from at least hdc - if lastDisk < 2 { - lastDisk = 2 - } - - if config.NetdevConfig == "" { - qemuArgs = append(qemuArgs, "-net", "none") - } else { - mac := generateMAC() - if config.Arch == "s390x" { - qemuArgs = append(qemuArgs, "-device", "virtio-net-ccw,netdev=t0,mac="+mac.String()) - } else { - qemuArgs = append(qemuArgs, "-device", "virtio-net-pci,netdev=t0,mac="+mac.String()) - } - forwardings, err := buildQemuForwardings(config.PublishedPorts) - if err != nil { - log.Error(err) - } - qemuArgs = append(qemuArgs, "-netdev", config.NetdevConfig+forwardings) - } - - if config.GUI != true { - qemuArgs = append(qemuArgs, "-nographic") - } - - if config.USB == true { - qemuArgs = append(qemuArgs, "-usb") - } - for _, d := range config.Devices { - qemuArgs = append(qemuArgs, "-device", d) - } - - return config, qemuArgs -} - -func discoverBinaries(config QemuConfig) (QemuConfig, error) { - if config.QemuImgPath != "" { - return config, nil - } - - qemuBinPath := "qemu-system-" + config.Arch - qemuImgPath := "qemu-img" - - var err error - config.QemuBinPath, err = exec.LookPath(qemuBinPath) - if err != nil { - return config, fmt.Errorf("Unable to find %s within the $PATH", qemuBinPath) - } - - config.QemuImgPath, err = exec.LookPath(qemuImgPath) - if err != nil { - return config, fmt.Errorf("Unable to find %s within the $PATH", qemuImgPath) - } - - return config, nil -} - -func buildQemuForwardings(publishFlags MultipleFlag) (string, error) { - if len(publishFlags) == 0 { - return "", nil - } - var forwardings string + var publishedPorts []PublishedPort for _, publish := range publishFlags { p, err := NewPublishedPort(publish) if err != nil { - return "", err + log.Fatal(err) } - - hostPort := p.Host - guestPort := p.Guest - - forwardings = fmt.Sprintf("%s,hostfwd=%s::%d-:%d", forwardings, p.Protocol, hostPort, guestPort) + publishedPorts = append(publishedPorts, p) } - - return forwardings, nil -} - -func buildDockerForwardings(publishedPorts []string) ([]string, error) { - pmap := []string{} - for _, port := range publishedPorts { - s, err := NewPublishedPort(port) - if err != nil { - return nil, err - } - pmap = append(pmap, "-p", fmt.Sprintf("%d:%d/%s", s.Host, s.Guest, s.Protocol)) + opts := []qemu.Option{ + qemu.WithDisks(disks...), + qemu.WithAccel(accel), + qemu.WithArch(arch), + qemu.WithCPUs(cpus), + qemu.WithMemory(mem), + qemu.WithNetworking(networking), + qemu.WithStdin(os.Stdin), + qemu.WithStdout(os.Stdout), + qemu.WithStderr(os.Stderr), + } + if enableGUI { + opts = append(opts, qemu.WithGUI()) + } + if qemuDetached { + opts = append(opts, qemu.WithDetached()) + } + if err := qemu.Run(cmd.Context(), path, opts...); err != nil { + log.Fatal(err) } - return pmap, nil -} - -// QemuConfig contains the config for Qemu -type QemuConfig struct { - Path string - GUI bool - Disks Disks - FWPath string - Arch string - CPUs uint - Memory uint - Accel string - Detached bool - QemuBinPath string - QemuImgPath string - PublishedPorts []string - NetdevConfig string - UUID uuid.UUID - USB bool - Devices []string } func haveKVM() bool { _, err := os.Stat("/dev/kvm") return !os.IsNotExist(err) } - -func generateMAC() net.HardwareAddr { - mac := make([]byte, 6) - n, err := rand.Read(mac) - if err != nil { - log.WithError(err).Fatal("failed to generate random mac address") - } - if n != 6 { - log.WithError(err).Fatalf("generated %d bytes for random mac address", n) - } - mac[0] &^= 0x01 // Clear multicast bit - mac[0] |= 0x2 // Set locally administered bit - return net.HardwareAddr(mac) -} diff --git a/cmd/d2vm/run/util.go b/cmd/d2vm/run/util.go index 9d84e26..0c71180 100644 --- a/cmd/d2vm/run/util.go +++ b/cmd/d2vm/run/util.go @@ -28,6 +28,8 @@ import ( "time" "golang.org/x/crypto/ssh" + + "go.linka.cloud/d2vm/pkg/qemu" ) //go:embed sparsecat-linux-amd64 @@ -188,15 +190,8 @@ func ConvertMBtoGB(i int) int { return (i + (1024 - i%1024)) / 1024 } -// DiskConfig is the config for a disk -type DiskConfig struct { - Path string - Size int - Format string -} - // Disks is the type for a list of DiskConfig -type Disks []DiskConfig +type Disks []qemu.Disk func (l *Disks) String() string { return fmt.Sprint(*l) @@ -204,7 +199,7 @@ func (l *Disks) String() string { // Set is used by flag to configure value from CLI func (l *Disks) Set(value string) error { - d := DiskConfig{} + d := qemu.Disk{} s := strings.Split(value, ",") for _, p := range s { c := strings.SplitN(p, "=", 2) diff --git a/docs/content/reference/d2vm_run_qemu.md b/docs/content/reference/d2vm_run_qemu.md index f94cb62..0975a2b 100644 --- a/docs/content/reference/d2vm_run_qemu.md +++ b/docs/content/reference/d2vm_run_qemu.md @@ -12,9 +12,7 @@ d2vm run qemu [options] [image-path] [flags] --accel string Choose acceleration mode. Use 'tcg' to disable it. (default "hvf:tcg") --arch string Type of architecture to use, e.g. x86_64, aarch64, s390x (default "x86_64") --cpus uint Number of CPUs (default 1) - --data string String of metadata to pass to VM --detached Set qemu container to run in the background - --device multiple-flag Add USB host device(s). Format driver[,prop=value][,...] -- add device, like --device on the qemu command line. (default A multiple flag is a type of flag that can be repeated any number of times) --disk disk Disk config, may be repeated. [file=]path[,size=1G][,format=qcow2] (default []) --gui Set qemu to use video output instead of stdio -h, --help help for qemu @@ -22,7 +20,6 @@ d2vm run qemu [options] [image-path] [flags] --networking string Networking mode. Valid options are 'default', 'user', 'bridge[,name]', tap[,name] and 'none'. 'user' uses QEMUs userspace networking. 'bridge' connects to a preexisting bridge. 'tap' uses a prexisting tap device. 'none' disables networking.` (default "user") --publish multiple-flag Publish a vm's port(s) to the host (default []) (default A multiple flag is a type of flag that can be repeated any number of times) --qemu string Path to the qemu binary (otherwise look in $PATH) - --usb Enable USB controller ``` ### Options inherited from parent commands diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go new file mode 100644 index 0000000..9b81b93 --- /dev/null +++ b/e2e/e2e_test.go @@ -0,0 +1,144 @@ +// Copyright 2023 Linka Cloud All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bufio" + "bytes" + "context" + "io" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + require2 "github.com/stretchr/testify/require" + + "go.linka.cloud/d2vm" + "go.linka.cloud/d2vm/pkg/docker" + "go.linka.cloud/d2vm/pkg/qemu" +) + +type test struct { + name string + args []string +} + +var images = []string{ + "alpine:3.17", + "ubuntu:20.04", + "debian:11", + "centos:8", +} + +func TestConvert(t *testing.T) { + require := require2.New(t) + tests := []test{ + { + name: "single-partition", + }, + { + name: "split-boot", + args: []string{"--split-boot"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // t.Parallel() + dir := filepath.Join("/tmp", "d2vm-e2e", tt.name) + require.NoError(os.MkdirAll(dir, os.ModePerm)) + + defer os.RemoveAll(dir) + for _, i := range images { + t.Run(i, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // t.Parallel() + require := require2.New(t) + + out := filepath.Join(dir, strings.NewReplacer(":", "-", ".", "-").Replace(i)+".qcow2") + + if _, err := os.Stat(out); err == nil { + require.NoError(os.Remove(out)) + } + + require.NoError(docker.RunD2VM(ctx, d2vm.Image, d2vm.Version, dir, dir, "convert", append([]string{"-p", "root", "-o", "/out/" + filepath.Base(out), "-v", i}, tt.args...)...)) + + inr, inw := io.Pipe() + defer inr.Close() + outr, outw := io.Pipe() + defer outw.Close() + var success atomic.Bool + go func() { + time.AfterFunc(2*time.Minute, cancel) + defer inw.Close() + defer outr.Close() + login := []byte("login:") + password := []byte("Password:") + s := bufio.NewScanner(outr) + s.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) { + if i := bytes.Index(data, login); i >= 0 { + return i + len(login), login, nil + } + if i := bytes.Index(data, password); i >= 0 { + return i + len(password), password, nil + } + if atEOF { + return 0, nil, io.EOF + } + return 0, nil, nil + }) + for s.Scan() { + b := s.Bytes() + if bytes.Contains(b, login) { + t.Logf("vm ready") + t.Logf("sending login") + if _, err := inw.Write([]byte("root\n")); err != nil { + t.Logf("failed to write login: %v", err) + cancel() + } + } + if bytes.Contains(b, password) { + t.Logf("sending password") + if _, err := inw.Write([]byte("root\n")); err != nil { + t.Logf("failed to write password: %v", err) + cancel() + } + time.Sleep(time.Second) + if _, err := inw.Write([]byte("poweroff\n")); err != nil { + t.Logf("failed to write poweroff: %v", err) + cancel() + } + success.Store(true) + return + } + } + if err := s.Err(); err != nil { + t.Logf("failed to scan output: %v", err) + cancel() + } + }() + if err := qemu.Run(ctx, out, qemu.WithStdin(inr), qemu.WithStdout(io.MultiWriter(outw, os.Stdout)), qemu.WithStderr(io.Discard)); err != nil && !success.Load() { + t.Fatalf("failed to run qemu: %v", err) + } + }) + } + }) + } +} diff --git a/go.mod b/go.mod index 9c2d6c2..7cf74ca 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( go.linka.cloud/console v0.0.0-20220910100646-48f9f2b8843b go.uber.org/multierr v1.8.0 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 + golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e ) require ( @@ -60,7 +61,6 @@ require ( go.uber.org/atomic v1.7.0 // indirect golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect - golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect golang.org/x/text v0.3.7 // indirect google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect google.golang.org/grpc v1.43.0 // indirect diff --git a/pkg/docker/check_term_unix.go b/pkg/docker/check_term_unix.go new file mode 100644 index 0000000..fcf396d --- /dev/null +++ b/pkg/docker/check_term_unix.go @@ -0,0 +1,28 @@ +//go:build !windows + +// Copyright 2023 Linka Cloud All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package docker + +import ( + "os" + + "golang.org/x/sys/unix" +) + +func isInteractive() bool { + _, err := unix.IoctlGetWinsize(int(os.Stdout.Fd()), unix.TIOCGWINSZ) + return err == nil +} diff --git a/pkg/docker/check_term_windows.go b/pkg/docker/check_term_windows.go new file mode 100644 index 0000000..a043e9f --- /dev/null +++ b/pkg/docker/check_term_windows.go @@ -0,0 +1,36 @@ +//go:build windows + +// Copyright 2023 Linka Cloud All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package docker + +import ( + "os" + + "golang.org/x/sys/windows" +) + +func isInteractive() bool { + handle := windows.Handle(os.Stdout.Fd()) + var mode uint32 + if err := windows.GetConsoleMode(handle, &mode); err != nil { + return false + } + mode |= windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING + if err := windows.SetConsoleMode(handle, mode); err != nil { + return false + } + return true +} diff --git a/pkg/docker/docker.go b/pkg/docker/docker.go index a27809d..56b815b 100644 --- a/pkg/docker/docker.go +++ b/pkg/docker/docker.go @@ -130,7 +130,14 @@ func RunD2VM(ctx context.Context, image, version, in, out, cmd string, args ...s if version == "" { version = "latest" } - a := []string{ + a := []string{"run", "--rm"} + + interactive := isInteractive() + + if interactive { + a = append(a, "-i", "-t") + } + a = append(a, "--privileged", "-e", // yes... it is kind of a dirty hack @@ -145,6 +152,12 @@ func RunD2VM(ctx context.Context, image, version, in, out, cmd string, args ...s "/d2vm", fmt.Sprintf("%s:%s", image, version), cmd, + ) + c := exec.CommandContext(ctx, "docker", append(a, args...)...) + if interactive { + c.Stdin = os.Stdin } - return RunInteractiveAndRemove(ctx, append(a, args...)...) + c.Stdout = os.Stdout + c.Stderr = os.Stderr + return c.Run() } diff --git a/pkg/qemu/config.go b/pkg/qemu/config.go new file mode 100644 index 0000000..bfa73d4 --- /dev/null +++ b/pkg/qemu/config.go @@ -0,0 +1,141 @@ +// Copyright 2023 Linka Cloud All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package qemu + +import ( + "io" + + "github.com/google/uuid" +) + +type Option func(c *config) + +type Disk struct { + Path string + Size int + Format string +} + +type PublishedPort struct { + Guest uint16 + Host uint16 + Protocol string +} + +// config contains the config for Qemu +type config struct { + path string + uuid uuid.UUID + gui bool + disks []Disk + networking string + arch string + cpus uint + memory uint + accel string + detached bool + qemuBinPath string + qemuImgPath string + publishedPorts []PublishedPort + netdevConfig string + + stdin io.Reader + stdout io.Writer + stderr io.Writer +} + +func WithGUI() Option { + return func(c *config) { + c.gui = true + } +} + +func WithDisks(disks ...Disk) Option { + return func(c *config) { + c.disks = disks + } +} + +func WithNetworking(networking string) Option { + return func(c *config) { + c.networking = networking + } +} + +func WithArch(arch string) Option { + return func(c *config) { + c.arch = arch + } +} + +func WithCPUs(cpus uint) Option { + return func(c *config) { + c.cpus = cpus + } +} + +func WithMemory(memory uint) Option { + return func(c *config) { + c.memory = memory + } +} + +func WithAccel(accel string) Option { + return func(c *config) { + c.accel = accel + } +} + +func WithDetached() Option { + return func(c *config) { + c.detached = true + } +} + +func WithQemuBinPath(path string) Option { + return func(c *config) { + c.qemuBinPath = path + } +} + +func WithQemuImgPath(path string) Option { + return func(c *config) { + c.qemuImgPath = path + } +} + +func WithPublishedPorts(ports ...PublishedPort) Option { + return func(c *config) { + c.publishedPorts = ports + } +} + +func WithStdin(r io.Reader) Option { + return func(c *config) { + c.stdin = r + } +} + +func WithStdout(w io.Writer) Option { + return func(c *config) { + c.stdout = w + } +} + +func WithStderr(w io.Writer) Option { + return func(c *config) { + c.stderr = w + } +} diff --git a/pkg/qemu/qemu.go b/pkg/qemu/qemu.go new file mode 100644 index 0000000..289e2a7 --- /dev/null +++ b/pkg/qemu/qemu.go @@ -0,0 +1,363 @@ +package qemu + +import ( + "context" + "crypto/rand" + "fmt" + "net" + "os" + "os/exec" + "runtime" + "strconv" + "strings" + + "github.com/google/uuid" + log "github.com/sirupsen/logrus" +) + +const ( + NetworkingNone = "none" + NetworkingUser = "user" + NetworkingTap = "tap" + NetworkingBridge = "bridge" + NetworkingDefault = NetworkingUser +) + +var ( + defaultArch string + defaultAccel string +) + +func init() { + switch runtime.GOARCH { + case "arm64": + defaultArch = "aarch64" + case "amd64": + defaultArch = "x86_64" + case "s390x": + defaultArch = "s390x" + } + switch { + case runtime.GOARCH == "s390x": + defaultAccel = "kvm" + case haveKVM(): + defaultAccel = "kvm:tcg" + case runtime.GOOS == "darwin": + defaultAccel = "hvf:tcg" + } +} + +func Run(ctx context.Context, path string, opts ...Option) error { + config := &config{} + + for _, o := range opts { + o(config) + } + + config.path = path + + // Generate UUID, so that /sys/class/dmi/id/product_uuid is populated + config.uuid = uuid.New() + // These envvars override the corresponding command line + // options. So this must remain after the `flags.Parse` above. + // accel = GetStringValue("LINUXKIT_QEMU_ACCEL", accel, "") + + if config.arch == "" { + config.arch = defaultArch + } + + if config.accel == "" { + config.accel = defaultAccel + } + + if _, err := os.Stat(config.path); err != nil { + return err + } + + if config.cpus == 0 { + config.cpus = 1 + } + + if config.memory == 0 { + config.memory = 1024 + } + + for i, d := range config.disks { + id := "" + if i != 0 { + id = strconv.Itoa(i) + } + if d.Size != 0 && d.Format == "" { + d.Format = "qcow2" + } + if d.Size != 0 && d.Path == "" { + d.Path = "disk" + id + ".img" + } + if d.Path == "" { + return fmt.Errorf("disk specified with no size or name") + } + config.disks[i] = d + } + + config.disks = append([]Disk{{Path: config.path}}, config.disks...) + + if config.networking == "" || config.networking == "default" { + dflt := NetworkingDefault + config.networking = dflt + } + netMode := strings.SplitN(config.networking, ",", 2) + + switch netMode[0] { + case NetworkingUser: + config.netdevConfig = "user,id=t0" + case NetworkingTap: + if len(netMode) != 2 { + return fmt.Errorf("Not enough arguments for %q networking mode", NetworkingTap) + } + if len(config.publishedPorts) != 0 { + return fmt.Errorf("Port publishing requires %q networking mode", NetworkingUser) + } + config.netdevConfig = fmt.Sprintf("tap,id=t0,ifname=%s,script=no,downscript=no", netMode[1]) + case NetworkingBridge: + if len(netMode) != 2 { + return fmt.Errorf("Not enough arguments for %q networking mode", NetworkingBridge) + } + if len(config.publishedPorts) != 0 { + return fmt.Errorf("Port publishing requires %q networking mode", NetworkingUser) + } + config.netdevConfig = fmt.Sprintf("bridge,id=t0,br=%s", netMode[1]) + case NetworkingNone: + if len(config.publishedPorts) != 0 { + return fmt.Errorf("Port publishing requires %q networking mode", NetworkingUser) + } + config.netdevConfig = "" + default: + return fmt.Errorf("Invalid networking mode: %s", netMode[0]) + } + + if err := config.discoverBinaries(); err != nil { + log.Fatal(err) + } + + return config.runQemuLocal(ctx) +} + +func (c *config) runQemuLocal(ctx context.Context) (err error) { + var args []string + args, err = c.buildQemuCmdline() + if err != nil { + return err + } + + for _, d := range c.disks { + // If disk doesn't exist then create one + if _, err := os.Stat(d.Path); err != nil { + if os.IsNotExist(err) { + log.Debugf("Creating new qemu disk [%s] format %s", d.Path, d.Format) + qemuImgCmd := exec.Command(c.qemuImgPath, "create", "-f", d.Format, d.Path, fmt.Sprintf("%dM", d.Size)) + log.Debugf("%v", qemuImgCmd.Args) + if err := qemuImgCmd.Run(); err != nil { + return fmt.Errorf("Error creating disk [%s] format %s: %s", d.Path, d.Format, err.Error()) + } + } else { + return err + } + } else { + log.Infof("Using existing disk [%s] format %s", d.Path, d.Format) + } + } + + // Detached mode is only supported in a container. + if c.detached == true { + return fmt.Errorf("Detached mode is only supported when running in a container, not locally") + } + + qemuCmd := exec.CommandContext(ctx, c.qemuBinPath, args...) + // If verbosity is enabled print out the full path/arguments + log.Debugf("%v", qemuCmd.Args) + + // If we're not using a separate window then link the execution to stdin/out + if c.gui == true { + qemuCmd.Stdin = nil + qemuCmd.Stdout = nil + qemuCmd.Stderr = nil + } else { + qemuCmd.Stdin = c.stdin + qemuCmd.Stdout = c.stdout + qemuCmd.Stderr = c.stderr + } + + return qemuCmd.Run() +} + +func (c *config) buildQemuCmdline() ([]string, error) { + // Iterate through the flags and build arguments + var qemuArgs []string + qemuArgs = append(qemuArgs, "-smp", fmt.Sprintf("%d", c.cpus)) + qemuArgs = append(qemuArgs, "-m", fmt.Sprintf("%d", c.memory)) + qemuArgs = append(qemuArgs, "-uuid", c.uuid.String()) + + // Need to specify the vcpu type when running qemu on arm64 platform, for security reason, + // the vcpu should be "host" instead of other names such as "cortex-a53"... + if c.arch == "aarch64" { + if runtime.GOARCH == "arm64" { + qemuArgs = append(qemuArgs, "-cpu", "host") + } else { + qemuArgs = append(qemuArgs, "-cpu", "cortex-a57") + } + } + + // goArch is the GOARCH equivalent of config.Arch + var goArch string + switch c.arch { + case "s390x": + goArch = "s390x" + case "aarch64": + goArch = "arm64" + case "x86_64": + goArch = "amd64" + default: + return nil, fmt.Errorf("%s is an unsupported architecture.", c.arch) + } + + if goArch != runtime.GOARCH { + log.Infof("Disable acceleration as %s != %s", c.arch, runtime.GOARCH) + c.accel = "" + } + + if c.accel != "" { + switch c.arch { + case "s390x": + qemuArgs = append(qemuArgs, "-machine", fmt.Sprintf("s390-ccw-virtio,accel=%s", c.accel)) + case "aarch64": + gic := "" + // VCPU supports less PA bits (36) than requested by the memory map (40) + highmem := "highmem=off," + if runtime.GOOS == "linux" { + // gic-version=host requires KVM, which implies Linux + gic = "gic_version=host," + highmem = "" + } + qemuArgs = append(qemuArgs, "-machine", fmt.Sprintf("virt,%s%saccel=%s", gic, highmem, c.accel)) + default: + qemuArgs = append(qemuArgs, "-machine", fmt.Sprintf("q35,accel=%s", c.accel)) + } + } else { + switch c.arch { + case "s390x": + qemuArgs = append(qemuArgs, "-machine", "s390-ccw-virtio") + case "aarch64": + qemuArgs = append(qemuArgs, "-machine", "virt") + default: + qemuArgs = append(qemuArgs, "-machine", "q35") + } + } + + // rng-random does not work on macOS + // Temporarily disable it until fixed upstream. + if runtime.GOOS != "darwin" { + rng := "rng-random,id=rng0" + if runtime.GOOS == "linux" { + rng = rng + ",filename=/dev/urandom" + } + if c.arch == "s390x" { + qemuArgs = append(qemuArgs, "-object", rng, "-device", "virtio-rng-ccw,rng=rng0") + } else { + qemuArgs = append(qemuArgs, "-object", rng, "-device", "virtio-rng-pci,rng=rng0") + } + } + + var lastDisk int + for i, d := range c.disks { + index := i + if d.Format != "" { + qemuArgs = append(qemuArgs, "-drive", "file="+d.Path+",format="+d.Format+",index="+strconv.Itoa(index)+",media=disk") + } else { + qemuArgs = append(qemuArgs, "-drive", "file="+d.Path+",index="+strconv.Itoa(index)+",media=disk") + } + lastDisk = index + } + + // Ensure CDROMs start from at least hdc + if lastDisk < 2 { + lastDisk = 2 + } + + if c.netdevConfig != "" { + mac := generateMAC() + if c.arch == "s390x" { + qemuArgs = append(qemuArgs, "-device", "virtio-net-ccw,netdev=t0,mac="+mac.String()) + } else { + qemuArgs = append(qemuArgs, "-device", "virtio-net-pci,netdev=t0,mac="+mac.String()) + } + forwardings, err := buildQemuForwardings(c.publishedPorts) + if err != nil { + log.Error(err) + } + qemuArgs = append(qemuArgs, "-netdev", c.netdevConfig+forwardings) + } else { + qemuArgs = append(qemuArgs, "-net", "none") + } + + if c.gui != true { + qemuArgs = append(qemuArgs, "-nographic") + } + + return qemuArgs, nil +} + +func (c *config) discoverBinaries() error { + if c.qemuImgPath != "" { + return nil + } + + qemuBinPath := "qemu-system-" + c.arch + qemuImgPath := "qemu-img" + + var err error + c.qemuBinPath, err = exec.LookPath(qemuBinPath) + if err != nil { + return fmt.Errorf("Unable to find %s within the $PATH", qemuBinPath) + } + + c.qemuImgPath, err = exec.LookPath(qemuImgPath) + if err != nil { + return fmt.Errorf("Unable to find %s within the $PATH", qemuImgPath) + } + + return nil +} + +func buildQemuForwardings(publishedPorts []PublishedPort) (string, error) { + if len(publishedPorts) == 0 { + return "", nil + } + var forwardings string + for _, p := range publishedPorts { + hostPort := p.Host + guestPort := p.Guest + + forwardings = fmt.Sprintf("%s,hostfwd=%s::%d-:%d", forwardings, p.Protocol, hostPort, guestPort) + } + + return forwardings, nil +} + +func haveKVM() bool { + _, err := os.Stat("/dev/kvm") + return !os.IsNotExist(err) +} + +func generateMAC() net.HardwareAddr { + mac := make([]byte, 6) + n, err := rand.Read(mac) + if err != nil { + log.WithError(err).Fatal("failed to generate random mac address") + } + if n != 6 { + log.WithError(err).Fatalf("generated %d bytes for random mac address", n) + } + mac[0] &^= 0x01 // Clear multicast bit + mac[0] |= 0x2 // Set locally administered bit + return mac +}