linka-cloud
/
d2vm
mirror of https://github.com/linka-cloud/d2vm.git
2
0
Fork 0
Code Issues Releases Activity

run/qemu: remove usb and device flags 

refactor: move qemu to its own package
tests: implements end to end tests for the convert command with the following images: alpine:3.17, ubuntu:20.04, debian:11, centos:8

Signed-off-by: Adphi <philippe.adrien.nousse@gmail.com>
This commit is contained in:
Adphi 2023-02-23 11:37:40 +01:00 committed by Adphi
parent 532ee3f1a3
commit 1970ac19e4
12 changed files with 761 additions and 365 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.dockerignore
View File

@ -7,3 +7,4 @@ bin
dist
images
examples/build
e2e

5
Makefile
View File

@ -64,7 +64,10 @@ docker-run:
.PHONY: tests
tests:
@go generate ./...
@go list ./...| xargs go test -exec sudo -count=1 -timeout 20m -v
@go list .| xargs go test -exec sudo -count=1 -timeout 20m -v
e2e: docker-build .build
@go test -v -exec sudo -count=1 -timeout 60m -ldflags "-X '$(MODULE).Version=$(VERSION)' -X '$(MODULE).BuildDate=$(shell date)'" ./e2e
docs-up-to-date:
@$(MAKE) cli-docs

373
cmd/d2vm/run/qemu.go
View File

@ -1,18 +1,13 @@
package run
import (
"crypto/rand"
"fmt"
"net"
"os"
"os/exec"
"runtime"
"strconv"
"strings"
"github.com/google/uuid"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"go.linka.cloud/d2vm/pkg/qemu"
)
const (
@ -37,8 +32,6 @@ var (
qemuDetached bool
networking string
publishFlags MultipleFlag
deviceFlags MultipleFlag
usbEnabled bool
QemuCmd = &cobra.Command{
Use: "qemu [options] [image-path]",
@ -71,7 +64,6 @@ func init() {
// Paths and settings for disks
flags.Var(&disks, "disk", "Disk config, may be repeated. [file=]path[,size=1G][,format=qcow2]")
flags.StringVar(&data, "data", "", "String of metadata to pass to VM")
// VM configuration
flags.StringVar(&accel, "accel", defaultAccel, "Choose acceleration mode. Use 'tcg' to disable it.")
@ -87,362 +79,45 @@ func init() {
flags.StringVar(&networking, "networking", qemuNetworkingDefault, "Networking mode. Valid options are 'default', 'user', 'bridge[,name]', tap[,name] and 'none'. 'user' uses QEMUs userspace networking. 'bridge' connects to a preexisting bridge. 'tap' uses a prexisting tap device. 'none' disables networking.`")
flags.Var(&publishFlags, "publish", "Publish a vm's port(s) to the host (default [])")
// USB devices
flags.BoolVar(&usbEnabled, "usb", false, "Enable USB controller")
flags.Var(&deviceFlags, "device", "Add USB host device(s). Format driver[,prop=value][,...] -- add device, like --device on the qemu command line.")
}
func Qemu(cmd *cobra.Command, args []string) {
// Generate UUID, so that /sys/class/dmi/id/product_uuid is populated
vmUUID := uuid.New()
// These envvars override the corresponding command line
// options. So this must remain after the `flags.Parse` above.
accel = GetStringValue("LINUXKIT_QEMU_ACCEL", accel, "")
path := args[0]
if _, err := os.Stat(path); err != nil {
log.Fatal(err)
}
for i, d := range disks {
id := ""
if i != 0 {
id = strconv.Itoa(i)
}
if d.Size != 0 && d.Format == "" {
d.Format = "qcow2"
}
if d.Size != 0 && d.Path == "" {
d.Path = "disk" + id + ".img"
}
if d.Path == "" {
log.Fatalf("disk specified with no size or name")
}
disks[i] = d
}
disks = append(Disks{DiskConfig{Path: path}}, disks...)
if networking == "" || networking == "default" {
dflt := qemuNetworkingDefault
networking = dflt
}
netMode := strings.SplitN(networking, ",", 2)
var netdevConfig string
switch netMode[0] {
case qemuNetworkingUser:
netdevConfig = "user,id=t0"
case qemuNetworkingTap:
if len(netMode) != 2 {
log.Fatalf("Not enough arguments for %q networking mode", qemuNetworkingTap)
}
if len(publishFlags) != 0 {
log.Fatalf("Port publishing requires %q networking mode", qemuNetworkingUser)
}
netdevConfig = fmt.Sprintf("tap,id=t0,ifname=%s,script=no,downscript=no", netMode[1])
case qemuNetworkingBridge:
if len(netMode) != 2 {
log.Fatalf("Not enough arguments for %q networking mode", qemuNetworkingBridge)
}
if len(publishFlags) != 0 {
log.Fatalf("Port publishing requires %q networking mode", qemuNetworkingUser)
}
netdevConfig = fmt.Sprintf("bridge,id=t0,br=%s", netMode[1])
case qemuNetworkingNone:
if len(publishFlags) != 0 {
log.Fatalf("Port publishing requires %q networking mode", qemuNetworkingUser)
}
netdevConfig = ""
default:
log.Fatalf("Invalid networking mode: %s", netMode[0])
}
config := QemuConfig{
Path: path,
GUI: enableGUI,
Disks: disks,
Arch: arch,
CPUs: cpus,
Memory: mem,
Accel: accel,
Detached: qemuDetached,
QemuBinPath: qemuCmd,
PublishedPorts: publishFlags,
NetdevConfig: netdevConfig,
UUID: vmUUID,
USB: usbEnabled,
Devices: deviceFlags,
}
config, err := discoverBinaries(config)
if err != nil {
log.Fatal(err)
}
if err = runQemuLocal(config); err != nil {
log.Fatal(err.Error())
}
}
func runQemuLocal(config QemuConfig) error {
var args []string
config, args = buildQemuCmdline(config)
for _, d := range config.Disks {
// If disk doesn't exist then create one
if _, err := os.Stat(d.Path); err != nil {
if os.IsNotExist(err) {
log.Debugf("Creating new qemu disk [%s] format %s", d.Path, d.Format)
qemuImgCmd := exec.Command(config.QemuImgPath, "create", "-f", d.Format, d.Path, fmt.Sprintf("%dM", d.Size))
log.Debugf("%v", qemuImgCmd.Args)
if err := qemuImgCmd.Run(); err != nil {
return fmt.Errorf("Error creating disk [%s] format %s: %s", d.Path, d.Format, err.Error())
}
} else {
return err
}
} else {
log.Infof("Using existing disk [%s] format %s", d.Path, d.Format)
}
}
// Detached mode is only supported in a container.
if config.Detached == true {
return fmt.Errorf("Detached mode is only supported when running in a container, not locally")
}
qemuCmd := exec.Command(config.QemuBinPath, args...)
// If verbosity is enabled print out the full path/arguments
log.Debugf("%v", qemuCmd.Args)
// If we're not using a separate window then link the execution to stdin/out
if config.GUI != true {
qemuCmd.Stdin = os.Stdin
qemuCmd.Stdout = os.Stdout
qemuCmd.Stderr = os.Stderr
}
return qemuCmd.Run()
}
func buildQemuCmdline(config QemuConfig) (QemuConfig, []string) {
// Iterate through the flags and build arguments
var qemuArgs []string
qemuArgs = append(qemuArgs, "-smp", fmt.Sprintf("%d", config.CPUs))
qemuArgs = append(qemuArgs, "-m", fmt.Sprintf("%d", config.Memory))
qemuArgs = append(qemuArgs, "-uuid", config.UUID.String())
// Need to specify the vcpu type when running qemu on arm64 platform, for security reason,
// the vcpu should be "host" instead of other names such as "cortex-a53"...
if config.Arch == "aarch64" {
if runtime.GOARCH == "arm64" {
qemuArgs = append(qemuArgs, "-cpu", "host")
} else {
qemuArgs = append(qemuArgs, "-cpu", "cortex-a57")
}
}
// goArch is the GOARCH equivalent of config.Arch
var goArch string
switch config.Arch {
case "s390x":
goArch = "s390x"
case "aarch64":
goArch = "arm64"
case "x86_64":
goArch = "amd64"
default:
log.Fatalf("%s is an unsupported architecture.", config.Arch)
}
if goArch != runtime.GOARCH {
log.Infof("Disable acceleration as %s != %s", config.Arch, runtime.GOARCH)
config.Accel = ""
}
if config.Accel != "" {
switch config.Arch {
case "s390x":
qemuArgs = append(qemuArgs, "-machine", fmt.Sprintf("s390-ccw-virtio,accel=%s", config.Accel))
case "aarch64":
gic := ""
// VCPU supports less PA bits (36) than requested by the memory map (40)
highmem := "highmem=off,"
if runtime.GOOS == "linux" {
// gic-version=host requires KVM, which implies Linux
gic = "gic_version=host,"
highmem = ""
}
qemuArgs = append(qemuArgs, "-machine", fmt.Sprintf("virt,%s%saccel=%s", gic, highmem, config.Accel))
default:
qemuArgs = append(qemuArgs, "-machine", fmt.Sprintf("q35,accel=%s", config.Accel))
}
} else {
switch config.Arch {
case "s390x":
qemuArgs = append(qemuArgs, "-machine", "s390-ccw-virtio")
case "aarch64":
qemuArgs = append(qemuArgs, "-machine", "virt")
default:
qemuArgs = append(qemuArgs, "-machine", "q35")
}
}
// rng-random does not work on macOS
// Temporarily disable it until fixed upstream.
if runtime.GOOS != "darwin" {
rng := "rng-random,id=rng0"
if runtime.GOOS == "linux" {
rng = rng + ",filename=/dev/urandom"
}
if config.Arch == "s390x" {
qemuArgs = append(qemuArgs, "-object", rng, "-device", "virtio-rng-ccw,rng=rng0")
} else {
qemuArgs = append(qemuArgs, "-object", rng, "-device", "virtio-rng-pci,rng=rng0")
}
}
var lastDisk int
for i, d := range config.Disks {
index := i
if d.Format != "" {
qemuArgs = append(qemuArgs, "-drive", "file="+d.Path+",format="+d.Format+",index="+strconv.Itoa(index)+",media=disk")
} else {
qemuArgs = append(qemuArgs, "-drive", "file="+d.Path+",index="+strconv.Itoa(index)+",media=disk")
}
lastDisk = index
}
// Ensure CDROMs start from at least hdc
if lastDisk < 2 {
lastDisk = 2
}
if config.NetdevConfig == "" {
qemuArgs = append(qemuArgs, "-net", "none")
} else {
mac := generateMAC()
if config.Arch == "s390x" {
qemuArgs = append(qemuArgs, "-device", "virtio-net-ccw,netdev=t0,mac="+mac.String())
} else {
qemuArgs = append(qemuArgs, "-device", "virtio-net-pci,netdev=t0,mac="+mac.String())
}
forwardings, err := buildQemuForwardings(config.PublishedPorts)
if err != nil {
log.Error(err)
}
qemuArgs = append(qemuArgs, "-netdev", config.NetdevConfig+forwardings)
}
if config.GUI != true {
qemuArgs = append(qemuArgs, "-nographic")
}
if config.USB == true {
qemuArgs = append(qemuArgs, "-usb")
}
for _, d := range config.Devices {
qemuArgs = append(qemuArgs, "-device", d)
}
return config, qemuArgs
}
func discoverBinaries(config QemuConfig) (QemuConfig, error) {
if config.QemuImgPath != "" {
return config, nil
}
qemuBinPath := "qemu-system-" + config.Arch
qemuImgPath := "qemu-img"
var err error
config.QemuBinPath, err = exec.LookPath(qemuBinPath)
if err != nil {
return config, fmt.Errorf("Unable to find %s within the $PATH", qemuBinPath)
}
config.QemuImgPath, err = exec.LookPath(qemuImgPath)
if err != nil {
return config, fmt.Errorf("Unable to find %s within the $PATH", qemuImgPath)
}
return config, nil
}
func buildQemuForwardings(publishFlags MultipleFlag) (string, error) {
if len(publishFlags) == 0 {
return "", nil
}
var forwardings string
var publishedPorts []PublishedPort
for _, publish := range publishFlags {
p, err := NewPublishedPort(publish)
if err != nil {
return "", err
log.Fatal(err)
}
hostPort := p.Host
guestPort := p.Guest
forwardings = fmt.Sprintf("%s,hostfwd=%s::%d-:%d", forwardings, p.Protocol, hostPort, guestPort)
publishedPorts = append(publishedPorts, p)
}
return forwardings, nil
}
func buildDockerForwardings(publishedPorts []string) ([]string, error) {
pmap := []string{}
for _, port := range publishedPorts {
s, err := NewPublishedPort(port)
if err != nil {
return nil, err
}
pmap = append(pmap, "-p", fmt.Sprintf("%d:%d/%s", s.Host, s.Guest, s.Protocol))
opts := []qemu.Option{
qemu.WithDisks(disks...),
qemu.WithAccel(accel),
qemu.WithArch(arch),
qemu.WithCPUs(cpus),
qemu.WithMemory(mem),
qemu.WithNetworking(networking),
qemu.WithStdin(os.Stdin),
qemu.WithStdout(os.Stdout),
qemu.WithStderr(os.Stderr),
}
if enableGUI {
opts = append(opts, qemu.WithGUI())
}
if qemuDetached {
opts = append(opts, qemu.WithDetached())
}
if err := qemu.Run(cmd.Context(), path, opts...); err != nil {
log.Fatal(err)
}
return pmap, nil
}
// QemuConfig contains the config for Qemu
type QemuConfig struct {
Path string
GUI bool
Disks Disks
FWPath string
Arch string
CPUs uint
Memory uint
Accel string
Detached bool
QemuBinPath string
QemuImgPath string
PublishedPorts []string
NetdevConfig string
UUID uuid.UUID
USB bool
Devices []string
}
func haveKVM() bool {
_, err := os.Stat("/dev/kvm")
return !os.IsNotExist(err)
}
func generateMAC() net.HardwareAddr {
mac := make([]byte, 6)
n, err := rand.Read(mac)
if err != nil {
log.WithError(err).Fatal("failed to generate random mac address")
}
if n != 6 {
log.WithError(err).Fatalf("generated %d bytes for random mac address", n)
}
mac[0] &^= 0x01 // Clear multicast bit
mac[0] |= 0x2 // Set locally administered bit
return net.HardwareAddr(mac)
}

13
cmd/d2vm/run/util.go
View File

@ -28,6 +28,8 @@ import (
"time"
"golang.org/x/crypto/ssh"
"go.linka.cloud/d2vm/pkg/qemu"
)
//go:embed sparsecat-linux-amd64
@ -188,15 +190,8 @@ func ConvertMBtoGB(i int) int {
return (i + (1024 - i%1024)) / 1024
}
// DiskConfig is the config for a disk
type DiskConfig struct {
Path string
Size int
Format string
}
// Disks is the type for a list of DiskConfig
type Disks []DiskConfig
type Disks []qemu.Disk
func (l *Disks) String() string {
return fmt.Sprint(*l)
@ -204,7 +199,7 @@ func (l *Disks) String() string {
// Set is used by flag to configure value from CLI
func (l *Disks) Set(value string) error {
d := DiskConfig{}
d := qemu.Disk{}
s := strings.Split(value, ",")
for _, p := range s {
c := strings.SplitN(p, "=", 2)

3
docs/content/reference/d2vm_run_qemu.md
View File

@ -12,9 +12,7 @@ d2vm run qemu [options] [image-path] [flags]
--accel string Choose acceleration mode. Use 'tcg' to disable it. (default "hvf:tcg")
--arch string Type of architecture to use, e.g. x86_64, aarch64, s390x (default "x86_64")
--cpus uint Number of CPUs (default 1)
--data string String of metadata to pass to VM
--detached Set qemu container to run in the background
--device multiple-flag Add USB host device(s). Format driver[,prop=value][,...] -- add device, like --device on the qemu command line. (default A multiple flag is a type of flag that can be repeated any number of times)
--disk disk Disk config, may be repeated. [file=]path[,size=1G][,format=qcow2] (default [])
--gui Set qemu to use video output instead of stdio
-h, --help help for qemu
@ -22,7 +20,6 @@ d2vm run qemu [options] [image-path] [flags]
--networking string Networking mode. Valid options are 'default', 'user', 'bridge[,name]', tap[,name] and 'none'. 'user' uses QEMUs userspace networking. 'bridge' connects to a preexisting bridge. 'tap' uses a prexisting tap device. 'none' disables networking.` (default "user")
--publish multiple-flag Publish a vm's port(s) to the host (default []) (default A multiple flag is a type of flag that can be repeated any number of times)
--qemu string Path to the qemu binary (otherwise look in $PATH)
--usb Enable USB controller
```
### Options inherited from parent commands

144
e2e/e2e_test.go Normal file
View File

@ -0,0 +1,144 @@
// Copyright 2023 Linka Cloud All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package e2e
import (
"bufio"
"bytes"
"context"
"io"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
require2 "github.com/stretchr/testify/require"
"go.linka.cloud/d2vm"
"go.linka.cloud/d2vm/pkg/docker"
"go.linka.cloud/d2vm/pkg/qemu"
)
type test struct {
name string
args []string
}
var images = []string{
"alpine:3.17",
"ubuntu:20.04",
"debian:11",
"centos:8",
}
func TestConvert(t *testing.T) {
require := require2.New(t)
tests := []test{
{
name: "single-partition",
},
{
name: "split-boot",
args: []string{"--split-boot"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// t.Parallel()
dir := filepath.Join("/tmp", "d2vm-e2e", tt.name)
require.NoError(os.MkdirAll(dir, os.ModePerm))
defer os.RemoveAll(dir)
for _, i := range images {
t.Run(i, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// t.Parallel()
require := require2.New(t)
out := filepath.Join(dir, strings.NewReplacer(":", "-", ".", "-").Replace(i)+".qcow2")
if _, err := os.Stat(out); err == nil {
require.NoError(os.Remove(out))
}
require.NoError(docker.RunD2VM(ctx, d2vm.Image, d2vm.Version, dir, dir, "convert", append([]string{"-p", "root", "-o", "/out/" + filepath.Base(out), "-v", i}, tt.args...)...))
inr, inw := io.Pipe()
defer inr.Close()
outr, outw := io.Pipe()
defer outw.Close()
var success atomic.Bool
go func() {
time.AfterFunc(2*time.Minute, cancel)
defer inw.Close()
defer outr.Close()
login := []byte("login:")
password := []byte("Password:")
s := bufio.NewScanner(outr)
s.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
if i := bytes.Index(data, login); i >= 0 {
return i + len(login), login, nil
}
if i := bytes.Index(data, password); i >= 0 {
return i + len(password), password, nil
}
if atEOF {
return 0, nil, io.EOF
}
return 0, nil, nil
})
for s.Scan() {
b := s.Bytes()
if bytes.Contains(b, login) {
t.Logf("vm ready")
t.Logf("sending login")
if _, err := inw.Write([]byte("root\n")); err != nil {
t.Logf("failed to write login: %v", err)
cancel()
}
}
if bytes.Contains(b, password) {
t.Logf("sending password")
if _, err := inw.Write([]byte("root\n")); err != nil {
t.Logf("failed to write password: %v", err)
cancel()
}
time.Sleep(time.Second)
if _, err := inw.Write([]byte("poweroff\n")); err != nil {
t.Logf("failed to write poweroff: %v", err)
cancel()
}
success.Store(true)
return
}
}
if err := s.Err(); err != nil {
t.Logf("failed to scan output: %v", err)
cancel()
}
}()
if err := qemu.Run(ctx, out, qemu.WithStdin(inr), qemu.WithStdout(io.MultiWriter(outw, os.Stdout)), qemu.WithStderr(io.Discard)); err != nil && !success.Load() {
t.Fatalf("failed to run qemu: %v", err)
}
})
}
})
}
}

2
go.mod
View File

@ -19,6 +19,7 @@ require (
go.linka.cloud/console v0.0.0-20220910100646-48f9f2b8843b
go.uber.org/multierr v1.8.0
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
)
require (
@ -60,7 +61,6 @@ require (
go.uber.org/atomic v1.7.0 // indirect
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
google.golang.org/grpc v1.43.0 // indirect

28
pkg/docker/check_term_unix.go Normal file
View File

@ -0,0 +1,28 @@
//go:build !windows
// Copyright 2023 Linka Cloud All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package docker
import (
"os"
"golang.org/x/sys/unix"
)
func isInteractive() bool {
_, err := unix.IoctlGetWinsize(int(os.Stdout.Fd()), unix.TIOCGWINSZ)
return err == nil
}

36
pkg/docker/check_term_windows.go Normal file
View File

@ -0,0 +1,36 @@
//go:build windows
// Copyright 2023 Linka Cloud All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package docker
import (
"os"
"golang.org/x/sys/windows"
)
func isInteractive() bool {
handle := windows.Handle(os.Stdout.Fd())
var mode uint32
if err := windows.GetConsoleMode(handle, &mode); err != nil {
return false
}
mode |= windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING
if err := windows.SetConsoleMode(handle, mode); err != nil {
return false
}
return true
}

17
pkg/docker/docker.go
View File

@ -130,7 +130,14 @@ func RunD2VM(ctx context.Context, image, version, in, out, cmd string, args ...s
if version == "" {
version = "latest"
}
a := []string{
a := []string{"run", "--rm"}
interactive := isInteractive()
if interactive {
a = append(a, "-i", "-t")
}
a = append(a,
"--privileged",
"-e",
// yes... it is kind of a dirty hack
@ -145,6 +152,12 @@ func RunD2VM(ctx context.Context, image, version, in, out, cmd string, args ...s
"/d2vm",
fmt.Sprintf("%s:%s", image, version),
cmd,
)
c := exec.CommandContext(ctx, "docker", append(a, args...)...)
if interactive {
c.Stdin = os.Stdin
}
return RunInteractiveAndRemove(ctx, append(a, args...)...)
c.Stdout = os.Stdout
c.Stderr = os.Stderr
return c.Run()
}

141
pkg/qemu/config.go Normal file
View File

@ -0,0 +1,141 @@
// Copyright 2023 Linka Cloud All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package qemu
import (
"io"
"github.com/google/uuid"
)
type Option func(c *config)
type Disk struct {
Path string
Size int
Format string
}
type PublishedPort struct {
Guest uint16
Host uint16
Protocol string
}
// config contains the config for Qemu
type config struct {
path string
uuid uuid.UUID
gui bool
disks []Disk
networking string
arch string
cpus uint
memory uint
accel string
detached bool
qemuBinPath string
qemuImgPath string
publishedPorts []PublishedPort
netdevConfig string
stdin io.Reader
stdout io.Writer
stderr io.Writer
}
func WithGUI() Option {
return func(c *config) {
c.gui = true
}
}
func WithDisks(disks ...Disk) Option {
return func(c *config) {
c.disks = disks
}
}
func WithNetworking(networking string) Option {
return func(c *config) {
c.networking = networking
}
}
func WithArch(arch string) Option {
return func(c *config) {
c.arch = arch
}
}
func WithCPUs(cpus uint) Option {
return func(c *config) {
c.cpus = cpus
}
}
func WithMemory(memory uint) Option {
return func(c *config) {
c.memory = memory
}
}
func WithAccel(accel string) Option {
return func(c *config) {
c.accel = accel
}
}
func WithDetached() Option {
return func(c *config) {
c.detached = true
}
}
func WithQemuBinPath(path string) Option {
return func(c *config) {
c.qemuBinPath = path
}
}
func WithQemuImgPath(path string) Option {
return func(c *config) {
c.qemuImgPath = path
}
}
func WithPublishedPorts(ports ...PublishedPort) Option {
return func(c *config) {
c.publishedPorts = ports
}
}
func WithStdin(r io.Reader) Option {
return func(c *config) {
c.stdin = r
}
}
func WithStdout(w io.Writer) Option {
return func(c *config) {
c.stdout = w
}
}
func WithStderr(w io.Writer) Option {
return func(c *config) {
c.stderr = w
}
}

363
pkg/qemu/qemu.go Normal file
View File

@ -0,0 +1,363 @@
package qemu
import (
"context"
"crypto/rand"
"fmt"
"net"
"os"
"os/exec"
"runtime"
"strconv"
"strings"
"github.com/google/uuid"
log "github.com/sirupsen/logrus"
)
const (
NetworkingNone = "none"
NetworkingUser = "user"
NetworkingTap = "tap"
NetworkingBridge = "bridge"
NetworkingDefault = NetworkingUser
)
var (
defaultArch string
defaultAccel string
)
func init() {
switch runtime.GOARCH {
case "arm64":
defaultArch = "aarch64"
case "amd64":
defaultArch = "x86_64"
case "s390x":
defaultArch = "s390x"
}
switch {
case runtime.GOARCH == "s390x":
defaultAccel = "kvm"
case haveKVM():
defaultAccel = "kvm:tcg"
case runtime.GOOS == "darwin":
defaultAccel = "hvf:tcg"
}
}
func Run(ctx context.Context, path string, opts ...Option) error {
config := &config{}
for _, o := range opts {
o(config)
}
config.path = path
// Generate UUID, so that /sys/class/dmi/id/product_uuid is populated
config.uuid = uuid.New()
// These envvars override the corresponding command line
// options. So this must remain after the `flags.Parse` above.
// accel = GetStringValue("LINUXKIT_QEMU_ACCEL", accel, "")
if config.arch == "" {
config.arch = defaultArch
}
if config.accel == "" {
config.accel = defaultAccel
}
if _, err := os.Stat(config.path); err != nil {
return err
}
if config.cpus == 0 {
config.cpus = 1
}
if config.memory == 0 {
config.memory = 1024
}
for i, d := range config.disks {
id := ""
if i != 0 {
id = strconv.Itoa(i)
}
if d.Size != 0 && d.Format == "" {
d.Format = "qcow2"
}
if d.Size != 0 && d.Path == "" {
d.Path = "disk" + id + ".img"
}
if d.Path == "" {
return fmt.Errorf("disk specified with no size or name")
}
config.disks[i] = d
}
config.disks = append([]Disk{{Path: config.path}}, config.disks...)
if config.networking == "" || config.networking == "default" {
dflt := NetworkingDefault
config.networking = dflt
}
netMode := strings.SplitN(config.networking, ",", 2)
switch netMode[0] {
case NetworkingUser:
config.netdevConfig = "user,id=t0"
case NetworkingTap:
if len(netMode) != 2 {
return fmt.Errorf("Not enough arguments for %q networking mode", NetworkingTap)
}
if len(config.publishedPorts) != 0 {
return fmt.Errorf("Port publishing requires %q networking mode", NetworkingUser)
}
config.netdevConfig = fmt.Sprintf("tap,id=t0,ifname=%s,script=no,downscript=no", netMode[1])
case NetworkingBridge:
if len(netMode) != 2 {
return fmt.Errorf("Not enough arguments for %q networking mode", NetworkingBridge)
}
if len(config.publishedPorts) != 0 {
return fmt.Errorf("Port publishing requires %q networking mode", NetworkingUser)
}
config.netdevConfig = fmt.Sprintf("bridge,id=t0,br=%s", netMode[1])
case NetworkingNone:
if len(config.publishedPorts) != 0 {
return fmt.Errorf("Port publishing requires %q networking mode", NetworkingUser)
}
config.netdevConfig = ""
default:
return fmt.Errorf("Invalid networking mode: %s", netMode[0])
}
if err := config.discoverBinaries(); err != nil {
log.Fatal(err)
}
return config.runQemuLocal(ctx)
}
func (c *config) runQemuLocal(ctx context.Context) (err error) {
var args []string
args, err = c.buildQemuCmdline()
if err != nil {
return err
}
for _, d := range c.disks {
// If disk doesn't exist then create one
if _, err := os.Stat(d.Path); err != nil {
if os.IsNotExist(err) {
log.Debugf("Creating new qemu disk [%s] format %s", d.Path, d.Format)
qemuImgCmd := exec.Command(c.qemuImgPath, "create", "-f", d.Format, d.Path, fmt.Sprintf("%dM", d.Size))
log.Debugf("%v", qemuImgCmd.Args)
if err := qemuImgCmd.Run(); err != nil {
return fmt.Errorf("Error creating disk [%s] format %s: %s", d.Path, d.Format, err.Error())
}
} else {
return err
}
} else {
log.Infof("Using existing disk [%s] format %s", d.Path, d.Format)
}
}
// Detached mode is only supported in a container.
if c.detached == true {
return fmt.Errorf("Detached mode is only supported when running in a container, not locally")
}
qemuCmd := exec.CommandContext(ctx, c.qemuBinPath, args...)
// If verbosity is enabled print out the full path/arguments
log.Debugf("%v", qemuCmd.Args)
// If we're not using a separate window then link the execution to stdin/out
if c.gui == true {
qemuCmd.Stdin = nil
qemuCmd.Stdout = nil
qemuCmd.Stderr = nil
} else {
qemuCmd.Stdin = c.stdin
qemuCmd.Stdout = c.stdout
qemuCmd.Stderr = c.stderr
}
return qemuCmd.Run()
}
func (c *config) buildQemuCmdline() ([]string, error) {
// Iterate through the flags and build arguments
var qemuArgs []string
qemuArgs = append(qemuArgs, "-smp", fmt.Sprintf("%d", c.cpus))
qemuArgs = append(qemuArgs, "-m", fmt.Sprintf("%d", c.memory))
qemuArgs = append(qemuArgs, "-uuid", c.uuid.String())
// Need to specify the vcpu type when running qemu on arm64 platform, for security reason,
// the vcpu should be "host" instead of other names such as "cortex-a53"...
if c.arch == "aarch64" {
if runtime.GOARCH == "arm64" {
qemuArgs = append(qemuArgs, "-cpu", "host")
} else {
qemuArgs = append(qemuArgs, "-cpu", "cortex-a57")
}
}
// goArch is the GOARCH equivalent of config.Arch
var goArch string
switch c.arch {
case "s390x":
goArch = "s390x"
case "aarch64":
goArch = "arm64"
case "x86_64":
goArch = "amd64"
default:
return nil, fmt.Errorf("%s is an unsupported architecture.", c.arch)
}
if goArch != runtime.GOARCH {
log.Infof("Disable acceleration as %s != %s", c.arch, runtime.GOARCH)
c.accel = ""
}
if c.accel != "" {
switch c.arch {
case "s390x":
qemuArgs = append(qemuArgs, "-machine", fmt.Sprintf("s390-ccw-virtio,accel=%s", c.accel))
case "aarch64":
gic := ""
// VCPU supports less PA bits (36) than requested by the memory map (40)
highmem := "highmem=off,"
if runtime.GOOS == "linux" {
// gic-version=host requires KVM, which implies Linux
gic = "gic_version=host,"
highmem = ""
}
qemuArgs = append(qemuArgs, "-machine", fmt.Sprintf("virt,%s%saccel=%s", gic, highmem, c.accel))
default:
qemuArgs = append(qemuArgs, "-machine", fmt.Sprintf("q35,accel=%s", c.accel))
}
} else {
switch c.arch {
case "s390x":
qemuArgs = append(qemuArgs, "-machine", "s390-ccw-virtio")
case "aarch64":
qemuArgs = append(qemuArgs, "-machine", "virt")
default:
qemuArgs = append(qemuArgs, "-machine", "q35")
}
}
// rng-random does not work on macOS
// Temporarily disable it until fixed upstream.
if runtime.GOOS != "darwin" {
rng := "rng-random,id=rng0"
if runtime.GOOS == "linux" {
rng = rng + ",filename=/dev/urandom"
}
if c.arch == "s390x" {
qemuArgs = append(qemuArgs, "-object", rng, "-device", "virtio-rng-ccw,rng=rng0")
} else {
qemuArgs = append(qemuArgs, "-object", rng, "-device", "virtio-rng-pci,rng=rng0")
}
}
var lastDisk int
for i, d := range c.disks {
index := i
if d.Format != "" {
qemuArgs = append(qemuArgs, "-drive", "file="+d.Path+",format="+d.Format+",index="+strconv.Itoa(index)+",media=disk")
} else {
qemuArgs = append(qemuArgs, "-drive", "file="+d.Path+",index="+strconv.Itoa(index)+",media=disk")
}
lastDisk = index
}
// Ensure CDROMs start from at least hdc
if lastDisk < 2 {
lastDisk = 2
}
if c.netdevConfig != "" {
mac := generateMAC()
if c.arch == "s390x" {
qemuArgs = append(qemuArgs, "-device", "virtio-net-ccw,netdev=t0,mac="+mac.String())
} else {
qemuArgs = append(qemuArgs, "-device", "virtio-net-pci,netdev=t0,mac="+mac.String())
}
forwardings, err := buildQemuForwardings(c.publishedPorts)
if err != nil {
log.Error(err)
}
qemuArgs = append(qemuArgs, "-netdev", c.netdevConfig+forwardings)
} else {
qemuArgs = append(qemuArgs, "-net", "none")
}
if c.gui != true {
qemuArgs = append(qemuArgs, "-nographic")
}
return qemuArgs, nil
}
func (c *config) discoverBinaries() error {
if c.qemuImgPath != "" {
return nil
}
qemuBinPath := "qemu-system-" + c.arch
qemuImgPath := "qemu-img"
var err error
c.qemuBinPath, err = exec.LookPath(qemuBinPath)
if err != nil {
return fmt.Errorf("Unable to find %s within the $PATH", qemuBinPath)
}
c.qemuImgPath, err = exec.LookPath(qemuImgPath)
if err != nil {
return fmt.Errorf("Unable to find %s within the $PATH", qemuImgPath)
}
return nil
}
func buildQemuForwardings(publishedPorts []PublishedPort) (string, error) {
if len(publishedPorts) == 0 {
return "", nil
}
var forwardings string
for _, p := range publishedPorts {
hostPort := p.Host
guestPort := p.Guest
forwardings = fmt.Sprintf("%s,hostfwd=%s::%d-:%d", forwardings, p.Protocol, hostPort, guestPort)
}
return forwardings, nil
}
func haveKVM() bool {
_, err := os.Stat("/dev/kvm")
return !os.IsNotExist(err)
}
func generateMAC() net.HardwareAddr {
mac := make([]byte, 6)
n, err := rand.Read(mac)
if err != nil {
log.WithError(err).Fatal("failed to generate random mac address")
}
if n != 6 {
log.WithError(err).Fatalf("generated %d bytes for random mac address", n)
}
mac[0] &^= 0x01 // Clear multicast bit
mac[0] |= 0x2 // Set locally administered bit
return mac
}