smartxworks · scuzhanglei · Aug 4, 2022
diff --git a/cmd/virt-prerunner/main.go b/cmd/virt-prerunner/main.go
@@ -191,6  191,26 @@ func buildVMConfig(ctx context.Context, vm *virtv1alpha1.VirtualMachine) (*cloud
 		}
 	}
 
 	var resourceIndexes = map[string]int{}
 	getResourceIndex := func(resourceName string) int {
 		index := resourceIndexes[resourceName]
 		resourceIndexes[resourceName] = index   1
 		return index
 	}
 
 	for _, gpu := range vm.Spec.Instance.GPUs {
 		pciAddresses := strings.Split(os.Getenv(gpu.ResourcePCIAddressEnvVarName), ",")
 		index := getResourceIndex(gpu.ResourceName)
 		if index >= len(pciAddresses) {
 			return nil, fmt.Errorf("failed to get PCI address for %s", gpu.Name)
 		}
 		gpuDeviceConfig := cloudhypervisor.DeviceConfig{
 			Id:   gpu.Name,
 			Path: fmt.Sprintf("/sys/bus/pci/devices/%s", pciAddresses[index]),
 		}
 		vmConfig.Devices = append(vmConfig.Devices, &gpuDeviceConfig)
 	}
 
 	return &vmConfig, nil
 }
 

diff --git a/deploy/crd/virt.virtink.smartx.com_virtualmachines.yaml b/deploy/crd/virt.virtink.smartx.com_virtualmachines.yaml
@@ -883,6  883,20 @@ spec:
                       - name
                       type: object
                     type: array
                   gpus:
                     items:
                       properties:
                         name:
                           type: string
                         resourceName:
                           type: string
                         resourcePCIAddressEnvVarName:
                           type: string
                       required:
                       - name
                       - resourceName
                       type: object
                     type: array
                   interfaces:
                     items:
                       properties:

diff --git a/pkg/apis/virt/v1alpha1/types.go b/pkg/apis/virt/v1alpha1/types.go
@@ -57,6  57,7 @@ type Instance struct {
 	Kernel     *Kernel     `json:"kernel,omitempty"`
 	Disks      []Disk      `json:"disks,omitempty"`
 	Interfaces []Interface `json:"interfaces,omitempty"`
 	GPUs       []GPU       `json:"gpus,omitempty"`
 }
 
 type CPU struct {
@@ -95,6  96,12 @@ type InterfaceBridge struct {
 type InterfaceSRIOV struct {
 }
 
 type GPU struct {
 	Name                         string `json:"name"`
 	ResourceName                 string `json:"resourceName"`
 	ResourcePCIAddressEnvVarName string `json:"resourcePCIAddressEnvVarName,omitempty"`
 }
 
 type Volume struct {
 	Name         string `json:"name"`
 	VolumeSource `json:",inline"`

diff --git a/pkg/apis/virt/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/virt/v1alpha1/zz_generated.deepcopy.go
diff --git a/pkg/controller/vm_controller.go b/pkg/controller/vm_controller.go
@@ -470,6  470,10 @@ func (r *VMReconciler) buildVMPod(ctx context.Context, vm *virtv1alpha1.VirtualM
 		vmPod.Annotations["k8s.v1.cni.cncf.io/networks"] = string(networksJSON)
 	}
 
 	for _, gpu := range vm.Spec.Instance.GPUs {
 		incrementContainerResource(&vmPod.Spec.Containers[0], gpu.ResourceName)
 	}
 
 	return &vmPod, nil
 }
 

diff --git a/pkg/controller/vm_webhook.go b/pkg/controller/vm_webhook.go
@@ -6,6  6,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
 	"strings"
 
 	"github.com/r3labs/diff/v2"
 	admissionv1 "k8s.io/api/admission/v1"
@@ -84,6  85,16 @@ func MutateVM(ctx context.Context, vm *virtv1alpha1.VirtualMachine, oldVM *virtv
 			}
 		}
 	}
 
 	for i := range vm.Spec.Instance.GPUs {
 		if vm.Spec.Instance.GPUs[i].ResourcePCIAddressEnvVarName == "" {
 			if strings.Contains(vm.Spec.Instance.GPUs[i].ResourceName, "nvidia.com/") {
 				// https://github.com/NVIDIA/kubevirt-gpu-device-plugin/blob/f2f291647189859946a4fbb61d4e1812d86861f7/pkg/device_plugin/generic_device_plugin.go#L48
 				deviceName := strings.TrimPrefix(vm.Spec.Instance.GPUs[i].ResourceName, "nvidia.com/")
 				vm.Spec.Instance.GPUs[i].ResourcePCIAddressEnvVarName = fmt.Sprintf("PCI_RESOURCE_NVIDIA_COM_%s", deviceName)
 			}
 		}
 	}
 	return nil
 }
 
@@ -210,6  221,16 @@ func ValidateInstance(ctx context.Context, instance *virtv1alpha1.Instance, fiel
 		errs = append(errs, ValidateInterface(ctx, &iface, fieldPath)...)
 	}
 
 	gpuNames := map[string]struct{}{}
 	for i, gpu := range instance.GPUs {
 		fieldPath := fieldPath.Child("gpus").Index(i)
 		if _, ok := gpuNames[gpu.Name]; ok {
 			errs = append(errs, field.Duplicate(fieldPath.Child("name"), gpu.Name))
 		}
 		gpuNames[gpu.Name] = struct{}{}
 		errs = append(errs, ValidateGPU(ctx, &gpu, fieldPath)...)
 	}
 
 	return errs
 }
 
@@ -336,6  357,19 @@ func ValidateCIDR(cidr string, capacity int, fieldPath *field.Path) field.ErrorL
 	return errs
 }
 
 func ValidateGPU(ctx context.Context, gpu *virtv1alpha1.GPU, fieldPath *field.Path) field.ErrorList {
 	var errs field.ErrorList
 	if gpu == nil {
 		errs = append(errs, field.Required(fieldPath, ""))
 		return errs
 	}
 
 	if gpu.ResourcePCIAddressEnvVarName == "" {
 		errs = append(errs, field.Required(fieldPath.Child("resourcePCIAddressEnvVarName"), ""))
 	}
 	return errs
 }
 
 func ValidateVolume(ctx context.Context, volume *virtv1alpha1.Volume, fieldPath *field.Path) field.ErrorList {
 	var errs field.ErrorList
 	if volume == nil {

diff --git a/pkg/controller/vm_webhook_test.go b/pkg/controller/vm_webhook_test.go
@@ -30,6  30,11 @@ func TestValidateVM(t *testing.T) {
 						Bridge: &virtv1alpha1.InterfaceBridge{},
 					},
 				}},
 				GPUs: []virtv1alpha1.GPU{{
 					Name:                         "gpu-1",
 					ResourceName:                 "nvidia.com/Tesla_01",
 					ResourcePCIAddressEnvVarName: "PCI_RESOURCE_NVIDIA_COM_Tesla_01",
 				}},
 			},
 			Volumes: []virtv1alpha1.Volume{{
 				Name: "vol-1",
@@ -117,6  122,27 @@ func TestValidateVM(t *testing.T) {
 		}(),
 		invalidFields: []string{"spec.instance.interfaces[0].sriov"},
 	}, {
 		vm: func() *virtv1alpha1.VirtualMachine {
 			vm := validVM.DeepCopy()
 			vm.Spec.Instance.GPUs[0].Name = ""
 			return vm
 		}(),
 		invalidFields: []string{"spec.instance.gpus[0].name"},
 	}, {
 		vm: func() *virtv1alpha1.VirtualMachine {
 			vm := validVM.DeepCopy()
 			vm.Spec.Instance.GPUs[0].ResourceName = ""
 			return vm
 		}(),
 		invalidFields: []string{"spec.instance.gpus[0].resourceName"},
 	}, {
 		vm: func() *virtv1alpha1.VirtualMachine {
 			vm := validVM.DeepCopy()
 			vm.Spec.Instance.GPUs[0].ResourcePCIAddressEnvVarName = ""
 			return vm
 		}(),
 		invalidFields: []string{"spec.instance.gpus[0].resourcePCIAddressEnvVarName"},
 	}, {
 
 		vm: func() *virtv1alpha1.VirtualMachine {
 			vm := validVM.DeepCopy()
@@ -168,3  194,51 @@ func TestValidateVM(t *testing.T) {
 		}
 	}
 }
 
 func TestMutateVM(t *testing.T) {
 	vm := &virtv1alpha1.VirtualMachine{
 		Spec: virtv1alpha1.VirtualMachineSpec{
 			Instance: virtv1alpha1.Instance{
 				Interfaces: []virtv1alpha1.Interface{{
 					Name: "net-1",
 					InterfaceBindingMethod: virtv1alpha1.InterfaceBindingMethod{
 						Bridge: &virtv1alpha1.InterfaceBridge{},
 					},
 				}},
 				GPUs: []virtv1alpha1.GPU{{
 					Name:         "gpu-1",
 					ResourceName: "nvidia.com/Tesla_01",
 				}, {
 					Name:         "gpu-2",
 					ResourceName: "amd.com/S7100",
 				}},
 			},
 		},
 	}
 
 	tests := []struct {
 		vm     *virtv1alpha1.VirtualMachine
 		assert func()
 	}{{
 		vm: vm,
 		assert: func() {
 			assert.NotNil(t, vm.Spec.Instance.Interfaces[0].InterfaceBindingMethod.Bridge)
 		},
 	}, {
 		vm: vm,
 		assert: func() {
 			assert.Equal(t, vm.Spec.Instance.GPUs[0].ResourcePCIAddressEnvVarName, "PCI_RESOURCE_NVIDIA_COM_Tesla_01")
 		},
 	}, {
 		vm: vm,
 		assert: func() {
 			assert.Equal(t, vm.Spec.Instance.GPUs[1].ResourcePCIAddressEnvVarName, "")
 		},
 	}}
 
 	for _, tc := range tests {
 		err := MutateVM(context.Background(), tc.vm, nil)
 		assert.Nil(t, err)
 		tc.assert()
 	}
 }
diff --git a/samples/ubuntu-gpu.yaml b/samples/ubuntu-gpu.yaml
@@ -0,0  1,30 @@
 apiVersion: virt.virtink.smartx.com/v1alpha1
 kind: VirtualMachine
 metadata:
   name: ubuntu-gpu
 spec:
   instance:
     memory:
       size: 1Gi
     disks:
       - name: ubuntu
       - name: cloud-init
     interfaces:
       - name: pod
     gpus:
       - name: gpu1
         resourceName: nvidia.com/XXXX #TODO
   volumes:
     - name: ubuntu
       containerDisk:
         image: smartxworks/virtink-container-disk-ubuntu
     - name: cloud-init
       cloudInit:
         userData: |-
           #cloud-config
           password: password
           chpasswd: { expire: False }
           ssh_pwauth: True
   networks:
     - name: pod
       pod: {}