feat: Live migration

cmd/virt-controller/main.go

@@ -75,8 +75,18 @@ func main() {
 		os.Exit(1)
 	}
 
+	if err = (&controller.VMMReconciler{
+		Client:   mgr.GetClient(),
+		Scheme:   mgr.GetScheme(),
+		Recorder: mgr.GetEventRecorderFor("virt-controller"),
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "VMM")
+		os.Exit(1)
+	}
+
 	mgr.GetWebhookServer().Register("/mutate-v1alpha1-virtualmachine", &webhook.Admission{Handler: &controller.VMMutator{}})
 	mgr.GetWebhookServer().Register("/validate-v1alpha1-virtualmachine", &webhook.Admission{Handler: &controller.VMValidator{}})
+	mgr.GetWebhookServer().Register("/validate-v1alpha1-virtualmachinemigration", &webhook.Admission{Handler: &controller.VMMValidator{Client: mgr.GetClient()}})
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")

cmd/virt-daemon/main.go

@@ -13,6 +13,7 @@ import (
 
 	virtv1alpha1 "github.com/smartxworks/virtink/pkg/apis/virt/v1alpha1"
 	"github.com/smartxworks/virtink/pkg/daemon"
+	"github.com/smartxworks/virtink/pkg/daemon/tcpproxy"
 )
 
 var (
@@ -50,10 +51,12 @@ func main() {
 	}
 
 	if err = (&daemon.VMReconciler{
-		Client:   mgr.GetClient(),
-		Scheme:   mgr.GetScheme(),
-		Recorder: mgr.GetEventRecorderFor("virt-daemon"),
-		NodeName: os.Getenv("NODE_NAME"),
+		Client:        mgr.GetClient(),
+		Scheme:        mgr.GetScheme(),
+		Recorder:      mgr.GetEventRecorderFor("virt-daemon"),
+		NodeName:      os.Getenv("NODE_NAME"),
+		NodeIP:        os.Getenv("NODE_IP"),
+		RelayProvider: tcpproxy.NewRelayProvider(),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "VM")
 		os.Exit(1)

cmd/virt-prerunner/main.go

@@ -5,6 +5,7 @@ import (
 	_ "embed"
 	"encoding/base64"
 	"encoding/json"
+	"flag"
 	"fmt"
 	"log"
 	"net"
@@ -27,9 +28,15 @@ import (
 )
 
 func main() {
-	vmJSON, err := base64.StdEncoding.DecodeString(os.Args[1])
+	var vmData string
+	var receiveMigration bool
+	flag.StringVar(&vmData, "vm-data", vmData, "Base64 encoded VM json data")
+	flag.BoolVar(&receiveMigration, "receive-migration", receiveMigration, "Receive migration instead of starting a new VM")
+	flag.Parse()
+
+	vmJSON, err := base64.StdEncoding.DecodeString(vmData)
 	if err != nil {
-		log.Fatalf("Failed to decode VM JSON: %s", err)
+		log.Fatalf("Failed to decode VM data: %s", err)
 	}
 
 	var vm virtv1alpha1.VirtualMachine
@@ -41,6 +48,11 @@ func main() {
 	if err != nil {
 		log.Fatalf("Failed to build VM config: %s", err)
 	}
+	if receiveMigration {
+		cloudHypervisorCmd := []string{"cloud-hypervisor", "--api-socket", "/var/run/virtink/ch.sock"}
+		fmt.Println(strings.Join(cloudHypervisorCmd, " "))
+		return
+	}
 
 	cloudHypervisorCmd := []string{"cloud-hypervisor", "--api-socket", "/var/run/virtink/ch.sock", "--console", "pty", "--serial", "tty"}
 	cloudHypervisorCmd = append(cloudHypervisorCmd, "--kernel", vmConfig.Payload.Kernel)

deploy/crd/virt.virtink.smartx.com_virtualmachinemigrations.yaml

@@ -0,0 +1,78 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.9.0
+  creationTimestamp: null
+  name: virtualmachinemigrations.virt.virtink.smartx.com
+spec:
+  group: virt.virtink.smartx.com
+  names:
+    kind: VirtualMachineMigration
+    listKind: VirtualMachineMigrationList
+    plural: virtualmachinemigrations
+    shortNames:
+    - vmm
+    singular: virtualmachinemigration
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - jsonPath: .spec.vmName
+      name: VM
+      type: string
+    - jsonPath: .status.sourceNodeName
+      name: Source
+      type: string
+    - jsonPath: .status.targetNodeName
+      name: Target
+      type: string
+    - jsonPath: .status.phase
+      name: Status
+      type: string
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            properties:
+              vmName:
+                type: string
+            required:
+            - vmName
+            type: object
+          status:
+            properties:
+              phase:
+                enum:
+                - Pending
+                - Scheduling
+                - Scheduled
+                - TargetReady
+                - Running
+                - Sent
+                - Succeeded
+                - Failed
+                type: string
+              sourceNodeName:
+                type: string
+              targetNodeName:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

deploy/crd/virt.virtink.smartx.com_virtualmachines.yaml

@@ -1387,6 +1387,108 @@ spec:
           status:
             description: VirtualMachineStatus is the status for a VirtualMachine resource
             properties:
+              conditions:
+                items:
+                  description: "Condition contains details for one aspect of the current
+                    state of this API Resource. --- This struct is intended for direct
+                    use as an array at the field path .status.conditions.  For example,
+                    type FooStatus struct{ // Represents the observations of a foo's
+                    current state. // Known .status.conditions.type are: \"Available\",
+                    \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge
+                    // +listType=map // +listMapKey=type Conditions []metav1.Condition
+                    `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\"
+                    protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }"
+                  properties:
+                    lastTransitionTime:
+                      description: lastTransitionTime is the last time the condition
+                        transitioned from one status to another. This should be when
+                        the underlying condition changed.  If that is not known, then
+                        using the time when the API field changed is acceptable.
+                      format: date-time
+                      type: string
+                    message:
+                      description: message is a human readable message indicating
+                        details about the transition. This may be an empty string.
+                      maxLength: 32768
+                      type: string
+                    observedGeneration:
+                      description: observedGeneration represents the .metadata.generation
+                        that the condition was set based upon. For instance, if .metadata.generation
+                        is currently 12, but the .status.conditions[x].observedGeneration
+                        is 9, the condition is out of date with respect to the current
+                        state of the instance.
+                      format: int64
+                      minimum: 0
+                      type: integer
+                    reason:
+                      description: reason contains a programmatic identifier indicating
+                        the reason for the condition's last transition. Producers
+                        of specific condition types may define expected values and
+                        meanings for this field, and whether the values are considered
+                        a guaranteed API. The value should be a CamelCase string.
+                        This field may not be empty.
+                      maxLength: 1024
+                      minLength: 1
+                      pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      type: string
+                    status:
+                      description: status of the condition, one of True, False, Unknown.
+                      enum:
+                      - "True"
+                      - "False"
+                      - Unknown
+                      type: string
+                    type:
+                      description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                        --- Many .condition.type values are consistent across resources
+                        like Available, but because arbitrary conditions can be useful
+                        (see .node.status.conditions), the ability to deconflict is
+                        important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
+                      maxLength: 316
+                      pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - message
+                  - reason
+                  - status
+                  - type
+                  type: object
+                type: array
+              migration:
+                properties:
+                  phase:
+                    enum:
+                    - Pending
+                    - Scheduling
+                    - Scheduled
+                    - TargetReady
+                    - Running
+                    - Sent
+                    - Succeeded
+                    - Failed
+                    type: string
+                  targetNodeIP:
+                    type: string
+                  targetNodeName:
+                    type: string
+                  targetNodePort:
+                    type: integer
+                  targetVMPodName:
+                    type: string
+                  targetVMPodUID:
+                    description: UID is a type that holds unique ID values, including
+                      UUIDs.  Because we don't ONLY use UUIDs, this is an alias to
+                      string.  Being a type captures intent and helps make sure that
+                      UIDs and names do not get conflated.
+                    type: string
+                  uid:
+                    description: UID is a type that holds unique ID values, including
+                      UUIDs.  Because we don't ONLY use UUIDs, this is an alias to
+                      string.  Being a type captures intent and helps make sure that
+                      UIDs and names do not get conflated.
+                    type: string
+                type: object
               nodeName:
                 type: string
               phase:

deploy/kustomization.yaml

@@ -1,5 +1,6 @@
 resources:
   - crd/virt.virtink.smartx.com_virtualmachines.yaml
+  - crd/virt.virtink.smartx.com_virtualmachinemigrations.yaml
   - namespace.yaml
   - virt-controller
   - virt-daemon

deploy/virt-controller/manifests-patch.yaml

@@ -23,3 +23,8 @@ webhooks:
       service:
         name: virt-controller
         namespace: virtink-system
+  - name: validate.virtualmachinemigration.v1alpha1.virt.virtink.smartx.com
+    clientConfig:
+      service:
+        name: virt-controller
+        namespace: virtink-system

deploy/virt-controller/manifests.yaml

@@ -54,3 +54,24 @@ webhooks:
     resources:
     - virtualmachines
   sideEffects: None
+- admissionReviewVersions:
+  - v1
+  - v1beta1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-v1alpha1-virtualmachinemigration
+  failurePolicy: Fail
+  name: validate.virtualmachinemigration.v1alpha1.virt.virtink.smartx.com
+  rules:
+  - apiGroups:
+    - virt.virtink.smartx.com
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - virtualmachinemigrations
+  sideEffects: None

deploy/virt-controller/role.yaml

@@ -73,6 +73,22 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - virt.virtink.smartx.com
+  resources:
+  - virtualmachinemigrations
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - virt.virtink.smartx.com
+  resources:
+  - virtualmachinemigrations/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - virt.virtink.smartx.com
   resources:

deploy/virt-daemon/cert-issuer.yaml

@@ -0,0 +1,7 @@
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: virt-daemon-cert-issuer
+  namespace: virtink-system
+spec:
+  selfSigned: {}

deploy/virt-daemon/cert.yaml

@@ -0,0 +1,13 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: virt-daemon-cert
+  namespace: virtink-system
+spec:
+  issuerRef:
+    kind: Issuer
+    name: virt-daemon-cert-issuer
+  dnsNames:
+    - virt-daemon.virtink-system.svc
+    - virt-daemon.virtink-system.svc.cluster.local
+  secretName: virt-daemon-cert

deploy/virt-daemon/daemonset.yaml

@@ -21,12 +21,23 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+            - name: NODE_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
           args:
             - --zap-time-encoding=iso8601
           volumeMounts:
             - name: kubelet-pods
               mountPath: /var/lib/kubelet/pods
+            - name: cert
+              mountPath: /var/lib/virtink/daemon/cert
+              readOnly: true
       volumes:
         - name: kubelet-pods
           hostPath:
             path: /var/lib/kubelet/pods
+        - name: cert
+          secret:
+            secretName: virt-daemon-cert
+            defaultMode: 420

deploy/virt-daemon/kustomization.yaml

@@ -3,3 +3,5 @@ resources:
   - rolebinding.yaml
   - role.yaml
   - sa.yaml
+  - cert.yaml
+  - cert-issuer.yaml

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/subgraph/libmacouflage v0.0.1
 	github.com/vishvananda/netlink v1.1.0
+	inet.af/tcpproxy v0.0.0-20220326234310-be3ee21c9fa0
 	k8s.io/api v0.24.1
 	k8s.io/apimachinery v0.24.1
 	k8s.io/apiserver v0.24.1