Merge pull request #39 from sp-yduck/node-image

sp-yduck · web-flow · commit bdd43c849e4b · 2023-07-10T21:19:11.000+09:00
updates on node images
diff --git a/README.md b/README.md
@@ -61,6 +61,18 @@ make delete-workload-cluster
 
 - Supports custom cloud-config (user data). CAPPX uses ssh for bootstrapping nodes so it can applies custom cloud-config that can not be achieved by only Proxmox API.
 
+### Node Images
+
+CAPPX is compatible with `qcow2` image. You can build your own node image and use it for `ProxmoxMachine`.
+
+CAPPX relies on a few prerequisites which have to be already installed in the used operating system images, e.g. a container runtime, kubelet, kubeadm,.. .
+
+To build your custom node image, you can use [kubernetes-sigs/image-builder](https://github.com/kubernetes-sigs/image-builder) project. 
+
+Also there are some available out-of-box images published other communities such as [Metal3](https://github.com/metal3-io). For example https://artifactory.nordix.org/ui/native/metal3/images/. Example MD can be found [metal3-ubuntu2204-k8s127.yaml](examples/machine_deployment/metal3-ubuntu2204-k8s127.yaml).
+
+If it isn't possible to pre-install those prerequisites in the image, you can always deploy and execute some custom scripts through the `ProxmoxMachine.spec.cloudInit` or `KubeadmConfig.spec.preKubeadmCommands` etc. . Example MD can be found [ubuntu2204.yaml](examples/machine_deployment/ubuntu2204.yaml).
+
 ## Compatibility
 
 ### Proxmox-VE REST API
diff --git a/api/v1beta1/cloudinit_types.go b/api/v1beta1/cloudinit_types.go
@@ -14,10 +14,10 @@ type User struct {
 	ChPasswd       ChPasswd     `yaml:"chpasswd,omitempty" json:"-"`
 	Users          []string     `yaml:"users,omitempty" json:"-"`
 	Password       string       `yaml:"password,omitempty" json:"password,omitempty"`
-	Packages       []string     `yaml:"packages,omitempty" json:"-"`
+	Packages       []string     `yaml:"packages,omitempty" json:"packages,omitempty"`
 	PackageUpgrade bool         `yaml:"package_upgrade,omitempty" json:"-"`
-	WriteFiles     []WriteFiles `yaml:"write_files,omitempty" json:"-"`
-	RunCmd         []string     `yaml:"runcmd,omitempty" json:"-"`
+	WriteFiles     []WriteFiles `yaml:"write_files,omitempty" json:"writeFiles,omitempty"`
+	RunCmd         []string     `yaml:"runcmd,omitempty" json:"runCmd,omitempty"`
 }
 
 type GrowPart struct {
@@ -31,8 +31,8 @@ type ChPasswd struct {
 }
 
 type WriteFiles struct {
-	Path        string `yaml:"path,omitempty" json:"-"`
-	Owner       string `yaml:"owner,omitempty" json:"-"`
-	Permissions string `yaml:"permissions,omitempty" json:"-"`
-	Content     string `yaml:"content,omitempty" json:"-"`
+	Path        string `yaml:"path,omitempty" json:"path,omitempty"`
+	Owner       string `yaml:"owner,omitempty" json:"owner,omitempty"`
+	Permissions string `yaml:"permissions,omitempty" json:"permissions,omitempty"`
+	Content     string `yaml:"content,omitempty" json:"content,omitempty"`
 }
diff --git a/cloud/scope/remote.go b/cloud/scope/remote.go
@@ -28,7 +28,7 @@ func NewSSHClient(addr, user, password string) (*SSHClient, error) {
 	return &SSHClient{conn, password}, nil
 }
 
-func (conn *SSHClient) RunCommand(cmd string, stdin ...string) (string, error) {
+func (conn *SSHClient) RunCommand(cmd string) (string, error) {
 	session, err := conn.NewSession()
 	if err != nil {
 		return "", err
diff --git a/cloud/services/compute/instance/cloudinit.go b/cloud/services/compute/instance/cloudinit.go
@@ -8,7 +8,6 @@ import (
 
 	infrav1 "github.com/sp-yduck/cluster-api-provider-proxmox/api/v1beta1"
 	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/cloudinit"
-	"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/scope"
 )
 
 const (
@@ -89,71 +88,10 @@ func userSnippetPath(vmName string) string {
 	return fmt.Sprintf(userSnippetPathFormat, vmName)
 }
 
-// DEPRECATED : cicustom should be set via API
-func ApplyCICustom(vmid int, vmName, storageName, ciType string, ssh scope.SSHClient) error {
-	if !cloudinit.IsValidType(ciType) {
-		return errors.Errorf("invalid cloud init type: %s", ciType)
-	}
-	cicustom := fmt.Sprintf("%s=%s:snippets/%s-%s.yml", ciType, storageName, vmName, ciType)
-	out, err := ssh.RunCommand(fmt.Sprintf("qm set %d --cicustom '%s'", vmid, cicustom))
-	if err != nil {
-		return errors.Errorf("ssh command error : %s : %v", out, err)
-	}
-	return nil
-}
-
-// to do : remove these cloud-config
 func baseUserData(vmName string) *infrav1.User {
 	return &infrav1.User{
-		GrowPart:       infrav1.GrowPart{Mode: "auto", Devices: []string{"/"}, IgnoreGrowrootDisabled: false},
-		HostName:       vmName,
-		ManageEtcHosts: true,
-		ChPasswd:       infrav1.ChPasswd{Expire: "False"},
-		Users:          []string{"default"},
-		Packages:       []string{"socat", "conntrack"},
-		PackageUpgrade: true,
-		WriteFiles: []infrav1.WriteFiles{
-			{
-				Path:        "/etc/modules-load.d/k8s.conf",
-				Owner:       "root:root",
-				Permissions: "0640",
-				Content:     "overlay\nbr_netfilter",
-			},
-			{
-				Path:        "/etc/sysctl.d/k8s.conf",
-				Owner:       "root:root",
-				Permissions: "0640",
-				Content: `net.bridge.bridge-nf-call-iptables  = 1
-net.bridge.bridge-nf-call-ip6tables = 1
-net.ipv4.ip_forward                 = 1`,
-			},
-		},
-		RunCmd: []string{
-			"modprobe overlay",
-			"modprobe br_netfilter",
-			"sysctl --system",
-			`mkdir -p /usr/local/bin`,
-			`curl -L "https://github.com/containerd/containerd/releases/download/v1.7.2/containerd-1.7.2-linux-amd64.tar.gz" | tar Cxvz "/usr/local"`,
-			`curl -L "https://raw.githubusercontent.com/containerd/containerd/main/containerd.service" -o /etc/systemd/system/containerd.service`,
-			"mkdir -p /etc/containerd",
-			"containerd config default > /etc/containerd/config.toml",
-			"sed 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml -i",
-			"systemctl daemon-reload",
-			"systemctl enable --now containerd",
-			"mkdir -p /usr/local/sbin",
-			`curl -L "https://github.com/opencontainers/runc/releases/download/v1.1.7/runc.amd64" -o /usr/local/sbin/runc`,
-			"chmod 755 /usr/local/sbin/runc",
-			"mkdir -p /opt/cni/bin",
-			`curl -L "https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz" | tar -C "/opt/cni/bin" -xz`,
-			`curl -L "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.27.0/crictl-v1.27.0-linux-amd64.tar.gz" | tar -C "/usr/local/bin" -xz`,
-			`curl -L --remote-name-all https://dl.k8s.io/release/v1.26.5/bin/linux/amd64/kubeadm -o /usr/local/bin/kubeadm`,
-			`chmod +x /usr/local/bin/kubeadm`,
-			`curl -L --remote-name-all https://dl.k8s.io/release/v1.26.5/bin/linux/amd64/kubelet -o /usr/local/bin/kubelet`,
-			`chmod +x /usr/local/bin/kubelet`,
-			`curl -sSL "https://raw.githubusercontent.com/kubernetes/release/v0.15.1/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service" | sed "s:/usr/bin:/usr/local/bin:g" | tee /etc/systemd/system/kubelet.service`,
-			`mkdir -p /etc/systemd/system/kubelet.service.d`,
-			`curl -sSL "https://raw.githubusercontent.com/kubernetes/release/v0.15.1/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf" | sed "s:/usr/bin:/usr/local/bin:g" | tee /etc/systemd/system/kubelet.service.d/10-kubeadm.conf`,
-			"systemctl enable kubelet.service",
-		},
+		HostName: vmName,
+		Packages: []string{"qemu-guest-agent"},
+		RunCmd:   []string{"systemctl start qemu-guest-agent"},
 	}
 }
diff --git a/cloud/services/compute/instance/image.go b/cloud/services/compute/instance/image.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"path"
+	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/sp-yduck/proxmox/pkg/service/node/vm"
@@ -53,7 +54,19 @@ func SetCloudImage(ctx context.Context, vmid int, storage infrav1.Storage, image
 		return errors.Errorf("failed to download image: %s : %v", out, err)
 	}
 
-	// to do: should confirm if the checksum matchies
+	// checksum
+	if image.Checksum != "" {
+		cscmd, err := findValidChecksumCommand(*image.ChecksumType)
+		if err != nil {
+			return err
+		}
+		cmd := fmt.Sprintf("%s --check -", cscmd)
+		stdin := fmt.Sprintf("%s %s", image.Checksum, rawImageFilePath)
+		out, err = ssh.RunWithStdin(cmd, stdin)
+		if err != nil {
+			return errors.Errorf("failed to confirm checksum: %s : %v", out, err)
+		}
+	}
 
 	destPath := fmt.Sprintf("%s/images/%d/vm-%d-disk-0.raw", storage.Path, vmid, vmid)
 	out, err = ssh.RunCommand(fmt.Sprintf("/usr/bin/qemu-img convert -O raw %s %s", rawImageFilePath, destPath))
@@ -62,3 +75,15 @@ func SetCloudImage(ctx context.Context, vmid int, storage infrav1.Storage, image
 	}
 	return nil
 }
+
+func findValidChecksumCommand(csType string) (string, error) {
+	csType = strings.ToLower(csType)
+	switch csType {
+	case "sha256", "sha256sum":
+		return "sha256sum", nil
+	case "md5", "md5sum":
+		return "md5sum", nil
+	default:
+		return "", errors.Errorf("checksum type %s is not supported", csType)
+	}
+}
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxmachines.yaml
@@ -65,10 +65,31 @@ spec:
                 properties:
                   user:
                     properties:
+                      packages:
+                        items:
+                          type: string
+                        type: array
                       password:
                         type: string
+                      runCmd:
+                        items:
+                          type: string
+                        type: array
                       user:
                         type: string
+                      writeFiles:
+                        items:
+                          properties:
+                            content:
+                              type: string
+                            owner:
+                              type: string
+                            path:
+                              type: string
+                            permissions:
+                              type: string
+                          type: object
+                        type: array
                     type: object
                 type: object
               failureDomain:
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxmachinetemplates.yaml
@@ -89,10 +89,31 @@ spec:
                         properties:
                           user:
                             properties:
+                              packages:
+                                items:
+                                  type: string
+                                type: array
                               password:
                                 type: string
+                              runCmd:
+                                items:
+                                  type: string
+                                type: array
                               user:
                                 type: string
+                              writeFiles:
+                                items:
+                                  properties:
+                                    content:
+                                      type: string
+                                    owner:
+                                      type: string
+                                    path:
+                                      type: string
+                                    permissions:
+                                      type: string
+                                  type: object
+                                type: array
                             type: object
                         type: object
                       failureDomain:
diff --git a/examples/machine_deployment/metal3-ubuntu2204-k8s127.yaml b/examples/machine_deployment/metal3-ubuntu2204-k8s127.yaml
@@ -0,0 +1,44 @@
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  name: ${CLUSTER_NAME}-md-2
+spec:
+  clusterName: ${CLUSTER_NAME}
+  replicas: 1
+  selector:
+    matchLabels: null
+  template:
+    spec:
+      bootstrap:
+        configRef:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: KubeadmConfigTemplate
+          name: ${CLUSTER_NAME}-md-2
+      clusterName: ${CLUSTER_NAME}
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: ProxmoxMachineTemplate
+        name: ${CLUSTER_NAME}-md-2
+      version: v1.27.1
+
+---
+
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-2
+spec:
+  template:
+    spec: {}
+
+---
+
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: ProxmoxMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-2
+spec:
+  template:
+    spec:
+      image:
+        url: https://artifactory.nordix.org/artifactory/metal3/images/k8s_v1.27.1/UBUNTU_22.04_NODE_IMAGE_K8S_v1.27.1.qcow2
diff --git a/examples/machine_deployment/ubuntu2204.yaml b/examples/machine_deployment/ubuntu2204.yaml
@@ -0,0 +1,89 @@
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  name: ${CLUSTER_NAME}-md-1
+spec:
+  clusterName: ${CLUSTER_NAME}
+  replicas: 1
+  selector:
+    matchLabels: null
+  template:
+    spec:
+      bootstrap:
+        configRef:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: KubeadmConfigTemplate
+          name: ${CLUSTER_NAME}-md-1
+      clusterName: ${CLUSTER_NAME}
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: ProxmoxMachineTemplate
+        name: ${CLUSTER_NAME}-md-1
+      version: v1.26.5
+
+---
+
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: KubeadmConfigTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-1
+spec:
+  template:
+    spec: {}
+
+---
+
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: ProxmoxMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-1
+spec:
+  template:
+    spec:
+      image:
+        url: https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64-disk-kvm.img
+        checksum: 86e996f35732d26cd8b0888c46c4309d4d3b04eb6980378cf82b4d3eb2796549
+        checksumType: sha256
+      cloudInit:
+        user:
+          packages:
+          - socat
+          - conntrack
+          writeFiles:
+          - path: /etc/modules-load.d/k8s.conf
+            owner: root:root
+            permissions: "0640"
+            content: overlay\nbr_netfilter
+          - path: /etc/sysctl.d/k8s.conf
+            owner: root:root
+            permissions: "0640"
+            content: |
+              net.bridge.bridge-nf-call-iptables  = 1
+              net.bridge.bridge-nf-call-ip6tables = 1
+              net.ipv4.ip_forward                 = 1
+          runCmd: 
+          - "modprobe overlay"
+          - "modprobe br_netfilter"
+          - "sysctl --system"
+          - "mkdir -p /usr/local/bin"
+          - curl -L "https://github.com/containerd/containerd/releases/download/v1.7.2/containerd-1.7.2-linux-amd64.tar.gz" | tar Cxvz "/usr/local"
+          - curl -L "https://raw.githubusercontent.com/containerd/containerd/main/containerd.service" -o /etc/systemd/system/containerd.service
+          - "mkdir -p /etc/containerd"
+          - "containerd config default > /etc/containerd/config.toml"
+          - "sed 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml -i"
+          - "systemctl daemon-reload"
+          - "systemctl enable --now containerd"
+          - "mkdir -p /usr/local/sbin"
+          - curl -L "https://github.com/opencontainers/runc/releases/download/v1.1.7/runc.amd64" -o /usr/local/sbin/runc
+          - "chmod 755 /usr/local/sbin/runc"
+          - "mkdir -p /opt/cni/bin"
+          - curl -L "https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz" | tar -C "/opt/cni/bin" -xz
+          - curl -L "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.27.0/crictl-v1.27.0-linux-amd64.tar.gz" | tar -C "/usr/local/bin" -xz
+          - curl -L --remote-name-all https://dl.k8s.io/release/v1.26.5/bin/linux/amd64/kubeadm -o /usr/local/bin/kubeadm
+          - chmod +x /usr/local/bin/kubeadm
+          - curl -L --remote-name-all https://dl.k8s.io/release/v1.26.5/bin/linux/amd64/kubelet -o /usr/local/bin/kubelet
+          - chmod +x /usr/local/bin/kubelet
+          - curl -sSL "https://raw.githubusercontent.com/kubernetes/release/v0.15.1/cmd/kubepkg/templates/latest/deb/kubelet/lib/systemd/system/kubelet.service" | sed "s:/usr/bin:/usr/local/bin:g" | tee /etc/systemd/system/kubelet.service
+          - mkdir -p /etc/systemd/system/kubelet.service.d
+          - curl -sSL "https://raw.githubusercontent.com/kubernetes/release/v0.15.1/cmd/kubepkg/templates/latest/deb/kubeadm/10-kubeadm.conf" | sed "s:/usr/bin:/usr/local/bin:g" | tee /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+          - "systemctl enable kubelet.service"
diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml

Original file line number	Diff line number	Diff line change
`@@ -28,7 +28,7 @@ func NewSSHClient(addr, user, password string) (*SSHClient, error) {`
`28`	`28`	`return &SSHClient{conn, password}, nil`
`29`	`29`	`}`
`30`	`30`
`31`		`-func (conn *SSHClient) RunCommand(cmd string, stdin ...string) (string, error) {`
	`31`	`+func (conn *SSHClient) RunCommand(cmd string) (string, error) {`
`32`	`32`	`session, err := conn.NewSession()`
`33`	`33`	`if err != nil {`
`34`	`34`	`return "", err`