Skip to content

Commit 7bb55d1

Browse files
authored
Merge pull request #7 from sp-yduck/v0.2.0
V0.2.0
2 parents 9cae206 + a717c3c commit 7bb55d1

22 files changed

+389
-185
lines changed

README.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,14 @@ metadata:
6060
type: Opaque
6161
```
6262
63+
3. Check your Cluster & Machines !!
64+
65+
Once CAPP reconciled your `ProxmoxCluster`/`ProxmoxMachine`, you can see `READY=true` for `ProxmoxCluster` and `STATUS=running` for `ProxmoxMachine`.
66+
67+
![kubectl-get-proxmox-cluster](./logos/k-get-proxmoxcluster.PNG)
68+
69+
![kubectl-get-proxmox-machine](./logos/k-get-proxmoxmachine.PNG)
70+
6371
## Fetures
6472

6573
- No need to prepare vm templates. You can specify any vm image in `ProxmoxMachine.Spec.Image`.

api/v1beta1/proxmoxcluster_types.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,10 @@ type ProxmoxClusterStatus struct {
5959

6060
//+kubebuilder:object:root=true
6161
//+kubebuilder:subresource:status
62-
// +kubebuilder:printcolumn:name="ControlPlaneEndpoint",type=string,JSONPath=`.spec.controlPlaneEndpoint.host`
63-
// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.ready`
62+
// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.ready",description="Cluster infrastructure is ready for ProxmoxMachine"
63+
// +kubebuilder:printcolumn:name="Proxmox-Server",type="string",JSONPath=".spec.serverRef.endpoint",description="Server is the address of the Proxmox API endpoint."
64+
// +kubebuilder:printcolumn:name="ControlPlane",type="string",JSONPath=".spec.controlPlaneEndpoint.host",description="kube-apiserver Endpoint"
65+
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of Machine"
6466

6567
// ProxmoxCluster is the Schema for the proxmoxclusters API
6668
type ProxmoxCluster struct {

api/v1beta1/proxmoxmachine_types.go

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@ type ProxmoxMachineSpec struct {
4242
// +optional
4343
Node string `json:"node,omitempty"`
4444

45+
// VMID is proxmox qemu's id
46+
// +optional
47+
VMID *int `json:"vmID,omitempty"`
48+
4549
// Image is the image to be provisioned
4650
Image Image `json:"image"`
4751

@@ -74,7 +78,7 @@ type ProxmoxMachineStatus struct {
7478
// FailureMessage
7579
FailureMessage *string `json:"failureMessage,omitempty"`
7680

77-
// Addresses contains the AWS instance associated addresses.
81+
// Addresses
7882
Addresses []clusterv1.MachineAddress `json:"addresses,omitempty"`
7983

8084
// Conditions
@@ -87,9 +91,12 @@ type ProxmoxMachineStatus struct {
8791

8892
//+kubebuilder:object:root=true
8993
//+kubebuilder:subresource:status
90-
// +kubebuilder:printcolumn:name="Addresses",type=string,JSONPath=`.status.addresses`
94+
// +kubebuilder:printcolumn:name="Cluster",type="string",JSONPath=".metadata.labels.cluster\\.x-k8s\\.io/cluster-name",description="Cluster to which this VSphereMachine belongs"
95+
// +kubebuilder:printcolumn:name="Machine",type="string",JSONPath=".metadata.ownerReferences[?(@.kind==\"Machine\")].name",description="Machine object which owns with this ProxmoxMachine",priority=1
96+
// +kubebuilder:printcolumn:name="vmid",type=string,JSONPath=`.spec.vmID`,priority=1
9197
// +kubebuilder:printcolumn:name="ProviderID",type=string,JSONPath=`.spec.providerID`
92-
// +kubebuilder:printcolumn:name="InstanceStatus",type=string,JSONPath=`.status.instanceStatus`
98+
// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.instanceStatus`
99+
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since creation of Machine"
93100

94101
// ProxmoxMachine is the Schema for the proxmoxmachines API
95102
type ProxmoxMachine struct {

api/v1beta1/zz_generated.deepcopy.go

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

cloud/interfaces.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ type MachineGetter interface {
5353
// IsControlPlane() bool
5454
// ControlPlaneGroupName() string
5555
NodeName() string
56-
GetInstanceID() *string
56+
GetBiosUUID() *string
5757
GetImage() infrav1.Image
5858
GetProviderID() string
5959
GetBootstrapData() (string, error)
@@ -62,13 +62,15 @@ type MachineGetter interface {
6262
GetCloudInit() infrav1.CloudInit
6363
GetNetwork() infrav1.Network
6464
GetHardware() infrav1.Hardware
65+
GetVMID() *int
6566
}
6667

6768
// MachineSetter is an interface which can set machine information.
6869
type MachineSetter interface {
69-
SetProviderID(node string, vmid int) error
70+
SetProviderID(uuid string) error
7071
SetInstanceStatus(v infrav1.InstanceStatus)
7172
SetNodeName(name string)
73+
SetVMID(vmid int)
7274
// SetFailureMessage(v error)
7375
// SetFailureReason(v capierrors.MachineStatusError)
7476
// SetAnnotation(key, value string)

cloud/providerid/providerid.go

Lines changed: 16 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -2,50 +2,41 @@ package providerid
22

33
import (
44
"fmt"
5-
"path"
6-
"strconv"
75

86
"github.com/pkg/errors"
97
)
108

11-
const Prefix = "proxmox://"
9+
const (
10+
Prefix = "proxmox://"
11+
UUIDFormat = `[a-f\d]{8}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{12}`
12+
)
1213

1314
type ProviderID interface {
14-
Node() string
15-
VMID() int
15+
UUID() string
1616
fmt.Stringer
1717
}
1818

1919
type providerID struct {
20-
// proxmox node name
21-
node string
22-
// proxmox vmid
23-
vmid int
20+
uuid string
2421
}
2522

26-
func New(node string, vmid int) (ProviderID, error) {
27-
if node == "" {
28-
return nil, errors.New("location required for provider id")
29-
}
30-
if vmid == 0 {
31-
return nil, errors.New("vmid required for provider id")
23+
func New(uuid string) (ProviderID, error) {
24+
if uuid == "" {
25+
return nil, errors.New("uuid is required for provider id")
3226
}
3327

28+
// to do: validate uuid
29+
3430
return &providerID{
35-
node: node,
36-
vmid: vmid,
31+
uuid: uuid,
3732
}, nil
3833
}
3934

40-
func (p *providerID) Node() string {
41-
return p.node
42-
}
43-
44-
func (p *providerID) VMID() int {
45-
return p.vmid
35+
func (p *providerID) UUID() string {
36+
return p.uuid
4637
}
4738

4839
func (p *providerID) String() string {
49-
// provider ID : proxmox://<node name>/<vmid>
50-
return Prefix + path.Join(p.node, strconv.Itoa(p.vmid))
40+
// provider ID : proxmox://<bios-uuid>
41+
return Prefix + p.uuid
5142
}

cloud/scope/machine.go

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -142,12 +142,11 @@ func (m *MachineScope) SetInstanceStatus(v infrav1.InstanceStatus) {
142142
m.ProxmoxMachine.Status.InstanceStatus = &v
143143
}
144144

145-
func (m *MachineScope) GetInstanceID() *string {
145+
func (m *MachineScope) GetBiosUUID() *string {
146146
parsed, err := noderefutil.NewProviderID(m.GetProviderID())
147147
if err != nil {
148148
return nil
149149
}
150-
// instance id == vmid
151150
return pointer.StringPtr(parsed.ID())
152151
}
153152

@@ -158,6 +157,10 @@ func (m *MachineScope) GetProviderID() string {
158157
return ""
159158
}
160159

160+
func (m *MachineScope) GetVMID() *int {
161+
return m.ProxmoxMachine.Spec.VMID
162+
}
163+
161164
func (m *MachineScope) GetImage() infrav1.Image {
162165
return m.ProxmoxMachine.Spec.Image
163166
}
@@ -182,15 +185,19 @@ func (m *MachineScope) GetHardware() infrav1.Hardware {
182185
}
183186

184187
// SetProviderID sets the ProxmoxMachine providerID in spec.
185-
func (m *MachineScope) SetProviderID(node string, vmid int) error {
186-
providerid, err := providerid.New(node, vmid)
188+
func (m *MachineScope) SetProviderID(uuid string) error {
189+
providerid, err := providerid.New(uuid)
187190
if err != nil {
188191
return err
189192
}
190193
m.ProxmoxMachine.Spec.ProviderID = pointer.StringPtr(providerid.String())
191194
return nil
192195
}
193196

197+
func (m *MachineScope) SetVMID(vmid int) {
198+
m.ProxmoxMachine.Spec.VMID = &vmid
199+
}
200+
194201
func (m *MachineScope) SetReady() {
195202
m.ProxmoxMachine.Status.Ready = true
196203
}

cloud/services/compute/instance/cloudinit.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"fmt"
55

66
"github.com/pkg/errors"
7+
"k8s.io/klog/v2"
78

89
infrav1 "github.com/sp-yduck/cluster-api-provider-proxmox/api/v1beta1"
910
"github.com/sp-yduck/cluster-api-provider-proxmox/cloud/cloudinit"
@@ -43,6 +44,8 @@ func reconcileCloudInitUser(vmid int, vmName, storageName, bootstrap string, con
4344
return err
4445
}
4546

47+
klog.Info(configYaml)
48+
4649
// to do: should be set via API
4750
// to do: storage path
4851
out, err := ssh.RunWithStdin(fmt.Sprintf("tee /var/lib/vz/%s/snippets/%s-user.yml", storageName, vmName), configYaml)
@@ -101,7 +104,7 @@ net.ipv4.ip_forward = 1`,
101104
`curl -L "https://raw.githubusercontent.com/containerd/containerd/main/containerd.service" -o /etc/systemd/system/containerd.service`,
102105
"mkdir -p /etc/containerd",
103106
"containerd config default > /etc/containerd/config.toml",
104-
"sed 's/SystemdCgroup = false/SystemdCgroup = true/g /etc/containerd/config.toml",
107+
"sed 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml -i",
105108
"systemctl daemon-reload",
106109
"systemctl enable --now containerd",
107110
"mkdir -p /usr/local/sbin",
Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
package instance
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"math/rand"
7+
"net/url"
8+
"strings"
9+
"time"
10+
11+
"github.com/pkg/errors"
12+
"github.com/sp-yduck/proxmox/pkg/api"
13+
"github.com/sp-yduck/proxmox/pkg/service/node"
14+
"github.com/sp-yduck/proxmox/pkg/service/node/vm"
15+
"sigs.k8s.io/controller-runtime/pkg/log"
16+
17+
infrav1 "github.com/sp-yduck/cluster-api-provider-proxmox/api/v1beta1"
18+
)
19+
20+
func (s *Service) reconcileQEMU(ctx context.Context) (*vm.VirtualMachine, error) {
21+
log := log.FromContext(ctx)
22+
log.Info("Reconciling QEMU")
23+
24+
nodeName := s.scope.NodeName()
25+
vmid := s.scope.GetVMID()
26+
vm, err := s.getQEMU(nodeName, vmid)
27+
if err == nil { // if vm is found, return it
28+
return vm, nil
29+
}
30+
if !IsNotFound(err) {
31+
log.Error(err, fmt.Sprintf("failed to get vm: node=%s,vmid=%d", nodeName, *vmid))
32+
return nil, err
33+
}
34+
35+
// no vm found, create new one
36+
return s.createQEMU(ctx, nodeName, vmid)
37+
}
38+
39+
func (s *Service) getQEMU(nodeName string, vmid *int) (*vm.VirtualMachine, error) {
40+
if vmid != nil && nodeName != "" {
41+
node, err := s.GetNode(nodeName)
42+
if err != nil {
43+
return nil, err
44+
}
45+
return node.VirtualMachine(*vmid)
46+
}
47+
return nil, api.ErrNotFound
48+
}
49+
50+
func (s *Service) createQEMU(ctx context.Context, nodeName string, vmid *int) (*vm.VirtualMachine, error) {
51+
log := log.FromContext(ctx)
52+
53+
var node *node.Node
54+
var err error
55+
56+
// get node
57+
if nodeName != "" {
58+
node, err = s.GetNode(nodeName)
59+
if err != nil {
60+
log.Error(err, fmt.Sprintf("failed to get node %s", nodeName))
61+
return nil, err
62+
}
63+
} else {
64+
// temp solution
65+
node, err = s.GetRandomNode()
66+
if err != nil {
67+
log.Error(err, "failed to get random node")
68+
return nil, err
69+
}
70+
s.scope.SetNodeName(node.Node)
71+
}
72+
73+
// (for multiple node proxmox cluster support)
74+
// to do : set ssh client for specific node
75+
76+
// if vmid is empty, generate new vmid
77+
if vmid == nil {
78+
nextid, err := s.GetNextID()
79+
if err != nil {
80+
log.Error(err, "failed to get available vmid")
81+
return nil, err
82+
}
83+
vmid = &nextid
84+
}
85+
86+
// create vm
87+
vmoption := generateVMOptions(s.scope.Name(), s.scope.GetStorage().Name, s.scope.GetNetwork(), s.scope.GetHardware())
88+
vm, err := node.CreateVirtualMachine(*vmid, vmoption)
89+
if err != nil {
90+
log.Error(err, "failed to create virtual machine")
91+
return nil, err
92+
}
93+
s.scope.SetVMID(*vmid)
94+
return vm, nil
95+
}
96+
97+
func (s *Service) GetNextID() (int, error) {
98+
return s.client.NextID()
99+
}
100+
101+
func (s *Service) GetNodes() ([]*node.Node, error) {
102+
return s.client.Nodes()
103+
}
104+
105+
func (s *Service) GetNode(name string) (*node.Node, error) {
106+
return s.client.Node(name)
107+
}
108+
109+
// GetRandomNode returns a node chosen randomly
110+
func (s *Service) GetRandomNode() (*node.Node, error) {
111+
nodes, err := s.GetNodes()
112+
if err != nil {
113+
return nil, err
114+
}
115+
if len(nodes) <= 0 {
116+
return nil, errors.Errorf("no nodes found")
117+
}
118+
src := rand.NewSource(time.Now().Unix())
119+
r := rand.New(src)
120+
return nodes[r.Intn(len(nodes))], nil
121+
}
122+
123+
func generateVMOptions(vmName, storageName string, network infrav1.Network, hardware infrav1.Hardware) vm.VirtualMachineCreateOptions {
124+
vmoptions := vm.VirtualMachineCreateOptions{
125+
Agent: "enabled=1",
126+
Cores: hardware.CPU,
127+
Memory: hardware.Memory,
128+
Name: vmName,
129+
NameServer: network.NameServer,
130+
Boot: "order=scsi0",
131+
Ide: vm.Ide{Ide2: fmt.Sprintf("file=%s:cloudinit,media=cdrom", storageName)},
132+
CiCustom: fmt.Sprintf("user=%s:snippets/%s-user.yml", storageName, vmName),
133+
IPConfig: vm.IPConfig{IPConfig0: network.IPConfig.String()},
134+
OSType: vm.L26,
135+
Net: vm.Net{Net0: "model=virtio,bridge=vmbr0,firewall=1"},
136+
Scsi: vm.Scsi{Scsi0: fmt.Sprintf("file=%s:8", storageName)},
137+
ScsiHw: vm.VirtioScsiPci,
138+
SearchDomain: network.SearchDomain,
139+
Serial: vm.Serial{Serial0: "socket"},
140+
VGA: "serial0",
141+
}
142+
return vmoptions
143+
}
144+
145+
// URL encodes the ssh keys
146+
func sshKeyUrlEncode(keys string) (encodedKeys string) {
147+
encodedKeys = url.PathEscape(keys + "\n")
148+
encodedKeys = strings.Replace(encodedKeys, "+", "%2B", -1)
149+
encodedKeys = strings.Replace(encodedKeys, "@", "%40", -1)
150+
encodedKeys = strings.Replace(encodedKeys, "=", "%3D", -1)
151+
return
152+
}

0 commit comments

Comments
 (0)