diff --git a/Dockerfile b/Dockerfile index 8ffbc85ca..050a3d733 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,12 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM registry.k8s.io/build-image/debian-base:bookworm-v1.0.6 +FROM debian:stable-slim ARG ARCH ARG binary=./bin/${ARCH}/nfsplugin COPY ${binary} /nfsplugin -RUN apt update && apt upgrade -y && apt-mark unhold libcap2 && clean-install ca-certificates mount nfs-common netbase +RUN apt update && apt upgrade -y && apt-mark unhold libcap2 && apt-get install -y --reinstall --purge ca-certificates mount nfs-common netbase krb5-user lsb-base bash -ENTRYPOINT ["/nfsplugin"] +RUN cat > /etc/default/nfs-common < /usr/local/bin/entry.sh <<'EOF' +#!/bin/sh +set -x + +if [ "$1" = "true" ]; then + shift 1 + service rpcbind start + service nfs-common start + sleep 5 +fi + +/nfsplugin $@ +EOF +RUN chmod +x /usr/local/bin/entry.sh + +ENTRYPOINT ["entry.sh"] diff --git a/Makefile b/Makefile index de5708af4..3956f6820 100644 --- a/Makefile +++ b/Makefile @@ -131,8 +131,9 @@ endif .PHONY: install-nfs-server install-nfs-server: kubectl apply -f ./deploy/example/nfs-provisioner/nfs-server.yaml + kubectl apply -f ./deploy/example/nfs-provisioner/nfs-krb-server.yaml kubectl delete secret mount-options -n default --ignore-not-found - kubectl create secret generic mount-options --from-literal mountOptions="nfsvers=4.1" -n default + kubectl create secret generic mount-options --from-literal mountOptions="nfsvers=4.1" --from-literal krb-pwd='password!' --from-file=krb5.conf=./test/krb5.conf -n default .PHONY: install-helm install-helm: diff --git a/charts/latest/csi-driver-nfs/templates/csi-nfs-controller.yaml b/charts/latest/csi-driver-nfs/templates/csi-nfs-controller.yaml index f4b9dd7c7..bf7ce9158 100644 --- a/charts/latest/csi-driver-nfs/templates/csi-nfs-controller.yaml +++ b/charts/latest/csi-driver-nfs/templates/csi-nfs-controller.yaml @@ -174,6 +174,7 @@ spec: allowPrivilegeEscalation: true imagePullPolicy: {{ .Values.image.nfs.pullPolicy }} args: + - 'true' # needed to distinguish controller from node - "--v={{ .Values.controller.logLevel }}" - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" diff --git a/deploy/csi-nfs-controller.yaml b/deploy/csi-nfs-controller.yaml index 234970d3c..79d43c6a7 100644 --- a/deploy/csi-nfs-controller.yaml +++ b/deploy/csi-nfs-controller.yaml @@ -157,6 +157,7 @@ spec: allowPrivilegeEscalation: true imagePullPolicy: IfNotPresent args: + - "true" - "-v=5" - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" diff --git a/deploy/example/nfs-provisioner/nfs-krb-server.yaml b/deploy/example/nfs-provisioner/nfs-krb-server.yaml new file mode 100644 index 000000000..5ef6b8503 --- /dev/null +++ b/deploy/example/nfs-provisioner/nfs-krb-server.yaml @@ -0,0 +1,93 @@ +--- +kind: Service +apiVersion: v1 +metadata: + name: nfs-krb-server + namespace: default + labels: + app: nfs-krb-server +spec: + type: ClusterIP # use "LoadBalancer" to get a public ip + selector: + app: nfs-krb-server + ports: + - name: tcp-2049 + port: 2049 + protocol: TCP + - name: udp-111 + port: 111 + protocol: UDP + - name: tcp-111 + port: 111 + protocol: TCP + - name: tcp-88 + port: 88 + protocol: TCP + - name: tcp-749 + port: 749 + protocol: TCP +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: nfs-krb-server + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: nfs-krb-server + template: + metadata: + name: nfs-krb-server + labels: + app: nfs-krb-server + spec: + nodeSelector: + kubernetes.io/os: linux + containers: + - name: nfs-krb-server + image: docker.io/thealmightydrawingtablet/nfs-krb:alpine + imagePullPolicy: Always + command: + - bash + - -c + - ./init.sh & sleep 10; ls /var/log; tail -f /var/log/messages /var/log/rpc-gssd.log + env: + - name: SHARED_DIRECTORY + value: /srv/shared + - name: NFS_KRB_REALM + value: NFS-KRB-SERVER.DEFAULT.SVC.CLUSTER.LOCAL + - name: NFS_KRB_PRINC + value: nfs/nfs-krb-server.default.svc.cluster.local + - name: NFS_KRB_PWD + valueFrom: + secretKeyRef: + name: mount-options + key: krb-pwd + volumeMounts: + - mountPath: /srv/shared + name: nfs-vol + securityContext: + privileged: true + ports: + - name: tcp-2049 + containerPort: 2049 + protocol: TCP + - name: udp-111 + containerPort: 111 + protocol: UDP + - name: tcp-111 + containerPort: 111 + protocol: TCP + - name: tcp-88 + containerPort: 88 + protocol: TCP + - name: tcp-749 + containerPort: 749 + protocol: TCP + volumes: + - name: nfs-vol + hostPath: + path: /srv/nfs-krb-vol # modify this to specify another path to store nfs share data + type: DirectoryOrCreate diff --git a/deploy/example/nfs-provisioner/nfs-server.yaml b/deploy/example/nfs-provisioner/nfs-server.yaml index 54c16a036..a4629d565 100644 --- a/deploy/example/nfs-provisioner/nfs-server.yaml +++ b/deploy/example/nfs-provisioner/nfs-server.yaml @@ -35,13 +35,13 @@ spec: app: nfs-server spec: nodeSelector: - "kubernetes.io/os": linux + kubernetes.io/os: linux containers: - name: nfs-server image: itsthenetwork/nfs-server-alpine:latest env: - name: SHARED_DIRECTORY - value: "/exports" + value: /exports volumeMounts: - mountPath: /exports name: nfs-vol diff --git a/deploy/example/nfs-provisioner/nginx-pod.yaml b/deploy/example/nfs-provisioner/nginx-pod.yaml index 6766ad060..71752aa55 100644 --- a/deploy/example/nfs-provisioner/nginx-pod.yaml +++ b/deploy/example/nfs-provisioner/nginx-pod.yaml @@ -9,11 +9,9 @@ metadata: spec: capacity: storage: 10Gi - accessModes: - - ReadWriteOnce + accessModes: [ReadWriteOnce] persistentVolumeReclaimPolicy: Delete - mountOptions: - - nfsvers=4.1 + mountOptions: [nfsvers=4.1] csi: driver: nfs.csi.k8s.io # volumeHandle format: {nfs-server-address}#{sub-dir-name}#{share-name} @@ -29,13 +27,12 @@ metadata: name: pvc-nginx namespace: default spec: - accessModes: - - ReadWriteOnce + accessModes: [ReadWriteOnce] resources: requests: storage: 10Gi volumeName: pv-nginx - storageClassName: "" + storageClassName: '' --- apiVersion: v1 kind: Pod diff --git a/pkg/nfs/controllerserver.go b/pkg/nfs/controllerserver.go index b9eb4f27d..808bead1b 100644 --- a/pkg/nfs/controllerserver.go +++ b/pkg/nfs/controllerserver.go @@ -140,6 +140,9 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol case pvcNamespaceKey: case pvcNameKey: case pvNameKey: + case paramKrbPrincipal: + case paramKrbPasswordSecret: + case paramKrbConf: // no op case mountPermissionsField: if v != "" { @@ -168,7 +171,7 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol volCap = req.GetVolumeCapabilities()[0] } // Mount nfs base share so we can create a subdirectory - if err = cs.internalMount(ctx, nfsVol, parameters, volCap); err != nil { + if err = cs.internalMount(ctx, nfsVol, parameters, volCap, req.GetSecrets()); err != nil { return nil, status.Errorf(codes.Internal, "failed to mount nfs server: %v", err) } defer func() { @@ -241,7 +244,7 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol } // mount nfs base share so we can delete the subdirectory volCap := getVolumeCapabilityFromSecret(volumeID, req.GetSecrets()) - if err = cs.internalMount(ctx, nfsVol, nil, volCap); err != nil { + if err = cs.internalMount(ctx, nfsVol, nil, volCap, req.GetSecrets()); err != nil { return nil, status.Errorf(codes.Internal, "failed to mount nfs server: %v", err) } defer func() { @@ -365,7 +368,7 @@ func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS } snapVol := volumeFromSnapshot(snapshot) volCap := getVolumeCapabilityFromSecret(req.GetSourceVolumeId(), req.GetSecrets()) - if err = cs.internalMount(ctx, snapVol, req.GetParameters(), volCap); err != nil { + if err = cs.internalMount(ctx, snapVol, req.GetParameters(), volCap, req.GetSecrets()); err != nil { return nil, status.Errorf(codes.Internal, "failed to mount snapshot nfs server: %v", err) } defer func() { @@ -381,7 +384,7 @@ func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS return nil, err } - if err = cs.internalMount(ctx, srcVol, req.GetParameters(), volCap); err != nil { + if err = cs.internalMount(ctx, srcVol, req.GetParameters(), volCap, req.GetSecrets()); err != nil { return nil, status.Errorf(codes.Internal, "failed to mount src nfs server: %v", err) } defer func() { @@ -436,7 +439,7 @@ func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS volCap := getVolumeCapabilityFromSecret(req.SnapshotId, req.GetSecrets()) vol := volumeFromSnapshot(snap) - if err = cs.internalMount(ctx, vol, nil, volCap); err != nil { + if err = cs.internalMount(ctx, vol, nil, volCap, req.GetSecrets()); err != nil { return nil, status.Errorf(codes.Internal, "failed to mount nfs server for snapshot deletion: %v", err) } defer func() { @@ -475,7 +478,7 @@ func (cs *ControllerServer) ControllerExpandVolume(_ context.Context, req *csi.C } // Mount nfs server at base-dir -func (cs *ControllerServer) internalMount(ctx context.Context, vol *nfsVolume, volumeContext map[string]string, volCap *csi.VolumeCapability) error { +func (cs *ControllerServer) internalMount(ctx context.Context, vol *nfsVolume, volumeContext map[string]string, volCap *csi.VolumeCapability, secrets map[string]string) error { if volCap == nil { volCap = &csi.VolumeCapability{ AccessType: &csi.VolumeCapability_Mount{ @@ -504,6 +507,7 @@ func (cs *ControllerServer) internalMount(ctx context.Context, vol *nfsVolume, v VolumeContext: volContext, VolumeCapability: volCap, VolumeId: vol.id, + Secrets: secrets, }) return err } @@ -533,7 +537,7 @@ func (cs *ControllerServer) copyFromSnapshot(ctx context.Context, req *csi.Creat volCap = req.GetVolumeCapabilities()[0] } - if err = cs.internalMount(ctx, snapVol, nil, volCap); err != nil { + if err = cs.internalMount(ctx, snapVol, nil, volCap, req.GetSecrets()); err != nil { return status.Errorf(codes.Internal, "failed to mount src nfs server for snapshot volume copy: %v", err) } defer func() { @@ -541,7 +545,7 @@ func (cs *ControllerServer) copyFromSnapshot(ctx context.Context, req *csi.Creat klog.Warningf("failed to unmount src nfs server after snapshot volume copy: %v", err) } }() - if err = cs.internalMount(ctx, dstVol, nil, volCap); err != nil { + if err = cs.internalMount(ctx, dstVol, nil, volCap, req.GetSecrets()); err != nil { return status.Errorf(codes.Internal, "failed to mount dst nfs server for snapshot volume copy: %v", err) } defer func() { @@ -582,7 +586,7 @@ func (cs *ControllerServer) copyFromVolume(ctx context.Context, req *csi.CreateV if len(req.GetVolumeCapabilities()) > 0 { volCap = req.GetVolumeCapabilities()[0] } - if err = cs.internalMount(ctx, srcVol, nil, volCap); err != nil { + if err = cs.internalMount(ctx, srcVol, nil, volCap, req.GetSecrets()); err != nil { return status.Errorf(codes.Internal, "failed to mount src nfs server: %v", err) } defer func() { @@ -590,7 +594,7 @@ func (cs *ControllerServer) copyFromVolume(ctx context.Context, req *csi.CreateV klog.Warningf("failed to unmount nfs server: %v", err) } }() - if err = cs.internalMount(ctx, dstVol, nil, volCap); err != nil { + if err = cs.internalMount(ctx, dstVol, nil, volCap, req.GetSecrets()); err != nil { return status.Errorf(codes.Internal, "failed to mount dst nfs server: %v", err) } defer func() { diff --git a/pkg/nfs/nfs.go b/pkg/nfs/nfs.go index 919d078d1..ef2dc3a72 100644 --- a/pkg/nfs/nfs.go +++ b/pkg/nfs/nfs.go @@ -73,8 +73,15 @@ const ( // The base directory must be a direct child of the root directory. // The root directory is omitted from the string, for example: // "base" instead of "/base" - paramShare = "share" - paramSubDir = "subdir" + paramShare = "share" + paramSubDir = "subdir" + // Kerberos principal to use when mounting with `-o sec=krb5*` + paramKrbPrincipal = "authprincipal" + // name of a secret containing the Kerberos password to use when authenticating + paramKrbPasswordSecret = "authpasswordsecret" + // name of a secret containing the contents of a krb5.conf file with + // realm and/or KDC information + paramKrbConf = "authkrbconf" paramOnDelete = "ondelete" mountOptionsField = "mountoptions" mountPermissionsField = "mountpermissions" diff --git a/pkg/nfs/nodeserver.go b/pkg/nfs/nodeserver.go index ac28dd22e..329ff575d 100644 --- a/pkg/nfs/nodeserver.go +++ b/pkg/nfs/nodeserver.go @@ -17,8 +17,10 @@ limitations under the License. package nfs import ( + "bytes" "fmt" "os" + "os/exec" "strconv" "strings" "time" @@ -42,7 +44,7 @@ type NodeServer struct { } // NodePublishVolume mount the volume -func (ns *NodeServer) NodePublishVolume(_ context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { +func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { volCap := req.GetVolumeCapability() if volCap == nil { return nil, status.Error(codes.InvalidArgument, "Volume capability missing in request") @@ -68,6 +70,7 @@ func (ns *NodeServer) NodePublishVolume(_ context.Context, req *csi.NodePublishV } var server, baseDir, subDir string + var krbPwd, krbPrinc, krbConf string subDirReplaceMap := map[string]string{} mountPermissions := ns.Driver.mountPermissions @@ -79,6 +82,16 @@ func (ns *NodeServer) NodePublishVolume(_ context.Context, req *csi.NodePublishV baseDir = v case paramSubDir: subDir = v + case paramKrbPrincipal: + krbPrinc = v + case paramKrbPasswordSecret: + if v != "" { + krbPwd = req.GetSecrets()[v] + } + case paramKrbConf: + if v != "" { + krbConf = req.GetSecrets()[v] + } case pvcNamespaceKey: subDirReplaceMap[pvcNamespaceMetadata] = v case pvcNameKey: @@ -130,8 +143,38 @@ func (ns *NodeServer) NodePublishVolume(_ context.Context, req *csi.NodePublishV return &csi.NodePublishVolumeResponse{}, nil } + if krbConf != "" { + if err = os.WriteFile("/etc/krb5.conf", []byte(krbConf), 0775); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + } klog.V(2).Infof("NodePublishVolume: volumeID(%v) source(%s) targetPath(%s) mountflags(%v)", volumeID, source, targetPath, mountOptions) execFunc := func() error { + if krbPrinc != "" && krbPwd != "" { + klog.V(3).Infof("Setting up kerberos auth with principal '%s' and password '%s'", krbPrinc, krbPwd) + _, err := os.Stat("/etc/krb5.keytab") + // initialize keytab if it doesn't exist + if err != nil && os.IsNotExist(err) { + cmd := exec.CommandContext(ctx, "ktutil") + cmd.Stdin = bytes.NewBufferString(fmt.Sprintf("addent -p %s -password -k 1 -f\n%s\nwkt /etc/krb5.keytab", krbPrinc, krbPwd)) + if err := cmd.Run(); err != nil { + klog.Errorf("error running 'ktutil': %+v", err) + return err + } + } + // obtain kerberos TGT + cmd := exec.CommandContext(ctx, "kinit", krbPrinc) + cmd.Stdin = bytes.NewBufferString(krbPwd + "\n") + if err := cmd.Run(); err != nil { + klog.Errorf("error running 'kinit': %+v", err) + return err + } + // initialize credentials from keytab + cmd = exec.CommandContext(ctx, "kinit", "-k", krbPrinc) + if err := cmd.Run(); err != nil { + klog.Warningf("error running 'kinit -k', but soldiering on: %+v", err) + } + } return ns.mounter.Mount(source, targetPath, "nfs", mountOptions) } timeoutFunc := func() error { return fmt.Errorf("time out") } diff --git a/test/e2e/dynamic_provisioning_test.go b/test/e2e/dynamic_provisioning_test.go index 542e449d1..c91752f70 100644 --- a/test/e2e/dynamic_provisioning_test.go +++ b/test/e2e/dynamic_provisioning_test.go @@ -52,6 +52,29 @@ var _ = ginkgo.Describe("Dynamic Provisioning", func() { }) testDriver = driver.InitNFSDriver() + ginkgo.It("should create a volume with kerberos auth", func(ctx ginkgo.SpecContext) { + pods := []testsuites.PodDetails{ + { + Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data", + Volumes: []testsuites.VolumeDetails{ + { + ClaimSize: "1Gi", + VolumeMount: testsuites.VolumeMountDetails{ + NameGenerate: "test-volume-", + MountPathGenerate: "/mnt/test-", + }, + MountOptions: []string{"sec=krb5", "noresvport", "nfsvers=4"}, + }, + }, + }, + } + test := testsuites.DynamicallyProvisionedVolumeWithKerberosAuth{ + Pods: pods, + StorageClassParameters: krbStorageClassParameters, + Driver: testDriver, + } + test.Run(ctx, cs, ns) + }) ginkgo.It("should create a volume on demand with mount options", func(ctx ginkgo.SpecContext) { pods := []testsuites.PodDetails{ { diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 906320270..056f1e9cf 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -92,6 +92,17 @@ var ( "mountPermissions": "0755", "onDelete": "archive", } + krbStorageClassParameters = map[string]string{ + "server": "nfs-krb-server.default.svc.cluster.local", + "share": "/srv/shared", + "csi.storage.k8s.io/provisioner-secret-namespace": "default", + "csi.storage.k8s.io/provisioner-secret-name": "mount-options", + "mountPermissions": "0755", + "authKrbConf": "krb5.conf", + "authPasswordSecret": "krb-pwd", + "authPrincipal": "nfs/nfs-krb-server.default.svc.cluster.local@NFS-KRB-SERVER.DEFAULT.SVC.CLUSTER.LOCAL", + } + controllerServer *nfs.ControllerServer ) diff --git a/test/e2e/testsuites/dynamically_provisioned_krb_auth_volume.go b/test/e2e/testsuites/dynamically_provisioned_krb_auth_volume.go new file mode 100644 index 000000000..aa03fa493 --- /dev/null +++ b/test/e2e/testsuites/dynamically_provisioned_krb_auth_volume.go @@ -0,0 +1,46 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testsuites + +import ( + "context" + + "github.com/kubernetes-csi/csi-driver-nfs/test/e2e/driver" + "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" + clientset "k8s.io/client-go/kubernetes" +) + +type DynamicallyProvisionedVolumeWithKerberosAuth struct { + Driver driver.DynamicPVTestDriver + Pods []PodDetails + StorageClassParameters map[string]string +} + +func (t *DynamicallyProvisionedVolumeWithKerberosAuth) Run(ctx context.Context, client clientset.Interface, namespace *v1.Namespace) { + for _, pod := range t.Pods { + tpod, cleanup := pod.SetupWithDynamicVolumes(ctx, client, namespace, t.Driver, t.StorageClassParameters) + for i := range cleanup { + defer cleanup[i](ctx) + } + ginkgo.By("deploying the pod") + tpod.Create(ctx) + defer tpod.Cleanup(ctx) + ginkgo.By("checking that the pods command exits with no error") + tpod.WaitForSuccess(ctx) + } +} diff --git a/test/krb5.conf b/test/krb5.conf new file mode 100644 index 000000000..1f0cd43de --- /dev/null +++ b/test/krb5.conf @@ -0,0 +1,13 @@ +[libdefaults] +default_realm = NFS-KRB-SERVER.DEFAULT.SVC.CLUSTER.LOCAL + +[realms] +NFS-KRB-SERVER.DEFAULT.SVC.CLUSTER.LOCAL = { + kdc = nfs-krb-server.default.svc.cluster.local + admin_server = nfs-krb-server.default.svc.cluster.local +} + +[logging] +kdc = FILE:/var/log/krb5kdc.log +admin_server = FILE:/var/log/kadmin.log +default = FILE:/var/log/krb5lib.log