Merge pull request #54 from ZeroMagic/e2e

test: add e2e test framework

Author: andyzhangx

Gopkg.lock

@@ -56,7 +56,7 @@
 
 [[constraint]]
   name = "k8s.io/kubernetes"
-  version = "v1.15.0"
+  version = "=v1.15.0"
 
 [[override]]
   name = "gopkg.in/fsnotify.v1"
@@ -101,3 +101,7 @@
 [[override]]
   name = "github.com/Azure/azure-sdk-for-go"
   version = "v21.3.0"
+
+[[override]]
+  name = "github.com/kubernetes-csi/external-snapshotter"
+  version = "=v1.0.1"

Gopkg.toml

@@ -33,6 +33,8 @@ integration-test:
 	test/integration/run-tests-all-clouds.sh
 sanity-test:
 	test/sanity/run-tests-all-clouds.sh
+e2e-test:
+	test/e2e/run-test.sh
 blobfuse:
 	if [ ! -d ./vendor ]; then dep ensure -vendor-only; fi
 	CGO_ENABLED=0 GOOS=linux go build -a -ldflags ${LDFLAGS} -o _output/blobfuseplugin ./pkg/blobfuseplugin

Makefile

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -uo pipefail
+
+kubectl apply -f deploy/crd-csi-driver-registry.yaml
+kubectl apply -f deploy/crd-csi-node-info.yaml
+kubectl apply -f deploy/rbac-csi-blobfuse-controller.yaml
+kubectl apply -f deploy/csi-blobfuse-controller.yaml
+kubectl apply -f deploy/csi-blobfuse-node.yaml

deploy/install-driver.sh

@@ -0,0 +1,10 @@
+## End to End Test
+
+## Run E2E tests Locally
+### Prerequisite
+Make sure a kubernetes cluster is set up and kubeconfig is under $HOME/.kube/config
+
+### Run test
+```
+make e2e-test
+```

test/e2e/README.md

@@ -0,0 +1,92 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package driver
+
+import (
+	"fmt"
+
+	blobfuse "github.com/csi-driver/blobfuse-csi-driver/pkg/blobfuse"
+	"github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1alpha1"
+	v1 "k8s.io/api/core/v1"
+	storagev1 "k8s.io/api/storage/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// Implement DynamicPVTestDriver interface
+type blobFuseCSIDriver struct {
+	driverName string
+}
+
+// InitBlobFuseCSIDriver returns blobFuseCSIDriver that implemnts DynamicPVTestDriver interface
+func InitBlobFuseCSIDriver() PVTestDriver {
+	return &blobFuseCSIDriver{
+		driverName: blobfuse.DriverName,
+	}
+}
+
+func (d *blobFuseCSIDriver) GetDynamicProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass {
+	provisioner := d.driverName
+	generateName := fmt.Sprintf("%s-%s-dynamic-sc-", namespace, provisioner)
+	return getStorageClass(generateName, provisioner, parameters, mountOptions, reclaimPolicy, bindingMode, nil)
+}
+
+func (d *blobFuseCSIDriver) GetVolumeSnapshotClass(namespace string) *v1alpha1.VolumeSnapshotClass {
+	provisioner := d.driverName
+	generateName := fmt.Sprintf("%s-%s-dynamic-sc-", namespace, provisioner)
+	return getVolumeSnapshotClass(generateName, provisioner)
+}
+
+func (d *blobFuseCSIDriver) GetPersistentVolume(volumeID string, fsType string, size string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, namespace string) *v1.PersistentVolume {
+	provisioner := d.driverName
+	generateName := fmt.Sprintf("%s-%s-preprovsioned-pv-", namespace, provisioner)
+	// Default to Retain ReclaimPolicy for pre-provisioned volumes
+	pvReclaimPolicy := v1.PersistentVolumeReclaimRetain
+	if reclaimPolicy != nil {
+		pvReclaimPolicy = *reclaimPolicy
+	}
+	return &v1.PersistentVolume{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: generateName,
+			Namespace:    namespace,
+			// TODO remove if https://github.com/kubernetes-csi/external-provisioner/issues/202 is fixed
+			Annotations: map[string]string{
+				"pv.kubernetes.io/provisioned-by": provisioner,
+			},
+		},
+		Spec: v1.PersistentVolumeSpec{
+			AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
+			Capacity: v1.ResourceList{
+				v1.ResourceName(v1.ResourceStorage): resource.MustParse(size),
+			},
+			PersistentVolumeReclaimPolicy: pvReclaimPolicy,
+			PersistentVolumeSource: v1.PersistentVolumeSource{
+				CSI: &v1.CSIPersistentVolumeSource{
+					Driver:       provisioner,
+					VolumeHandle: volumeID,
+					FSType:       fsType,
+				},
+			},
+		},
+	}
+}
+
+func GetParameters() map[string]string {
+	return map[string]string{
+		"skuName": "Standard_LRS",
+	}
+}

test/e2e/driver/blobfuse_csi_driver.go

@@ -0,0 +1,95 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package driver
+
+import (
+	"github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1alpha1"
+	v1 "k8s.io/api/core/v1"
+	storagev1 "k8s.io/api/storage/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	VolumeSnapshotClassKind = "VolumeSnapshotClass"
+	SnapshotAPIVersion      = "snapshot.storage.k8s.io/v1alpha1"
+)
+
+type PVTestDriver interface {
+	DynamicPVTestDriver
+	PreProvisionedVolumeTestDriver
+	VolumeSnapshotTestDriver
+}
+
+// DynamicPVTestDriver represents an interface for a CSI driver that supports DynamicPV
+type DynamicPVTestDriver interface {
+	// GetDynamicProvisionStorageClass returns a StorageClass dynamic provision Persistent Volume
+	GetDynamicProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass
+}
+
+// PreProvisionedVolumeTestDriver represents an interface for a CSI driver that supports pre-provisioned volume
+type PreProvisionedVolumeTestDriver interface {
+	// GetPersistentVolume returns a PersistentVolume with pre-provisioned volumeHandle
+	GetPersistentVolume(volumeID string, fsType string, size string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, namespace string) *v1.PersistentVolume
+}
+
+type VolumeSnapshotTestDriver interface {
+	GetVolumeSnapshotClass(namespace string) *v1alpha1.VolumeSnapshotClass
+}
+
+func getStorageClass(
+	generateName string,
+	provisioner string,
+	parameters map[string]string,
+	mountOptions []string,
+	reclaimPolicy *v1.PersistentVolumeReclaimPolicy,
+	bindingMode *storagev1.VolumeBindingMode,
+	allowedTopologies []v1.TopologySelectorTerm,
+) *storagev1.StorageClass {
+	if reclaimPolicy == nil {
+		defaultReclaimPolicy := v1.PersistentVolumeReclaimDelete
+		reclaimPolicy = &defaultReclaimPolicy
+	}
+	if bindingMode == nil {
+		defaultBindingMode := storagev1.VolumeBindingImmediate
+		bindingMode = &defaultBindingMode
+	}
+	return &storagev1.StorageClass{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: generateName,
+		},
+		Provisioner:       provisioner,
+		Parameters:        parameters,
+		MountOptions:      mountOptions,
+		ReclaimPolicy:     reclaimPolicy,
+		VolumeBindingMode: bindingMode,
+		AllowedTopologies: allowedTopologies,
+	}
+}
+
+func getVolumeSnapshotClass(generateName string, provisioner string) *v1alpha1.VolumeSnapshotClass {
+	return &v1alpha1.VolumeSnapshotClass{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       VolumeSnapshotClassKind,
+			APIVersion: SnapshotAPIVersion,
+		},
+
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: generateName,
+		},
+		Snapshotter: provisioner,
+	}
+}

test/e2e/driver/driver.go

@@ -0,0 +1,76 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/csi-driver/blobfuse-csi-driver/pkg/blobfuse"
+	"github.com/csi-driver/blobfuse-csi-driver/test/e2e/driver"
+	"github.com/csi-driver/blobfuse-csi-driver/test/e2e/testsuites"
+	. "github.com/onsi/ginkgo"
+	v1 "k8s.io/api/core/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+var _ = Describe("Dynamic Provisioning", func() {
+	f := framework.NewDefaultFramework("blobfuse")
+
+	var (
+		cs         clientset.Interface
+		ns         *v1.Namespace
+		testDriver driver.PVTestDriver
+	)
+
+	nodeid := os.Getenv("nodeid")
+	blobfuseDriver := blobfuse.NewDriver(nodeid)
+	endpoint := "unix:///tmp/csi.sock"
+
+	go func() {
+		blobfuseDriver.Run(endpoint)
+	}()
+
+	BeforeEach(func() {
+		cs = f.ClientSet
+		ns = f.Namespace
+	})
+
+	testDriver = driver.InitBlobFuseCSIDriver()
+	It(fmt.Sprintf("should create a volume on demand"), func() {
+		pods := []testsuites.PodDetails{
+			{
+				Cmd: "echo 'hello world' > /mnt/test-1/data && grep 'hello world' /mnt/test-1/data",
+				Volumes: []testsuites.VolumeDetails{
+					{
+						ClaimSize: "10Gi",
+						VolumeMount: testsuites.VolumeMountDetails{
+							NameGenerate:      "test-volume-",
+							MountPathGenerate: "/mnt/test-",
+						},
+					},
+				},
+			},
+		}
+		test := testsuites.DynamicallyProvisionedCmdVolumeTest{
+			CSIDriver: testDriver,
+			Pods:      pods,
+		}
+		test.Run(cs, ns)
+	})
+})

test/e2e/dynamic_provisioning.go

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -uo pipefail
+
+kubectl get daemonsets csi-blobfuse-node -n kube-system; checkBlobFuseDriver=$?;
+if [ $checkBlobFuseDriver -ne 0 ]; then
+    echo "BlobFuse csi driver daemonset not found";
+    echo "Installing BlobFuse csi driver";
+    deploy/install-driver.sh
+    echo "BlobFuse csi driver installed";    
+fi
+
+# Fetching ginkgo for running the test
+GO111MODULE=off go get -u github.com/onsi/ginkgo/ginkgo
+# Exporting KUBECONFIG path
+export KUBECONFIG=$HOME/.kube/config
+# Running the e2e test
+ginkgo test/e2e
+# Checking for test status
+TEST_PASS=$?
+if [[ $TEST_PASS -ne 0 ]]; then
+    exit 1
+fi