dataset add storageSize

Signed-off-by: wangshulei098 <[email protected]>

api/v1alpha1/dataset_types.go

@@ -20,6 +20,7 @@ import (
 	"strings"
 
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	// "github.com/rook/rook/pkg/apis/rook.io/v1"
@@ -169,6 +170,10 @@ type DatasetSpec struct {
 	// SharedEncryptOptions is the encryptOption to all mount
 	// +optional
 	SharedEncryptOptions []EncryptOption `json:"sharedEncryptOptions,omitempty"`
+
+	// StorageCapacity is the storage size of generated PVC and PV
+	// +optional
+	StorageCapacity resource.Quantity `json:"storageCapacity,omitempty"`
 }
 
 // Runtime describes a runtime to be used to support dataset

charts/fluid/fluid/crds/data.fluid.io_datasets.yaml

@@ -333,6 +333,14 @@ spec:
                   type: string
                 description: SharedOptions is the options to all mount
                 type: object
+              storageCapacity:
+                anyOf:
+                - type: integer
+                - type: string
+                description: StorageCapacity is the storage size of generated PVC
+                  and PV
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
               tolerations:
                 description: If specified, the pod's tolerations.
                 items:

config/crd/bases/data.fluid.io_datasets.yaml

@@ -333,6 +333,14 @@ spec:
                   type: string
                 description: SharedOptions is the options to all mount
                 type: object
+              storageCapacity:
+                anyOf:
+                - type: integer
+                - type: string
+                description: StorageCapacity is the storage size of generated PVC
+                  and PV
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
               tolerations:
                 description: If specified, the pod's tolerations.
                 items:

pkg/ddc/efc/create_volume.go

@@ -26,7 +26,6 @@ import (
 	volumehelper "github.com/fluid-cloudnative/fluid/pkg/utils/dataset/volume"
 	"github.com/fluid-cloudnative/fluid/pkg/utils/kubeclient"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -82,6 +81,11 @@ func (e *EFCEngine) createPersistentVolumeForRuntime(runtime base.RuntimeInfoInt
 	if err != nil {
 		return err
 	}
+
+	storageCapacity, err := utils.GetStorageCapacityOfDataset(e.Client, runtime.GetName(), runtime.GetNamespace())
+	if err != nil {
+		return err
+	}
 
 	pvName := runtime.GetPersistentVolumeName()
 
@@ -103,8 +107,9 @@ func (e *EFCEngine) createPersistentVolumeForRuntime(runtime base.RuntimeInfoInt
 			Spec: corev1.PersistentVolumeSpec{
 				AccessModes: accessModes,
 				Capacity: corev1.ResourceList{
-					corev1.ResourceName(corev1.ResourceStorage): resource.MustParse("100Pi"),
+					corev1.ResourceName(corev1.ResourceStorage): storageCapacity,
 				},
+
 				StorageClassName: common.FluidStorageClass,
 				PersistentVolumeSource: corev1.PersistentVolumeSource{
 					CSI: &corev1.CSIPersistentVolumeSource{

pkg/utils/dataset.go

@@ -23,6 +23,7 @@ import (
 	datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1"
 	"github.com/fluid-cloudnative/fluid/pkg/common"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -71,6 +72,20 @@ func GetAccessModesOfDataset(client client.Client, name, namespace string) (acce
 
 }
 
+func GetStorageCapacityOfDataset(client client.Client, name, namespace string) (storageCapacity resource.Quantity, err error) {
+	dataset, err := GetDataset(client, name, namespace)
+	if err != nil {
+		return storageCapacity, err
+	}
+
+	storageCapacity = dataset.Spec.StorageCapacity
+	if storageCapacity.IsZero() {
+		storageCapacity = resource.MustParse("100Pi")
+	}
+
+	return storageCapacity, err
+}
+
 // IsTargetPathUnderFluidNativeMounts checks if targetPath is a subpath under some given native mount point.
 // We check this for the reason that native mount points need extra metadata sync alluxioOperations.
 func IsTargetPathUnderFluidNativeMounts(targetPath string, dataset datav1alpha1.Dataset) bool {

pkg/utils/dataset/volume/create.go

@@ -23,7 +23,6 @@ import (
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 
@@ -45,6 +44,11 @@ func CreatePersistentVolumeForRuntime(client client.Client,
 		return err
 	}
 
+	storageCapacity, err := utils.GetStorageCapacityOfDataset(client, runtime.GetName(), runtime.GetNamespace())
+	if err != nil {
+		return err
+	}
+
 	pvName := runtime.GetPersistentVolumeName()
 
 	found, err := kubeclient.IsPersistentVolumeExist(client, pvName, common.ExpectedFluidAnnotations)
@@ -65,7 +69,7 @@ func CreatePersistentVolumeForRuntime(client client.Client,
 			Spec: corev1.PersistentVolumeSpec{
 				AccessModes: accessModes,
 				Capacity: corev1.ResourceList{
-					corev1.ResourceName(corev1.ResourceStorage): resource.MustParse("100Pi"),
+					corev1.ResourceName(corev1.ResourceStorage): storageCapacity,
 				},
 				StorageClassName: common.FluidStorageClass,
 				PersistentVolumeSource: corev1.PersistentVolumeSource{
@@ -179,6 +183,11 @@ func CreatePersistentVolumeClaimForRuntime(client client.Client,
 		return err
 	}
 
+	storageCapacity, err := utils.GetStorageCapacityOfDataset(client, runtime.GetName(), runtime.GetNamespace())
+	if err != nil {
+		return err
+	}
+
 	found, err := kubeclient.IsPersistentVolumeClaimExist(client, runtime.GetName(), runtime.GetNamespace(), common.ExpectedFluidAnnotations)
 	if err != nil {
 		return err
@@ -204,7 +213,7 @@ func CreatePersistentVolumeClaimForRuntime(client client.Client,
 				AccessModes:      accessModes,
 				Resources: corev1.ResourceRequirements{
 					Requests: corev1.ResourceList{
-						corev1.ResourceName(corev1.ResourceStorage): resource.MustParse("100Pi"),
+						corev1.ResourceName(corev1.ResourceStorage): storageCapacity,
 					},
 				},
 			},

pkg/utils/dataset_test.go

@@ -23,6 +23,7 @@ import (
 	datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1"
 	"github.com/fluid-cloudnative/fluid/pkg/utils/fake"
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 )
@@ -174,6 +175,63 @@ func TestGetAccessModesOfDataset(t *testing.T) {
 	}
 }
 
+func TestGetStorageCapacityOfDataset(t *testing.T) {
+
+	testCases := map[string]struct {
+		name                string
+		getName             string
+		namespace           string
+		storageCapacity     resource.Quantity
+		wantStorageCapacity resource.Quantity
+		notFound            bool
+	}{
+		"test get dataset storage capacity case 1": {
+			name:                "dataset-1",
+			getName:             "dataset-1",
+			notFound:            false,
+			namespace:           "default",
+			storageCapacity:     resource.Quantity{},
+			wantStorageCapacity: resource.MustParse("100Pi"),
+		},
+		"test get dataset storage capacity case 2": {
+			name:                "dataset-1",
+			getName:             "dataset-1",
+			notFound:            false,
+			namespace:           "default",
+			storageCapacity:     resource.MustParse("1Gi"),
+			wantStorageCapacity: resource.MustParse("1Gi"),
+		},
+		"test get dataset storage capacity case 3": {
+			name:                "dataset-1",
+			getName:             "dataset-1-notexist",
+			notFound:            true,
+			namespace:           "default",
+			storageCapacity:     resource.Quantity{},
+			wantStorageCapacity: resource.Quantity{},
+		},
+	}
+
+	for k, item := range testCases {
+		dataset := mockDatasetWithStorageCapacity(item.name, item.namespace, item.storageCapacity)
+		s := runtime.NewScheme()
+		s.AddKnownTypes(datav1alpha1.GroupVersion, dataset)
+
+		fakeClient := fake.NewFakeClientWithScheme(s, dataset)
+
+		gotStorageCapacity, err := GetStorageCapacityOfDataset(fakeClient, item.getName, item.namespace)
+
+		if item.notFound {
+			if err == nil {
+				t.Errorf("%s check failure,want err but got nil", k)
+			}
+		} else {
+			if !reflect.DeepEqual(gotStorageCapacity, item.wantStorageCapacity) {
+				t.Errorf("%s check failure, want:%v,got:%v", k, item.wantStorageCapacity, gotStorageCapacity)
+			}
+		}
+	}
+}
+
 func TestIsTargetPathUnderFluidNativeMounts(t *testing.T) {
 	testCases := map[string]struct {
 		targetPath   string
@@ -315,6 +373,19 @@ func mockDatasetWithAccessModel(name, ns string, accessModel []v1.PersistentVolu
 	return dataset
 }
 
+func mockDatasetWithStorageCapacity(name, ns string, storageCapacity resource.Quantity) *datav1alpha1.Dataset {
+	dataset := &datav1alpha1.Dataset{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: ns,
+		},
+		Spec: datav1alpha1.DatasetSpec{
+			StorageCapacity: storageCapacity,
+		},
+	}
+	return dataset
+}
+
 func mockDatasetWithCondition(name, ns string, conditions []datav1alpha1.DatasetCondition) *datav1alpha1.Dataset {
 	dataset := &datav1alpha1.Dataset{
 		ObjectMeta: metav1.ObjectMeta{