diff --git a/api/v1alpha1/dataset_types.go b/api/v1alpha1/dataset_types.go index 91524bc89e1..431b4923c51 100644 --- a/api/v1alpha1/dataset_types.go +++ b/api/v1alpha1/dataset_types.go @@ -20,6 +20,7 @@ import ( "strings" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" // "github.com/rook/rook/pkg/apis/rook.io/v1" @@ -169,6 +170,10 @@ type DatasetSpec struct { // SharedEncryptOptions is the encryptOption to all mount // +optional SharedEncryptOptions []EncryptOption `json:"sharedEncryptOptions,omitempty"` + + // StorageCapacity is the storage size of generated PVC and PV + // +optional + StorageCapacity resource.Quantity `json:"storageCapacity,omitempty"` } // Runtime describes a runtime to be used to support dataset diff --git a/api/v1alpha1/openapi_generated.go b/api/v1alpha1/openapi_generated.go index ee6da1a58a5..79c802b9b48 100644 --- a/api/v1alpha1/openapi_generated.go +++ b/api/v1alpha1/openapi_generated.go @@ -2034,11 +2034,18 @@ func schema_fluid_cloudnative_fluid_api_v1alpha1_DatasetSpec(ref common.Referenc }, }, }, + "storageCapacity": { + SchemaProps: spec.SchemaProps{ + Description: "StorageCapacity is the storage size of generated PVC and PV", + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, }, }, }, Dependencies: []string{ - "github.com/fluid-cloudnative/fluid/api/v1alpha1.CacheableNodeAffinity", "github.com/fluid-cloudnative/fluid/api/v1alpha1.DataRestoreLocation", "github.com/fluid-cloudnative/fluid/api/v1alpha1.EncryptOption", "github.com/fluid-cloudnative/fluid/api/v1alpha1.Mount", "github.com/fluid-cloudnative/fluid/api/v1alpha1.Runtime", "github.com/fluid-cloudnative/fluid/api/v1alpha1.User", "k8s.io/api/core/v1.Toleration"}, + "github.com/fluid-cloudnative/fluid/api/v1alpha1.CacheableNodeAffinity", "github.com/fluid-cloudnative/fluid/api/v1alpha1.DataRestoreLocation", "github.com/fluid-cloudnative/fluid/api/v1alpha1.EncryptOption", "github.com/fluid-cloudnative/fluid/api/v1alpha1.Mount", "github.com/fluid-cloudnative/fluid/api/v1alpha1.Runtime", "github.com/fluid-cloudnative/fluid/api/v1alpha1.User", "k8s.io/api/core/v1.Toleration", "k8s.io/apimachinery/pkg/api/resource.Quantity"}, } } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index e1aae2528e6..3e407a63264 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -950,6 +950,7 @@ func (in *DatasetSpec) DeepCopyInto(out *DatasetSpec) { *out = make([]EncryptOption, len(*in)) copy(*out, *in) } + out.StorageCapacity = in.StorageCapacity.DeepCopy() } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DatasetSpec. diff --git a/charts/fluid/fluid/crds/data.fluid.io_datasets.yaml b/charts/fluid/fluid/crds/data.fluid.io_datasets.yaml index f8b91dbbad5..deb9e910d9a 100644 --- a/charts/fluid/fluid/crds/data.fluid.io_datasets.yaml +++ b/charts/fluid/fluid/crds/data.fluid.io_datasets.yaml @@ -333,6 +333,14 @@ spec: type: string description: SharedOptions is the options to all mount type: object + storageCapacity: + anyOf: + - type: integer + - type: string + description: StorageCapacity is the storage size of generated PVC + and PV + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true tolerations: description: If specified, the pod's tolerations. items: diff --git a/config/crd/bases/data.fluid.io_datasets.yaml b/config/crd/bases/data.fluid.io_datasets.yaml index f8b91dbbad5..deb9e910d9a 100644 --- a/config/crd/bases/data.fluid.io_datasets.yaml +++ b/config/crd/bases/data.fluid.io_datasets.yaml @@ -333,6 +333,14 @@ spec: type: string description: SharedOptions is the options to all mount type: object + storageCapacity: + anyOf: + - type: integer + - type: string + description: StorageCapacity is the storage size of generated PVC + and PV + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true tolerations: description: If specified, the pod's tolerations. items: diff --git a/pkg/ddc/efc/create_volume.go b/pkg/ddc/efc/create_volume.go index b1a8cba0b58..827746d3d21 100644 --- a/pkg/ddc/efc/create_volume.go +++ b/pkg/ddc/efc/create_volume.go @@ -26,7 +26,6 @@ import ( volumehelper "github.com/fluid-cloudnative/fluid/pkg/utils/dataset/volume" "github.com/fluid-cloudnative/fluid/pkg/utils/kubeclient" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -82,6 +81,11 @@ func (e *EFCEngine) createPersistentVolumeForRuntime(runtime base.RuntimeInfoInt if err != nil { return err } + + storageCapacity, err := utils.GetStorageCapacityOfDataset(e.Client, runtime.GetName(), runtime.GetNamespace()) + if err != nil { + return err + } pvName := runtime.GetPersistentVolumeName() @@ -103,8 +107,9 @@ func (e *EFCEngine) createPersistentVolumeForRuntime(runtime base.RuntimeInfoInt Spec: corev1.PersistentVolumeSpec{ AccessModes: accessModes, Capacity: corev1.ResourceList{ - corev1.ResourceName(corev1.ResourceStorage): resource.MustParse("100Pi"), + corev1.ResourceName(corev1.ResourceStorage): storageCapacity, }, + StorageClassName: common.FluidStorageClass, PersistentVolumeSource: corev1.PersistentVolumeSource{ CSI: &corev1.CSIPersistentVolumeSource{ diff --git a/pkg/utils/dataset.go b/pkg/utils/dataset.go index 02ad375b781..36bf74f0d4b 100644 --- a/pkg/utils/dataset.go +++ b/pkg/utils/dataset.go @@ -23,6 +23,7 @@ import ( datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1" "github.com/fluid-cloudnative/fluid/pkg/common" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" @@ -71,6 +72,20 @@ func GetAccessModesOfDataset(client client.Client, name, namespace string) (acce } +func GetStorageCapacityOfDataset(client client.Client, name, namespace string) (storageCapacity resource.Quantity, err error) { + dataset, err := GetDataset(client, name, namespace) + if err != nil { + return storageCapacity, err + } + + storageCapacity = dataset.Spec.StorageCapacity + if storageCapacity.IsZero() { + storageCapacity = resource.MustParse("100Pi") + } + + return storageCapacity, err +} + // IsTargetPathUnderFluidNativeMounts checks if targetPath is a subpath under some given native mount point. // We check this for the reason that native mount points need extra metadata sync alluxioOperations. func IsTargetPathUnderFluidNativeMounts(targetPath string, dataset datav1alpha1.Dataset) bool { diff --git a/pkg/utils/dataset/volume/create.go b/pkg/utils/dataset/volume/create.go index 4ea848b3030..5201817c869 100644 --- a/pkg/utils/dataset/volume/create.go +++ b/pkg/utils/dataset/volume/create.go @@ -23,7 +23,6 @@ import ( "github.com/go-logr/logr" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -45,6 +44,11 @@ func CreatePersistentVolumeForRuntime(client client.Client, return err } + storageCapacity, err := utils.GetStorageCapacityOfDataset(client, runtime.GetName(), runtime.GetNamespace()) + if err != nil { + return err + } + pvName := runtime.GetPersistentVolumeName() found, err := kubeclient.IsPersistentVolumeExist(client, pvName, common.ExpectedFluidAnnotations) @@ -65,7 +69,7 @@ func CreatePersistentVolumeForRuntime(client client.Client, Spec: corev1.PersistentVolumeSpec{ AccessModes: accessModes, Capacity: corev1.ResourceList{ - corev1.ResourceName(corev1.ResourceStorage): resource.MustParse("100Pi"), + corev1.ResourceName(corev1.ResourceStorage): storageCapacity, }, StorageClassName: common.FluidStorageClass, PersistentVolumeSource: corev1.PersistentVolumeSource{ @@ -179,6 +183,11 @@ func CreatePersistentVolumeClaimForRuntime(client client.Client, return err } + storageCapacity, err := utils.GetStorageCapacityOfDataset(client, runtime.GetName(), runtime.GetNamespace()) + if err != nil { + return err + } + found, err := kubeclient.IsPersistentVolumeClaimExist(client, runtime.GetName(), runtime.GetNamespace(), common.ExpectedFluidAnnotations) if err != nil { return err @@ -204,7 +213,7 @@ func CreatePersistentVolumeClaimForRuntime(client client.Client, AccessModes: accessModes, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceName(corev1.ResourceStorage): resource.MustParse("100Pi"), + corev1.ResourceName(corev1.ResourceStorage): storageCapacity, }, }, }, diff --git a/pkg/utils/dataset_test.go b/pkg/utils/dataset_test.go index eae518b36c0..7ecc5298892 100644 --- a/pkg/utils/dataset_test.go +++ b/pkg/utils/dataset_test.go @@ -23,6 +23,7 @@ import ( datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1" "github.com/fluid-cloudnative/fluid/pkg/utils/fake" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -174,6 +175,63 @@ func TestGetAccessModesOfDataset(t *testing.T) { } } +func TestGetStorageCapacityOfDataset(t *testing.T) { + + testCases := map[string]struct { + name string + getName string + namespace string + storageCapacity resource.Quantity + wantStorageCapacity resource.Quantity + notFound bool + }{ + "test get dataset storage capacity case 1": { + name: "dataset-1", + getName: "dataset-1", + notFound: false, + namespace: "default", + storageCapacity: resource.Quantity{}, + wantStorageCapacity: resource.MustParse("100Pi"), + }, + "test get dataset storage capacity case 2": { + name: "dataset-1", + getName: "dataset-1", + notFound: false, + namespace: "default", + storageCapacity: resource.MustParse("1Gi"), + wantStorageCapacity: resource.MustParse("1Gi"), + }, + "test get dataset storage capacity case 3": { + name: "dataset-1", + getName: "dataset-1-notexist", + notFound: true, + namespace: "default", + storageCapacity: resource.Quantity{}, + wantStorageCapacity: resource.Quantity{}, + }, + } + + for k, item := range testCases { + dataset := mockDatasetWithStorageCapacity(item.name, item.namespace, item.storageCapacity) + s := runtime.NewScheme() + s.AddKnownTypes(datav1alpha1.GroupVersion, dataset) + + fakeClient := fake.NewFakeClientWithScheme(s, dataset) + + gotStorageCapacity, err := GetStorageCapacityOfDataset(fakeClient, item.getName, item.namespace) + + if item.notFound { + if err == nil { + t.Errorf("%s check failure,want err but got nil", k) + } + } else { + if !reflect.DeepEqual(gotStorageCapacity, item.wantStorageCapacity) { + t.Errorf("%s check failure, want:%v,got:%v", k, item.wantStorageCapacity, gotStorageCapacity) + } + } + } +} + func TestIsTargetPathUnderFluidNativeMounts(t *testing.T) { testCases := map[string]struct { targetPath string @@ -315,6 +373,19 @@ func mockDatasetWithAccessModel(name, ns string, accessModel []v1.PersistentVolu return dataset } +func mockDatasetWithStorageCapacity(name, ns string, storageCapacity resource.Quantity) *datav1alpha1.Dataset { + dataset := &datav1alpha1.Dataset{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + Spec: datav1alpha1.DatasetSpec{ + StorageCapacity: storageCapacity, + }, + } + return dataset +} + func mockDatasetWithCondition(name, ns string, conditions []datav1alpha1.DatasetCondition) *datav1alpha1.Dataset { dataset := &datav1alpha1.Dataset{ ObjectMeta: metav1.ObjectMeta{