Skip to content

Commit

Permalink
Remove Storage PipelineResources
Browse files Browse the repository at this point in the history
This commit removes the Storage Resources support.
This PR removes `github.com/tektoncd/pipeline/pkg/apis/resource/v1alpha1/storage`,
`pkg/artifacts` and the respective test cases.
  • Loading branch information
JeromeJu committed Feb 8, 2023
1 parent 0fc3b1a commit 52dec61
Show file tree
Hide file tree
Showing 30 changed files with 66 additions and 2,419 deletions.
96 changes: 0 additions & 96 deletions docs/resources.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,6 @@ For example:
- [Resource types](#resource-types)
- [Git Resource](#git-resource)
- [Image Resource](#image-resource)
- [Storage Resource](#storage-resource)
- [GCS Storage Resource](#gcs-storage-resource)
- [Why Aren't PipelineResources in Beta?](#why-aren-t-pipelineresources-in-beta)

## Syntax
Expand Down Expand Up @@ -540,100 +538,6 @@ status:
If the `index.json` file is not produced, the image digest will not be included
in the `taskRun` output.

### Storage Resource

The `storage` resource represents blob storage, that contains either an object
or directory. Adding the storage resource as an input to a `Task` will download
the blob and allow the `Task` to perform the required actions on the contents of
the blob.

Only blob storage type
[Google Cloud Storage](https://cloud.google.com/storage/)(gcs) is supported as
of now via [GCS storage resource](#gcs-storage-resource).

#### GCS Storage Resource

The `gcs` storage resource points to
[Google Cloud Storage](https://cloud.google.com/storage/) blob.

To create a GCS type of storage resource using the `PipelineResource` CRD:

```yaml
apiVersion: tekton.dev/v1alpha1
kind: PipelineResource
metadata:
name: wizzbang-storage
namespace: default
spec:
type: storage
params:
- name: type
value: gcs
- name: location
value: gs://some-bucket
- name: dir
value: "y" # This can have any value to be considered "true"
```

Params that can be added are the following:

1. `location`: represents the location of the blob storage.
1. `type`: represents the type of blob storage. For GCS storage resource this
value should be set to `gcs`.
1. `dir`: represents whether the blob storage is a directory or not. By default
a storage artifact is not considered a directory.

- If the artifact is a directory then `-r`(recursive) flag is used, to
copy all files under the source directory to a GCS bucket. Eg: `gsutil
cp -r source_dir/* gs://some-bucket`
- If an artifact is a single file like a zip or tar, then the copy will be
only 1 level deep(not recursive). It will not trigger a copy of sub
directories in the source directory. Eg: `gsutil cp source.tar
gs://some-bucket.tar`.

Private buckets can also be configured as storage resources. To access GCS
private buckets, service accounts with correct permissions are required. The
`secrets` field on the storage resource is used for configuring this
information. Below is an example on how to create a storage resource with a
service account.

1. Refer to the
[official documentation](https://cloud.google.com/compute/docs/access/service-accounts)
on how to create service accounts and configuring
[IAM permissions](https://cloud.google.com/storage/docs/access-control/iam-permissions)
to access buckets.

1. Create a Kubernetes secret from a downloaded service account json key

```bash
kubectl create secret generic bucket-sa --from-file=./service_account.json
```

1. To access the GCS private bucket environment variable
[`GOOGLE_APPLICATION_CREDENTIALS`](https://cloud.google.com/docs/authentication/production)
should be set, so apply the above created secret to the GCS storage resource
under the `fieldName` key.

```yaml
apiVersion: tekton.dev/v1alpha1
kind: PipelineResource
metadata:
name: wizzbang-storage
namespace: default
spec:
type: storage
params:
- name: type
value: gcs
- name: location
value: gs://some-private-bucket
- name: dir
value: "y"
secrets:
- fieldName: GOOGLE_APPLICATION_CREDENTIALS
secretName: bucket-sa
secretKey: service_account.json
```

--------------------------------------------------------------------------------

Expand Down
106 changes: 0 additions & 106 deletions examples/v1beta1/pipelineruns/output-pipelinerun.yaml

This file was deleted.

26 changes: 0 additions & 26 deletions examples/v1beta1/taskruns/gcs-resource.yaml

This file was deleted.

4 changes: 0 additions & 4 deletions pkg/apis/config/store_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,6 @@ import (
func TestStoreLoadWithContext(t *testing.T) {
defaultConfig := test.ConfigMapFromTestFile(t, "config-defaults")
featuresConfig := test.ConfigMapFromTestFile(t, "feature-flags-all-flags-set")
artifactBucketConfig := test.ConfigMapFromTestFile(t, "config-artifact-bucket")
artifactPVCConfig := test.ConfigMapFromTestFile(t, "config-artifact-pvc")
metricsConfig := test.ConfigMapFromTestFile(t, "config-observability")
trustedresourcesConfig := test.ConfigMapFromTestFile(t, "config-trusted-resources")
spireConfig := test.ConfigMapFromTestFile(t, "config-spire")
Expand All @@ -54,8 +52,6 @@ func TestStoreLoadWithContext(t *testing.T) {
store := config.NewStore(logtesting.TestLogger(t))
store.OnConfigChanged(defaultConfig)
store.OnConfigChanged(featuresConfig)
store.OnConfigChanged(artifactBucketConfig)
store.OnConfigChanged(artifactPVCConfig)
store.OnConfigChanged(metricsConfig)
store.OnConfigChanged(trustedresourcesConfig)
store.OnConfigChanged(spireConfig)
Expand Down
3 changes: 0 additions & 3 deletions pkg/apis/pipeline/v1beta1/resource_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,6 @@ const (
// PipelineResourceTypeGit indicates that this source is a GitHub repo.
PipelineResourceTypeGit PipelineResourceType = resource.PipelineResourceTypeGit

// PipelineResourceTypeStorage indicates that this source is a storage blob resource.
PipelineResourceTypeStorage PipelineResourceType = resource.PipelineResourceTypeStorage

// PipelineResourceTypeImage indicates that this source is a docker Image.
PipelineResourceTypeImage PipelineResourceType = resource.PipelineResourceTypeImage
)
Expand Down
3 changes: 0 additions & 3 deletions pkg/apis/resource/resource.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
resourcev1alpha1 "github.com/tektoncd/pipeline/pkg/apis/resource/v1alpha1"
"github.com/tektoncd/pipeline/pkg/apis/resource/v1alpha1/git"
"github.com/tektoncd/pipeline/pkg/apis/resource/v1alpha1/image"
"github.com/tektoncd/pipeline/pkg/apis/resource/v1alpha1/storage"
)

// FromType returns an instance of the correct PipelineResource object type which can be
Expand All @@ -36,8 +35,6 @@ func FromType(name string, r *resourcev1alpha1.PipelineResource, images pipeline
return git.NewResource(name, images.GitImage, r)
case resourcev1alpha1.PipelineResourceTypeImage:
return image.NewResource(name, r)
case resourcev1alpha1.PipelineResourceTypeStorage:
return storage.NewResource(name, images, r)
}
return nil, fmt.Errorf("%s is an invalid or unimplemented PipelineResource", r.Spec.Type)
}
8 changes: 2 additions & 6 deletions pkg/apis/resource/v1alpha1/pipeline_resource_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,18 +28,14 @@ type PipelineResourceType = string
var (
// AllowedOutputResources are the resource types that can be used as outputs
AllowedOutputResources = map[PipelineResourceType]bool{
PipelineResourceTypeStorage: true,
PipelineResourceTypeGit: true,
PipelineResourceTypeGit: true,
}
)

const (
// PipelineResourceTypeGit indicates that this source is a GitHub repo.
PipelineResourceTypeGit PipelineResourceType = "git"

// PipelineResourceTypeStorage indicates that this source is a storage blob resource.
PipelineResourceTypeStorage PipelineResourceType = "storage"

// PipelineResourceTypeImage indicates that this source is a docker Image.
PipelineResourceTypeImage PipelineResourceType = "image"

Expand All @@ -48,7 +44,7 @@ const (
)

// AllResourceTypes can be used for validation to check if a provided Resource type is one of the known types.
var AllResourceTypes = []PipelineResourceType{PipelineResourceTypeGit, PipelineResourceTypeStorage, PipelineResourceTypeImage}
var AllResourceTypes = []PipelineResourceType{PipelineResourceTypeGit, PipelineResourceTypeImage}

// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
Expand Down
23 changes: 0 additions & 23 deletions pkg/apis/resource/v1alpha1/pipelineresource_validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ package v1alpha1

import (
"context"
"strings"

"github.com/tektoncd/pipeline/pkg/apis/validate"
"k8s.io/apimachinery/pkg/api/equality"
Expand All @@ -43,28 +42,6 @@ func (rs *PipelineResourceSpec) Validate(ctx context.Context) *apis.FieldError {
if equality.Semantic.DeepEqual(rs, &PipelineResourceSpec{}) {
return apis.ErrMissingField("spec.type")
}
if rs.Type == PipelineResourceTypeStorage {
foundTypeParam := false
var location string
for _, param := range rs.Params {
switch {
case strings.EqualFold(param.Name, "type"):
if !AllowedStorageType(param.Value) {
return apis.ErrInvalidValue(param.Value, "spec.params.type")
}
foundTypeParam = true
case strings.EqualFold(param.Name, "Location"):
location = param.Value
}
}

if !foundTypeParam {
return apis.ErrMissingField("spec.params.type")
}
if location == "" {
return apis.ErrMissingField("spec.params.location")
}
}

for _, allowedType := range AllResourceTypes {
if allowedType == rs.Type {
Expand Down
Loading

0 comments on commit 52dec61

Please sign in to comment.