kovagoadi/hunfabric
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

E2E tests for GCS

Browse Source
This commit is contained in:
Wiktor Niesiobędzki
2023-10-31 09:22:34 +00:00
parent 03bf0b15b3
commit 1fbd018f5f
11 changed files with 93 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
modules/gcs/README.md
View File

@@ -5,27 +5,55 @@
```hcl ```hcl
module "bucket" { module "bucket" {
source = "./fabric/modules/gcs" source = "./fabric/modules/gcs"
project_id = "myproject" project_id = var.project_id
prefix = "test" prefix = var.prefix
name = "my-bucket" name = "my-bucket"
versioning = true versioning = true
labels = { labels = {
cost-center = "devops" cost-center = "devops"
} }
} }
# tftest modules=1 resources=1 inventory=simple.yaml # tftest modules=1 resources=1 inventory=simple.yaml e2e
``` ```
### Example with Cloud KMS ### Example with Cloud KMS
```hcl ```hcl
module "project" {
source = "./fabric/modules/project"
name = var.project_id
project_create = false
}
module "kms" {
source = "./fabric/modules/kms"
project_id = var.project_id
keyring = {
location = "europe" # location of the KMS must match location of the bucket
name = "test"
}
keys = {
bucket_key = {
iam_bindings = {
bucket_key_iam = {
members = ["serviceAccount:${module.project.service_accounts.robots.storage}"]
role = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
}
}
}
}
}
module "bucket" { module "bucket" {
source = "./fabric/modules/gcs" source = "./fabric/modules/gcs"
project_id = "myproject" project_id = var.project_id
prefix = var.prefix
name = "my-bucket" name = "my-bucket"
encryption_key = "my-encryption-key" encryption_key = module.kms.keys.bucket_key.id
location = "EU"
} }
# tftest modules=1 resources=1 inventory=cmek.yaml
# tftest skip e2e
``` ```
### Example with retention policy and logging ### Example with retention policy and logging
@@ -33,7 +61,8 @@ module "bucket" {
```hcl ```hcl
module "bucket" { module "bucket" {
source = "./fabric/modules/gcs" source = "./fabric/modules/gcs"
project_id = "myproject" project_id = var.project_id
prefix = var.prefix
name = "my-bucket" name = "my-bucket"
retention_policy = { retention_policy = {
retention_period = 100 retention_period = 100
@@ -52,7 +81,8 @@ module "bucket" {
```hcl ```hcl
module "bucket" { module "bucket" {
source = "./fabric/modules/gcs" source = "./fabric/modules/gcs"
project_id = "myproject" project_id = var.project_id
prefix = var.prefix
name = "my-bucket" name = "my-bucket"
lifecycle_rules = { lifecycle_rules = {
lr-0 = { lr-0 = {
@@ -66,26 +96,33 @@ module "bucket" {
} }
} }
} }
# tftest modules=1 resources=1 inventory=lifecycle.yaml # tftest modules=1 resources=1 inventory=lifecycle.yaml e2e
``` ```
### Minimal example with GCS notifications ### Minimal example with GCS notifications
```hcl ```hcl
module "project" {
source = "./fabric/modules/project"
name = var.project_id
project_create = false
}
module "bucket-gcs-notification" { module "bucket-gcs-notification" {
source = "./fabric/modules/gcs" source = "./fabric/modules/gcs"
project_id = "myproject" project_id = var.project_id
prefix = var.prefix
name = "my-bucket" name = "my-bucket"
notification_config = { notification_config = {
enabled = true enabled = true
payload_format = "JSON_API_V1" payload_format = "JSON_API_V1"
sa_email = "service-<project-number>@gs-project-accounts.iam.gserviceaccount.com" # GCS SA email must be passed or fetched from projects module. sa_email = module.project.service_accounts.robots.storage
topic_name = "gcs-notification-topic" topic_name = "gcs-notification-topic"
event_types = ["OBJECT_FINALIZE"] event_types = ["OBJECT_FINALIZE"]
custom_attributes = {} custom_attributes = {}
} }
} }
# tftest modules=1 resources=4 inventory=notification.yaml # tftest skip e2e
``` ```
### Example with object upload ### Example with object upload
@@ -93,17 +130,18 @@ module "bucket-gcs-notification" {
```hcl ```hcl
module "bucket" { module "bucket" {
source = "./fabric/modules/gcs" source = "./fabric/modules/gcs"
project_id = "myproject" project_id = var.project_id
prefix = var.prefix
name = "my-bucket" name = "my-bucket"
objects_to_upload = { objects_to_upload = {
sample-data = { sample-data = {
name = "example-file.csv" name = "example-file.csv"
source = "data/example-file.csv" source = "assets/example-file.csv"
content_type = "text/csv" content_type = "text/csv"
} }
} }
} }
# tftest modules=1 resources=2 inventory=object-upload.yaml # tftest modules=1 resources=2 inventory=object-upload.yaml e2e
``` ```
### Examples of IAM ### Examples of IAM
@@ -111,24 +149,26 @@ module "bucket" {
```hcl ```hcl
module "bucket" { module "bucket" {
source = "./fabric/modules/gcs" source = "./fabric/modules/gcs"
project_id = "myproject" project_id = var.project_id
prefix = var.prefix
name = "my-bucket" name = "my-bucket"
iam = { iam = {
"roles/storage.admin" = ["group:storage@example.com"] "roles/storage.admin" = ["group:${var.group_email}"]
} }
} }
# tftest modules=1 resources=2 inventory=iam-authoritative.yaml # tftest modules=1 resources=2 inventory=iam-authoritative.yaml e2e
``` ```
```hcl ```hcl
module "bucket" { module "bucket" {
source = "./fabric/modules/gcs" source = "./fabric/modules/gcs"
project_id = "myproject" project_id = var.project_id
prefix = var.prefix
name = "my-bucket" name = "my-bucket"
iam_bindings = { iam_bindings = {
storage-admin-with-delegated_roles = { storage-admin-with-delegated_roles = {
role = "roles/storage.admin" role = "roles/storage.admin"
members = ["group:storage@example.com"] members = ["group:${var.group_email}"]
condition = { condition = {
title = "delegated-role-grants" title = "delegated-role-grants"
expression = format( expression = format(
@@ -144,18 +184,19 @@ module "bucket" {
} }
} }
} }
# tftest modules=1 resources=2 inventory=iam-bindings.yaml # tftest modules=1 resources=2 inventory=iam-bindings.yaml e2e
``` ```
```hcl ```hcl
module "bucket" { module "bucket" {
source = "./fabric/modules/gcs" source = "./fabric/modules/gcs"
project_id = "myproject" project_id = var.project_id
prefix = var.prefix
name = "my-bucket" name = "my-bucket"
iam_bindings_additive = { iam_bindings_additive = {
storage-admin-with-delegated_roles = { storage-admin-with-delegated_roles = {
role = "roles/storage.admin" role = "roles/storage.admin"
member = "group:storage@example.com" member = "group:${var.group_email}"
condition = { condition = {
title = "delegated-role-grants" title = "delegated-role-grants"
expression = format( expression = format(
@@ -171,7 +212,7 @@ module "bucket" {
} }
} }
} }
# tftest modules=1 resources=2 inventory=iam-bindings-additive.yaml # tftest modules=1 resources=2 inventory=iam-bindings-additive.yaml e2e
``` ```
<!-- BEGIN TFDOC --> <!-- BEGIN TFDOC -->
## Variables ## Variables

1
tests/modules/gcs/assets/example-file.csv Normal file
View File

@@ -0,0 +1 @@
example,file
1 example file

8
tests/modules/gcs/examples/cmek.yaml
View File

@@ -14,10 +14,10 @@
values: values:
module.bucket.google_storage_bucket.bucket: module.bucket.google_storage_bucket.bucket:
encryption: # encryption: __missing__
- default_kms_key_name: my-encryption-key # - default_kms_key_name:
name: my-bucket name: test-my-bucket
project: myproject project: project-id
counts: counts:
google_storage_bucket: 1 google_storage_bucket: 1

8
tests/modules/gcs/examples/iam-authoritative.yaml
View File

@@ -24,8 +24,8 @@ values:
lifecycle_rule: [] lifecycle_rule: []
location: EU location: EU
logging: [] logging: []
name: my-bucket name: test-my-bucket
project: myproject project: project-id
requester_pays: null requester_pays: null
retention_policy: [] retention_policy: []
storage_class: MULTI_REGIONAL storage_class: MULTI_REGIONAL
@@ -36,10 +36,10 @@ values:
autoclass: autoclass:
- enabled: false - enabled: false
module.bucket.google_storage_bucket_iam_binding.authoritative["roles/storage.admin"]: module.bucket.google_storage_bucket_iam_binding.authoritative["roles/storage.admin"]:
bucket: my-bucket bucket: test-my-bucket
condition: [] condition: []
members: members:
- group:storage@example.com - group:organization-admins@example.org
role: roles/storage.admin role: roles/storage.admin
counts: counts:

8
tests/modules/gcs/examples/iam-bindings-additive.yaml
View File

@@ -24,8 +24,8 @@ values:
lifecycle_rule: [] lifecycle_rule: []
location: EU location: EU
logging: [] logging: []
name: my-bucket name: test-my-bucket
project: myproject project: project-id
requester_pays: null requester_pays: null
retention_policy: [] retention_policy: []
storage_class: MULTI_REGIONAL storage_class: MULTI_REGIONAL
@@ -36,12 +36,12 @@ values:
autoclass: autoclass:
- enabled: false - enabled: false
module.bucket.google_storage_bucket_iam_member.bindings["storage-admin-with-delegated_roles"]: module.bucket.google_storage_bucket_iam_member.bindings["storage-admin-with-delegated_roles"]:
bucket: my-bucket bucket: test-my-bucket
condition: condition:
- description: null - description: null
expression: api.getAttribute('iam.googleapis.com/modifiedGrantsByRole', []).hasOnly(['roles/storage.objectAdmin','roles/storage.objectViewer']) expression: api.getAttribute('iam.googleapis.com/modifiedGrantsByRole', []).hasOnly(['roles/storage.objectAdmin','roles/storage.objectViewer'])
title: delegated-role-grants title: delegated-role-grants
member: group:storage@example.com member: group:organization-admins@example.org
role: roles/storage.admin role: roles/storage.admin
counts: counts:

8
tests/modules/gcs/examples/iam-bindings.yaml
View File

@@ -24,8 +24,8 @@ values:
lifecycle_rule: [] lifecycle_rule: []
location: EU location: EU
logging: [] logging: []
name: my-bucket name: test-my-bucket
project: myproject project: project-id
requester_pays: null requester_pays: null
retention_policy: [] retention_policy: []
storage_class: MULTI_REGIONAL storage_class: MULTI_REGIONAL
@@ -36,13 +36,13 @@ values:
autoclass: autoclass:
- enabled: false - enabled: false
module.bucket.google_storage_bucket_iam_binding.bindings["storage-admin-with-delegated_roles"]: module.bucket.google_storage_bucket_iam_binding.bindings["storage-admin-with-delegated_roles"]:
bucket: my-bucket bucket: test-my-bucket
condition: condition:
- description: null - description: null
expression: api.getAttribute('iam.googleapis.com/modifiedGrantsByRole', []).hasOnly(['roles/storage.objectAdmin','roles/storage.objectViewer']) expression: api.getAttribute('iam.googleapis.com/modifiedGrantsByRole', []).hasOnly(['roles/storage.objectAdmin','roles/storage.objectViewer'])
title: delegated-role-grants title: delegated-role-grants
members: members:
- group:storage@example.com - group:organization-admins@example.org
role: roles/storage.admin role: roles/storage.admin
counts: counts:

4
tests/modules/gcs/examples/lifecycle.yaml
View File

@@ -29,8 +29,8 @@ values:
matches_suffix: [] matches_suffix: []
noncurrent_time_before: '' noncurrent_time_before: ''
num_newer_versions: null num_newer_versions: null
name: my-bucket name: test-my-bucket
project: myproject project: project-id
counts: counts:
google_storage_bucket: 1 google_storage_bucket: 1

6
tests/modules/gcs/examples/notification.yaml
View File

@@ -16,10 +16,10 @@ values:
module.bucket-gcs-notification.google_pubsub_topic.topic[0]: {} module.bucket-gcs-notification.google_pubsub_topic.topic[0]: {}
module.bucket-gcs-notification.google_pubsub_topic_iam_binding.binding[0]: {} module.bucket-gcs-notification.google_pubsub_topic_iam_binding.binding[0]: {}
module.bucket-gcs-notification.google_storage_bucket.bucket: module.bucket-gcs-notification.google_storage_bucket.bucket:
name: my-bucket name: test-my-bucket
project: myproject project: project-id
module.bucket-gcs-notification.google_storage_notification.notification[0]: module.bucket-gcs-notification.google_storage_notification.notification[0]:
bucket: my-bucket bucket: test-my-bucket
event_types: event_types:
- OBJECT_FINALIZE - OBJECT_FINALIZE
payload_format: JSON_API_V1 payload_format: JSON_API_V1

6
tests/modules/gcs/examples/object-upload.yaml
View File

@@ -14,11 +14,11 @@
values: values:
module.bucket.google_storage_bucket.bucket: module.bucket.google_storage_bucket.bucket:
name: my-bucket name: test-my-bucket
project: myproject project: project-id
module.bucket.google_storage_bucket_object.objects["sample-data"]: module.bucket.google_storage_bucket_object.objects["sample-data"]:
name: example-file.csv name: example-file.csv
source: data/example-file.csv source: assets/example-file.csv
content_type: text/csv content_type: text/csv
counts: counts:

4
tests/modules/gcs/examples/retention-logging.yaml
View File

@@ -16,8 +16,8 @@ values:
module.bucket.google_storage_bucket.bucket: module.bucket.google_storage_bucket.bucket:
logging: logging:
- log_bucket: log-bucket - log_bucket: log-bucket
name: my-bucket name: test-my-bucket
project: myproject project: project-id
retention_policy: retention_policy:
- is_locked: true - is_locked: true
retention_period: 100 retention_period: 100

2
tests/modules/gcs/examples/simple.yaml
View File

@@ -26,7 +26,7 @@ values:
location: EU location: EU
logging: [] logging: []
name: test-my-bucket name: test-my-bucket
project: myproject project: project-id
requester_pays: null requester_pays: null
retention_policy: [] retention_policy: []
storage_class: MULTI_REGIONAL storage_class: MULTI_REGIONAL