kovagoadi/hunfabric
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support for cloud logging buckets

Browse Source
This commit is contained in:
Julio Castillo
2021-03-03 14:19:08 +01:00
parent 0f469a22a1
commit ad68fc4dfa
23 changed files with 847 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/modules/folder/fixture/variables.tf
View File

@@ -61,6 +61,7 @@ variable "logging_sinks" {
filter = string
iam = bool
include_children = bool
exclusions = map(string)
}))
default = {}
}

183
tests/modules/folder/test_plan_logging.py
View File

@@ -18,18 +18,19 @@ import pytest
from collections import Counter
FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture')
FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixture")
def test_sinks(plan_runner):
"Test folder-level sinks."
logging_sinks = """ {
"Test folder-level sinks."
logging_sinks = """ {
warning = {
type = "gcs"
destination = "mybucket"
filter = "severity=WARNING"
iam = true
include_children = true
exclusions = {}
}
info = {
type = "bigquery"
@@ -37,6 +38,7 @@ def test_sinks(plan_runner):
filter = "severity=INFO"
iam = true
include_children = true
exclusions = {}
}
notice = {
type = "pubsub"
@@ -44,72 +46,123 @@ def test_sinks(plan_runner):
filter = "severity=NOTICE"
iam = true
include_children = false
exclusions = {}
}
debug = {
type = "logging"
destination = "projects/myproject/locations/global/buckets/mybucket"
filter = "severity=DEBUG"
iam = true
include_children = false
exclusions = {
no-compute = "logName:compute"
no-container = "logName:container"
}
}
}
"""
_, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks)
assert len(resources) == 7
_, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks)
assert len(resources) == 8
resource_types = Counter([r['type'] for r in resources])
assert resource_types == {
'google_bigquery_dataset_iam_binding': 1,
'google_folder': 1,
'google_logging_folder_sink': 3,
'google_pubsub_topic_iam_binding': 1,
'google_storage_bucket_iam_binding': 1
}
sinks = [r for r in resources
if r['type'] == 'google_logging_folder_sink']
assert sorted([r['index'] for r in sinks]) == [
'info',
'notice',
'warning',
]
values = [(r['index'], r['values']['filter'], r['values']['destination'],
r['values']['include_children'])
for r in sinks]
assert sorted(values) == [
('info',
'severity=INFO',
'bigquery.googleapis.com/projects/myproject/datasets/mydataset',
True),
('notice',
'severity=NOTICE',
'pubsub.googleapis.com/projects/myproject/topics/mytopic',
False),
('warning', 'severity=WARNING', 'storage.googleapis.com/mybucket', True)]
resource_types = Counter([r["type"] for r in resources])
assert resource_types == {
"google_bigquery_dataset_iam_binding": 1,
"google_folder": 1,
"google_logging_folder_sink": 4,
"google_pubsub_topic_iam_binding": 1,
"google_storage_bucket_iam_binding": 1,
}
sinks = [r for r in resources if r["type"] == "google_logging_folder_sink"]
assert sorted([r["index"] for r in sinks]) == [
"debug",
"info",
"notice",
"warning",
]
values = [
(
r["index"],
r["values"]["filter"],
r["values"]["destination"],
r["values"]["include_children"],
)
for r in sinks
]
assert sorted(values) == [
(
"debug",
"severity=DEBUG",
"logging.googleapis.com/projects/myproject/locations/global/buckets/mybucket",
False,
),
(
"info",
"severity=INFO",
"bigquery.googleapis.com/projects/myproject/datasets/mydataset",
True,
),
(
"notice",
"severity=NOTICE",
"pubsub.googleapis.com/projects/myproject/topics/mytopic",
False,
),
("warning", "severity=WARNING", "storage.googleapis.com/mybucket", True),
]
bindings = [r for r in resources if "binding" in r["type"]]
values = [(r["index"], r["type"], r["values"]["role"]) for r in bindings]
assert sorted(values) == [
("info", "google_bigquery_dataset_iam_binding", "roles/bigquery.dataEditor"),
("notice", "google_pubsub_topic_iam_binding", "roles/pubsub.publisher"),
("warning", "google_storage_bucket_iam_binding", "roles/storage.objectCreator"),
]
exclusions = [(r["index"], r["values"]["exclusions"]) for r in sinks]
assert sorted(exclusions) == [
(
"debug",
[
{
"description": None,
"disabled": False,
"filter": "logName:compute",
"name": "no-compute",
},
{
"description": None,
"disabled": False,
"filter": "logName:container",
"name": "no-container",
},
],
),
("info", []),
("notice", []),
("warning", []),
]
bindings = [r for r in resources
if 'binding' in r['type']]
values = [(r['index'], r['type'], r['values']['role'])
for r in bindings]
assert sorted(values) == [
('info', 'google_bigquery_dataset_iam_binding', 'roles/bigquery.dataEditor'),
('notice', 'google_pubsub_topic_iam_binding', 'roles/pubsub.publisher'),
('warning', 'google_storage_bucket_iam_binding', 'roles/storage.objectCreator')
]
def test_exclusions(plan_runner):
"Test folder-level logging exclusions."
logging_exclusions = (
'{'
'exclusion1 = "resource.type=gce_instance", '
'exclusion2 = "severity=NOTICE", '
'}'
)
_, resources = plan_runner(FIXTURES_DIR,
logging_exclusions=logging_exclusions)
assert len(resources) == 3
exclusions = [r for r in resources
if r['type'] == 'google_logging_folder_exclusion']
assert sorted([r['index'] for r in exclusions]) == [
'exclusion1',
'exclusion2',
]
values = [(r['index'], r['values']['filter']) for r in exclusions]
assert sorted(values) == [
('exclusion1', 'resource.type=gce_instance'),
('exclusion2', 'severity=NOTICE')
]
"Test folder-level logging exclusions."
logging_exclusions = (
"{"
'exclusion1 = "resource.type=gce_instance", '
'exclusion2 = "severity=NOTICE", '
"}"
)
_, resources = plan_runner(FIXTURES_DIR, logging_exclusions=logging_exclusions)
assert len(resources) == 3
exclusions = [
r for r in resources if r["type"] == "google_logging_folder_exclusion"
]
assert sorted([r["index"] for r in exclusions]) == [
"exclusion1",
"exclusion2",
]
values = [(r["index"], r["values"]["filter"]) for r in exclusions]
assert sorted(values) == [
("exclusion1", "resource.type=gce_instance"),
("exclusion2", "severity=NOTICE"),
]

13
tests/modules/logging_bucket/__init__.py Normal file
View File

@@ -0,0 +1,13 @@
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

24
tests/modules/logging_bucket/fixture/main.tf Normal file
View File

@@ -0,0 +1,24 @@
/**
* Copyright 2021 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
module "test" {
source = "../../../../modules/logging-bucket"
parent_type = var.parent_type
parent = var.parent
id = var.id
retention = var.retention
location = var.location
}

42
tests/modules/logging_bucket/fixture/variables.tf Normal file
View File

@@ -0,0 +1,42 @@
/**
* Copyright 2021 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
variable "parent" {
type = string
}
variable "parent_type" {
type = string
validation {
condition = contains(["project", "folder", "organization", "billing_account"], var.parent_type)
error_message = "Parent type must be project, folder, organization or billing_account."
}
}
variable "location" {
type = string
default = "global"
}
variable "id" {
type = string
default = "mybucket"
}
variable "retention" {
type = number
default = 30
}

86
tests/modules/logging_bucket/test_plan.py Normal file
View File

@@ -0,0 +1,86 @@
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import pytest
FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixture")
def test_project_logging_bucket(plan_runner):
"Test project logging bucket."
_, resources = plan_runner(FIXTURES_DIR, parent_type="project", parent="myproject")
assert len(resources) == 1
resource = resources[0]
assert resource["type"] == "google_logging_project_bucket_config"
assert resource["values"] == {
"bucket_id": "mybucket",
"project": "myproject",
"location": "global",
"retention_days": 30,
}
def test_folder_logging_bucket(plan_runner):
"Test project logging bucket."
_, resources = plan_runner(
FIXTURES_DIR, parent_type="folder", parent="folders/0123456789"
)
assert len(resources) == 1
resource = resources[0]
assert resource["type"] == "google_logging_folder_bucket_config"
assert resource["values"] == {
"bucket_id": "mybucket",
"folder": "folders/0123456789",
"location": "global",
"retention_days": 30,
}
def test_organization_logging_bucket(plan_runner):
"Test project logging bucket."
_, resources = plan_runner(
FIXTURES_DIR, parent_type="organization", parent="organizations/0123456789"
)
assert len(resources) == 1
resource = resources[0]
assert resource["type"] == "google_logging_organization_bucket_config"
assert resource["values"] == {
"bucket_id": "mybucket",
"organization": "organizations/0123456789",
"location": "global",
"retention_days": 30,
}
def test_billing_account_logging_bucket(plan_runner):
"Test project logging bucket."
_, resources = plan_runner(
FIXTURES_DIR, parent_type="billing_account", parent="0123456789"
)
assert len(resources) == 1
resource = resources[0]
assert resource["type"] == "google_logging_billing_account_bucket_config"
assert resource["values"] == {
"bucket_id": "mybucket",
"billing_account": "0123456789",
"location": "global",
"retention_days": 30,
}

1
tests/modules/organization/fixture/variables.tf
View File

@@ -81,6 +81,7 @@ variable "logging_sinks" {
filter = string
iam = bool
include_children = bool
exclusions = map(string)
}))
default = {}
}

181
tests/modules/organization/test_plan_logging.py
View File

@@ -18,18 +18,19 @@ import pytest
from collections import Counter
FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture')
FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixture")
def test_sinks(plan_runner):
"Test folder-level sinks."
logging_sinks = """ {
"Test folder-level sinks."
logging_sinks = """ {
warning = {
type = "gcs"
destination = "mybucket"
filter = "severity=WARNING"
iam = true
include_children = true
exclusions = {}
}
info = {
type = "bigquery"
@@ -37,6 +38,7 @@ def test_sinks(plan_runner):
filter = "severity=INFO"
iam = true
include_children = true
exclusions = {}
}
notice = {
type = "pubsub"
@@ -44,71 +46,122 @@ def test_sinks(plan_runner):
filter = "severity=NOTICE"
iam = true
include_children = false
exclusions = {}
}
debug = {
type = "logging"
destination = "projects/myproject/locations/global/buckets/mybucket"
filter = "severity=DEBUG"
iam = true
include_children = false
exclusions = {
no-compute = "logName:compute"
no-container = "logName:container"
}
}
}
"""
_, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks)
assert len(resources) == 6
_, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks)
assert len(resources) == 7
resource_types = Counter([r['type'] for r in resources])
assert resource_types == {
'google_bigquery_dataset_iam_binding': 1,
'google_logging_organization_sink': 3,
'google_pubsub_topic_iam_binding': 1,
'google_storage_bucket_iam_binding': 1
}
sinks = [r for r in resources
if r['type'] == 'google_logging_organization_sink']
assert sorted([r['index'] for r in sinks]) == [
'info',
'notice',
'warning',
]
values = [(r['index'], r['values']['filter'], r['values']['destination'],
r['values']['include_children'])
for r in sinks]
assert sorted(values) == [
('info',
'severity=INFO',
'bigquery.googleapis.com/projects/myproject/datasets/mydataset',
True),
('notice',
'severity=NOTICE',
'pubsub.googleapis.com/projects/myproject/topics/mytopic',
False),
('warning', 'severity=WARNING', 'storage.googleapis.com/mybucket', True)]
resource_types = Counter([r["type"] for r in resources])
assert resource_types == {
"google_bigquery_dataset_iam_binding": 1,
"google_logging_organization_sink": 4,
"google_pubsub_topic_iam_binding": 1,
"google_storage_bucket_iam_binding": 1,
}
sinks = [r for r in resources if r["type"] == "google_logging_organization_sink"]
assert sorted([r["index"] for r in sinks]) == [
"debug",
"info",
"notice",
"warning",
]
values = [
(
r["index"],
r["values"]["filter"],
r["values"]["destination"],
r["values"]["include_children"],
)
for r in sinks
]
assert sorted(values) == [
(
"debug",
"severity=DEBUG",
"logging.googleapis.com/projects/myproject/locations/global/buckets/mybucket",
False,
),
(
"info",
"severity=INFO",
"bigquery.googleapis.com/projects/myproject/datasets/mydataset",
True,
),
(
"notice",
"severity=NOTICE",
"pubsub.googleapis.com/projects/myproject/topics/mytopic",
False,
),
("warning", "severity=WARNING", "storage.googleapis.com/mybucket", True),
]
bindings = [r for r in resources if "binding" in r["type"]]
values = [(r["index"], r["type"], r["values"]["role"]) for r in bindings]
assert sorted(values) == [
("info", "google_bigquery_dataset_iam_binding", "roles/bigquery.dataEditor"),
("notice", "google_pubsub_topic_iam_binding", "roles/pubsub.publisher"),
("warning", "google_storage_bucket_iam_binding", "roles/storage.objectCreator"),
]
exclusions = [(r["index"], r["values"]["exclusions"]) for r in sinks]
assert sorted(exclusions) == [
(
"debug",
[
{
"description": None,
"disabled": False,
"filter": "logName:compute",
"name": "no-compute",
},
{
"description": None,
"disabled": False,
"filter": "logName:container",
"name": "no-container",
},
],
),
("info", []),
("notice", []),
("warning", []),
]
bindings = [r for r in resources
if 'binding' in r['type']]
values = [(r['index'], r['type'], r['values']['role'])
for r in bindings]
assert sorted(values) == [
('info', 'google_bigquery_dataset_iam_binding', 'roles/bigquery.dataEditor'),
('notice', 'google_pubsub_topic_iam_binding', 'roles/pubsub.publisher'),
('warning', 'google_storage_bucket_iam_binding', 'roles/storage.objectCreator')
]
def test_exclusions(plan_runner):
"Test folder-level logging exclusions."
logging_exclusions = (
'{'
'exclusion1 = "resource.type=gce_instance", '
'exclusion2 = "severity=NOTICE", '
'}'
)
_, resources = plan_runner(FIXTURES_DIR,
logging_exclusions=logging_exclusions)
assert len(resources) == 2
exclusions = [r for r in resources
if r['type'] == 'google_logging_organization_exclusion']
assert sorted([r['index'] for r in exclusions]) == [
'exclusion1',
'exclusion2',
]
values = [(r['index'], r['values']['filter']) for r in exclusions]
assert sorted(values) == [
('exclusion1', 'resource.type=gce_instance'),
('exclusion2', 'severity=NOTICE')
]
"Test folder-level logging exclusions."
logging_exclusions = (
"{"
'exclusion1 = "resource.type=gce_instance", '
'exclusion2 = "severity=NOTICE", '
"}"
)
_, resources = plan_runner(FIXTURES_DIR, logging_exclusions=logging_exclusions)
assert len(resources) == 2
exclusions = [
r for r in resources if r["type"] == "google_logging_organization_exclusion"
]
assert sorted([r["index"] for r in exclusions]) == [
"exclusion1",
"exclusion2",
]
values = [(r["index"], r["values"]["filter"]) for r in exclusions]
assert sorted(values) == [
("exclusion1", "resource.type=gce_instance"),
("exclusion2", "severity=NOTICE"),
]

10
tests/modules/project/fixture/variables.tf
View File

@@ -96,10 +96,12 @@ variable "services" {
variable "logging_sinks" {
type = map(object({
destination = string
type = string
filter = string
iam = bool
destination = string
type = string
filter = string
iam = bool
exclusions = map(string)
unique_writer = bool
}))
default = {}
}

199
tests/modules/project/test_plan_logging.py
View File

@@ -18,92 +18,151 @@ import pytest
from collections import Counter
FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture')
FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixture")
def test_sinks(plan_runner):
"Test folder-level sinks."
logging_sinks = """ {
"Test folder-level sinks."
logging_sinks = """ {
warning = {
type = "gcs"
destination = "mybucket"
filter = "severity=WARNING"
iam = true
type = "gcs"
destination = "mybucket"
filter = "severity=WARNING"
iam = true
exclusions = {}
unique_writer = false
}
info = {
type = "bigquery"
destination = "projects/myproject/datasets/mydataset"
filter = "severity=INFO"
iam = true
exclusions = {}
unique_writer = false
}
notice = {
type = "pubsub"
destination = "projects/myproject/topics/mytopic"
filter = "severity=NOTICE"
iam = true
type = "pubsub"
destination = "projects/myproject/topics/mytopic"
filter = "severity=NOTICE"
iam = true
exclusions = {}
unique_writer = false
}
debug = {
type = "logging"
destination = "projects/myproject/locations/global/buckets/mybucket"
filter = "severity=DEBUG"
iam = true
exclusions = {
no-compute = "logName:compute"
no-container = "logName:container"
}
unique_writer = true
}
}
"""
_, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks)
assert len(resources) == 7
_, resources = plan_runner(FIXTURES_DIR, logging_sinks=logging_sinks)
assert len(resources) == 8
resource_types = Counter([r['type'] for r in resources])
assert resource_types == {
'google_bigquery_dataset_iam_binding': 1,
'google_logging_project_sink': 3,
'google_project': 1,
'google_pubsub_topic_iam_binding': 1,
'google_storage_bucket_iam_binding': 1
}
sinks = [r for r in resources
if r['type'] == 'google_logging_project_sink']
assert sorted([r['index'] for r in sinks]) == [
'info',
'notice',
'warning',
]
values = [(r['index'], r['values']['filter'], r['values']['destination'])
for r in sinks]
assert sorted(values) == [
('info',
'severity=INFO',
'bigquery.googleapis.com/projects/myproject/datasets/mydataset'),
('notice',
'severity=NOTICE',
'pubsub.googleapis.com/projects/myproject/topics/mytopic'),
('warning', 'severity=WARNING', 'storage.googleapis.com/mybucket')]
resource_types = Counter([r["type"] for r in resources])
assert resource_types == {
"google_bigquery_dataset_iam_binding": 1,
"google_logging_project_sink": 4,
"google_project": 1,
"google_pubsub_topic_iam_binding": 1,
"google_storage_bucket_iam_binding": 1,
}
sinks = [r for r in resources if r["type"] == "google_logging_project_sink"]
assert sorted([r["index"] for r in sinks]) == [
"debug",
"info",
"notice",
"warning",
]
values = [
(
r["index"],
r["values"]["filter"],
r["values"]["destination"],
r["values"]["unique_writer_identity"],
)
for r in sinks
]
assert sorted(values) == [
(
"debug",
"severity=DEBUG",
"logging.googleapis.com/projects/myproject/locations/global/buckets/mybucket",
True,
),
(
"info",
"severity=INFO",
"bigquery.googleapis.com/projects/myproject/datasets/mydataset",
False,
),
(
"notice",
"severity=NOTICE",
"pubsub.googleapis.com/projects/myproject/topics/mytopic",
False,
),
("warning", "severity=WARNING", "storage.googleapis.com/mybucket", False),
]
bindings = [r for r in resources if "binding" in r["type"]]
values = [(r["index"], r["type"], r["values"]["role"]) for r in bindings]
assert sorted(values) == [
("info", "google_bigquery_dataset_iam_binding", "roles/bigquery.dataEditor"),
("notice", "google_pubsub_topic_iam_binding", "roles/pubsub.publisher"),
("warning", "google_storage_bucket_iam_binding", "roles/storage.objectCreator"),
]
exclusions = [(r["index"], r["values"]["exclusions"]) for r in sinks]
assert sorted(exclusions) == [
(
"debug",
[
{
"description": None,
"disabled": False,
"filter": "logName:compute",
"name": "no-compute",
},
{
"description": None,
"disabled": False,
"filter": "logName:container",
"name": "no-container",
},
],
),
("info", []),
("notice", []),
("warning", []),
]
bindings = [r for r in resources
if 'binding' in r['type']]
values = [(r['index'], r['type'], r['values']['role'])
for r in bindings]
assert sorted(values) == [
('info', 'google_bigquery_dataset_iam_binding', 'roles/bigquery.dataEditor'),
('notice', 'google_pubsub_topic_iam_binding', 'roles/pubsub.publisher'),
('warning', 'google_storage_bucket_iam_binding', 'roles/storage.objectCreator')
]
def test_exclusions(plan_runner):
"Test folder-level logging exclusions."
logging_exclusions = (
'{'
'exclusion1 = "resource.type=gce_instance", '
'exclusion2 = "severity=NOTICE", '
'}'
)
_, resources = plan_runner(FIXTURES_DIR,
logging_exclusions=logging_exclusions)
assert len(resources) == 3
exclusions = [r for r in resources
if r['type'] == 'google_logging_project_exclusion']
assert sorted([r['index'] for r in exclusions]) == [
'exclusion1',
'exclusion2',
]
values = [(r['index'], r['values']['filter']) for r in exclusions]
assert sorted(values) == [
('exclusion1', 'resource.type=gce_instance'),
('exclusion2', 'severity=NOTICE')
]
"Test folder-level logging exclusions."
logging_exclusions = (
"{"
'exclusion1 = "resource.type=gce_instance", '
'exclusion2 = "severity=NOTICE", '
"}"
)
_, resources = plan_runner(FIXTURES_DIR, logging_exclusions=logging_exclusions)
assert len(resources) == 3
exclusions = [
r for r in resources if r["type"] == "google_logging_project_exclusion"
]
assert sorted([r["index"] for r in exclusions]) == [
"exclusion1",
"exclusion2",
]
values = [(r["index"], r["values"]["filter"]) for r in exclusions]
assert sorted(values) == [
("exclusion1", "resource.type=gce_instance"),
("exclusion2", "severity=NOTICE"),
]