feat: Added modules/notifications for S3 bucket notifications (#31)

.pre-commit-config.yaml

 repos:
   - repo: git://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.27.0
+    rev: v1.30.0
     hooks:
       - id: terraform_fmt
       - id: terraform_docs
   - repo: git://github.com/pre-commit/pre-commit-hooks
-    rev: v2.5.0
+    rev: v3.1.0
     hooks:
       - id: check-merge-conflict

README.md

@@ -6,6 +6,7 @@ This type of resources are supported:
 
 * [S3 Bucket](https://www.terraform.io/docs/providers/aws/r/s3_bucket.html)
 * [S3 Bucket Policy](https://www.terraform.io/docs/providers/aws/r/s3_bucket_policy.html)
+* [S3 Bucket Notification](https://www.terraform.io/docs/providers/aws/r/s3_bucket_notification.html) - use [modules/notification](https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/tree/master/modules/notification) to configure notifications to Lambda functions, SQS queues, and SNS topics.
 
 These features of S3 bucket configurations are supported:
 
@@ -75,26 +76,35 @@ module "s3_bucket" {
 
 * [Complete](https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/tree/master/examples/complete) - Complete S3 bucket with most of supported features enabled
 * [Cross-Region Replication](https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/tree/master/examples/s3-replication) - S3 bucket with Cross-Region Replication (CRR) enabled
+- [S3 bucket Notifications](https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/tree/master/examples/notification) - S3 bucket notifications to Lambda functions, SQS queues, and SNS topics.
+
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| terraform | ~> 0.12.6 |
+| aws | ~> 2.35 |
+
 ## Providers
 
 | Name | Version |
 |------|---------|
-| aws | n/a |
+| aws | ~> 2.35 |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
-|------|-------------|------|---------|:-----:|
-| acceleration\_status | (Optional) Sets the accelerate configuration of an existing bucket. Can be Enabled or Suspended. | `string` | n/a | yes |
+|------|-------------|------|---------|:--------:|
+| acceleration\_status | (Optional) Sets the accelerate configuration of an existing bucket. Can be Enabled or Suspended. | `string` | `null` | no |
 | acl | (Optional) The canned ACL to apply. Defaults to 'private'. | `string` | `"private"` | no |
 | attach\_elb\_log\_delivery\_policy | Controls if S3 bucket should have ELB log delivery policy attached | `bool` | `false` | no |
 | attach\_policy | Controls if S3 bucket should have bucket policy attached (set to `true` to use value of `policy` as bucket policy) | `bool` | `false` | no |
 | block\_public\_acls | Whether Amazon S3 should block public ACLs for this bucket. | `bool` | `false` | no |
 | block\_public\_policy | Whether Amazon S3 should block public bucket policies for this bucket. | `bool` | `false` | no |
-| bucket | (Optional, Forces new resource) The name of the bucket. If omitted, Terraform will assign a random, unique name. | `string` | n/a | yes |
-| bucket\_prefix | (Optional, Forces new resource) Creates a unique bucket name beginning with the specified prefix. Conflicts with bucket. | `string` | n/a | yes |
+| bucket | (Optional, Forces new resource) The name of the bucket. If omitted, Terraform will assign a random, unique name. | `string` | `null` | no |
+| bucket\_prefix | (Optional, Forces new resource) Creates a unique bucket name beginning with the specified prefix. Conflicts with bucket. | `string` | `null` | no |
 | cors\_rule | Map containing a rule of Cross-Origin Resource Sharing. | `any` | `{}` | no |
 | create\_bucket | Controls if S3 bucket should be created | `bool` | `true` | no |
 | force\_destroy | (Optional, Default:false ) A boolean that indicates all objects should be deleted from the bucket so that the bucket can be destroyed without error. These objects are not recoverable. | `bool` | `false` | no |
@@ -102,10 +112,10 @@ module "s3_bucket" {
 | lifecycle\_rule | List of maps containing configuration of object lifecycle management. | `any` | `[]` | no |
 | logging | Map containing access bucket logging configuration. | `map(string)` | `{}` | no |
 | object\_lock\_configuration | Map containing S3 object locking configuration. | `any` | `{}` | no |
-| policy | (Optional) A valid bucket policy JSON document. Note that if the policy document is not specific enough (but still valid), Terraform may view the policy as constantly changing in a terraform plan. In this case, please make sure you use the verbose/specific version of the policy. For more information about building AWS IAM policy documents with Terraform, see the AWS IAM Policy Document Guide. | `string` | n/a | yes |
-| region | (Optional) If specified, the AWS region this bucket should reside in. Otherwise, the region used by the callee. | `string` | n/a | yes |
+| policy | (Optional) A valid bucket policy JSON document. Note that if the policy document is not specific enough (but still valid), Terraform may view the policy as constantly changing in a terraform plan. In this case, please make sure you use the verbose/specific version of the policy. For more information about building AWS IAM policy documents with Terraform, see the AWS IAM Policy Document Guide. | `string` | `null` | no |
+| region | (Optional) If specified, the AWS region this bucket should reside in. Otherwise, the region used by the callee. | `string` | `null` | no |
 | replication\_configuration | Map containing cross-region replication configuration. | `any` | `{}` | no |
-| request\_payer | (Optional) Specifies who should bear the cost of Amazon S3 data transfer. Can be either BucketOwner or Requester. By default, the owner of the S3 bucket would incur the costs of any data transfer. See Requester Pays Buckets developer guide for more information. | `string` | n/a | yes |
+| request\_payer | (Optional) Specifies who should bear the cost of Amazon S3 data transfer. Can be either BucketOwner or Requester. By default, the owner of the S3 bucket would incur the costs of any data transfer. See Requester Pays Buckets developer guide for more information. | `string` | `null` | no |
 | restrict\_public\_buckets | Whether Amazon S3 should restrict public bucket policies for this bucket. | `bool` | `false` | no |
 | server\_side\_encryption\_configuration | Map containing server-side encryption configuration. | `any` | `{}` | no |
 | tags | (Optional) A mapping of tags to assign to the bucket. | `map(string)` | `{}` | no |

examples/complete/README.md

@@ -24,6 +24,10 @@ $ terraform apply
 Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+No requirements.
+
 ## Providers
 
 | Name | Version |

examples/notification/README.md

+# S3 bucket notifications to Lambda functions, SQS queues, and SNS topics
+
+Configuration in this directory creates S3 bucket notifications to all supported destinations.
+
+## Usage
+
+To run this example you need to execute:
+
+```bash
+$ terraform init
+$ terraform plan
+$ terraform apply
+```
+
+Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+No requirements.
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| aws | n/a |
+| random | n/a |
+
+## Inputs
+
+No input.
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| this\_s3\_bucket\_arn | The ARN of the bucket. Will be of format arn:aws:s3:::bucketname. |
+| this\_s3\_bucket\_bucket\_domain\_name | The bucket domain name. Will be of format bucketname.s3.amazonaws.com. |
+| this\_s3\_bucket\_bucket\_regional\_domain\_name | The bucket region-specific domain name. The bucket domain name including the region name, please refer here for format. Note: The AWS CloudFront allows specifying S3 region-specific endpoint when creating S3 origin, it will prevent redirect issues from CloudFront to S3 Origin URL. |
+| this\_s3\_bucket\_hosted\_zone\_id | The Route 53 Hosted Zone ID for this bucket's region. |
+| this\_s3\_bucket\_id | The name of the bucket. |
+| this\_s3\_bucket\_region | The AWS region this bucket resides in. |
+| this\_s3\_bucket\_website\_domain | The domain of the website endpoint, if the bucket is configured with a website. If not, this will be an empty string. This is used to create Route 53 alias records. |
+| this\_s3\_bucket\_website\_endpoint | The website endpoint, if the bucket is configured with a website. If not, this will be an empty string. |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/notification/main.tf

+locals {
+  bucket_name = "s3-bucket-${random_pet.this.id}"
+}
+
+resource "random_pet" "this" {
+  length = 2
+}
+
+module "s3_bucket" {
+  source = "../../"
+
+  bucket        = local.bucket_name
+  force_destroy = true
+}
+
+module "lambda_function1" {
+  source = "terraform-aws-modules/cloudwatch/aws//examples/fixtures/aws_lambda_function"
+}
+
+module "lambda_function2" {
+  source = "terraform-aws-modules/cloudwatch/aws//examples/fixtures/aws_lambda_function"
+}
+
+module "sns_topic1" {
+  source = "terraform-aws-modules/cloudwatch/aws//examples/fixtures/aws_sns_topic"
+}
+
+module "sns_topic2" {
+  source = "terraform-aws-modules/cloudwatch/aws//examples/fixtures/aws_sns_topic"
+}
+
+resource "aws_sqs_queue" "this" {
+  count = 2
+  name  = "${random_pet.this.id}-${count.index}"
+}
+
+module "all_notifications" {
+  source = "../../modules/notification"
+
+  bucket = module.s3_bucket.this_s3_bucket_id
+  create = false
+
+  // Common error - Error putting S3 notification configuration: InvalidArgument: Configuration is ambiguously defined. Cannot have overlapping suffixes in two rules if the prefixes are overlapping for the same event type.
+
+  lambda_notifications = {
+    lambda1 = {
+      lambda_function_arn = module.lambda_function1.this_lambda_function_arn
+      events              = ["s3:ObjectCreated:Put"]
+      filter_prefix       = "prefix/"
+      filter_suffix       = ".json"
+    }
+
+    lambda2 = {
+      lambda_function_arn = module.lambda_function2.this_lambda_function_arn
+      events              = ["s3:ObjectCreated:Post"]
+    }
+  }
+
+  sqs_notifications = {
+    sqs1 = {
+      queue_arn     = aws_sqs_queue.this[0].arn
+      events        = ["s3:ObjectCreated:Put"]
+      filter_prefix = "prefix2/"
+      filter_suffix = ".txt"
+
+      //      queue_id =  aws_sqs_queue.this[0].id // optional
+    }
+
+    sqs2 = {
+      queue_arn = aws_sqs_queue.this[1].arn
+      events    = ["s3:ObjectCreated:Copy"]
+    }
+  }
+
+  sns_notifications = {
+    sns1 = {
+      topic_arn     = module.sns_topic1.this_sns_topic_arn
+      events        = ["s3:ObjectRemoved:Delete"]
+      filter_prefix = "prefix3/"
+      filter_suffix = ".csv"
+    }
+
+    sns2 = {
+      topic_arn = module.sns_topic2.this_sns_topic_arn
+      events    = ["s3:ObjectRemoved:DeleteMarkerCreated"]
+    }
+  }
+
+}

examples/notification/outputs.tf

+output "this_s3_bucket_id" {
+  description = "The name of the bucket."
+  value       = module.s3_bucket.this_s3_bucket_id
+}
+
+output "this_s3_bucket_arn" {
+  description = "The ARN of the bucket. Will be of format arn:aws:s3:::bucketname."
+  value       = module.s3_bucket.this_s3_bucket_arn
+}
+
+output "this_s3_bucket_bucket_domain_name" {
+  description = "The bucket domain name. Will be of format bucketname.s3.amazonaws.com."
+  value       = module.s3_bucket.this_s3_bucket_bucket_domain_name
+}
+
+output "this_s3_bucket_bucket_regional_domain_name" {
+  description = "The bucket region-specific domain name. The bucket domain name including the region name, please refer here for format. Note: The AWS CloudFront allows specifying S3 region-specific endpoint when creating S3 origin, it will prevent redirect issues from CloudFront to S3 Origin URL."
+  value       = module.s3_bucket.this_s3_bucket_bucket_regional_domain_name
+}
+
+output "this_s3_bucket_hosted_zone_id" {
+  description = "The Route 53 Hosted Zone ID for this bucket's region."
+  value       = module.s3_bucket.this_s3_bucket_hosted_zone_id
+}
+
+output "this_s3_bucket_region" {
+  description = "The AWS region this bucket resides in."
+  value       = module.s3_bucket.this_s3_bucket_region
+}
+
+output "this_s3_bucket_website_endpoint" {
+  description = "The website endpoint, if the bucket is configured with a website. If not, this will be an empty string."
+  value       = module.s3_bucket.this_s3_bucket_website_endpoint
+}
+
+output "this_s3_bucket_website_domain" {
+  description = "The domain of the website endpoint, if the bucket is configured with a website. If not, this will be an empty string. This is used to create Route 53 alias records. "
+  value       = module.s3_bucket.this_s3_bucket_website_domain
+}

examples/s3-replication/README.md

@@ -17,6 +17,10 @@ $ terraform apply
 Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+No requirements.
+
 ## Providers
 
 | Name | Version |

modules/notification/README.md

+# S3 bucket notification
+
+Creates S3 bucket notification resource with all supported types of deliveries: AWS Lambda, SQS Queue, SNS Topic.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+No requirements.
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| aws | n/a |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| bucket | Name of S3 bucket to use | `string` | `""` | no |
+| bucket\_arn | ARN of S3 bucket to use in policies | `string` | `null` | no |
+| create | Whether to create this resource or not? | `bool` | `true` | no |
+| lambda\_notifications | Map of S3 bucket notifications to Lambda function | `any` | `{}` | no |
+| sns\_notifications | Map of S3 bucket notifications to SNS topic | `any` | `{}` | no |
+| sqs\_notifications | Map of S3 bucket notifications to SQS queue | `any` | `{}` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| this\_s3\_bucket\_notification\_id | ID of S3 bucket |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

modules/notification/main.tf

+locals {
+  bucket_arn = coalesce(var.bucket_arn, "arn:aws:s3:::${var.bucket}")
+
+  // Convert from "arn:aws:sqs:eu-west-1:835367859851:bold-starling-0" into "https://sqs.eu-west-1.amazonaws.com/835367859851/bold-starling-0" if queue_id was not specified
+  // queue_url used in aws_sqs_queue_policy is not the same as arn which is used in all other places
+  queue_ids = { for k, v in var.sqs_notifications : k => format("https://%s.%s.amazonaws.com/%s/%s", data.aws_arn.queue[k].service, data.aws_arn.queue[k].region, data.aws_arn.queue[k].account, data.aws_arn.queue[k].resource) if lookup(v, "queue_id", "") == "" }
+}
+
+resource "aws_s3_bucket_notification" "this" {
+  count = var.create && (length(var.lambda_notifications) > 0 || length(var.sqs_notifications) > 0 || length(var.sns_notifications) > 0) ? 1 : 0
+
+  bucket = var.bucket
+
+  dynamic "lambda_function" {
+    for_each = var.lambda_notifications
+
+    content {
+      id                  = lambda_function.key
+      lambda_function_arn = lambda_function.value.lambda_function_arn
+      events              = lambda_function.value.events
+      filter_prefix       = lookup(lambda_function.value, "filter_prefix", null)
+      filter_suffix       = lookup(lambda_function.value, "filter_suffix", null)
+    }
+  }
+
+  dynamic "queue" {
+    for_each = var.sqs_notifications
+
+    content {
+      id            = queue.key
+      queue_arn     = queue.value.queue_arn
+      events        = queue.value.events
+      filter_prefix = lookup(queue.value, "filter_prefix", null)
+      filter_suffix = lookup(queue.value, "filter_suffix", null)
+    }
+  }
+
+  dynamic "topic" {
+    for_each = var.sns_notifications
+
+    content {
+      id            = topic.key
+      topic_arn     = topic.value.topic_arn
+      events        = topic.value.events
+      filter_prefix = lookup(topic.value, "filter_prefix", null)
+      filter_suffix = lookup(topic.value, "filter_suffix", null)
+    }
+  }
+
+  depends_on = [
+    aws_lambda_permission.allow,
+    aws_sqs_queue_policy.allow,
+    aws_sns_topic_policy.allow,
+  ]
+}
+
+// Lambda
+resource "aws_lambda_permission" "allow" {
+  for_each = var.lambda_notifications
+
+  statement_id_prefix = "AllowLambdaS3BucketNotification-"
+  action              = "lambda:InvokeFunction"
+  function_name       = each.value.lambda_function_arn
+  principal           = "s3.amazonaws.com"
+  source_arn          = local.bucket_arn
+}
+
+// SQS Queue
+data "aws_arn" "queue" {
+  for_each = var.sqs_notifications
+
+  arn = each.value.queue_arn
+}
+
+data "aws_iam_policy_document" "sqs" {
+  for_each = var.sqs_notifications
+
+  statement {
+    sid = "AllowSQSS3BucketNotification"
+
+    effect = "Allow"
+
+    actions = [
+      "sqs:SendMessage",
+    ]
+
+    principals {
+      type        = "Service"
+      identifiers = ["s3.amazonaws.com"]
+    }
+
+    resources = [each.value.queue_arn]
+
+    condition {
+      test     = "ArnEquals"
+      variable = "aws:SourceArn"
+      values   = [local.bucket_arn]
+    }
+  }
+}
+
+resource "aws_sqs_queue_policy" "allow" {
+  for_each = var.sqs_notifications
+
+  queue_url = lookup(each.value, "queue_id", lookup(local.queue_ids, each.key, null))
+  policy    = data.aws_iam_policy_document.sqs[each.key].json
+}
+
+// SNS Topic
+data "aws_iam_policy_document" "sns" {
+  for_each = var.sns_notifications
+
+  statement {
+    sid = "AllowSNSS3BucketNotification"
+
+    effect = "Allow"
+
+    actions = [
+      "sns:Publish",
+    ]
+
+    principals {
+      type        = "Service"
+      identifiers = ["s3.amazonaws.com"]
+    }
+
+    resources = [each.value.topic_arn]
+
+    condition {
+      test     = "ArnEquals"
+      variable = "aws:SourceArn"
+      values   = [local.bucket_arn]
+    }
+  }
+}
+
+resource "aws_sns_topic_policy" "allow" {
+  for_each = var.sns_notifications
+
+  arn    = each.value.topic_arn
+  policy = data.aws_iam_policy_document.sns[each.key].json
+}

modules/notification/outputs.tf

+output "this_s3_bucket_notification_id" {
+  description = "ID of S3 bucket"
+  value       = element(concat(aws_s3_bucket_notification.this.*.id, [""]), 0)
+}

modules/notification/variables.tf

+variable "create" {
+  description = "Whether to create this resource or not?"
+  type        = bool
+  default     = true
+}
+
+variable "bucket" {
+  description = "Name of S3 bucket to use"
+  type        = string
+  default     = ""
+}
+
+variable "bucket_arn" {
+  description = "ARN of S3 bucket to use in policies"
+  type        = string
+  default     = null
+}
+
+variable "lambda_notifications" {
+  description = "Map of S3 bucket notifications to Lambda function"
+  type        = any # map(map(any)) is better, but Terraform 0.12.25 panics
+  default     = {}
+}
+
+variable "sqs_notifications" {
+  description = "Map of S3 bucket notifications to SQS queue"
+  type        = any # map(map(any)) is better, but Terraform 0.12.25 panics
+  default     = {}
+}
+
+variable "sns_notifications" {
+  description = "Map of S3 bucket notifications to SNS topic"
+  type        = any # map(map(any)) is better, but Terraform 0.12.25 panics
+  default     = {}
+}