Use Security Group rules instead of inline rule blocks. Fix example. Fix...

Use Security Group rules instead of inline rule blocks. Fix example. Fix README. Update to latest versions of modules. Change region for tests (#54)

* Use Security Group rules instead of inline rule blocks. Fix example. Fix README. Update to latest versions of modules. Change region for tests

* Use Security Group rules instead of inline rule blocks. Fix example. Fix README. Update to latest versions of modules. Change region for tests

* Use Security Group rules instead of inline rule blocks. Fix example. Fix README. Update to latest versions of modules. Change region for tests

README.yaml

@@ -49,38 +49,62 @@ description: |-
 # How to use this project
 usage: |-
 
+  For a complete example, see [examples/complete](examples/complete).
+
+  For automated tests of the complete example using [bats](https://github.com/bats-core/bats-core) and [Terratest](https://github.com/gruntwork-io/terratest) (which tests and deploys the example on AWS), see [test](test).
+
   ```hcl
-  // Generate a random string for auth token, no special chars
-  resource "random_string" "auth_token" {
-    length = 64
-    special = false
-  }
-
-  module "example_redis" {
-    source          = "git::https://github.com/cloudposse/terraform-aws-elasticache-redis.git?ref=master"
-    namespace       = "eg"
-    stage           = "dev"
-    name            = "redis"
-    zone_id         = var.route53_zone_id
-    security_groups = [var.security_group_id]
-
-    auth_token                   = random_string.auth_token.result
-    vpc_id                       = var.vpc_id
-    subnets                      = var.private_subnets
-    maintenance_window           = "wed:03:00-wed:04:00"
-    cluster_size                 = 2
-    instance_type                = "cache.t2.micro"
-    engine_version               = "4.0.10"
-    alarm_cpu_threshold_percent  = var.cache_alarm_cpu_threshold_percent
-    alarm_memory_threshold_bytes = var.cache_alarm_memory_threshold_bytes
-    apply_immediately            = true
-    availability_zones           = var.availability_zones
-    automatic_failover           = false
-  }
-
-  output "auth_token" {
-    value = random_string.auth_token.result
-  }
+    provider "aws" {
+      region = var.region
+    }
+
+    module "vpc" {
+      source     = "git::https://github.com/cloudposse/terraform-aws-vpc.git?ref=tags/0.8.1"
+      namespace  = var.namespace
+      stage      = var.stage
+      name       = var.name
+      cidr_block = "172.16.0.0/16"
+    }
+
+    module "subnets" {
+      source               = "git::https://github.com/cloudposse/terraform-aws-dynamic-subnets.git?ref=tags/0.18.1"
+      availability_zones   = var.availability_zones
+      namespace            = var.namespace
+      stage                = var.stage
+      name                 = var.name
+      vpc_id               = module.vpc.vpc_id
+      igw_id               = module.vpc.igw_id
+      cidr_block           = module.vpc.vpc_cidr_block
+      nat_gateway_enabled  = true
+      nat_instance_enabled = false
+    }
+
+    module "redis" {
+      source                     = "git::https://github.com/cloudposse/terraform-aws-elasticache-redis.git?ref=master"
+      availability_zones         = var.availability_zones
+      namespace                  = var.namespace
+      stage                      = var.stage
+      name                       = var.name
+      zone_id                    = var.zone_id
+      vpc_id                     = module.vpc.vpc_id
+      allowed_security_groups    = [module.vpc.vpc_default_security_group_id]
+      subnets                    = module.subnets.private_subnet_ids
+      cluster_size               = var.cluster_size
+      instance_type              = var.instance_type
+      apply_immediately          = true
+      automatic_failover         = false
+      engine_version             = var.engine_version
+      family                     = var.family
+      at_rest_encryption_enabled = var.at_rest_encryption_enabled
+      transit_encryption_enabled = var.transit_encryption_enabled
+
+      parameter = [
+        {
+          name  = "notify-keyspace-events"
+          value = "lK"
+        }
+      ]
+    }
   ```
 
 examples: |-

docs/terraform.md

@@ -5,17 +5,23 @@
 | alarm_actions | Alarm action list | list(string) | `<list>` | no |
 | alarm_cpu_threshold_percent | CPU threshold alarm level | number | `75` | no |
 | alarm_memory_threshold_bytes | Ram threshold alarm level | number | `10000000` | no |
+| allowed_cidr_blocks | List of CIDR blocks that are allowed ingress to the cluster's Security Group created in the module | list(string) | `<list>` | no |
+| allowed_security_groups | List of Security Group IDs that are allowed ingress to the cluster's Security Group created in the module | list(string) | `<list>` | no |
 | apply_immediately | Apply changes immediately | bool | `true` | no |
 | at_rest_encryption_enabled | Enable encryption at rest | bool | `false` | no |
 | attributes | Additional attributes (_e.g._ "1") | list(string) | `<list>` | no |
-| auth_token | Auth token for password protecting redis, `transit_encryption_enabled` must be set to `true`. Password must be longer than 16 chars | string | `` | no |
-| automatic_failover | Automatic failover (Not available for T1/T2 instances) | bool | `false` | no |
+| auth_token | Auth token for password protecting redis, `transit_encryption_enabled` must be set to `true`. Password must be longer than 16 chars | string | `null` | no |
+| automatic_failover_enabled | Automatic failover (Not available for T1/T2 instances) | bool | `false` | no |
 | availability_zones | Availability zone IDs | list(string) | `<list>` | no |
-| cluster_size | Count of nodes in cluster | number | `1` | no |
+| cluster_mode_enabled | Flag to enable/disable creation of a native redis cluster. `automatic_failover_enabled` must be set to `true`. Only 1 `cluster_mode` block is allowed | bool | `false` | no |
+| cluster_mode_num_node_groups | Number of node groups (shards) for this Redis replication group. Changing this number will trigger an online resizing operation before other settings modifications | number | `0` | no |
+| cluster_mode_replicas_per_node_group | Number of replica nodes in each node group. Valid values are 0 to 5. Changing this number will force a new resource | number | `0` | no |
+| cluster_size | Number of nodes in cluster | number | `1` | no |
 | delimiter | Delimiter between `name`, `namespace`, `stage` and `attributes` | string | `-` | no |
 | elasticache_subnet_group_name | Subnet group name for the ElastiCache instance | string | `` | no |
 | enabled | Set to false to prevent the module from creating any resources | bool | `true` | no |
 | engine_version | Redis engine version | string | `4.0.10` | no |
+| existing_security_groups | List of existing Security Group IDs to place the cluster into. Set `use_existing_security_groups` to `true` to enable using `existing_security_groups` as Security Groups for the cluster | list(string) | `<list>` | no |
 | family | Redis family | string | `redis4.0` | no |
 | instance_type | Elastic cache instance type | string | `cache.t2.micro` | no |
 | maintenance_window | Maintenance window | string | `wed:03:00-wed:04:00` | no |
@@ -26,13 +32,13 @@
 | parameter | A list of Redis parameters to apply. Note that parameters may differ from one Redis family to another | object | `<list>` | no |
 | port | Redis port | number | `6379` | no |
 | replication_group_id | Replication group ID with the following constraints:  A name must contain from 1 to 20 alphanumeric characters or hyphens.   The first character must be a letter.   A name cannot end with a hyphen or contain two consecutive hyphens. | string | `` | no |
-| security_groups | Security Group IDs | list(string) | `<list>` | no |
 | snapshot_retention_limit | The number of days for which ElastiCache will retain automatic cache cluster snapshots before deleting them. | number | `0` | no |
 | snapshot_window | The daily time range (in UTC) during which ElastiCache will begin taking a daily snapshot of your cache cluster. | string | `06:30-07:30` | no |
 | stage | Stage (e.g. `prod`, `dev`, `staging`) | string | `` | no |
 | subnets | Subnet IDs | list(string) | `<list>` | no |
 | tags | Additional tags (_e.g._ map("BusinessUnit","ABC") | map(string) | `<map>` | no |
 | transit_encryption_enabled | Enable TLS | bool | `true` | no |
+| use_existing_security_groups | Flag to enable/disable creation of Security Group in the module. Set to `true` to disable Security Group creation and provide a list of existing security Group IDs in `existing_security_groups` to place the cluster into | bool | `false` | no |
 | vpc_id | VPC ID | string | - | yes |
 | zone_id | Route53 DNS Zone ID | string | `` | no |
 
@@ -40,7 +46,8 @@
 
 | Name | Description |
 |------|-------------|
-| host | Redis host |
+| endpoint | Redis primary endpoint |
+| host | Redis hostname |
 | id | Redis cluster ID |
 | port | Redis port |
 | security_group_id | Security group ID |

examples/basic/main.tf

-module "elasticsearch" {
-  source                  = "../../"
-  namespace               = "eg"
-  stage                   = "dev"
-  name                    = "es"
-  dns_zone_id             = "Z14EN2YD427LRQ"
-  security_groups         = ["sg-XXXXXXXXX", "sg-YYYYYYYY"]
-  vpc_id                  = "vpc-XXXXXXXXX"
-  subnet_ids              = ["subnet-XXXXXXXXX", "subnet-YYYYYYYY"]
-  zone_awareness_enabled  = "true"
-  elasticsearch_version   = "6.5"
-  instance_type           = "t2.small.elasticsearch"
-  instance_count          = 4
-  iam_role_arns           = ["arn:aws:iam::XXXXXXXXX:role/ops", "arn:aws:iam::XXXXXXXXX:role/dev"]
-  iam_actions             = ["es:ESHttpGet", "es:ESHttpPut", "es:ESHttpPost"]
-  encrypt_at_rest_enabled = "true"
-  kibana_subdomain_name   = "kibana-es"
-
-  advanced_options = {
-    rest.action.multi.allow_explicit_index = "true"
-  }
-}

examples/complete/fixtures.us-west-1.tfvars → examples/complete/fixtures.us-east-2.tfvars

-region = "us-west-1"
+region = "us-east-2"
 
-availability_zones = ["us-west-1b", "us-west-1c"]
+availability_zones = ["us-east-2a", "us-east-2b"]
 
 namespace = "eg"
 

examples/complete/main.tf

@@ -3,7 +3,7 @@ provider "aws" {
 }
 
 module "vpc" {
-  source     = "git::https://github.com/cloudposse/terraform-aws-vpc.git?ref=tags/0.7.0"
+  source     = "git::https://github.com/cloudposse/terraform-aws-vpc.git?ref=tags/0.8.1"
   namespace  = var.namespace
   stage      = var.stage
   name       = var.name
@@ -11,7 +11,7 @@ module "vpc" {
 }
 
 module "subnets" {
-  source               = "git::https://github.com/cloudposse/terraform-aws-dynamic-subnets.git?ref=tags/0.16.0"
+  source               = "git::https://github.com/cloudposse/terraform-aws-dynamic-subnets.git?ref=tags/0.18.1"
   availability_zones   = var.availability_zones
   namespace            = var.namespace
   stage                = var.stage
@@ -31,12 +31,12 @@ module "redis" {
   name                       = var.name
   zone_id                    = var.zone_id
   vpc_id                     = module.vpc.vpc_id
-  security_groups            = [module.vpc.vpc_default_security_group_id]
+  allowed_security_groups    = [module.vpc.vpc_default_security_group_id]
   subnets                    = module.subnets.private_subnet_ids
   cluster_size               = var.cluster_size
   instance_type              = var.instance_type
   apply_immediately          = true
-  automatic_failover         = false
+  automatic_failover_enabled = false
   engine_version             = var.engine_version
   family                     = var.family
   at_rest_encryption_enabled = var.at_rest_encryption_enabled

examples/complete/outputs.tf

@@ -23,7 +23,12 @@ output "cluster_security_group_id" {
   description = "Cluster Security Group ID"
 }
 
+output "cluster_endpoint" {
+  value       = module.redis.endpoint
+  description = "Redis primary endpoint"
+}
+
 output "cluster_host" {
   value       = module.redis.host
-  description = "Redis host"
+  description = "Redis hostname"
 }

examples/complete/variables.tf

@@ -25,7 +25,7 @@ variable "name" {
 
 variable "cluster_size" {
   type        = number
-  description = "Count of nodes in cluster"
+  description = "Number of nodes in cluster"
 }
 
 variable "instance_type" {

main.tf

 module "label" {
-  source     = "git::https://github.com/cloudposse/terraform-null-label.git?ref=tags/0.14.1"
+  source     = "git::https://github.com/cloudposse/terraform-null-label.git?ref=tags/0.16.0"
   enabled    = var.enabled
   namespace  = var.namespace
   name       = var.name
@@ -13,25 +13,43 @@ module "label" {
 # Security Group Resources
 #
 resource "aws_security_group" "default" {
-  count  = var.enabled ? 1 : 0
+  count  = var.enabled && var.use_existing_security_groups == false ? 1 : 0
   vpc_id = var.vpc_id
   name   = module.label.id
+  tags   = module.label.tags
+}
 
-  ingress {
-    from_port       = var.port # Redis
-    to_port         = var.port
-    protocol        = "tcp"
-    security_groups = var.security_groups
-  }
+resource "aws_security_group_rule" "egress" {
+  count             = var.enabled && var.use_existing_security_groups == false ? 1 : 0
+  description       = "Allow all egress traffic"
+  from_port         = 0
+  to_port           = 0
+  protocol          = "-1"
+  cidr_blocks       = ["0.0.0.0/0"]
+  security_group_id = join("", aws_security_group.default.*.id)
+  type              = "egress"
+}
 
-  egress {
-    from_port   = 0
-    to_port     = 0
-    protocol    = "-1"
-    cidr_blocks = ["0.0.0.0/0"]
-  }
+resource "aws_security_group_rule" "ingress_security_groups" {
+  count                    = var.enabled && var.use_existing_security_groups == false ? length(var.allowed_security_groups) : 0
+  description              = "Allow inbound traffic from existing Security Groups"
+  from_port                = var.port
+  to_port                  = var.port
+  protocol                 = "tcp"
+  source_security_group_id = var.allowed_security_groups[count.index]
+  security_group_id        = join("", aws_security_group.default.*.id)
+  type                     = "ingress"
+}
 
-  tags = module.label.tags
+resource "aws_security_group_rule" "ingress_cidr_blocks" {
+  count             = var.enabled && var.use_existing_security_groups == false && length(var.allowed_cidr_blocks) > 0 ? 1 : 0
+  description       = "Allow inbound traffic from CIDR blocks"
+  from_port         = var.port
+  to_port           = var.port
+  protocol          = "tcp"
+  cidr_blocks       = var.allowed_cidr_blocks
+  security_group_id = join("", aws_security_group.default.*.id)
+  type              = "ingress"
 }
 
 locals {
@@ -61,7 +79,7 @@ resource "aws_elasticache_parameter_group" "default" {
 resource "aws_elasticache_replication_group" "default" {
   count = var.enabled ? 1 : 0
 
-  auth_token                    = var.auth_token
+  auth_token                    = var.transit_encryption_enabled ? var.auth_token : null
   replication_group_id          = var.replication_group_id == "" ? module.label.id : var.replication_group_id
   replication_group_description = module.label.id
   node_type                     = var.instance_type
@@ -69,9 +87,9 @@ resource "aws_elasticache_replication_group" "default" {
   port                          = var.port
   parameter_group_name          = join("", aws_elasticache_parameter_group.default.*.name)
   availability_zones            = slice(var.availability_zones, 0, var.cluster_size)
-  automatic_failover_enabled    = var.automatic_failover
+  automatic_failover_enabled    = var.automatic_failover_enabled
   subnet_group_name             = local.elasticache_subnet_group_name
-  security_group_ids            = [join("", aws_security_group.default.*.id)]
+  security_group_ids            = var.use_existing_security_groups ? var.existing_security_groups : [join("", aws_security_group.default.*.id)]
   maintenance_window            = var.maintenance_window
   notification_topic_arn        = var.notification_topic_arn
   engine_version                = var.engine_version
@@ -81,6 +99,14 @@ resource "aws_elasticache_replication_group" "default" {
   snapshot_retention_limit      = var.snapshot_retention_limit
 
   tags = module.label.tags
+
+  dynamic "cluster_mode" {
+    for_each = var.cluster_mode_enabled ? ["true"] : []
+    content {
+      replicas_per_node_group = var.cluster_mode_replicas_per_node_group
+      num_node_groups         = var.cluster_mode_num_node_groups
+    }
+  }
 }
 
 #

outputs.tf

@@ -13,13 +13,12 @@ output "port" {
   description = "Redis port"
 }
 
+output "endpoint" {
+  value       = join("", aws_elasticache_replication_group.default.*.primary_endpoint_address)
+  description = "Redis primary endpoint"
+}
+
 output "host" {
-  value = coalesce(
-    module.dns.hostname,
-    join(
-      "",
-      aws_elasticache_replication_group.default.*.primary_endpoint_address
-    )
-  )
-  description = "Redis host"
+  value       = module.dns.hostname
+  description = "Redis hostname"
 }

test/src/examples_complete_test.go

@@ -16,7 +16,7 @@ func TestExamplesComplete(t *testing.T) {
 		TerraformDir: "../../examples/complete",
 		Upgrade:      true,
 		// Variables to pass to our Terraform code using -var-file options
-		VarFiles: []string{"fixtures.us-west-1.tfvars"},
+		VarFiles: []string{"fixtures.us-east-2.tfvars"},
 	}
 
 	// At the end of the test, run `terraform destroy` to clean up any resources that were created
@@ -33,12 +33,12 @@ func TestExamplesComplete(t *testing.T) {
 	// Run `terraform output` to get the value of an output variable
 	privateSubnetCidrs := terraform.OutputList(t, terraformOptions, "private_subnet_cidrs")
 	// Verify we're getting back the outputs we expect
-	assert.Equal(t, []string{"172.16.0.0/18", "172.16.64.0/18"}, privateSubnetCidrs)
+	assert.Equal(t, []string{"172.16.0.0/19", "172.16.32.0/19"}, privateSubnetCidrs)
 
 	// Run `terraform output` to get the value of an output variable
 	publicSubnetCidrs := terraform.OutputList(t, terraformOptions, "public_subnet_cidrs")
 	// Verify we're getting back the outputs we expect
-	assert.Equal(t, []string{"172.16.128.0/18", "172.16.192.0/18"}, publicSubnetCidrs)
+	assert.Equal(t, []string{"172.16.96.0/19", "172.16.128.0/19"}, publicSubnetCidrs)
 
 	// Run `terraform output` to get the value of an output variable
 	clusterHost := terraform.Output(t, terraformOptions, "cluster_host")

variables.tf

@@ -21,10 +21,28 @@ variable "name" {
   description = "Name of the application"
 }
 
-variable "security_groups" {
+variable "use_existing_security_groups" {
+  type        = bool
+  description = "Flag to enable/disable creation of Security Group in the module. Set to `true` to disable Security Group creation and provide a list of existing security Group IDs in `existing_security_groups` to place the cluster into"
+  default     = false
+}
+
+variable "existing_security_groups" {
   type        = list(string)
   default     = []
-  description = "Security Group IDs"
+  description = "List of existing Security Group IDs to place the cluster into. Set `use_existing_security_groups` to `true` to enable using `existing_security_groups` as Security Groups for the cluster"
+}
+
+variable "allowed_security_groups" {
+  type        = list(string)
+  default     = []
+  description = "List of Security Group IDs that are allowed ingress to the cluster's Security Group created in the module"
+}
+
+variable "allowed_cidr_blocks" {
+  type        = list(string)
+  default     = []
+  description = "List of CIDR blocks that are allowed ingress to the cluster's Security Group created in the module"
 }
 
 variable "vpc_id" {
@@ -53,7 +71,7 @@ variable "maintenance_window" {
 variable "cluster_size" {
   type        = number
   default     = 1
-  description = "Count of nodes in cluster"
+  description = "Number of nodes in cluster"
 }
 
 variable "port" {
@@ -138,7 +156,7 @@ variable "apply_immediately" {
   description = "Apply changes immediately"
 }
 
-variable "automatic_failover" {
+variable "automatic_failover_enabled" {
   type        = bool
   default     = false
   description = "Automatic failover (Not available for T1/T2 instances)"
@@ -177,7 +195,7 @@ variable "tags" {
 variable "auth_token" {
   type        = string
   description = "Auth token for password protecting redis, `transit_encryption_enabled` must be set to `true`. Password must be longer than 16 chars"
-  default     = ""
+  default     = null
 }
 
 variable "replication_group_id" {
@@ -197,3 +215,21 @@ variable "snapshot_retention_limit" {
   description = "The number of days for which ElastiCache will retain automatic cache cluster snapshots before deleting them."
   default     = 0
 }
+
+variable "cluster_mode_enabled" {
+  type        = bool
+  description = "Flag to enable/disable creation of a native redis cluster. `automatic_failover_enabled` must be set to `true`. Only 1 `cluster_mode` block is allowed"
+  default     = false
+}
+
+variable "cluster_mode_replicas_per_node_group" {
+  type        = number
+  description = "Number of replica nodes in each node group. Valid values are 0 to 5. Changing this number will force a new resource"
+  default     = 0
+}
+
+variable "cluster_mode_num_node_groups" {
+  type        = number
+  description = "Number of node groups (shards) for this Redis replication group. Changing this number will trigger an online resizing operation before other settings modifications"
+  default     = 0
+}