Use service module and leverage IMDSv2 (closes #12)

Author: jbeemster

README.md

@@ -168,43 +168,40 @@ module "pipeline_rds" {
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0.0 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.45.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.72.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.45.0 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.72.0 |
 
 ## Modules
 
 | Name | Source | Version |
 |------|--------|---------|
 | <a name="module_instance_type_metrics"></a> [instance\_type\_metrics](#module\_instance\_type\_metrics) | snowplow-devops/ec2-instance-type-metrics/aws | 0.1.2 |
 | <a name="module_kcl_autoscaling"></a> [kcl\_autoscaling](#module\_kcl\_autoscaling) | snowplow-devops/dynamodb-autoscaling/aws | 0.2.0 |
-| <a name="module_tags"></a> [tags](#module\_tags) | snowplow-devops/tags/aws | 0.2.0 |
-| <a name="module_telemetry"></a> [telemetry](#module\_telemetry) | snowplow-devops/telemetry/snowplow | 0.3.0 |
+| <a name="module_service"></a> [service](#module\_service) | snowplow-devops/service-ec2/aws | 0.1.0 |
+| <a name="module_telemetry"></a> [telemetry](#module\_telemetry) | snowplow-devops/telemetry/snowplow | 0.4.0 |
 
 ## Resources
 
 | Name | Type |
 |------|------|
-| [aws_autoscaling_group.asg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group) | resource |
 | [aws_cloudwatch_log_group.log_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
 | [aws_dynamodb_table.kcl](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dynamodb_table) | resource |
 | [aws_iam_instance_profile.instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource |
 | [aws_iam_policy.iam_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_role.iam_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_iam_role_policy_attachment.policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
-| [aws_launch_configuration.lc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_configuration) | resource |
 | [aws_security_group.sg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource |
 | [aws_security_group_rule.egress_tcp_443](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
 | [aws_security_group_rule.egress_tcp_80](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
 | [aws_security_group_rule.egress_tcp_server_rds](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
 | [aws_security_group_rule.egress_udp_123](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
 | [aws_security_group_rule.ingress_tcp_22](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
 | [aws_security_group_rule.rds_egress_tcp_webserver](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
-| [aws_ami.amazon_linux_2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
 | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
 | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
 
@@ -232,6 +229,7 @@ module "pipeline_rds" {
 | <a name="input_custom_iglu_resolvers"></a> [custom\_iglu\_resolvers](#input\_custom\_iglu\_resolvers) | The custom Iglu Resolvers that will be used by Enrichment to resolve and validate events | <pre>list(object({<br>    name            = string<br>    priority        = number<br>    uri             = string<br>    api_key         = string<br>    vendor_prefixes = list(string)<br>  }))</pre> | `[]` | no |
 | <a name="input_db_max_connections"></a> [db\_max\_connections](#input\_db\_max\_connections) | The maximum number of connections to the backing database | `number` | `10` | no |
 | <a name="input_default_iglu_resolvers"></a> [default\_iglu\_resolvers](#input\_default\_iglu\_resolvers) | The default Iglu Resolvers that will be used by Enrichment to resolve and validate events | <pre>list(object({<br>    name            = string<br>    priority        = number<br>    uri             = string<br>    api_key         = string<br>    vendor_prefixes = list(string)<br>  }))</pre> | <pre>[<br>  {<br>    "api_key": "",<br>    "name": "Iglu Central",<br>    "priority": 10,<br>    "uri": "http://iglucentral.com",<br>    "vendor_prefixes": []<br>  },<br>  {<br>    "api_key": "",<br>    "name": "Iglu Central - Mirror 01",<br>    "priority": 20,<br>    "uri": "http://mirror01.iglucentral.com",<br>    "vendor_prefixes": []<br>  }<br>]</pre> | no |
+| <a name="input_enable_auto_scaling"></a> [enable\_auto\_scaling](#input\_enable\_auto\_scaling) | Whether to enable auto-scaling policies for the service (WARN: ensure you have sufficient db\_connections available for the max number of nodes in the ASG) | `bool` | `true` | no |
 | <a name="input_iam_permissions_boundary"></a> [iam\_permissions\_boundary](#input\_iam\_permissions\_boundary) | The permissions boundary ARN to set on IAM roles created | `string` | `""` | no |
 | <a name="input_in_max_batch_size_checkpoint"></a> [in\_max\_batch\_size\_checkpoint](#input\_in\_max\_batch\_size\_checkpoint) | The maximum number events to process before checkpointing progress on the stream | `number` | `1000` | no |
 | <a name="input_in_max_batch_wait_checkpoint"></a> [in\_max\_batch\_wait\_checkpoint](#input\_in\_max\_batch\_wait\_checkpoint) | The maximum amount of time to wait before checkpointing progress on the stream | `string` | `"10 seconds"` | no |
@@ -244,6 +242,12 @@ module "pipeline_rds" {
 | <a name="input_kcl_write_min_capacity"></a> [kcl\_write\_min\_capacity](#input\_kcl\_write\_min\_capacity) | The minimum WRITE capacity for the KCL DynamoDB table | `number` | `1` | no |
 | <a name="input_max_size"></a> [max\_size](#input\_max\_size) | The maximum number of servers in this server-group | `number` | `2` | no |
 | <a name="input_min_size"></a> [min\_size](#input\_min\_size) | The minimum number of servers in this server-group | `number` | `1` | no |
+| <a name="input_scale_down_cooldown_sec"></a> [scale\_down\_cooldown\_sec](#input\_scale\_down\_cooldown\_sec) | Time (in seconds) until another scale-down action can occur | `number` | `600` | no |
+| <a name="input_scale_down_cpu_threshold_percentage"></a> [scale\_down\_cpu\_threshold\_percentage](#input\_scale\_down\_cpu\_threshold\_percentage) | The average CPU percentage that we must be below to scale-down | `number` | `20` | no |
+| <a name="input_scale_down_eval_minutes"></a> [scale\_down\_eval\_minutes](#input\_scale\_down\_eval\_minutes) | The number of consecutive minutes that we must be below the threshold to scale-down | `number` | `60` | no |
+| <a name="input_scale_up_cooldown_sec"></a> [scale\_up\_cooldown\_sec](#input\_scale\_up\_cooldown\_sec) | Time (in seconds) until another scale-up action can occur | `number` | `180` | no |
+| <a name="input_scale_up_cpu_threshold_percentage"></a> [scale\_up\_cpu\_threshold\_percentage](#input\_scale\_up\_cpu\_threshold\_percentage) | The average CPU percentage that must be exceeded to scale-up | `number` | `60` | no |
+| <a name="input_scale_up_eval_minutes"></a> [scale\_up\_eval\_minutes](#input\_scale\_up\_eval\_minutes) | The number of consecutive minutes that the threshold must be breached to scale-up | `number` | `5` | no |
 | <a name="input_ssh_ip_allowlist"></a> [ssh\_ip\_allowlist](#input\_ssh\_ip\_allowlist) | The list of CIDR ranges to allow SSH traffic from | `list(any)` | <pre>[<br>  "0.0.0.0/0"<br>]</pre> | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | The tags to append to this resource | `map(string)` | `{}` | no |
 | <a name="input_telemetry_enabled"></a> [telemetry\_enabled](#input\_telemetry\_enabled) | Whether or not to send telemetry information back to Snowplow Analytics Ltd | `bool` | `true` | no |

main.tf

@@ -26,7 +26,7 @@ data "aws_caller_identity" "current" {}
 
 module "telemetry" {
   source  = "snowplow-devops/telemetry/snowplow"
-  version = "0.3.0"
+  version = "0.4.0"
 
   count = var.telemetry_enabled ? 1 : 0
 
@@ -39,27 +39,6 @@ module "telemetry" {
   module_version   = local.module_version
 }
 
-data "aws_ami" "amazon_linux_2" {
-  most_recent = true
-
-  filter {
-    name   = "name"
-    values = ["amzn2-ami-hvm-*-x86_64-ebs"]
-  }
-
-  filter {
-    name   = "root-device-type"
-    values = ["ebs"]
-  }
-
-  filter {
-    name   = "virtualization-type"
-    values = ["hvm"]
-  }
-
-  owners = ["amazon"]
-}
-
 # --- DynamoDB: KCL Table
 
 resource "aws_dynamodb_table" "kcl" {
@@ -345,58 +324,30 @@ locals {
   })
 }
 
-resource "aws_launch_configuration" "lc" {
-  name_prefix = "${var.name}-"
+module "service" {
+  source  = "snowplow-devops/service-ec2/aws"
+  version = "0.1.0"
 
-  image_id             = var.amazon_linux_2_ami_id == "" ? data.aws_ami.amazon_linux_2.id : var.amazon_linux_2_ami_id
-  instance_type        = var.instance_type
-  key_name             = var.ssh_key_name
-  iam_instance_profile = aws_iam_instance_profile.instance_profile.name
-  security_groups      = [aws_security_group.sg.id]
-  user_data            = local.user_data
+  user_supplied_script = local.user_data
+  name                 = var.name
+  tags                 = local.tags
 
-  # Note: Required if deployed in a public subnet
+  amazon_linux_2_ami_id       = var.amazon_linux_2_ami_id
+  instance_type               = var.instance_type
+  ssh_key_name                = var.ssh_key_name
+  iam_instance_profile_name   = aws_iam_instance_profile.instance_profile.name
   associate_public_ip_address = var.associate_public_ip_address
-
-  root_block_device {
-    volume_type           = "gp2"
-    volume_size           = "10"
-    delete_on_termination = true
-    encrypted             = true
-  }
-
-  lifecycle {
-    create_before_destroy = true
-  }
-}
-
-module "tags" {
-  source  = "snowplow-devops/tags/aws"
-  version = "0.2.0"
-
-  tags = local.tags
-}
-
-resource "aws_autoscaling_group" "asg" {
-  name = var.name
-
-  max_size = var.max_size
-  min_size = var.min_size
-
-  launch_configuration = aws_launch_configuration.lc.name
-
-  health_check_grace_period = 300
-  health_check_type         = "EC2"
-
-  vpc_zone_identifier = var.subnet_ids
-
-  instance_refresh {
-    strategy = "Rolling"
-    preferences {
-      min_healthy_percentage = 90
-    }
-    triggers = ["tag"]
-  }
-
-  tags = module.tags.asg_tags
+  security_groups             = [aws_security_group.sg.id]
+
+  min_size   = var.min_size
+  max_size   = var.max_size
+  subnet_ids = var.subnet_ids
+
+  enable_auto_scaling                 = var.enable_auto_scaling
+  scale_up_cooldown_sec               = var.scale_up_cooldown_sec
+  scale_up_cpu_threshold_percentage   = var.scale_up_cpu_threshold_percentage
+  scale_up_eval_minutes               = var.scale_up_eval_minutes
+  scale_down_cooldown_sec             = var.scale_down_cooldown_sec
+  scale_down_cpu_threshold_percentage = var.scale_down_cpu_threshold_percentage
+  scale_down_eval_minutes             = var.scale_down_eval_minutes
 }

outputs.tf

@@ -1,10 +1,10 @@
 output "asg_id" {
-  value       = aws_autoscaling_group.asg.id
+  value       = module.service.asg_id
   description = "ID of the ASG"
 }
 
 output "asg_name" {
-  value       = aws_autoscaling_group.asg.name
+  value       = module.service.asg_name
   description = "Name of the ASG"
 }
 

templates/user-data.sh.tmpl

@@ -1,26 +1,5 @@
-#!/bin/bash
-set -e -x
-exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
-
-# -----------------------------------------------------------------------------
-#  BASE INSTALL
-# -----------------------------------------------------------------------------
-
 readonly CONFIG_DIR=/opt/snowplow/config
 
-function install_docker_ce() {
-  sudo yum install -y docker-20.10.4-1.amzn2
-  sudo systemctl enable docker
-  sudo systemctl start docker
-  sudo usermod -a -G docker ec2-user
-}
-
-function get_instance_id() {
-  curl --silent --location "http://169.254.169.254/latest/meta-data/instance-id/"
-}
-
-install_docker_ce
-
 sudo mkdir -p $${CONFIG_DIR}
 sudo cat << EOF > $${CONFIG_DIR}/postgres_loader.json
 ${config}

variables.tf

@@ -108,6 +108,50 @@ variable "java_opts" {
   type        = string
 }
 
+# --- Auto-scaling options
+
+variable "enable_auto_scaling" {
+  description = "Whether to enable auto-scaling policies for the service (WARN: ensure you have sufficient db_connections available for the max number of nodes in the ASG)"
+  default     = true
+  type        = bool
+}
+
+variable "scale_up_cooldown_sec" {
+  description = "Time (in seconds) until another scale-up action can occur"
+  default     = 180
+  type        = number
+}
+
+variable "scale_up_cpu_threshold_percentage" {
+  description = "The average CPU percentage that must be exceeded to scale-up"
+  default     = 60
+  type        = number
+}
+
+variable "scale_up_eval_minutes" {
+  description = "The number of consecutive minutes that the threshold must be breached to scale-up"
+  default     = 5
+  type        = number
+}
+
+variable "scale_down_cooldown_sec" {
+  description = "Time (in seconds) until another scale-down action can occur"
+  default     = 600
+  type        = number
+}
+
+variable "scale_down_cpu_threshold_percentage" {
+  description = "The average CPU percentage that we must be below to scale-down"
+  default     = 20
+  type        = number
+}
+
+variable "scale_down_eval_minutes" {
+  description = "The number of consecutive minutes that we must be below the threshold to scale-down"
+  default     = 60
+  type        = number
+}
+
 # --- Configuration options
 
 variable "in_stream_name" {

versions.tf

@@ -4,7 +4,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 3.45.0"
+      version = ">= 3.72.0"
     }
   }
 }