Never miss an alert with AWS Chatbot and AWS SSM – Incident Manager

terraform {
  required_providers {
    aws = {
      source                = "hashicorp/aws"
      version               = "~> 5.53.0"
      configuration_aliases = [aws, aws.us-east-1]
    }
    awscc = {
      source  = "hashicorp/awscc"
      version = "~> 1.2.0"
    }
  }
}

resource "awscc_chatbot_slack_channel_configuration" "chatbot_slack" {
  configuration_name = var.slack_channel_configuration_name
  iam_role_arn       = awscc_iam_role.chatbot_channel_role.arn
  slack_channel_id   = var.slack_channel_id
  slack_workspace_id = var.slack_workspace_id
  logging_level      = var.logging_level
  sns_topic_arns     = [aws_sns_topic.sns_topic_for_aws_chatbot_primary_region.arn, aws_sns_topic.sns_topic_for_aws_chatbot_us_east_1.arn]
  guardrail_policies = [
    "arn:aws:iam::aws:policy/PowerUserAccess"
  ]
}

resource "awscc_iam_role" "chatbot_channel_role" {
  role_name = "aws-chatbot-channel-role"
  assume_role_policy_document = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = "sts:AssumeRole"
        Effect = "Allow"
        Sid    = "Chatbot"
        Principal = {
          Service = "chatbot.amazonaws.com"
        }
      },
    ]
  })
  managed_policy_arns = [
    "arn:aws:iam::aws:policy/AWSResourceExplorerReadOnlyAccess",
    "arn:aws:iam::aws:policy/AWSIncidentManagerResolverAccess",
    "arn:aws:iam::aws:policy/AmazonQFullAccess",
    "arn:aws:iam::aws:policy/CloudWatchReadOnlyAccess",
    "arn:aws:iam::aws:policy/AWSSecurityHubFullAccess",
    "arn:aws:iam::aws:policy/AWSSupportAccess"
  ]
}

resource "aws_sns_topic" "sns_topic_for_aws_chatbot_primary_region" {
  #checkov:skip=CKV_AWS_26:
  name                           = "aws-chatbot-notifications"
  http_success_feedback_role_arn = aws_iam_role.delivery_status_logging_for_sns_topic.arn
  http_failure_feedback_role_arn = aws_iam_role.delivery_status_logging_for_sns_topic.arn
  tags = {
    Name    = "aws_chatbot_notifications"
    Service = "monitoring"
  }
}

resource "aws_sns_topic" "sns_topic_for_aws_chatbot_us_east_1" {
  #checkov:skip=CKV_AWS_26:
  provider                       = aws.us-east-1
  name                           = "aws-chatbot-notifications"
  http_success_feedback_role_arn = aws_iam_role.delivery_status_logging_for_sns_topic.arn
  http_failure_feedback_role_arn = aws_iam_role.delivery_status_logging_for_sns_topic.arn
  tags = {
    Name    = "aws_chatbot_notifications"
    Service = "monitoring"
  }
}

# Define SNS topic policy primary region
resource "aws_sns_topic_policy" "sns_topic_policy_for_aws_chatbot_primary_region" {
  arn    = aws_sns_topic.sns_topic_for_aws_chatbot_primary_region.arn
  policy = data.aws_iam_policy_document.sns_topic_policy_for_aws_chatbot_primary_region.json
}

# Define SNS topic policy primary region us-east-1
resource "aws_sns_topic_policy" "sns_topic_policy_for_aws_chatbot_us_east_1" {
  provider = aws.us-east-1
  arn      = aws_sns_topic.sns_topic_for_aws_chatbot_us_east_1.arn
  policy   = data.aws_iam_policy_document.sns_topic_policy_for_aws_chatbot_us_east_1.json
}

# IAM role for delivery_status_logging_for_sns_topic
resource "aws_iam_role" "delivery_status_logging_for_sns_topic" {
  name               = "aws-chatbot-delivery-status-logging"
  assume_role_policy = data.aws_iam_policy_document.sns_to_cw_logs_assume_role_policy.json
}

resource "aws_iam_policy" "delivery_status_logging_for_sns_topic_policy" {
  policy = data.aws_iam_policy_document.sns_to_cw_logs_policy.json
}

resource "aws_iam_role_policy_attachment" "delivery_status_logging_for_sns_topic_attachment" {
  role       = aws_iam_role.delivery_status_logging_for_sns_topic.name
  policy_arn = aws_iam_policy.delivery_status_logging_for_sns_topic_policy.arn
}

module "aws_chatbot_slack" {
  source = "git::https://github.com/haakond/terraform-aws-chatbot.git"
  providers = {
    aws           = aws
    aws.us-east-1 = aws.us-east-1
    awscc         = awscc
  }
  slack_channel_configuration_name = "slack-hed-aws-monitoring"
  slack_channel_id                 = "AABBCC001122DD88"
  slack_workspace_id               = "ABCD1234EFGH5678"
  logging_level                    = "INFO"
}

provider "aws" {
  region  = var.aws_region
  profile = var.profile_cicd
  assume_role {
    role_arn     = "arn:aws:iam::${var.aws_account_id}:role/${var.profile_cicd}"
    session_name = "SESSION_NAME"
    external_id  = "EXTERNAL_ID"
  }
}

provider "awscc" {
  region  = var.aws_region
  profile = var.profile_cicd
  assume_role = {
    role_arn     = "arn:aws:iam::${var.aws_account_id}:role/${var.profile_cicd}"
    session_name = "SESSION_NAME"
    external_id  = "EXTERNAL_ID"
  }
}

provider "aws" {
  alias   = "us-east-1"
  region  = "us-east-1"
  profile = var.profile_cicd
  assume_role {
    role_arn     = "arn:aws:iam::${var.aws_account_id}:role/${var.profile_cicd}"
    session_name = "SESSION_NAME"
    external_id  = "EXTERNAL_ID"
  }
}

# Define an additional provider for the us-east-1 region.
provider "aws" {
  alias   = "us-east-1"
  region  = "us-east-1"
  profile = var.profile_cicd
  assume_role {
    role_arn     = "arn:aws:iam::${var.aws_account_id}:role/${var.profile_cicd}"
    session_name = "SESSION_NAME"
    external_id  = "EXTERNAL_ID"
  }
}

# AWS Route 53 Health Checks and corresponding metrics reside in us-east-1.
resource "aws_route53_health_check" "hedrange_com_about" {
  provider           = aws.us-east-1
  fqdn               = "hedrange.com"
  port               = 443
  type               = "HTTPS"
  resource_path      = "/about/"
  failure_threshold  = "3"
  request_interval   = "30"
  measure_latency    = true
  invert_healthcheck = false
  regions            = ["us-east-1", "us-west-1", "eu-west-1"]
  tags = {
    Name = "health-check-blog"
  }
}

# Since AWS Route 53 Health Checks and corresponding metrics reside in us-east-1, so the alarm has to be provisioned there as well.
resource "aws_cloudwatch_metric_alarm" "healthcheck_hedrange_com_about" {
  provider            = aws.us-east-1
  alarm_name          = "health-check-blog-about"
  comparison_operator = "LessThanThreshold"
  evaluation_periods  = "3"
  metric_name         = "HealthCheckStatus"
  namespace           = "AWS/Route53"
  period              = "60"
  statistic           = "Minimum"
  threshold           = "1"
  alarm_description   = "CRITICAL - https://hedrange.com/about is unavailable!"
  dimensions = {
    HealthCheckId = aws_route53_health_check.hedrange_com_about.id
  }
  alarm_actions = [local.chatbot_sns_topic_arn_us_east_1]
  ok_actions    = [local.chatbot_sns_topic_arn_us_east_1]
  tags = {
    Name     = "alarm-health-check-blog",
    Severity = "CRITICAL"
  }
}

# Eventbridge rule with event pattern to catch high severity Security Hub findings, regardless of product.
resource "aws_cloudwatch_event_rule" "main_securityhub_event_rule" {
  name        = "aws-securityhub-rule"
  description = "Capture AWS Security Hub events"

  event_pattern = <<EOF
{
    "source": [
        "aws.securityhub"
     ],
    "detail-type": [
        "Security Hub Findings - Imported"
    ],
    "detail": {
        "findings": {
            "Severity": {
                "Label": ["CRITICAL", "HIGH"]
            }
        }
    }
}
EOF
}

resource "aws_cloudwatch_event_target" "main_securityhub_rule_target_sns_topic_for_aws_chatbot" {
  rule      = aws_cloudwatch_event_rule.main_securityhub_event_rule.name
  target_id = "SendToSNS"
  arn       = aws_sns_topic.sns_topic_for_aws_chatbot.arn
}

# Eventbridge rule with event pattern to AWS Health notifications
resource "aws_cloudwatch_event_rule" "health_event_rule" {
  name        = "aws-health-rule"
  description = "Capture AWS Health events"

  event_pattern = <<EOF
{
    "source": ["aws.health"],
    "detail-type": ["AWS Health Event"]
}
EOF
}

# EventBridge rule target to the SNS topic for AWS Chatbot
resource "aws_cloudwatch_event_target" "health_event_rule_target_sns_topic_for_aws_chatbot" {
  rule      = aws_cloudwatch_event_rule.health_event_rule.name
  target_id = "SendToSNS"
  arn       = aws_sns_topic.sns_topic_for_aws_chatbot.arn
}

resource "aws_ssmincidents_replication_set" "default" {
  region {
    name = local.current_region
  }
  region {
    name = var.replication_set_fallback_region
  }

  tags = {
    Name = "default"
  }
}

resource "aws_ssmcontacts_contact" "primary_contact" {
  alias        = var.primary_contact_alias
  display_name = var.primary_contact_display_name
  type         = "PERSONAL"

  tags = {
    key = "primary-contact"
  }
  depends_on = [aws_ssmincidents_replication_set.default]
}

resource "aws_ssmcontacts_contact_channel" "primary_contact_email" {
  contact_id = aws_ssmcontacts_contact.primary_contact.arn

  delivery_address {
    simple_address = var.primary_contact_email_address
  }

  name = "primary-contact-email"
  type = "EMAIL"
}

resource "aws_ssmcontacts_contact_channel" "primary_contact_sms" {
  contact_id = aws_ssmcontacts_contact.primary_contact.arn

  delivery_address {
    simple_address = var.primary_contact_phone_number
  }

  name = "primary-contact-sms"
  type = "SMS"
}

resource "aws_ssmcontacts_contact_channel" "primary_contact_voice" {
  contact_id = aws_ssmcontacts_contact.primary_contact.arn

  delivery_address {
    simple_address = var.primary_contact_phone_number
  }

  name = "primary-contact-voice"
  type = "VOICE"
}

resource "aws_ssmcontacts_plan" "primary_contact" {
  contact_id = aws_ssmcontacts_contact.primary_contact.arn

  stage {
    duration_in_minutes = 1

    target {
      channel_target_info {
        retry_interval_in_minutes = 5
        contact_channel_id        = aws_ssmcontacts_contact_channel.primary_contact_email.arn
      }
    }
  }
  stage {
    duration_in_minutes = 5

    target {
      channel_target_info {
        retry_interval_in_minutes = 5
        contact_channel_id        = aws_ssmcontacts_contact_channel.primary_contact_sms.arn
      }
    }
  }
  stage {
    duration_in_minutes = 10

    target {
      channel_target_info {
        retry_interval_in_minutes = 5
        contact_channel_id        = aws_ssmcontacts_contact_channel.primary_contact_voice.arn
      }
    }
  }
}

resource "awscc_ssmcontacts_contact" "oncall_schedule" {

  alias        = "default-schedule"
  display_name = "default-schedule"
  type         = "ONCALL_SCHEDULE"
  plan = [{
    rotation_ids = [aws_ssmcontacts_rotation.business_hours.id]
  }]
  depends_on = [aws_ssmincidents_replication_set.default]
}

resource "aws_ssmincidents_response_plan" "critical_incident" {
  name = "CRITICAL-INCIDENT"

  incident_template {
    title  = "CRITICAL-INCIDENT"
    impact = "1"
    incident_tags = {
      Name = "CRITICAL-INCIDENT"
    }

    summary = "Follow CRITICAL INCIDENT process."
  }

  display_name = "CRITICAL-INCIDENT"
  chat_channel = [var.chatbot_sns_topic_notification_arn]
  engagements  = [awscc_ssmcontacts_contact.oncall_schedule.arn]

  action {
    ssm_automation {
      document_name    = aws_ssm_document.critical_incident_runbook.arn
      role_arn         = aws_iam_role.service_role_for_ssm_incident_manager.arn
      document_version = "$LATEST"
      target_account   = "RESPONSE_PLAN_OWNER_ACCOUNT"
      parameter {
        name   = "Environment"
        values = ["Production"]
      }
      dynamic_parameters = {
        resources   = "INVOLVED_RESOURCES"
        incidentARN = "INCIDENT_RECORD_ARN"
      }
    }
  }

  tags = {
    Name = "critical-incident-response-plan"
  }

  depends_on = [aws_ssmincidents_replication_set.default]
}

resource "aws_ssmcontacts_rotation" "business_hours" {
  contact_ids = [
    aws_ssmcontacts_contact.primary_contact.arn
  ]

  name = "business-hours"

  recurrence {
    number_of_on_calls    = 1
    recurrence_multiplier = 1
    weekly_settings {
      day_of_week = "MON"
      hand_off_time {
        hour_of_day    = 09
        minute_of_hour = 00
      }
    }

    weekly_settings {
      day_of_week = "FRI"
      hand_off_time {
        hour_of_day    = 15
        minute_of_hour = 55
      }
    }

    shift_coverages {
      map_block_key = "MON"
      coverage_times {
        start {
          hour_of_day    = 08
          minute_of_hour = 30
        }
        end {
          hour_of_day    = 16
          minute_of_hour = 00
        }
      }
    }
    shift_coverages {
      map_block_key = "TUE"
      coverage_times {
        start {
          hour_of_day    = 08
          minute_of_hour = 30
        }
        end {
          hour_of_day    = 16
          minute_of_hour = 00
        }
      }
    }
    shift_coverages {
      map_block_key = "WED"
      coverage_times {
        start {
          hour_of_day    = 08
          minute_of_hour = 30
        }
        end {
          hour_of_day    = 16
          minute_of_hour = 00
        }
      }
    }
    shift_coverages {
      map_block_key = "THU"
      coverage_times {
        start {
          hour_of_day    = 08
          minute_of_hour = 30
        }
        end {
          hour_of_day    = 16
          minute_of_hour = 00
        }
      }
    }
    shift_coverages {
      map_block_key = "FRI"
      coverage_times {
        start {
          hour_of_day    = 08
          minute_of_hour = 30
        }
        end {
          hour_of_day    = 16
          minute_of_hour = 00
        }
      }
    }
  }

  start_time   = var.rotation_start_time
  time_zone_id = "Europe/Oslo"
  depends_on   = [aws_ssmincidents_replication_set.default]
}

resource "aws_ssm_document" "critical_incident_runbook" {
  name            = "critical_incident_runbook"
  document_type   = "Automation"
  document_format = "YAML"
  content         = <<DOC
#
# Original source: AWSIncidents-CriticalIncidentRunbookTemplate
#
# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of this
# software and associated documentation files (the "Software"), to deal in the Software
# without restriction, including without limitation the rights to use, copy, modify,
# merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
---
description: "This document is intended as a template for an incident response runbook in [Incident Manager](https://docs.aws.amazon.com/incident-manager/latest/userguide/index.html).\n\nFor optimal use, create your own automation document by copying the contents of this runbook template and customizing it for your scenario. Then, navigate to your [Response Plan](https://console.aws.amazon.com/systems-manager/incidents/response-plans/home) and associate it with your new automation document; your runbook is automatically started when an incident is created with the associated response plan. For more information, see [Incident Manager - Runbooks](https://docs.aws.amazon.com/incident-manager/latest/userguide/runbooks.html). \v\n\nSuggested customizations include:\n* Updating the text in each step to provide specific guidance and instructions, such as commands to run or links to relevant dashboards\n* Automating actions before triage or diagnosis to gather additional telemetry or diagnostics using aws:executeAwsApi\n* Automating actions in mitigation using aws:executeAutomation, aws:executeScript, or aws:invokeLambdaFunction\n"
schemaVersion: '0.3'
parameters:
  Environment:
    type: String
  incidentARN:
    type: String
  resources:
    type: String
mainSteps:
  - name: Triage
    action: 'aws:pause'
    inputs: {}
    description: |-
      **Determine customer impact**

      * View the **Metrics** tab of the incident or navigate to your [CloudWatch Dashboards](https://console.aws.amazon.com/cloudwatch/home#dashboards:) to find key performance indicators (KPIs) that show the extent of customer impact.
      * Use [CloudWatch Synthetics](https://console.aws.amazon.com/cloudwatch/home#synthetics:) and [Contributor Insights](https://console.aws.amazon.com/cloudwatch/home#contributorinsights:) to identify real-time failures in customer workflows.

      **Communicate customer impact**

      Update the following fields to accurately describe the incident:
      * **Title** - The title should be quickly recognizable by the team and specific to the particular incident.
      * **Summary** - The summary should contain the most important and up-to-date information to quickly onboard new responders to the incident.
      * **Impact** - Select one of the following impact ratings to describe the incident:
        * 1 – Critical impact, full application failure that impacts many to all customers.
        * 2 – High impact, partial application failure with impact to many customers.
        * 3 – Medium impact, the application is providing reduced service to many customers.
        * 4 – Low impact, the application is providing reduced service to few customers.
        * 5 – No impact, customers are not currently impacted but urgent action is needed to avoid impact.
  - name: Diagnosis
    action: 'aws:pause'
    inputs: {}
    description: |
      **Rollback**

      * Look for recent changes to the production environment that might have caused the incident. Engage the responsible team using the **Contacts** tab of the incident.
      * Rollback these changes if possible.

      **Locate failures**
      * Review metrics and alarms related to your [Application](https://console.aws.amazon.com/systems-manager/appmanager/applications). Add any related metrics and alarms to the **Metrics** tab of the incident.
      * Use [CloudWatch ServiceLens](https://console.aws.amazon.com/cloudwatch/home#servicelens:) to troubleshoot issues across multiple services.
      * Investigate the possibility of ongoing incidents across your organization. Check for known incidents and issues in AWS using [Personal Health Dashboard](https://console.aws.amazon.com/systems-manager/insights). Add related links to the **Related Items** tab of the incident.
      * Avoid going too deep in diagnosing the failure and focus on how to mitigate the customer impact. Update the **Timeline** tab of the incident when a possible diagnosis is identified.
  - name: Mitigation
    action: 'aws:pause'
    description: |-
      **Collaborate**
      * Communicate any changes or important information from the previous step to the members of the associated chat channel for this incident. Ask for input on possible ways to mitigate customer impact.
      * Engage additional contacts or teams using their escalation plan from the **Contacts** tab.
      * If necessary, prepare an emergency change request in [Change Manager](https://console.aws.amazon.com/systems-manager/change-manager).

      **Implement mitigation**
      * Consider re-routing customer traffic or throttling incoming requests to reduce customer impact.
      * Look for common runbooks in [Automation](https://console.aws.amazon.com/systems-manager/automation) or run commands using [Run Command](https://.console.aws.amazon.com/systems-manager/run-command).
      * Update the **Timeline** tab of the incident when a possible mitigation is identified. If needed, review the mitigation with others in the associated chat channel before proceeding.
    inputs: {}
  - name: Recovery
    action: 'aws:pause'
    inputs: {}
    description: |-
      **Monitor customer impact**
      * View the **Metrics** tab of the incident to monitor for recovery of your key performance indicators (KPIs).
      * Update the **Impact** field in the incident when customer impact has been reduced or resolved.

      **Identify action items**
      * Add entries in the **Timeline** tab of the incident to record key decisions and actions taken, including temporary mitigations that might have been implemented.
      * Create a **Post-Incident Analysis** when the incident is closed in order to identify and track action items in [OpsCenter](https://console.aws.amazon.com/systems-manager/opsitems).

DOC
}

# Use aws and awscc providers to provision SSM Incident Manager
# Ref. https://registry.terraform.io/providers/hashicorp/aws/latest/docs/guides/using-aws-with-awscc-provider
# Refers to output from the previous module provision for AWS Chatbot. 
module "ssm_incident_manager" {
  source = "git::https://github.com/haakond/terraform-aws-ssm-incident-manager.git?ref=dev"
  providers = {
    aws   = aws
    awscc = awscc
  }
  primary_contact_alias              = "primary-contact"
  primary_contact_display_name       = "Håkon Eriksen Drange"
  primary_contact_email_address      = "alpha.bravo@charlie-company.com"
  primary_contact_phone_number       = "+4799887766"
  chatbot_sns_topic_notification_arn = module.aws_chatbot_slack.chatbot_sns_topic_arn_primary_region
  rotation_start_time                = "2024-06-24T07:00:00+00:00"
}

provider "aws" {
  region  = var.aws_region
  profile = var.profile_cicd
  assume_role {
    role_arn     = "arn:aws:iam::${var.aws_account_id}:role/${var.profile_cicd}"
    session_name = "SESSION_NAME"
    external_id  = "EXTERNAL_ID"
  }
}

provider "awscc" {
  region  = var.aws_region
  profile = var.profile_cicd
  assume_role = {
    role_arn     = "arn:aws:iam::${var.aws_account_id}:role/${var.profile_cicd}"
    session_name = "SESSION_NAME"
    external_id  = "EXTERNAL_ID"
  }
}

resource "aws_cloudwatch_metric_alarm" "demo_ec2_instance_cpu_utilization" {
  alarm_name          = "CRITICAL-EC2-CPU_i-04f7af9ea5374c4d8"
  comparison_operator = "GreaterThanOrEqualToThreshold"
  evaluation_periods  = "3"
  metric_name         = "CPUUtilization"
  namespace           = "AWS/EC2"
  period              = "300"
  statistic           = "Average"
  threshold           = "80"
  alarm_description   = "CRITICAL CPU utilization for instance ID i-04f7af9ea5374c4d8!"
  dimensions = {
    InstanceId = "i-04f7af9ea5374c4d8"
  }
  alarm_actions = [module.aws_chatbot_slack.chatbot_sns_topic_arn_primary_region, module.ssm_incident_manager.critical_incident_response_plan_arn]
  ok_actions    = [module.aws_chatbot_slack.chatbot_sns_topic_arn_primary_region]
}