Skip to content

Commit 30523cd

Browse files
joshmyersjoshmyers
authored
Add ES cleanup module (#1)
* Add first cut of the ES cleanup module * Add working example usage This creates all the base resources necessary to test the elasticsearch-cleanup module. Unfortunately this requires a two stage apply as the `aws_route53_zone` resource fails to compute ID when passed into other modules. See: cloudposse/terraform-aws-elasticsearch#13 * Deploy Lambda artifact from S3 bucket Pull the deployment artifact from S3 using the new terraform-external-module-artifact module. This approach has the benefit of not checking in zip files into git, but the downside of more preamble when wanting to test this module as per the example docs. You need to know that the function should already be in S3 at a pre determined path. We need to think of proper deployment and promotion of these artifacts through different S3 buckets e.g. artifacts.testing.cloudposse.org -> artifacts.prod.cloudposse.org
1 parent dcd1601 commit 30523cd

File tree

13 files changed

+776
-11
lines changed

13 files changed

+776
-11
lines changed

‎.gitignore‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,18 @@
1-
# Compiled files
1+
# Local .terraform directories
2+
**/.terraform/*
3+
4+
# .tfstate files
25
*.tfstate
3-
*.tfstate.backup
4-
*.zip
6+
*.tfstate.*
7+
8+
# .tfvars files
9+
*.tfvars
510

6-
# Module directory
7-
.terraform
8-
.idea
9-
*.iml
11+
**/.idea
12+
**/*.iml
1013

11-
.build-harness
12-
build-harness
14+
**/.build-harness
15+
**/build-harness
1316
**/venv
17+
18+
*.zip

‎.travis.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@ addons:
55
- make
66
- curl
77

8+
env:
9+
global:
10+
- TERRAFORM_MODULE_NAME="$(basename $TRAVIS_REPO_SLUG)"
11+
812
install:
913
- make init
1014

@@ -14,3 +18,17 @@ script:
1418
- make terraform/get-modules
1519
- make terraform/lint
1620
- make terraform/validate
21+
- make build
22+
23+
deploy:
24+
# Deploy artifacts to S3
25+
- provider: s3
26+
region: "us-west-2"
27+
access_key_id: $AWS_ACCESS_KEY_ID
28+
secret_access_key: $AWS_SECRET_ACCESS_KEY
29+
bucket: artifacts.prod.cloudposse.org
30+
skip_cleanup: true
31+
upload-dir: $TERRAFORM_MODULE_NAME/$TRAVIS_COMMIT
32+
local-dir: artifacts
33+
on:
34+
all_branches: true

‎Makefile‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
SHELL := /bin/bash
1+
SHELL := /bin/bash
2+
LAMBDA_DIR := lambda
3+
DEPS_CONTAINER := alpine:3.8
24

35
# List of targets the `readme` target should call before generating the readme
46
export README_DEPS ?= docs/targets.md docs/terraform.md
@@ -7,4 +9,23 @@ export README_DEPS ?= docs/targets.md docs/terraform.md
79

810
## Lint terraform code
911
lint:
10-
$(SELF) terraform/install terraform/get-modules terraform/get-plugins terraform/lint terraform/validate
12+
$(SELF) terraform/install terraform/get-modules terraform/get-plugins terraform/lint terraform/validate
13+
14+
define docker
15+
docker run -it -v $(PWD)/$(LAMBDA_DIR)/:/code -w /code $(DEPS_CONTAINER) /bin/sh -c '$(1)'
16+
endef
17+
18+
## Install dependencies
19+
dependencies:
20+
@echo "==> Installing Lambda function dependencies..."
21+
@$(call docker, apk add --update py-pip && \
22+
pip install virtualenv && \
23+
virtualenv venv --always-copy && \
24+
source ./venv/bin/activate && \
25+
./venv/bin/pip install -qUr requirements.txt)
26+
27+
## Build Lambda function zip
28+
build: dependencies
29+
@echo "==> Building Lambda function zip..."
30+
@mkdir -p artifacts
31+
@cd $(LAMBDA_DIR) && zip -r ../artifacts/lambda.zip * && cd ../

‎README.yaml‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
---
2+
#
3+
# This is the canonical configuration for the `README.md`
4+
# Run `make readme` to rebuild the `README.md`
5+
#
6+
7+
# Name of this project
8+
name: terraform-aws-lambda-elasticsearch-cleanup
9+
10+
# Logo for this project
11+
#logo: docs/logo.png
12+
13+
# License of this project
14+
license: "APACHE2"
15+
16+
# Canonical GitHub repo
17+
github_repo: cloudposse/terraform-aws-lambda-elasticsearch-cleanup
18+
19+
# Badges to display
20+
badges:
21+
- name: "Build Status"
22+
image: "https://travis-ci.org/cloudposse/terraform-aws-lambda-elasticsearch-cleanup.svg?branch=master"
23+
url: "https://travis-ci.org/cloudposse/terraform-aws-lambda-elasticsearch-cleanup"
24+
- name: "Latest Release"
25+
image: "https://img.shields.io/github/release/cloudposse/terraform-aws-lambda-elasticsearch-cleanup.svg"
26+
url: "https://github.com/cloudposse/terraform-aws-lambda-elasticsearch-cleanup/releases/latest"
27+
- name: "Slack Community"
28+
image: "https://slack.cloudposse.com/badge.svg"
29+
url: "https://slack.cloudposse.com"
30+
31+
related:
32+
- name: "terraform-aws-vpc"
33+
description: "Terraform Module that defines a VPC with public/private subnets across multiple AZs with Internet Gateways"
34+
url: "https://github.com/cloudposse/terraform-aws-vpc"
35+
- name: "terraform-aws-dynamic-subnets"
36+
description: "Terraform module for dynamic subnets provisioning."
37+
url: "https://github.com/cloudposse/terraform-aws-dynamic-subnets"
38+
- name: "terraform-aws-elasticsearch"
39+
description: "Terraform module for AWS Elasticsearch provisioning."
40+
url: "https://github.com/cloudposse/terraform-aws-elasticsearch"
41+
42+
# Short description of this project
43+
description: |-
44+
Terraform module to provision a scheduled Lambda function which will
45+
delete old Elasticsearch indexes using [SigV4Auth](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) authentication. The
46+
lambda function can optionally send output to an SNS topic if the
47+
topic ARN is given. This module was largely inspired by
48+
[aws-lambda-es-cleanup](https://github.com/cloudreach/aws-lambda-es-cleanup)
49+
50+
# How to use this project
51+
usage: |-
52+
```hcl
53+
module "elasticsearch_cleanup" {
54+
source = "../"
55+
es_endpoint = "${module.elasticsearch.domain_endpoint}"
56+
es_domain_arn = "${module.elasticsearch.domain_arn}"
57+
es_security_group_id = "${module.elasticsearch.security_group_id}"
58+
vpc_id = "${module.vpc.vpc_id}"
59+
namespace = "example"
60+
stage = "dev"
61+
schedule = "rate(5 minutes)"
62+
}
63+
```
64+
65+
include:
66+
- "docs/targets.md"
67+
- "docs/terraform.md"
68+
69+
# Contributors to this project
70+
contributors:
71+
- name: "Josh Myers"
72+
github: "joshmyers"

‎docs/targets.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
## Makefile Targets
2+
```
3+
Available targets:
4+
5+
build Build Lambda function zip
6+
dependencies Install dependencies
7+
help Help screen
8+
help/all Display help for all targets
9+
help/short This help short screen
10+
lint Lint terraform code
11+
12+
```

‎docs/terraform.md‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
## Module: cloudposse/terraform-aws-lambda-elasticsearch-cleanup
2+
3+
This module creates a scheduled Lambda function which will delete old
4+
Elasticsearch indexes using SigV4Auth authentication. The lambda
5+
function can optionally send output to an SNS topic if the topic ARN
6+
is given
7+
8+
## Inputs
9+
10+
| Name | Description | Type | Default | Required |
11+
|------|-------------|:----:|:-----:|:-----:|
12+
| attributes | Additional attributes (e.g. `1`) | list | `<list>` | no |
13+
| delete_after | Number of days to preserve | string | `15` | no |
14+
| delimiter | Delimiter to be used between `namespace`, `stage`, `name` and `attributes` | string | `-` | no |
15+
| es_domain_arn | The Elasticsearch domain ARN | string | - | yes |
16+
| es_endpoint | The Elasticsearch endpoint for the Lambda function to connect to | string | - | yes |
17+
| es_security_group_id | The Elasticsearch cluster security group ID | string | - | yes |
18+
| index | Index/indices to process. Use a comma-separated list. Specify `all` to match every index except for `.kibana` | string | `all` | no |
19+
| index_format | Combined with 'index' variable and is used to evaluate the index age | string | `%Y.%m.%d` | no |
20+
| name | Solution name, e.g. 'app' or 'cluster' | string | `app` | no |
21+
| namespace | Namespace, which could be your organization name, e.g. 'eg' or 'cp' | string | - | yes |
22+
| python_version | The Python version to use | string | `2.7` | no |
23+
| schedule | CloudWatch Events rule schedule using cron or rate expression | string | `cron(0 3 * * ? *)` | no |
24+
| sns_arn | SNS ARN to pusblish alerts | string | `` | no |
25+
| stage | Stage, e.g. 'prod', 'staging', 'dev', or 'test' | string | - | yes |
26+
| subnet_ids | Subnet ids | list | - | yes |
27+
| tags | Additional tags (e.g. `map('BusinessUnit','XYZ')` | map | `<map>` | no |
28+
| timeout | Timeout for Lambda function in seconds | string | `300` | no |
29+
| vpc_id | The VPC ID for the Lambda function | string | - | yes |
30+
31+
## Outputs
32+
33+
| Name | Description |
34+
|------|-------------|
35+
| security_group_id | Security Group ID of the Lambda |
36+

‎example/main.tf‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
module "label" {
2+
source = "git::https://github.com/cloudposse/terraform-terraform-label.git?ref=tags/0.2.1"
3+
namespace = "${var.namespace}"
4+
name = "${var.name}"
5+
stage = "${var.stage}"
6+
delimiter = "${var.delimiter}"
7+
attributes = "${compact(concat(var.attributes, list("elasticsearch", "cleanup")))}"
8+
tags = "${var.tags}"
9+
enabled = "true"
10+
}
11+
12+
module "vpc" {
13+
source = "git::https://github.com/cloudposse/terraform-aws-vpc.git?ref=master"
14+
name = "${var.name}"
15+
namespace = "${var.namespace}"
16+
stage = "${var.stage}"
17+
tags = "${module.label.tags}"
18+
}
19+
20+
module "subnets" {
21+
source = "git::https://github.com/cloudposse/terraform-aws-dynamic-subnets.git?ref=master"
22+
name = "${var.name}"
23+
namespace = "${var.namespace}"
24+
stage = "${var.stage}"
25+
region = "us-west-2"
26+
vpc_id = "${module.vpc.vpc_id}"
27+
igw_id = "${module.vpc.igw_id}"
28+
cidr_block = "10.0.0.0/16"
29+
availability_zones = ["us-west-2a", "us-west-2b"]
30+
tags = "${module.label.tags}"
31+
}
32+
33+
module "elasticsearch" {
34+
source = "git::https://github.com/cloudposse/terraform-aws-elasticsearch.git?ref=master"
35+
name = "${var.name}"
36+
namespace = "${var.namespace}"
37+
stage = "${var.stage}"
38+
dns_zone_id = "Z3SO0TKDDQ0RGG"
39+
security_groups = []
40+
vpc_id = "${module.vpc.vpc_id}"
41+
subnet_ids = ["${module.subnets.public_subnet_ids}"]
42+
zone_awareness_enabled = "true"
43+
elasticsearch_version = "6.3"
44+
instance_type = "t2.small.elasticsearch"
45+
instance_count = 4
46+
kibana_subdomain_name = "kibana-es"
47+
encrypt_at_rest_enabled = "false"
48+
ebs_volume_size = 10
49+
iam_actions = ["es:*"]
50+
iam_role_arns = ["*"]
51+
create_iam_service_linked_role = "false"
52+
tags = "${module.label.tags}"
53+
}
54+
55+
module "elasticsearch_cleanup" {
56+
source = "../"
57+
es_endpoint = "${module.elasticsearch.domain_endpoint}"
58+
es_domain_arn = "${module.elasticsearch.domain_arn}"
59+
es_security_group_id = "${module.elasticsearch.security_group_id}"
60+
subnet_ids = ["${module.subnets.public_subnet_ids}"]
61+
vpc_id = "${module.vpc.vpc_id}"
62+
namespace = "${var.namespace}"
63+
stage = "${var.stage}"
64+
schedule = "${var.schedule}"
65+
tags = "${module.label.tags}"
66+
}

‎example/variables.tf‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
variable "schedule" {
2+
default = "rate(5 minutes)"
3+
}
4+
5+
variable "namespace" {
6+
type = "string"
7+
description = "Namespace, which could be your organization name, e.g. 'eg' or 'cp'"
8+
}
9+
10+
variable "stage" {
11+
type = "string"
12+
description = "Stage, e.g. 'prod', 'staging', 'dev', or 'test'"
13+
}
14+
15+
variable "environment" {
16+
type = "string"
17+
default = ""
18+
description = "Environment, e.g. 'testing', 'UAT'"
19+
}
20+
21+
variable "name" {
22+
type = "string"
23+
default = "app"
24+
description = "Solution name, e.g. 'app' or 'cluster'"
25+
}
26+
27+
variable "delimiter" {
28+
type = "string"
29+
default = "-"
30+
description = "Delimiter to be used between `name`, `namespace`, `stage`, etc."
31+
}
32+
33+
variable "attributes" {
34+
type = "list"
35+
default = []
36+
description = "Additional attributes (e.g. `1`)"
37+
}
38+
39+
variable "tags" {
40+
type = "map"
41+
default = {}
42+
description = "Additional tags (e.g. `map('BusinessUnit`,`XYZ`)"
43+
}

0 commit comments

Comments
 (0)