Merge pull request #3 from philips-labs/feature/runners-lambdas

Base setup for scaling and cleanup lambdas

Author: npalm

.ci/Dockerfile

@@ -0,0 +1,13 @@
+FROM node:12
+
+WORKDIR /lambda
+
+COPY . /lambda
+
+RUN apt-get update \
+    && apt-get install -y zip \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN yarn install \
+    && yarn run dist
+

.ci/build.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+lambaSrcDirs=("modules/runner-binaries-syncer/lambdas/runner-binaries-syncer" "modules/runners/lambdas/scale-runners" "modules/webhook/lambdas/webhook")
+repoRoot=$(dirname "${BASH_SOURCE[0]}")/..
+
+for lambdaDir in ${lambaSrcDirs[@]}; do
+    cd $repoRoot/${lambdaDir}
+    docker build -t lambda -f ../../../../.ci/Dockerfile .
+    docker create --name lambda lambda
+    zipName=$(basename "$PWD")
+    docker cp lambda:/lambda/${zipName}.zip ${zipName}.zip
+    docker rm lambda
+done

.github/workflows/lambda-scale-runners.yml

@@ -0,0 +1,26 @@
+name: Lambda Scale Runners
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - .github/workflows/lambda-scale-runners.yml
+      - "modules/runners/lambdas/scale-runners/**"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    container: node:12
+    defaults:
+      run:
+        working-directory: modules/runners/lambdas/scale-runners
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install dependencies
+        run: yarn install
+      - name: Run tests
+        run: yarn test
+      - name: Build distribution
+        run: yarn build

.github/workflows/terraform.yml

@@ -19,9 +19,10 @@ jobs:
       - name: "Checkout"
         uses: actions/checkout@v2
       - name: "Fake zip files" # Validate will fail if it cannot find the zip files
-        run:
-          touch modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/runner-binaries-syncer.zip
+        run: |
           touch modules/webhook/lambdas/webhook/webhook.zip
+          touch modules/runners/lambdas/scale-runners/scale-runners.zip
+          touch modules/runner-binaries-syncer/lambdas/runner-binaries-syncer/runner-binaries-syncer.zip
       - name: "Terraform Format"
         uses: hashicorp/terraform-github-actions@master
         with:

.gitignore

@@ -16,3 +16,4 @@ example/*.secrets*.tfvars
 *.zip
 *.gz
 *.tgz
+*.env

examples/default/main.tf

@@ -8,20 +8,28 @@ resource "random_password" "random" {
   length = 32
 }
 
-
 module "runners" {
   source = "../../"
 
   aws_region = local.aws_region
   vpc_id     = module.vpc.vpc_id
+  subnet_ids = module.vpc.private_subnets
 
   environment = local.environment
   tags = {
     Project = "ProjectX"
   }
 
-  github_app_webhook_secret = random_password.random.result
+  github_app = {
+    key_base64     = var.github_app_key_base64
+    id             = var.github_app_id
+    client_id      = var.github_app_client_id
+    client_secret  = var.github_app_client_secret
+    webhook_secret = random_password.random.result
+  }
 
+  enable_organization_runners = false
+  runner_extra_labels         = "default,example"
 }
 
 

examples/default/outputs.tf

@@ -1,17 +1,12 @@
-output "action_runners" {
+output "runners" {
   value = {
     runners = module.runners.runners
   }
 }
 
-
-output "lambda_binaries_syncer_name" {
-  value = module.runners.binaries_syncer.lambda.id
-}
-
-
-output "github_app_webhook_secret" {
-  value = random_password.random.result
+output "webhook" {
+  value = {
+    secret  = random_password.random.result
+    gateway = module.runners.webhook.gateway
+  }
 }
-
-

examples/default/providers.tf

@@ -1,5 +1,5 @@
 provider "aws" {
   region  = local.aws_region
-  version = "2.59"
+  version = "2.61"
 }
 

examples/default/variables.tf

@@ -0,0 +1,9 @@
+
+variable "github_app_key_base64" {}
+
+variable "github_app_id" {}
+
+variable "github_app_client_id" {}
+
+variable "github_app_client_secret" {}
+

main.tf

@@ -15,6 +15,7 @@ resource "random_string" "random" {
 resource "aws_sqs_queue" "queued_builds" {
   name                        = "${var.environment}-queued-builds.fifo"
   delay_seconds               = 30
+  visibility_timeout_seconds  = 60
   fifo_queue                  = true
   receive_wait_time_seconds   = 10
   content_based_deduplication = true
@@ -30,19 +31,27 @@ module "webhook" {
   tags        = local.tags
 
   sqs_build_queue           = aws_sqs_queue.queued_builds
-  github_app_webhook_secret = var.github_app_webhook_secret
+  github_app_webhook_secret = var.github_app.webhook_secret
 }
 
 module "runners" {
   source = "./modules/runners"
 
   aws_region  = var.aws_region
   vpc_id      = var.vpc_id
+  subnet_ids  = var.subnet_ids
   environment = var.environment
   tags        = local.tags
 
   s3_bucket_runner_binaries   = module.runner_binaries.bucket
   s3_location_runner_binaries = local.s3_action_runner_url
+
+  sqs                             = aws_sqs_queue.queued_builds
+  github_app                      = var.github_app
+  enable_organization_runners     = var.enable_organization_runners
+  scale_down_schedule_expression  = var.scale_down_schedule_expression
+  minimum_running_time_in_minutes = var.minimum_running_time_in_minutes
+  runner_extra_labels             = var.runner_extra_labels
 }
 
 module "runner_binaries" {