Merge pull request #13 from theodoresiu/log_slack_alerts

Hourly Audit Log Slack Alerts Example

Author: aaron-lane

examples/logs-slack-alerts/README.md

@@ -0,0 +1,30 @@
+# Logging Slack Alerts
+
+Disclaimer (8/1/2019): Test Coverage has currently not been added to this example and may be unintentionally broken by future releases to the module.
+
+This logging slack alerts example module schedules a job to run hourly queries of any errors which have occurred in logs which have been ingested into BigQuery. If any errors are found, the errors are sent as alerts to a slack webhook.
+
+Running this module requires log exports into BigQuery in the specified project/region, which is not handled by this example. 
+A good example of exported logging in BigQuery can be found in [Stackdriver Logging](https://cloud.google.com/logging/docs/export/).
+
+## Configure a Service Account
+
+If not using the default App Engine default service account (PROJECT_ID@appspot.gserviceaccount.com), which has the Editor role on the project, one can configure a service account for the cloud function which has the following IAM role - (roles/bigquery.dataViewer, roles/bigquery.jobUser). Additionally, the BigQuery API (https://bigquery.googleapis.com) needs to be enabled as well.
+
+
+[^]: (autogen_docs_start)
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| project\_id | The project ID to host the network in | string | n/a | yes |
+| region | The region the project is in (App Engine specific) | string | `"us-central1"` | no |
+| slack_webhook | The Slack webhook to send alerts | string | n/a | yes |
+| dataset_name | The BigQuery Dataset where exported logging is sent | string | n/a | yes |
+| audit_log_table | The BigQuery Table within the dataset where logging is sent | string | n/a | yes |
+| time_column | The column within the BQ Table representing logging time | string | n/a | yes |
+| error_message_column | The column within the BQ Table representing logging errors | string | n/a | yes |
+
+
+[^]: (autogen_docs_end)

examples/logs-slack-alerts/function_source/main.py

@@ -0,0 +1,80 @@
+"""
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+"""
+
+import os
+import logging
+import requests
+from google.cloud import bigquery
+
+BQ_CLIENT = bigquery.Client()
+logging.getLogger().setLevel(logging.INFO)
+
+VARIABLES = {
+    "SLACK_WEBHOOK_URL": os.getenv("SLACK_WEBHOOK_URL"),
+    "DATASET_NAME": os.getenv("DATASET_NAME"),
+    "AUDIT_LOG_TABLE":  os.getenv("AUDIT_LOG_TABLE"),
+    "TIME_COLUMN": os.getenv("TIME_COLUMN"),
+    "ERROR_MESSAGE_COLUMN": os.getenv("ERROR_MESSAGE_COLUMN")
+}
+QUERY = """
+WITH
+  errors AS (
+  SELECT
+    {ERROR_MESSAGE_COLUMN} AS error_message,
+    EXTRACT(HOUR FROM current_timestamp) as hr
+  FROM
+    {DATASET_NAME}.{AUDIT_LOG_TABLE}
+  WHERE
+    {ERROR_MESSAGE_COLUMN} IS NOT NULL
+    AND EXTRACT(HOUR
+    FROM
+      current_timestamp) = EXTRACT(HOUR
+    FROM
+     {TIME_COLUMN}))
+SELECT
+  error_message as Error,
+  hr,
+  COUNT(*) as Count
+FROM
+  errors
+GROUP BY
+  1,2
+""".format(**VARIABLES)
+
+def query_for_errors(pubsub_event, pubsub_context):
+    """
+    Cloud Function to query audit logs for errors
+    and send alerts to Slack Webhook
+    """
+    
+    logging.info("Running: %s", QUERY)
+    query_job = BQ_CLIENT.query(QUERY)
+
+    if list(query_job):
+        for row in list(query_job):
+            text = ("Alert: Error {0}... has occurred {1} times"
+                    "in the past hour - {2}:00 PST. "
+                    "Please file a bug ticket to have").format(
+                        (row["Error"][:500]),
+                        str(row["Count"]),
+                        str(row["hr"]))
+            logging.info("Posting to Slack: %s", text)
+            req = requests.post(url=VARIABLES['SLACK_WEBHOOK_URL'],
+                                data=str({"text": text}))
+            logging.info(req.text)
+
+if __name__ == "__main__":
+    query_for_errors(None, None)

examples/logs-slack-alerts/function_source/requirements.txt

@@ -0,0 +1,2 @@
+google-cloud-bigquery==1.14.0
+requests==2.21.0

examples/logs-slack-alerts/main.tf

@@ -0,0 +1,49 @@
+/**
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+provider "google-beta" {
+  version = "~> 2.1"
+  project = var.project_id
+  region  = var.region
+}
+
+module "log_slack_alerts_example" {
+  providers = {
+    google = google-beta
+  }
+
+  source                    = "../../"
+  project_id                = var.project_id
+  job_name                  = "logs_query"
+  job_description           = "Scheduled time to run audit query to check for errors"
+  job_schedule              = "55 * * * *"
+  function_entry_point      = "query_for_errors"
+  function_source_directory = "${path.module}/function_source"
+  function_name             = "logs_query_alerting"
+  function_description      = "Cloud Function to query audit logs for errors"
+  region                    = var.region
+  topic_name                = "logs_query_topic"
+  function_runtime          = "python37"
+
+  function_environment_variables = {
+    SLACK_WEBHOOK        = var.slack_webhook
+    DATASET_NAME         = var.dataset_name
+    AUDIT_LOG_TABLE      = var.audit_log_table
+    TIME_COLUMN          = var.time_column
+    ERROR_MESSAGE_COLUMN = var.error_message_column
+  }
+}
+

examples/logs-slack-alerts/variables.tf

@@ -0,0 +1,59 @@
+/**
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "The project ID to host the network in"
+  type = string
+}
+
+variable "job_schedule" {
+  description = "The cron schedule for triggering the cloud function"
+  type = string
+  default     = "55 * * * *"
+  
+}
+
+variable "slack_webhook" {
+  description = "Slack webhook to send alerts"
+  type = string
+}
+
+variable "dataset_name" {
+  description = "BigQuery Dataset where logs are sent"
+  type = string
+}
+
+variable "audit_log_table" {
+  description = "BigQuery Table where logs are sent"
+  type = string
+}
+
+variable "time_column" {
+  description = "BigQuery Column in audit log table representing logging time"
+  type = string
+}
+
+variable "error_message_column" {
+  description = "BigQuery Column in audit log table representing logging error"
+  type = string
+}
+
+variable "region" {
+  description = "The region the project is in (App Engine specific)"
+  type = string
+  default     = "us-central1"
+}
+