Merge branch 'master' into master

Author: michelp

README.md

@@ -1,6 +1,14 @@
+<a href="https://postgres.ai"><img src="https://img.shields.io/badge/Postgres-AI-orange.svg" alt="Postgres.AI" /></a>
+  &nbsp;
+[![GitHub code size in bytes](https://img.shields.io/github/languages/code-size/badges/shields.svg)](github.com/postgres-ai/nancy)
+  &nbsp;
+<img src="https://img.shields.io/docker/pulls/postgresmen/postgres-with-stuff.svg" />
+  &nbsp;
 [![CircleCI](https://circleci.com/gh/postgres-ai/nancy.svg?style=svg)](https://circleci.com/gh/postgres-ai/nancy)
 
-Description
+
+About
+<img width="122" alt="screen shot 2018-09-18 at 03 04 09" src="https://user-images.githubusercontent.com/1345402/45656700-8a987f00-baef-11e8-87b6-cccf8f65ee8f.png" align="right">
 ===
 Nancy helps to conduct automated database experiments.
 

docker/Dockerfile

@@ -24,7 +24,7 @@ RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys B97
       && apt-get install -y git postgresql-client-10 pspg pgreplay jq etcd libjson-xs-perl \
       && perl -MCPAN -e'install Text::CSV_XS' \
       && git clone https://github.com/NikolayS/postgres_dba.git /root/postgres_dba \
-      && git clone https://github.com/NikolayS/pgbadger.git /root/pgbadger
+      && git clone https://github.com/darold/pgbadger.git /root/pgbadger
 
 # additionally, install newer NodeJS, npm, Sqitch, and more
 RUN wget -q -S -O - https://deb.nodesource.com/setup_8.x | sudo bash \

nancy_run.sh

@@ -13,6 +13,7 @@ CURRENT_TS=$(date +%Y%m%d_%H%M%S%N_%Z)
 DOCKER_MACHINE="nancy-$CURRENT_TS"
 DOCKER_MACHINE="${DOCKER_MACHINE//_/-}"
 KEEP_ALIVE=0
+DURATION_WRKLD=""
 VERBOSE_OUTPUT_REDIRECT=" > /dev/null"
 EBS_SIZE_MULTIPLIER=5
 POSTGRES_VERSION_DEFAULT=10
@@ -1065,7 +1066,7 @@ if [[ "$RUN_ON" == "localhost" ]]; then
   if [[ -z ${CONTAINER_ID+x} ]]; then
     CONTAINER_HASH=$(docker run --name="pg_nancy_${CURRENT_TS}" \
       -v $TMP_PATH:/machine_home \
-      -dit "postgresmen/postgres-with-stuff:pg${PG_VERSION}" \
+      -dit "postgresmen/postgres-with-stuff:postgres${PG_VERSION}_pgbadger10" \
     )
   else
     CONTAINER_HASH="$CONTAINER_ID"
@@ -1388,7 +1389,9 @@ function apply_postgres_configuration() {
 #   None
 #######################################
 function prepare_start_workload() {
-  #Save before workload log
+  msg "Execute vacuumdb..."
+  docker_exec vacuumdb -U postgres $DB_NAME -j $CPU_CNT --analyze
+
   msg "Save prepaparation log"
   logpath=$( \
     docker_exec bash -c "psql -XtU postgres \
@@ -1398,17 +1401,16 @@ function prepare_start_workload() {
   docker_exec bash -c "mkdir $MACHINE_HOME/$ARTIFACTS_FILENAME"
   docker_exec bash -c "gzip -c $logpath > $MACHINE_HOME/$ARTIFACTS_FILENAME/postgresql.prepare.log.gz"
 
-  # Clear statistics and log
-  msg "Execute vacuumdb..."
-  docker_exec vacuumdb -U postgres $DB_NAME -j $CPU_CNT --analyze
+  msg "Reset pg_stat_*** and Postgres log"
+  docker_exec psql -U postgres $DB_NAME -c 'select pg_stat_reset(), pg_stat_statements_reset();' >/dev/null
   docker_exec bash -c "echo '' > /var/log/postgresql/postgresql-$PG_VERSION-main.log"
 }
 
 #######################################
 # Execute workload.
 # Globals:
 #   WORKLOAD_REAL, WORKLOAD_REAL_REPLAY_SPEED, WORKLOAD_CUSTOM_SQL, MACHINE_HOME,
-#   DB_NAME, VERBOSE_OUTPUT_REDIRECT, docker_exec alias
+#   DURATION_WRKLD, DB_NAME, VERBOSE_OUTPUT_REDIRECT, docker_exec alias
 # Arguments:
 #   None
 # Returns:
@@ -1439,6 +1441,7 @@ function execute_workload() {
   END_TIME=$(date +%s)
   DURATION=$(echo $((END_TIME-OP_START_TIME)) | awk '{printf "%d:%02d:%02d", $1/3600, ($1/60)%60, $1%60}')
   msg "Time taken to execute workload: $DURATION."
+  DURATION_WRKLD="$DURATION"
 }
 
 #######################################
@@ -1490,7 +1493,7 @@ function collect_results() {
     if [[ "$RUN_ON" == "localhost" ]]; then
       docker cp $CONTAINER_HASH:$MACHINE_HOME/$ARTIFACTS_FILENAME $ARTIFACTS_DESTINATION/
     elif [[ "$RUN_ON" == "aws" ]]; then
-      mkdir $ARTIFACTS_DESTINATION/$ARTIFACTS_FILENAME
+      mkdir -p $ARTIFACTS_DESTINATION/$ARTIFACTS_FILENAME
       docker-machine scp $DOCKER_MACHINE:/home/storage/$ARTIFACTS_FILENAME/* $ARTIFACTS_DESTINATION/$ARTIFACTS_FILENAME/
     else
       err "ASSERT: must not reach this point"
@@ -1531,17 +1534,24 @@ apply_ddl_undo_code
 
 END_TIME=$(date +%s)
 DURATION=$(echo $((END_TIME-START_TIME)) | awk '{printf "%d:%02d:%02d", $1/3600, ($1/60)%60, $1%60}')
-echo -e "$(date "+%Y-%m-%d %H:%M:%S"): Run done for $DURATION"
-echo -e "  JSON Report: $ARTIFACTS_DESTINATION/$ARTIFACTS_FILENAME/pgbadger.json"
-echo -e "  HTML Report: $ARTIFACTS_DESTINATION/$ARTIFACTS_FILENAME/pgbadger.html"
-echo -e "  Query log: $ARTIFACTS_DESTINATION/$ARTIFACTS_FILENAME/postgresql.workload.log.gz"
-echo -e "  Prepare log: $ARTIFACTS_DESTINATION/$ARTIFACTS_FILENAME/postgresql.prepare.log.gz"
-echo -e "  Postgresql configuration log: $ARTIFACTS_DESTINATION/$ARTIFACTS_FILENAME/postgresql.conf"
-
-echo -e "  -------------------------------------------"
-echo -e "  Workload summary:"
-echo -e "    Summarized query duration:\t" $(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json | jq '.overall_stat.queries_duration') " ms"
-echo -e "    Queries:\t\t\t" $( docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json | jq '.overall_stat.queries_number')
-echo -e "    Query groups:\t\t" $(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json | jq '.normalyzed_info| length')
-echo -e "    Errors:\t\t\t" $(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json | jq '.overall_stat.errors_number')
-echo -e "-------------------------------------------"
+msg "Done."
+echo -e "------------------------------------------------------------------------------"
+echo -e "Artifacts (collected in \"$ARTIFACTS_DESTINATION/$ARTIFACTS_FILENAME/\"):"
+echo -e "  Postgres config:    postgresql.conf"
+echo -e "  Postgres logs:      postgresql.prepare.log.gz (preparation),"
+echo -e "                      postgresql.workload.log.gz (workload)"
+echo -e "  pgBadger reports:   pgbadger.html (for humans),"
+echo -e "                      pgbadger.json (for robots)"
+echo -e "  Stat stapshots:     pg_stat_statements.csv,"
+echo -e "                      pg_stat_***.csv"
+echo -e "------------------------------------------------------------------------------"
+echo -e "Total execution time: $DURATION"
+echo -e "------------------------------------------------------------------------------"
+echo -e "Workload:"
+echo -e "  Execution time:     $DURATION_WRKLD"
+echo -e "  Total query time:   "$(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json | jq '.overall_stat.queries_duration') " ms"
+echo -e "  Queries:            "$(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json | jq '.overall_stat.queries_number')
+echo -e "  Query groups:       "$(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json | jq '.normalyzed_info | length')
+echo -e "  Errors:             "$(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json | jq '.overall_stat.errors_number')
+echo -e "  Errors groups:      "$(docker_exec cat $MACHINE_HOME/$ARTIFACTS_FILENAME/pgbadger.json | jq '.error_info | length')
+echo -e "------------------------------------------------------------------------------"