Merge pull request #111 from postgres-ai/dmius-use-ebs-vol

WIP: Use ebs volume with backup

Author: dmius

nancy_run.sh

@@ -19,6 +19,28 @@ EBS_SIZE_MULTIPLIER=5
 POSTGRES_VERSION_DEFAULT=10
 AWS_BLOCK_DURATION=0
 
+#######################################
+# Attach an EBS volume containing the database backup (made with pg_basebackup)
+# Globals:
+#   DOCKER_MACHINE, AWS_REGION, DB_EBS_VOLUME_ID
+# Arguments:
+#   None
+# Returns:
+#   None
+#######################################
+function attach_db_ebs_drive() {
+  docker-machine ssh $DOCKER_MACHINE "sudo sh -c \"mkdir /home/backup\""
+  docker-machine ssh $DOCKER_MACHINE "wget http://s3.amazonaws.com/ec2metadata/ec2-metadata"
+  docker-machine ssh $DOCKER_MACHINE "chmod u+x ec2-metadata"
+  local instance_id=$(docker-machine ssh $DOCKER_MACHINE ./ec2-metadata -i)
+  instance_id=${instance_id:13}
+  local attach_result=$(aws --region=$AWS_REGION ec2 attach-volume \
+    --device /dev/xvdc --volume-id $DB_EBS_VOLUME_ID --instance-id $instance_id)
+  sleep 10
+  docker-machine ssh $DOCKER_MACHINE sudo mount /dev/xvdc /home/backup
+  dbg $(docker-machine ssh $DOCKER_MACHINE "sudo df -h /dev/xvdc")
+}
+
 #######################################
 # Print a help
 # Globals:
@@ -150,6 +172,24 @@ function help() {
       - dump in \"custom\" format, made with 'pg_dump -Fc ..' ('*.pgdmp'),
     * sequence of SQL commands specified as in a form of plain text.
 
+  \033[1m--db-name\033[22m (string)
+
+  Name of database which must be tested. Name 'test' is internal used name,
+  so is not correct value.
+
+  \033[1m--db-ebs-volume-id\033[22m (string)
+
+  ID of an AWS EBS volume, containing the database backup (made with pg_basebackup).
+
+  In the volume's root directory, the following two files are expected:
+    - base.tar.gz
+    - pg_xlog.tar.gz for Postgres version up to 9.6 or pg_wal.tar.gz for Postgres 10+
+
+  The following command can be used to get such files:
+    'pg_basebackup -U postgres -zPFt -Z 5 -D /path/to/ebs/volume/root'
+  Here '-Z 5' means that level 5 to be used for compression, you can choose any value from 0 to 9.
+
+
   \033[1m--db-pgbench\033[22m (string)
 
   Initialize database for pgbench. Contains pgbench init arguments:
@@ -518,7 +558,9 @@ function check_cli_parameters() {
   [[ ! -z ${WORKLOAD_REAL+x} ]] && let workloads_count=$workloads_count+1
   [[ ! -z ${WORKLOAD_CUSTOM_SQL+x} ]] && let workloads_count=$workloads_count+1
   [[ ! -z ${WORKLOAD_PGBENCH+x} ]] && let workloads_count=$workloads_count+1
-  if [[ -z ${DB_PREPARED_SNAPSHOT+x} ]]  &&  [[ -z ${DB_DUMP+x} ]] &&  [[ -z ${DB_PGBENCH+x} ]]; then
+
+  if [[ -z ${DB_PREPARED_SNAPSHOT+x} ]]  &&  [[ -z ${DB_DUMP+x} ]] \
+    &&  [[ -z ${DB_PGBENCH+x} ]] && [[ -z ${DB_EBS_VOLUME_ID+x} ]]; then
     err "ERROR: The object (database) is not defined."
     exit 1
   fi
@@ -1079,6 +1121,8 @@ while [ $# -gt 0 ]; do
         AWS_ZONE="$2"; shift 2 ;;
     --aws-block-duration )
         AWS_BLOCK_DURATION=$2; shift 2 ;;
+    --db-ebs-volume-id )
+      DB_EBS_VOLUME_ID=$2; shift 2;;
 
     --s3cfg-path )
       S3_CFG_PATH="$2"; shift 2 ;;
@@ -1157,24 +1201,29 @@ elif [[ "$RUN_ON" == "aws" ]]; then
   msg "  To connect docker machine use:"
   msg "    docker-machine ssh $DOCKER_MACHINE"
 
+  if [[ "$RUN_ON" == "aws" ]] && [[ ! -z ${DB_EBS_VOLUME_ID+x} ]]; then
+    attach_db_ebs_drive
+  fi
+
   docker-machine ssh $DOCKER_MACHINE "sudo sh -c \"mkdir /home/storage\""
   if [[ "${AWS_EC2_TYPE:0:2}" == "i3" ]]; then
     msg "Using high-speed NVMe SSD disks"
     use_ec2_nvme_drive
   else
     msg "Use EBS volume"
     # Create new volume and attach them for non i3 instances if needed
-    if [ ! -z ${AWS_EBS_VOLUME_SIZE+x} ]; then
+    if [[ "$RUN_ON" == "aws" ]] && [[ ! -z ${AWS_EBS_VOLUME_SIZE+x} ]]; then
       use_ec2_ebs_drive $AWS_EBS_VOLUME_SIZE
     fi
   fi
 
   CONTAINER_HASH=$( \
     docker `docker-machine config $DOCKER_MACHINE` run \
       --name="pg_nancy_${CURRENT_TS}" \
+      --privileged \
       -v /home/ubuntu:/machine_home \
       -v /home/storage:/storage \
-      -v /home/basedump:/basedump \
+      -v /home/backup:/backup \
       -dit "postgresmen/postgres-with-stuff:pg${PG_VERSION}"
   )
   DOCKER_CONFIG=$(docker-machine config $DOCKER_MACHINE)
@@ -1190,6 +1239,61 @@ MACHINE_HOME="/machine_home/nancy_${CONTAINER_HASH}"
 alias docker_exec='docker $DOCKER_CONFIG exec -i ${CONTAINER_HASH} '
 get_system_characteristics
 
+#######################################
+# Extract the database backup from the attached EBS volume.
+# Globals:
+#   PG_VERSION
+# Arguments:
+#   None
+# Returns:
+#   None
+#######################################
+function cp_db_ebs_backup() {
+  # Here we think what postgress stopped
+  msg "Extract database backup from EBS volume"
+  docker_exec bash -c "rm -rf /var/lib/postgresql/9.6/main/*"
+
+  local op_start_time=$(date +%s)
+  docker_exec bash -c "rm -rf /var/lib/postgresql/$PG_VERSION/main/*"
+  local result=$(docker_exec bash -c "([[ -f /backup/base.tar.gz ]] \
+    && tar -C /storage/postgresql/$PG_VERSION/main/ -xzvf /backup/base.tar.gz) || true")
+  result=$(docker_exec bash -c "([[ -f /backup/base.tar ]] \
+    && tar -C /storage/postgresql/$PG_VERSION/main/ -xvf /backup/base.tar) || true")
+
+  result=$(docker_exec bash -c "([[ -f /backup/pg_xlog.tar.gz ]] \
+    && tar -C /storage/postgresql/$PG_VERSION/main/pg_xlog -xzvf /backup/pg_xlog.tar.gz) || true")
+  result=$(docker_exec bash -c "([[ -f /backup/pg_xlog.tar ]] \
+    && tar -C /storage/postgresql/$PG_VERSION/main/pg_xlog -xvf /backup/pg_xlog.tar) || true")
+
+  result=$(docker_exec bash -c "([[ -f /backup/pg_wal.tar.gz ]] \
+    && tar -C /storage/postgresql/$PG_VERSION/main/pg_xlog -xzvf /backup/pg_wal.tar.gz) || true")
+  result=$(docker_exec bash -c "([[ -f /backup/pg_wal.tar ]] \
+    && tar -C /storage/postgresql/$PG_VERSION/main/pg_wal -xvf /backup/pg_wal.tar) || true")
+
+  local end_time=$(date +%s)
+  local duration=$(echo $((end_time-op_start_time)) | awk '{printf "%d:%02d:%02d", $1/3600, ($1/60)%60, $1%60}')
+  msg "Time taken to extract database backup from EBS volume: $duration."
+
+  docker_exec bash -c "chown -R postgres:postgres /storage/postgresql/$PG_VERSION/main"
+  docker_exec bash -c "localedef -f UTF-8 -i en_US en_US.UTF-8"
+  docker_exec bash -c "localedef -f UTF-8 -i ru_RU ru_RU.UTF-8"
+}
+
+#######################################
+# Detach EBS volume
+# Globals:
+#   DOCKER_MACHINE, DB_EBS_VOLUME_ID, AWS_REGION
+# Arguments:
+#   None
+# Returns:
+#   None
+#######################################
+function dettach_db_ebs_drive() {
+  docker_exec bash -c "umount /backup"
+  docker-machine ssh $DOCKER_MACHINE sudo umount /home/backup
+  local dettach_result=$(aws --region=$AWS_REGION ec2 detach-volume --volume-id $DB_EBS_VOLUME_ID)
+}
+
 docker_exec bash -c "mkdir $MACHINE_HOME && chmod a+w $MACHINE_HOME"
 if [[ "$RUN_ON" == "aws" ]]; then
   docker-machine ssh $DOCKER_MACHINE "sudo chmod a+w /home/storage"
@@ -1201,6 +1305,12 @@ if [[ "$RUN_ON" == "aws" ]]; then
   sleep 2 # wait for postgres stopped
   docker_exec bash -c "sudo mv /var/lib/postgresql /storage/"
   docker_exec bash -c "ln -s /storage/postgresql /var/lib/postgresql"
+
+  if [[ ! -z ${DB_EBS_VOLUME_ID+x} ]]; then
+    cp_db_ebs_backup
+    dettach_db_ebs_drive
+  fi
+
   docker_exec bash -c "sudo /etc/init.d/postgresql start"
   sleep 2 # wait for postgres started
 fi
@@ -1608,6 +1718,12 @@ function collect_results() {
   msg "Time taken to generate and collect artifacts: $DURATION."
 }
 
+if [[ ! -z ${DB_EBS_VOLUME_ID+x} ]] && [[ ! "$DB_NAME" == "test" ]]; then
+  docker_exec bash -c "psql --set ON_ERROR_STOP=on -U postgres -c 'drop database if exists test;'"
+  docker_exec bash -c "psql --set ON_ERROR_STOP=on -U postgres -c 'alter database $DB_NAME rename to test;'"
+  DB_NAME=test
+fi
+
 [ ! -z ${S3_CFG_PATH+x} ] && copy_file $S3_CFG_PATH \
   && docker_exec cp $MACHINE_HOME/.s3cfg /root/.s3cfg
 [ ! -z ${DB_DUMP+x} ] && copy_file $DB_DUMP
@@ -1624,7 +1740,9 @@ sleep 2 # wait for postgres up&running
 
 apply_commands_after_container_init
 apply_sql_before_db_restore
-restore_dump
+if [[ -z ${DB_EBS_VOLUME_ID+x} ]]; then
+  restore_dump
+fi
 apply_sql_after_db_restore
 docker_exec bash -c "psql -U postgres $DB_NAME -b -c 'create extension if not exists pg_stat_statements;' $VERBOSE_OUTPUT_REDIRECT"
 pg_config_init