Allow pgbackrest clone method to use a separate configuration

Cloning from the same repository as is used from backups is usually a
bad idea.

Also allow specification recovery target information. Patroni will
remove recovery target settings created by pgbackrest restore.

Author: ants

bootstrap/clone_with_pgbackrest.py

@@ -17,47 +17,50 @@ def read_configuration():
     parser.add_argument('--recovery-target-time',
                         help='the timestamp up to which recovery will proceed (including time zone)',
                         dest='recovery_target_time_string')
-    parser.add_argument('--dry-run', action='store_true', help='find a matching backup and build the wal-e '
-                        'command to fetch that backup without running it')
+    parser.add_argument('--config-include-path',
+                        help='pgbackrest configuration directory')    
     args = parser.parse_args()
 
-    options = namedtuple('Options', 'name datadir recovery_target_time dry_run')
+    options = namedtuple('Options', 'name datadir recovery_target_time config_include_path')
     if args.recovery_target_time_string:
         recovery_target_time = parse(args.recovery_target_time_string)
         if recovery_target_time.tzinfo is None:
             raise Exception("recovery target time must contain a timezone")
     else:
         recovery_target_time = None
 
-    return options(args.scope, args.datadir, recovery_target_time, args.dry_run)
+    return options(args.scope, args.datadir, recovery_target_time, args.config_include_path)
 
 def run_clone_from_pgbackrest(options):
     env = os.environ.copy()
 
     pg_path_argument = "--pg1-path={0}".format(options.datadir)
 
+    pgbackrest_command = ['/usr/bin/pgbackrest', 'restore', '--stanza=db', pg_path_argument]
+
+    if options.config_include_path:
+        pgbackrest_command.extend(['--config-include-path=' + options.config_include_path])
+
     if options.recovery_target_time:
         target_time_argument = "--target={0}".format(options.recovery_target_time)
-        pgbackrest_command = ['/usr/bin/pgbackrest', '--stanza=db', '--type=time', target_time_argument, 'restore', pg_path_argument]
-    else:
-        pgbackrest_command = ['/usr/bin/pgbackrest', '--stanza=db', 'restore', pg_path_argument]
+        pgbackrest_command.extend(['--type=time', target_time_argument])
 
     logger.info("cloning cluster %s using %s", options.name, ' '.join(pgbackrest_command))
 
-    if not options.dry_run:
-        ret = subprocess.call(pgbackrest_command, env=env)
-        if ret != 0:
-            raise Exception("pgbackrest restore exited with exit code {0}".format(ret))
+    ret = subprocess.call(pgbackrest_command, env=env)
+    if ret != 0:
+        logger.error("pgbackrest restore exited with exit code {0}".format(ret))
+        return ret
 
     return 0
 
 def main():
-    options = read_configuration()
     try:
+        options = read_configuration()
         run_clone_from_pgbackrest(options)
     except Exception:
         logger.exception("Clone with pgbackrest failed")
         return 1
 
 if __name__ == '__main__':
-    sys.exit(main())
+    sys.exit(main())

scripts/configure_spilo.py

@@ -251,10 +251,24 @@ def deep_update(a, b):
   {{#CLONE_WITH_PGBACKREST}}
   method: clone_with_pgbackrest
   clone_with_pgbackrest:
-    command: python3 /scripts/clone_with_pgbackrest.py
+    command: python3 /scripts/clone_with_pgbackrest.py 
       --recovery-target-time="{{CLONE_TARGET_TIME}}"
+      --config-include-path="{{CLONE_PGBACKREST_CONFIG}}"
     recovery_conf: 
-        restore_command: pgbackrest --stanza=db archive-get %f "%p"
+        restore_command: pgbackrest --config-include-path="{{CLONE_PGBACKREST_CONFIG}}" --stanza=db archive-get %f "%p"
+        recovery_target_timeline: "{{CLONE_TARGET_TIMELINE}}"
+        {{#USE_PAUSE_AT_RECOVERY_TARGET}}
+        recovery_target_action: pause
+        {{/USE_PAUSE_AT_RECOVERY_TARGET}}
+        {{^USE_PAUSE_AT_RECOVERY_TARGET}}
+        recovery_target_action: promote
+        {{/USE_PAUSE_AT_RECOVERY_TARGET}}
+        {{#CLONE_TARGET_TIME}}
+        recovery_target_time: "{{CLONE_TARGET_TIME}}"
+        {{/CLONE_TARGET_TIME}}
+        {{^CLONE_TARGET_INCLUSIVE}}
+        recovery_target_inclusive: false
+        {{/CLONE_TARGET_INCLUSIVE}}
   {{/CLONE_WITH_PGBACKREST}}
   {{#CLONE_WITH_BASEBACKUP}}
   method: clone_with_basebackup
@@ -673,6 +687,9 @@ def get_placeholders(provider):
         else:
             logging.warning("Clone method is set to basebackup, but no 'CLONE_SCOPE' "
                             "or 'CLONE_HOST' or 'CLONE_USER' or 'CLONE_PASSWORD' specified")
+    elif placeholders['CLONE_METHOD'] == 'CLONE_WITH_PGBACKREST':
+        placeholders['CLONE_WITH_PGBACKREST'] = True
+        placeholders.setdefault('CLONE_PGBACKREST_CONFIG', '/etc/pgbackrest/clone-conf.d')
     else:
         if set_extended_wale_placeholders(placeholders, 'STANDBY_') == 'S3':
             placeholders.setdefault('STANDBY_USE_WALG', 'true')