Merge pull request #118 from timvaillancourt/1.0.1

1.0.1: Minor fixes for 1.0 release

Author: timvaillancourt

Dockerfile

@@ -2,7 +2,7 @@ FROM centos:centos7
 MAINTAINER Tim Vaillancourt <tim.vaillancourt@percona.com>
 RUN yum install -y https://www.percona.com/redir/downloads/percona-release/redhat/latest/percona-release-0.1-4.noarch.rpm epel-release && \
 	yum install -y Percona-Server-MongoDB-32-tools zbackup && yum clean all && \
-	curl -Lo /usr/bin/mongodb-consistent-backup https://github.com/Percona-Lab/mongodb_consistent_backup/releases/download/1.0.0/mongodb-consistent-backup.el7.centos.x86_64 && \
+	curl -Lo /usr/bin/mongodb-consistent-backup https://github.com/Percona-Lab/mongodb_consistent_backup/releases/download/1.0.1/mongodb-consistent-backup.el7.centos.x86_64 && \
 	chmod +x /usr/bin/mongodb-consistent-backup
 ENTRYPOINT ["mongodb-consistent-backup"]
 CMD ["--help"]

VERSION

@@ -1 +1 @@
-1.0.0
+1.0.1

mongodb_consistent_backup/Archive/Zbackup/Zbackup.py

@@ -4,6 +4,7 @@
 from select import select
 from subprocess import Popen, PIPE, call
 
+from mongodb_consistent_backup.Common import Lock
 from mongodb_consistent_backup.Errors import OperationError
 from mongodb_consistent_backup.Pipeline import Task
 
@@ -24,7 +25,9 @@ def __init__(self, manager, config, timer, base_dir, backup_dir, **kwargs):
         self.compression_method    = 'lzma'
         self.compression_supported = ['lzma']
 
-        self.zbackup_dir         = os.path.join(self.config.backup.location, self.backup_name, "mongodb-consistent-backup_zbackup")
+        self.base_dir            = os.path.join(self.config.backup.location, self.backup_name)
+        self.zbackup_dir         = os.path.join(self.base_dir, "mongodb-consistent-backup_zbackup")
+        self.zbackup_lock        = os.path.join(self.base_dir, "mongodb-consistent-backup_zbackup.lock")
         self.zbackup_backups     = os.path.join(self.zbackup_dir, "backups")
         self.zbackup_backup_path = os.path.join(self.zbackup_backups, "%s.tar" % self.backup_time)
         self.zbackup_bundles     = os.path.join(self.zbackup_dir, "bundles")
@@ -50,6 +53,10 @@ def init(self):
             else:
                 raise OperationError("ZBackup dir: %s is not a zbackup storage directory!" % self.zbackup_dir)
         else:
+            if not os.path.isdir(self.base_dir):
+                os.makedirs(self.base_dir)
+            lock = Lock(self.zbackup_lock)
+            lock.acquire()
             try:
                 cmd_line = [self.zbackup_binary]
                 if self.zbackup_passwd_file:
@@ -65,6 +72,8 @@ def init(self):
                     raise OperationError("ZBackup initialization failed! Exit code: %i" % exit_code)
             except Exception, e:
                 raise OperationError("Error creating ZBackup storage directory! Error: %s" % e)
+            finally:
+                lock.release()
 
     def version(self):
         if self._version:
@@ -148,6 +157,8 @@ def get_commands(self, base_dir, sub_dir):
 
     def run(self):
         if self.has_zbackup():
+            lock = Lock(self.zbackup_lock)
+            lock.acquire()
             try:
                 logging.info("Starting ZBackup version: %s (options: compression=%s, encryption=%s, threads=%i, cache_mb=%i)" %
                     (self.version(), self.compression(), self.encrypted, self.threads(), self.zbackup_cache_mb)
@@ -171,5 +182,6 @@ def run(self):
             finally:
                 self.running = False
                 self.stopped = True
+                lock.release()
         else:
             raise OperationError("Cannot find ZBackup at %s!" % self.zbackup_binary)

mongodb_consistent_backup/Backup/Mongodump/MongodumpThread.py

@@ -4,6 +4,7 @@
 
 from multiprocessing import Process
 from select import select
+from shutil import rmtree
 from signal import signal, SIGINT, SIGTERM, SIG_IGN
 from subprocess import Popen, PIPE
 
@@ -102,7 +103,7 @@ def run(self):
         mongodump_cmd = self.mongodump_cmd()
         try:
             if os.path.isdir(self.dump_dir):
-                os.removedirs(self.dump_dir)
+                rmtree(self.dump_dir)
             os.makedirs(self.dump_dir)
             logging.debug("Running mongodump cmd: %s" % mongodump_cmd)
             self._process = Popen(mongodump_cmd, stderr=PIPE)

mongodb_consistent_backup/Oplog/Tailer/Tailer.py

@@ -64,6 +64,8 @@ def run(self):
             }
             self.shards[shard]['thread'].start()
             while not oplog_state.get('running'):
+                if self.shards[shard]['thread'].exitcode:
+                    raise OperationError("Oplog tailer for %s failed with exit code %i!" % (mongo_uri, self.shards[shard]['thread'].exitcode))
                 sleep(0.5)
 
     def stop(self, kill=False, sleep_secs=0.5):

mongodb_consistent_backup/State.py

@@ -132,9 +132,9 @@ def load_backups(self):
         if os.path.isdir(self.base_dir):
             for subdir in os.listdir(self.base_dir):
                 try:
-                    if subdir == self.meta_name:
+                    bkp_path = os.path.join(self.base_dir, subdir)
+                    if subdir == self.meta_name or os.path.islink(bkp_path):
                         continue
-                    bkp_path   = os.path.join(self.base_dir, subdir)
                     state_path = os.path.join(bkp_path, self.meta_name)
                     state_file = os.path.join(state_path, "meta.bson")
                     done_path  = os.path.join(state_path, "done.bson")