> Typo, "revision_regex"
fixed at the attached patch.
> > + cmd = [ self.__svnadmin_path, "dump",
> > + "--incremental", "-r", revparam,
self.__repospath ]
>
> That doesn't look right. If self.__repospath can be a local path to a
> repository root, you shouldn't pass it as an argument to 'svn' a few
lines
> above.
If self.__repospath is a local path to a repository,
I don't pass it to svn. See get_head_rev().
Also, it'd be good practice to pass "--" in front of self.__repospath, but that
appears to be a preexisting problem in the code (i.e., not introduced by your
patch).
What is the purpose in passing "--"?
> ... and while at it, use r"" literals to avoid clashes with a
potential future
> "r\d" backslash escape sequence.
Do you worry about changing format change of 'svn log'
in later versions of svn?
If so, 'svn log' is not called because future versions of svn
supports 'svn youngest'.
I add --non-interactive option to the script.
[[[
add support for svnrdump to svn-backup-dumps.py
* tools/server-side/svn-backup-dumps.py
() : fix comment
() : update to ver 0.7
() : import urlparse
(SvnBackup.__init__) : fix message in SvnBackupException constructor.
(SvnBackup.__init__) : check whether the second parameter is a local
path or an URL.
(SvnBackup.__init__) : check whether --deltas option is specified when
the second parameter is an URL.
(SvnBackup.__init__) : set svn and svnrdump paths.
(SvnBackup.__init__) : set flags for --trust-server-cert and
--no-auth-cache.
(SvnBackup.exec_cmd_nt) : open null device and pass it to Popen
when printerr is False.
(SvnBackup.get_extra_param) : get extra option for svn and svnrdump.
always pass --non-interactive to
svn and svnrdump to make sure.
(SvnBackup.get_head_rev) : rename this to get_head_rev_for_local.
(SvnBackup.get_head_rev_for_url) : get HEAD revision for an URL.
(SvnBackup.get_head_rev) : call get_head_rev_for_local for a local
path, get_head_rev_for_url for an URL
(SvnBackup.create_dump) : invoke svnadmin for a local path,
svnrdump for an URL.
() : fix help message.
() : add --svnrdump-path and --svn-path.
() : add --trust-server-cert.
() : add --no-auth-cache.
() : add --non-interactive.
]]]
Index: tools/server-side/svn-backup-dumps.py
===================================================================
--- tools/server-side/svn-backup-dumps.py (revision 1511761)
+++ tools/server-side/svn-backup-dumps.py (working copy)
@@ -45,10 +45,10 @@
#
# 1. Create a full dump (revisions 0 to HEAD).
#
-# svn-backup-dumps.py <repos> <dumpdir>
+# svn-backup-dumps.py <repos or URL> <dumpdir>
#
-# <repos> Path to the repository.
-# <dumpdir> Directory for storing the dump file.
+# <repos or URL> Path or URL to the repository.
+# <dumpdir> Directory for storing the dump file.
#
# This creates a dump file named 'src.000000-NNNNNN.svndmp.gz'
# where NNNNNN is the revision number of HEAD.
@@ -56,11 +56,11 @@
#
# 2. Create incremental dumps containing at most N revisions.
#
-# svn-backup-dumps.py -c <count> <repos> <dumpdir>
+# svn-backup-dumps.py -c <count> <repos or URL> <dumpdir>
#
-# <count> Count of revisions per dump file.
-# <repos> Path to the repository.
-# <dumpdir> Directory for storing the dump file.
+# <count> Count of revisions per dump file.
+# <repos or URL> Path or URL to the repository.
+# <dumpdir> Directory for storing the dump file.
#
# When started the first time with a count of 1000 and if HEAD is
# at 2923 it creates the following files:
@@ -77,11 +77,11 @@
#
# 3. Create incremental single revision dumps (for use in post-commit).
#
-# svn-backup-dumps.py -r <revnr> <repos> <dumpdir>
+# svn-backup-dumps.py -r <revnr> <repos or URL> <dumpdir>
#
-# <revnr> A revision number.
-# <repos> Path to the repository.
-# <dumpdir> Directory for storing the dump file.
+# <revnr> A revision number.
+# <repos or URL> Path or URL to the repository.
+# <dumpdir> Directory for storing the dump file.
#
# This creates a dump file named 'src.NNNNNN.svndmp.gz' where
# NNNNNN is the revision number of HEAD.
@@ -89,10 +89,10 @@
#
# 4. Create incremental dumps relative to last dump
#
-# svn-backup-dumps.py -i <repos> <dumpdir>
+# svn-backup-dumps.py -i <repos or URL> <dumpdir>
#
-# <repos> Path to the repository.
-# <dumpdir> Directory for storing the dump file.
+# <repos or URL> Path or URL to the repository.
+# <dumpdir> Directory for storing the dump file.
#
# When if dumps are performed when HEAD is 2923,
# then when HEAD is 3045, is creates these files:
@@ -105,7 +105,7 @@
#
# svn-backup-dumps.py -z ...
#
-# ... More options, see 1-4, 7, 8.
+# ... More options, see 1-4, 7, 8.
#
#
# 6. Create bzipped dump files.
@@ -112,7 +112,7 @@
#
# svn-backup-dumps.py -b ...
#
-# ... More options, see 1-4, 7, 8.
+# ... More options, see 1-4, 7, 8.
#
#
# 7. Transfer the dumpfile to another host using ftp.
@@ -119,11 +119,11 @@
#
# svn-backup-dumps.py -t ftp:<host>:<user>:<password>:<path> ...
#
-# <host> Name of the FTP host.
-# <user> Username on the remote host.
-# <password> Password for the user.
-# <path> Subdirectory on the remote host.
-# ... More options, see 1-6.
+# <host> Name of the FTP host.
+# <user> Username on the remote host.
+# <password> Password for the user.
+# <path> Subdirectory on the remote host.
+# ... More options, see 1-6.
#
# If <path> contains the string '%r' it is replaced by the
# repository name (basename of the repository path).
@@ -133,11 +133,11 @@
#
# svn-backup-dumps.py -t smb:<share>:<user>:<password>:<path> ...
#
-# <share> Name of an SMB share in the form '//host/share'.
-# <user> Username on the remote host.
-# <password> Password for the user.
-# <path> Subdirectory of the share.
-# ... More options, see 1-6.
+# <share> Name of an SMB share in the form '//host/share'.
+# <user> Username on the remote host.
+# <password> Password for the user.
+# <path> Subdirectory of the share.
+# ... More options, see 1-6.
#
# If <path> contains the string '%r' it is replaced by the
# repository name (basename of the repository path).
@@ -149,7 +149,7 @@
# - improve documentation
#
-__version = "0.6"
+__version = "0.7"
import sys
import os
@@ -162,6 +162,7 @@ import re
from optparse import OptionParser
from ftplib import FTP
from subprocess import Popen, PIPE
+from urlparse import urlparse
try:
import bz2
@@ -284,29 +285,46 @@ class SvnBackup:
if len(args) != 3:
if len(args) < 3:
raise SvnBackupException("too few arguments, specify"
- " repospath and dumpdir.\nuse -h or"
+ " repospath (or URL) and
dumpdir.\nuse -h or"
" --help option to see help.")
else:
raise SvnBackupException("too many arguments, specify"
- " repospath and dumpdir only.\nuse"
+ " repospath (or URL) and dumpdir
only.\nuse"
" -h or --help option to see help.")
self.__repospath = args[1]
self.__dumpdir = args[2]
- # check repospath
- rpathparts = os.path.split(self.__repospath)
- if len(rpathparts[1]) == 0:
- # repospath without trailing slash
- self.__repospath = rpathparts[0]
- if not os.path.exists(self.__repospath):
- raise SvnBackupException("repos '%s' does not exist." %
self.__repospath)
- if not os.path.isdir(self.__repospath):
- raise SvnBackupException("repos '%s' is not a directory." %
self.__repospath)
- for subdir in [ "db", "conf", "hooks" ]:
- dir = os.path.join(self.__repospath, subdir)
- if not os.path.isdir(dir):
- raise SvnBackupException("repos '%s' is not a repository." %
self.__repospath)
- rpathparts = os.path.split(self.__repospath)
- self.__reposname = rpathparts[1]
+
+ # check whether TARGET is an URL or a local repospath
+ url = urlparse(self.__repospath)
+ if url.scheme is None:
+ # TARGET is a local repospath
+ self.__is_local_repos = True
+
+ # check repospath
+ rpathparts = os.path.split(self.__repospath)
+ if len(rpathparts[1]) == 0:
+ # repospath without trailing slash
+ self.__repospath = rpathparts[0]
+ if not os.path.exists(self.__repospath):
+ raise SvnBackupException("repos '%s' does not exist." %
self.__repospath)
+ if not os.path.isdir(self.__repospath):
+ raise SvnBackupException("repos '%s' is not a directory." %
self.__repospath)
+ for subdir in [ "db", "conf", "hooks" ]:
+ dir = os.path.join(self.__repospath, subdir)
+ if not os.path.isdir(dir):
+ raise SvnBackupException("repos '%s' is not a repository."
% self.__repospath)
+ rpathparts = os.path.split(self.__repospath)
+ self.__reposname = rpathparts[1]
+ else:
+ # TARGET is an URL
+ self.__is_local_repos = False
+
+ # remove trailing slash
+ self.__repospath = self.__repospath.rstrip('/')
+ # get repository name
+ rpathparts = self.__repospath.split('/')
+ self.__reposname = rpathparts[-1]
+
if self.__reposname in [ "", ".", ".." ]:
raise SvnBackupException("couldn't extract repos name from '%s'."
% self.__repospath)
# check dumpdir
@@ -321,7 +339,11 @@ class SvnBackup:
self.__deltas = options.deltas
self.__relative_incremental = options.relative_incremental
- # svnadmin/svnlook path
+ # svnrdump doesn't support --deltas option, so check it.
+ if self.__deltas and self.__is_local_repos is False:
+ raise SvnBackupException("svnrdump doen't support --deltas option")
+
+ # svnadmin/svnlook/svnrdump/svn path
self.__svnadmin_path = "svnadmin"
if options.svnadmin_path:
self.__svnadmin_path = options.svnadmin_path
@@ -328,7 +350,36 @@ class SvnBackup:
self.__svnlook_path = "svnlook"
if options.svnlook_path:
self.__svnlook_path = options.svnlook_path
+ self.__svnrdump_path = "svnrdump"
+ if options.svnrdump_path:
+ self.__svnrdump_path = options.svnrdump_path
+ self.__svn_path = "svn"
+ if options.svn_path:
+ self.__svn_path = options.svn_path
+ # username/password
+ self.__username = None
+ self.__password = None
+ if options.username:
+ self.__username = options.username
+ if options.password:
+ self.__password = options.password
+
+ # auth-cache
+ self.__no_auth_cache = None
+ if options.no_auth_cache:
+ self.__no_auth_cache = options.no_auth_cache
+
+ # server certificate
+ self.__trust_server_cert = None
+ if options.trust_server_cert:
+ self.__trust_server_cert = options.trust_server_cert
+
+ # non-interactive
+ self.__non_interactive = None
+ if options.trust_server_cert:
+ self.__non_interactive = options.non_interactive
+
# check compress option
self.__gzip_path = options.gzip_path
self.__bzip2_path = options.bzip2_path
@@ -414,8 +465,13 @@ class SvnBackup:
return (rc, bufout, buferr)
def exec_cmd_nt(self, cmd, output=None, printerr=False):
+ if printerr:
+ stderr_handle = None
+ else:
+ # open null device and discard stderr output by setting stderr
+ stderr_handle = open(os.devnull, 'w')
try:
- proc = Popen(cmd, stdout=PIPE, stderr=None, shell=False)
+ proc = Popen(cmd, stdout=PIPE, stderr=stderr_handle, shell=False)
except:
return (256, "", "Popen failed (%s ...):\n %s" % (cmd[0],
str(sys.exc_info()[1])))
@@ -432,7 +488,7 @@ class SvnBackup:
rc = proc.wait()
return (rc, bufout, buferr)
- def get_head_rev(self):
+ def get_head_rev_for_local(self):
cmd = [ self.__svnlook_path, "youngest", self.__repospath ]
r = self.exec_cmd(cmd)
if r[0] == 0 and len(r[2]) == 0:
@@ -441,6 +497,56 @@ class SvnBackup:
print(r[2])
return -1
+ def get_extra_param(self):
+ extra_param = []
+ if self.__username:
+ extra_param.append( "--username" )
+ extra_param.append( self.__username )
+ if self.__password:
+ extra_param.append( "--password" )
+ extra_param.append( self.__password )
+ if self.__trust_server_cert:
+ extra_param.append( "--trust-server-cert" )
+ extra_param.append( "--non-interactive" )
+ elif self.__non_interactive:
+ extra_param.append( "--non-interactive" )
+ if self.__no_auth_cache:
+ extra_param.append( "--no-auth-cache" )
+ return extra_param
+
+ def get_head_rev_for_url(self):
+ extra_param = self.get_extra_param()
+
+ # use 'svn yougest' to get the HEAD revision of URL
+ # 'svn yougest' is supported on subversion 1.9 or laster.
+ cmd = [ self.__svn_path, "youngest", self.__repospath ]
+ cmd[2:2] = extra_param
+ r = self.exec_cmd(cmd)
+ if r[0] == 0 and len(r[2]) == 0:
+ return int(r[1].strip())
+
+ # use 'svn log' to get the latest revision of URL
+ # it may be different from the HEAD revision.
+ cmd = [ self.__svn_path, "log", "-l 1", "-q", self.__repospath ]
+ cmd[2:2] = extra_param
+ r = self.exec_cmd(cmd)
+ if r[0] == 0 and len(r[2]) == 0:
+ revision_regex = re.compile("^r(\d+)")
+ # revision information is in the second line of 'svn log' output
+ lines = r[1].splitlines()
+ result = revision_regex.match(lines[1])
+ if result:
+ return int(result.group(1).strip())
+ else:
+ print(r[2])
+ return -1
+
+ def get_head_rev(self):
+ if self.__is_local_repos is True:
+ return self.get_head_rev_for_local()
+ else:
+ return self.get_head_rev_for_url()
+
def get_last_dumped_rev(self):
filename_regex = re.compile("(.+)\.\d+-(\d+)\.svndmp.*")
# start with -1 so the next one will be rev 0
@@ -534,8 +640,16 @@ class SvnBackup:
return True
else:
print("writing " + absfilename)
- cmd = [ self.__svnadmin_path, "dump",
- "--incremental", "-r", revparam, self.__repospath ]
+
+ # create command line for svnadmin/svnrdump
+ if self.__is_local_repos:
+ cmd = [ self.__svnadmin_path, "dump",
+ "--incremental", "-r", revparam, self.__repospath ]
+ else:
+ cmd = [ self.__svnrdump_path, "dump",
+ "--incremental", "-r", revparam, self.__repospath ]
+ extra_param = self.get_extra_param()
+ cmd[2:2] = extra_param
if self.__quiet:
cmd[2:2] = [ "-q" ]
if self.__deltas:
@@ -601,7 +715,8 @@ class SvnBackup:
if __name__ == "__main__":
- usage = "usage: svn-backup-dumps.py [options] repospath dumpdir"
+ usage = "usage: svn-backup-dumps.py [options] repospath dumpdir\n" \
+ " svn-backup-dumps.py [options] URL dumpdir"
parser = OptionParser(usage=usage, version="%prog "+__version)
if have_bz2:
parser.add_option("-b",
@@ -661,6 +776,35 @@ if __name__ == "__main__":
action="store", type="string",
dest="svnlook_path", default=None,
help="svnlook command path.")
+ parser.add_option("--svnrdump-path",
+ action="store", type="string",
+ dest="svnrdump_path", default=None,
+ help="svnrdump command path.")
+ parser.add_option("--svn-path",
+ action="store", type="string",
+ dest="svn_path", default=None,
+ help="svn command path.")
+ parser.add_option("--username",
+ action="store", type="string",
+ dest="username", default=None,
+ help="username")
+ parser.add_option("--password",
+ action="store", type="string",
+ dest="password", default=None,
+ help="password")
+ parser.add_option("--trust-server-cert",
+ action="store_true", default=False,
+ dest="trust_server_cert",
+ help="accept SSL server certificates from unknown"
+ "certificate authorities without prompting")
+ parser.add_option("--no-auth-cache",
+ action="store_true", default=False,
+ dest="no_auth_cache",
+ help="do not cache authentication tokens")
+ parser.add_option("--non-interactive",
+ action="store_true", default=False,
+ dest="non_interactive",
+ help="do no interactive prompting")
parser.add_option("--help-transfer",
action="store_true",
dest="help_transfer", default=False,
