#!/usr/bin/env python # # svn-import-releases # Copyright (C) 2005 Martin Blais # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA """svn-import-releases [] Take a list of directories, each representing a version of some files (like a checkout of a release of some software), and imports each of these sequentially over an existing checkout, registering the new fileset and creating a subversion release for every directory imported. This script can be used to convert a set of directories of files into a Subversion repository, in a way that is most natural to the way that it would have occurred if the repository would have been managed by Subversion the whole time. It is also a way to compress a repository (CVS, Subversion or otherwise), by re-creating it from specific moments in time. To compress a CVS repository, you can use cvs2svn to let it figure out the CVS tags and branches, then you can checkout the entire repository and select the specific tags that you would like to keep and import them using this script. The must point to a file that contains on each line and separated by whitespace: - the directory to import, an absolute path; - (optionally) the version tag to release it as (or '-'). If no tag is specified, do not release this directory, only create a commit for it; - (optionally) the destination directory, relative to the checkout root. Normally, each directory is copied over the root of the checkout, but this can be used to specify an alternate subdirectory to copy the files into; can be either of: 1. a path to an existing checkout directory; 2. a repository URL. In this case, the script will create or checkout that directory in the repository by using the appropriate cmdline option. If the contains releases to be made, the default URL to make the releases to will be /../tags (we assume that point to <...something>/trunk). You can specify an alternate subdirectory using the options. Notes ----- This script uses svn-copy-register to perform the copies. It directs this script and verifies the results in a specific way. """ __version__ = "Revision: 1.3 " __author__ = 'Martin Blais ' # stdlib imports. import os, tempfile, urlparse, re from subprocess import call, Popen, PIPE, STDOUT from os.path import * def parse_dirlist(fn): """ Parse the directory list and return a list of (dirname, version) pairs. This is a generator. """ for line in map(str.strip, open(fn, 'r')): # Skip empty lines and comments. if not line or line.startswith('#'): continue ll = line.split() if len(ll) == 1: dirn = ll[0] version, targetdir = None, None elif len(ll) == 2: dirn, version = ll targetdir = None elif len(ll) == 3: dirn, version, targetdir = ll else: raise RuntimeError( "Error: line '%s' does not follow format." % line) if version == '-': version = None yield dirn, version, targetdir def rmrf(fnodn): """ Delete the given directory and all its contents. """ if not exists(fnodn): return if isdir(fnodn): for root, dirs, files in os.walk(fnodn, topdown=False): for fn in files: os.remove(join(root, fn)) for dn in dirs: adn = join(root, dn) if islink(adn): os.remove(adn) else: os.rmdir(adn) os.rmdir(root) else: os.remove(fnodn) def svn(*args, **kwds): """ Call a subversion command, make sure it doesn't fail. """ noerror = kwds.pop('noerror', None) cmd = ('svn',) + args print '$$$', ' '.join(cmd) out = kwds.pop('out', None) if not out: kwds['stdout'] = kwds['stderr'] = PIPE p = Popen(cmd, **kwds) sout, serr = p.communicate() if p.returncode != 0 and not noerror: raise IOError("Error calling Subversion: %s\n%s\n\n%s" % (p.returncode, ' '.join(args), sout + serr)) return p.returncode def svn_makedirs(url): """ Creates all intermediate directories in the given Subversion repository path. """ # Create the necessary director(y|ies) in the repository. createlist = [] purl = list(urlparse.urlparse(url)) ppath = purl[2] while ppath != '/': purl[2] = ppath url = urlparse.urlunparse(purl) if svn('list', url, noerror=1) != 0: createlist.append(url) else: break ppath = dirname(ppath) for cdir in reversed(createlist): svn('mkdir', '--message=', cdir) def prepare_checkout(checkout, tempfiles): """ Prepare the checkout and repository URL and return them. After this function returns, you are insured of a valid, committed checkout directory for the repository URL. This processes the 'checkout' option according to the usage. """ # Create the checkout directory where we will apply the changes. if re.match('[a-zA-Z]+://', checkout): # The checkout is a URL: we need to checkout or create the directory and # checkout ourselves. svnurl = checkout svn_makedirs(svnurl) # Checkout the directory in a temporary location. tdir = tempfile.mkdtemp() tempfiles.append(tdir) checkout = join(tdir, 'checkout') svn('checkout', svnurl, checkout) else: checkout = abspath(checkout) # The checkout is an existing checked-out directory. Get the # repository URL for it. p = Popen(('svn', 'info', checkout,), stdout=PIPE, stderr=STDOUT) sout, serr = p.communicate() if p.returncode != 0: raise RuntimeError("Error: The checkout directory is not a valid " "subversion repository.") svnurl = re.search('^URL: (.*)$', sout, re.M).group(1) # Make sure that files in the checkout are already checked-in. p = Popen(('svn', 'status', checkout,), stdout=PIPE, stderr=STDOUT) sout, serr = p.communicate() if p.returncode != 0 or sout or serr: print p.returncode print sout print serr raise RuntimeError("Error: Some files are not committed in the " "checkout dir.") return checkout, svnurl def compare_dirs(d1, d2): """ Recursively compare the contents of two directories, ignore keyword changes. """ r = call(['diff', '-q', '-r', # Recursive silent diff # Ignore changes in CVS keywords. '-I', '\\$[^$]*\\$', # Ignore changes in CVS and Subversion private files. '--exclude=.svn', '--exclude=CVS', d1, d2]) return r def main(): import optparse parser = optparse.OptionParser(__doc__.strip()) parser.add_option('-v', '--verbose', action="store_true", help="Be verbose in your output.") parser.add_option('-i', '--ignore', action="append", default=['.svn'], help="Specify paths to ignore." "You can use this option many times.") parser.add_option('-R', '--releases-url', action='store', default='../tags', help="URL where releases are made, if needed.") parser.add_option('--no-delete-temp', action='store_true', help="Don't delete temporary files. " "This is used for debugging.") global opts opts, args = parser.parse_args() # Validate arguments. if len(args) != 2: parser.error( "You must specify a filename and a destination repository.") dirlistfn, checkout = args # Validate the the directories in the dirlist all exist. for d in parse_dirlist(dirlistfn): if not exists(d[0]): raise IOError("Directory '%s' does not exist." % d[0]) tempfiles = [] try: # Prepare the checkout directory. checkout, svnurl = prepare_checkout(checkout, tempfiles) print 'Checkout:', checkout print 'Repository URL:', svnurl # Check that the destination directory is not the root of the repository. p = Popen(['svn', 'info', svnurl], stdout=PIPE, stderr=PIPE) sout, serr = p.communicate() assert p.returncode == 0 reporoot = re.search('^Repository Root: (.*)$', sout, re.M) if svnurl == reporoot.group(1): raise SystemExit( "Error: you cannot import directly in the root of a repository.") # Compute the URL for the releases and make sure that the release path # is there. purl = list(urlparse.urlparse(svnurl)) purl[2] = normpath(join(purl[2], opts.releases_url)) tagsurl = urlparse.urlunparse(purl) print 'Releases URL:', tagsurl svn_makedirs(tagsurl) # Define big headers for tracing. def head(s): print '^^^^', s, '^' * 80 def bighead(s): print print '^^^^' print '^^^^', s print '^^^^' print # For each of the releases... for x, (dn, version, targetdir) in enumerate(parse_dirlist(dirlistfn)): bighead( str((dn, version, targetdir)) ) dest = targetdir and join(targetdir, checkout) or checkout head("Registering") # Copy and register the new files. cmd = ['svn-copy-register', dn, checkout] if opts.verbose: cmd[1:1] = ['--verbose'] print ' '.join(cmd) p = Popen(cmd, stdout=PIPE, stderr=PIPE) sout, serr = p.communicate() assert p.returncode == 0 for out in sout, serr: if out: for line in out.splitlines(): print ' ', line # Commit the files to the repository. head("Committing") if version: msg = 'Committing version %s' % version else: msg = 'Committing' msg += ' (imported from %s)' % dn svn('commit', '--message=%s' % msg, out=1, cwd=checkout) # # Check that the current HEAD is the same as the original directory. # head("Comparing") tmpdir = tempfile.mkdtemp() codir = join(tmpdir, 'co') svn('checkout', svnurl, codir) r = compare_dirs(dn, codir) assert r == 0 # Make a release on the repository (only if a version is specified). if version: head("Releasing") svn('copy', '--message=Releasing version %s' % version, svnurl, join(tagsurl, version)) finally: # Clean up temporary directories. if not opts.no_delete_temp: for temp in tempfiles: rmrf(temp) if __name__ == '__main__': main()