#!/usr/bin/env python # # svn-copy-register # Copyright (C) 2005 Martin Blais # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA """svn-copy-register [] Replicates the directory structure and files of into , performing the necessary additions and deletions to register the changes files in into Subversion. is assumed to be a Subversion checkout. Files that exist on both sides are diffed to figure out if there are changes to be copied. Be careful before importing, always check with a dry-run that the behaviour is what you think it should be. Even on a large codebase, it is worth at least just having a look. Example invocation:: svn-update-tree /home/blais/tmp/boost-cvs/boost-1.31.0 /3rdparty/boost-latest Warning: this is meant to be run under UNIX, it has not been tested under Windows. It probably won't work under Windows. If you want to fix it for that, I'll be happy to include the changes if you want to send them to me. Motivation ---------- I have been using a CVS repository as a backup mechanism for years. Sometimes I would tag versions and release the software for these versions. Individual diffs between those do not matter so much, but I would like to preserve the individual versions in the repository. cvs2svn keeps all the diffs and does not allow to compress the diffs in-between. This is why I wrote this script. (Note that another way to do this would have been to compress the diffs in the CVS repository, but I like to leverage the different versions that cvs2svn produces). What I did was to run cvs2svn on the CVS repository, and then to progressively merge the branches in order and commit everytime. The process that controls this will be available in a separate script. Limitations ----------- - The permissions are not set as Subversion properties (apart from the automatic detection that occurs); - Obviously, the dates are lost; """ ## FIXME: write script that extracts tags list from a file and does the commit, and verifies ## FIXME: can we automatically apply the .cvsignore files as properties on the ## subversion repository? FIXME: we want to remove the .cvsignore files too, after. __version__ = "Revision: 1.5 " __author__ = 'Martin Blais ' # stdlib imports. import os, sys, filecmp, shutil from subprocess import call, Popen, PIPE from os.path import * def copyfile(sfn, dfn): """ Copies a file. """ if opts.verbose: print ' $$$ copyfile', sfn, dfn dn = dirname(dfn) if not exists(dn): os.makedirs(dn) shutil.copyfile(sfn, dfn) def copytree(src, dst): """ Recursively copy files underneath a source directory tree, ignoring what has to be ignored. """ assert exists(src) and isdir(src) and not exists(dst) for root, dirs, files in os.walk(src): rroot = normpath(root[len(src)+1:]) for d in dirs: if d in opts.ignore: dirs.remove(d) # don't visit ignored directories # create output directory ddn = normpath(join(dst, rroot)) assert not exists(ddn) os.makedirs(ddn) # copy and create file elements for fn in files: sfn = normpath(join(root, fn)) dfn = normpath(join(ddn, fn)) copyfile(sfn, dfn) def rmrf(fnodn): """ Delete the given directory and all its contents. """ if not exists(fnodn): return if isdir(fnodn): for root, dirs, files in os.walk(fnodn, topdown=False): for fn in files: os.remove(join(root, fn)) for dn in dirs: adn = join(root, dn) if islink(adn): os.remove(adn) else: os.rmdir(adn) os.rmdir(root) else: os.remove(fnodn) class Effector: """ Interface for classes to apply changes on a repository. The methods in this class do not affect the files themselves otherwise. """ def isco(self, filename): "Return true if 'filename' is checked out." def checkout(self, filename): "Checkout the given or directory." def add(self, filename): "Register 'filename' in the repository." def remove(self, filename): "Remove 'filename' from the repository." def add_dir(self, dirname): "Register 'dirname' and all of its contents in the repository." def remove_dir(self, dirname): "Remove the given directory recursively." class DryRunEffector(Effector): """ Just print the actions that would be undertaken. """ def __init__(self, f): self.f = f self.pfx = ' >>> ' def isco(self, filename): return True # Files are always writable in svn. def checkout(self, filename): self.f.write(self.pfx + 'checkout %s' % filename) def add(self, filename): self.f.write(self.pfx + 'add %s' % filename) def remove(self, filename): self.f.write(self.pfx + 'remove %s' % filename) def add_dir(self, dirname): self.f.write(self.pfx + 'add_dir %s' % dirname) def remove_dir(self, dirname): self.f.write(self.pfx + 'remove_dir %s' % dirname) def svn(*args, **kwds): """ Call a subversion command, make sure it doesn't fail. """ noerror = kwds.pop('noerror', None) cmd = ('svn',) + args print '$$$', ' '.join(cmd) out = kwds.pop('out', None) if not out: kwds['stdout'] = kwds['stderr'] = PIPE p = Popen(cmd, **kwds) sout, serr = p.communicate() if p.returncode != 0 and not noerror: raise IOError("Error calling Subversion: %s\n%s\n\n%s" % (p.returncode, ' '.join(args), sout + serr)) if opts.verbose: for o in sout, serr: for line in o.splitlines(): print '>>>', line return p.returncode class SubversionEffector(Effector): """ Apply changes to a Subversion repository. """ def isco(self, filename): return True # Files are always writable in svn. def checkout(self, filename): pass # No need to explicitly checkout files or directories. def add(self, filename): return svn('add', filename) def remove(self, filename): return svn('remove', filename) def add_dir(self, dirname): return self.add(dirname) # Same as for files. Is recursive. def remove_dir(self, dirname): return self.remove(dirname) # Same as for files. Is recursive. def bhead(s1, s2): print '==================================================' print 'Comparing directories...' print ' ', s1 print ' ', s2 print '==================================================' print def shead(s): print '------------------------------' print s print def process(dc, eff): """ Process directory compare and proceed recursively. An instance of a dircmp (module filecmp) is given, and is visited recursively. 'eff' is a class that is used to perform the actual modifications on the given repository. """ bhead(dc.left, dc.right) if dc.common_funny or dc.funny_files: raise SystemExit("Error: funny files present: %s" % str(dc.common_funny + dc.funny_files)) def co_dest(co): "Insure that the destination directory is checked out." if not co: eff.checkout(dc.right) return 1 return co shead('Adding files') co = eff.isco(dc.right) for rfn in dc.left_only: sfn = normpath(join(dc.left, rfn)) dfn = normpath(join(dc.right, rfn)) if isfile(sfn): print '--- Adding file:', sfn assert not exists(dfn) co = co_dest(co) copyfile(sfn, dfn) eff.add(dfn) elif islink(sfn): print '--- Add link:', sfn co = co_dest(co) lfn = os.readlink(sfn) call(['ln', '-s', lfn, dfn]) eff.add(dfn) elif isdir(sfn): print '--- Add dir tree:', sfn co = co_dest(co) copytree(sfn, dfn) eff.add_dir(dfn) else: raise SystemExit("Error: file '%s' has unknown type." % fn) shead('Removing files') for rfn in dc.right_only: sfn = normpath(join(dc.left, rfn)) dfn = normpath(join(dc.right, rfn)) if isfile(dfn) or islink(dfn): if opts.no_delete: print '--- Ignoring destination file:', dfn else: print '--- Removing file:', dfn assert exists(dfn) and not exists(sfn) co = co_dest(co) os.remove(dfn) eff.remove(dfn) elif isdir(dfn): if opts.no_delete: print '--- Ignoring destination dir tree:', dfn else: print '--- Removing dir tree:', dfn co = co_dest(co) rmrf(dfn) eff.remove_dir(dfn) else: raise SystemExit("Error: file '%s' has unknown type." % fn) shead('Same files') for fn in dc.same_files: print '--- Ignoring file:', fn shead('Diff files') for fn in dc.diff_files: sfn = normpath(join(dc.left, fn)) dfn = normpath(join(dc.right, fn)) if isfile(sfn): print '--- Copying file:', sfn # Checkout the file if necessary before overwriting. if not co: eff.checkout(dfn) co = 1 copyfile(sfn, dfn) # Note: nothing special to be done to register a change. shead('Subdirs') for dc in dc.subdirs.itervalues(): process(dc, eff) def main(): import optparse parser = optparse.OptionParser(__doc__.strip()) parser.add_option('-v', '--verbose', action="store_true", help="Be verbose in your output.") parser.add_option('-n', '--dry-run', action="store_true", help="Don't really run the commands.") parser.add_option('-i', '--ignore', action="append", default=['.svn'], help="Specify paths to ignore." "You can use this option many times.") parser.add_option('-d', '--no-delete', action='store_true', help="Don't delete files on the destination side.") parser.add_option('-e', '--effector', action='store', type='choice', choices=['subversion'], default='subversion', help="Select the target backend.") global opts opts, args = parser.parse_args() # # Validate arguments. # if len(args) != 2: parser.error("You must specify source and destination trees.") src, dst = args # Check that the directories exist. dst = abspath(dst) for dn in src,: if not exists(dn): raise parser.error("Directory '%s' doesn't exist" % dn) # Check that the destination is a Subversion checkout. create = 0 parentdst = dst # Find a parent checkout from the destination path. parentsub = parentdst while parentdst != '/': if exists(parentdst): break create = 1 parentsub, parentdst = parentdst, dirname(parentdst) else: parser.error("Destination directory does not exist.") if opts.effector == 'subversion': ret = svn('info', parentdst) if ret != 0: parser.error("Destination directory is not a valid Subversion checkout") # # Run comparison and process # ## FIXME: don't we need to let 'hide' take its default value of '.' and '..' here? dc = filecmp.dircmp(src, dst, opts.ignore, []) if opts.dry_run: e = DryRunEffector(sys.stdout) elif opts.effector == 'subversion': e = SubversionEffector() if create: # If the destination does not exist, we simply copy it over and add it # and we're done. if not e.isco(parentdst): eff.checkout(parentdst) copytree(src, dst) # Register it in the checkout. e.add_dir(parentsub) else: # Otherwise we need to recursively process the directories. process(dc, e) if __name__ == '__main__': main()