From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (qmail 15478 invoked by alias); 18 Apr 2012 15:42:20 -0000 Received: (qmail 15174 invoked by uid 22791); 18 Apr 2012 15:42:09 -0000 X-SWARE-Spam-Status: No, hits=-3.2 required=5.0 tests=AWL,BAYES_00,KHOP_THREADED,T_FILL_THIS_FORM_SHORT,T_RP_MATCHES_RCVD X-Spam-Check-By: sourceware.org Received: from mx.meyering.net (HELO mx.meyering.net) (88.168.87.75) by sourceware.org (qpsmtpd/0.43rc1) with ESMTP; Wed, 18 Apr 2012 15:41:24 +0000 Received: from rho.meyering.net (localhost.localdomain [127.0.0.1]) by rho.meyering.net (Acme Bit-Twister) with ESMTP id 13DE2601B5; Wed, 18 Apr 2012 17:41:23 +0200 (CEST) From: Jim Meyering To: Cc: Subject: Re: FYI, cvs-to-git mirroring hiccup: just resolved In-Reply-To: <393A4D071DD24B45A370BEBB86AD05A201EFDDCD7F@MX37A.corp.emc.com> (mikhail terekhov's message of "Wed, 18 Apr 2012 11:17:26 -0400") References: <20120418063802.10092.qmail@sourceware.org> <87ehrl31xs.fsf@rho.meyering.net> <393A4D071DD24B45A370BEBB86AD05A201EFDDCD7F@MX37A.corp.emc.com> Date: Wed, 18 Apr 2012 15:42:00 -0000 Message-ID: <87sjg1rqsd.fsf@rho.meyering.net> MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="=-=-=" X-IsSubscribed: yes Mailing-List: contact gdb-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Subscribe: List-Archive: List-Post: List-Help: , Sender: gdb-owner@sourceware.org X-SW-Source: 2012-04/txt/msg00117.txt.bz2 --=-=-= Content-Type: text/plain Content-length: 800 mikhail.terekhov@emc.com wrote: >> That was because my cvs-to-git mirroring script was using >> /cvs/src/src/gdb/gdbinit.in,v (a witness that the configuration >> repo and modules are well-configured), yet that file has recently >> been cvs-removed. >> >> I've changed the mirroring config to use gdb-demangle.h,v instead, >> and am running the sync manually now to be sure. > > Jim, > > Is this cvs-to-git mirroring script available somewhere to learn from? There are two scripts. A driver I call mirror-sw (sw=sourceware, to contrast with another variant I've used for savannah-based projects) and another that it invokes to do the bulk of the work, called mirror-cvs-to-git. You may not want to try to learn from these scripts. They're not well documented. But for the record, here they are: --=-=-= Content-Type: text/x-sh Content-Disposition: inline; filename=mirror-sw Content-length: 4252 #!/bin/sh # sync a CVS-to-GIT mirror on sourceware.org # Take a package name and map that to the parameters needed to invoke # mirror-cvs-to-git. Then, push any changes from the temporary git # repository to the destination one. PATH=$HOME/bin:/usr/local/bin:/sbin:/usr/sbin:/bin:/usr/bin export PATH ME=$(basename "$0") die() { echo >&2 "$ME: $@"; exit 1; } filter_stderr() { # Usage: filter-stderr EGREP_REGEXP 'command' regex=$1 cmd=$2 (exec 3>&1; eval "$cmd" 2>&1 >&3 | grep -E -v "$regex" 1>&2) } verbose= case $# in 0) ;; *) case $1 in --verbose) verbose=--verbose; shift;; esac;; esac binutils_exclude=' blt cgen contrib dejagnu expat expect gdb itcl iwidgets libdecnumber libgloss libgui mmalloc newlib proto-toplev rda readline sid sim tcl tix tk utils winsup ' gdb_exclude=' binutils blt cgen contrib dejagnu elfcpp expat expect gas gdb/gdbtk gold gprof itcl iwidgets ld libgloss libgui mmalloc newlib proto-toplev rda sid tcl tix tk utils winsup ' newlib_exclude=' bfd binutils blt cgen cpu dejagnu elfcpp expat expect gas gdb gold gprof include intl itcl iwidgets ld libdecnumber libgui libiberty mmalloc opcodes proto-toplev rda readline sid sim tcl tix tk utils winsup ' repo=$1 case $repo in gdb) exclude=$gdb_exclude # CAUTION: m= # Note how here, m is not set, yet it is for binutils and newlib. cvs_mod=src url=/cvs/src f=gdb/gdb-demangle.h,v g=/git/gdb.git;; binutils) exclude=$binutils_exclude m=binutils cvs_mod=src url=/cvs/src f=binutils/arlex.l,v g=/git/binutils.git;; cluster) cvs_mod=cluster url=/cvs/cluster f=gfs/gfs_mkfs/mkfs_gfs.h,v g=/git/cluster.git;; newlib) exclude=$newlib_exclude m=newlib cvs_mod=src url=/cvs/src f=newlib/testsuite/lib/newlib.exp,v g=/git/newlib.git;; automake) cvs_mod=automake url=/cvs/automake f=lib/Automake/RuleDef.pm,v g=/git/automake.git;; dm|device-mapper) cvs_mod=device-mapper url=/cvs/dm f=dmsetup/dmsetup.c,v g=/git/dm.git ;; lvm|lvm2) cvs_mod=LVM2 url=/cvs/lvm2 f=tools/lvm.c,v g=/git/lvm2.git ;; libc) cvs_mod=libc url=sourceware.org::glibc-cvs f=manual/libc.texinfo,v g=/git/glibc.git ;; glibc-ports) cvs_mod=ports url=sourceware.org::glibc-cvs f=bare/brdinit.c,v g=/git/glibc-ports.git ;; *) die "Usage: $ME package_name" ;; esac exclude=$(printf %s "$exclude"|tr '\012' ' '|sed 's/ $//') test -z "$m" && m=$cvs_mod tmp_repo=$HOME/mirror-git-to-cvs/repo/$m mirror_cmd="mirror-cvs-to-git \ $verbose \ --exclude='$exclude' \ --rsync-url=$url \ --module=$cvs_mod \ --git-dir=$tmp_repo \ --state-dir=$HOME/mirror-git-to-cvs/.state/$m \ --key-file=$f \ --user-map=$HOME/mirror-git-to-cvs/git-user-map/$m" mirror_re=$(printf %s ' no patchset for tag raeburn |Skipping #CVSPS_NO_BRANCH |branch #CVSPS_NO_BRANCH not found |\* UNKNOWN LINE \* Branches: |revision .* is tagged but not present ' |tr -d '\012') filter_stderr "$mirror_re" "$mirror_cmd" # Push unconditionally. Otherwise, at least the initial # run fails to push. # case $? in # 0) exit 0 ;; # nothing rsync'd, no push required # 1) ;; # fall through # *) exit 1 ;; # esac push_re=$(printf %s ' Everything up-to-date |^To /git/\S+\.git$ |^ \x{7}\.\.\x{7} \S+ -> \S+$ |^updating .refs/heads/master.$ |^Branch \S+ already exists\.$ |^Removing duplicate objects:.*done\.$ |^Delta compression using up to 4 threads\.$ |^Generating pack\.\.\.$ |^Done counting \d+ objects\.$ |^Deltifying \d+ objects\.\.\.$ |^Total \d+ \(delta \d+\), |^refs/heads/master: \x{40} -> \x{40}$ |^ +\d+% \(1/\d+\) done |^ from \x{40}$ |^ to \x{40}$ |((Compress|Count|Writ)ing objects:) ' |tr -d '\012' \ |sed 's/\\S/[^[:space:]]/g;s/\\d/[0-9]/g;s/\\x/[[:xdigit:]]/g') case $repo in libc|glibc-ports) spec='refs/heads/*:refs/heads/cvs/*';; *) spec='refs/heads/*:refs/heads/*';; esac push_cmd="GIT_DIR=$tmp_repo/.git git push --force --tags $g +'$spec'" # Push to the public git repo. filter_stderr "$push_re" "$push_cmd" exit 0 --=-=-= Content-Type: text/x-sh Content-Disposition: inline; filename=mirror-cvs-to-git Content-length: 10067 #!/bin/sh # Sync a local git repository from a remote CVS repository. VERSION='2012-04-18 15:36' # UTC # The definition above must lie within the first 8 lines in order # for the Emacs time-stamp write hook (at end) to update it. # If you change this file with Emacs, please let the write hook # do its job. Otherwise, update this string manually. # Requirements: # - git and cvs, of course # - cvsps (used by git cvsimport) # - GNU find and xargs # FIXME: Add an option to handle *local* master CVS repo, # hence no need for the rsync run: just use original # FIXME: while the rsync copies any write lock files, sleep+repeat-up-to-N-times # Hmm... if a modified file is so new that it'd trigger the "too recent" # skip-changeset, then consider sleeping. But then we'd need to know # the cron-job frequency, or take out a lock. # Hmm... better (simpler), just remove the write-lock files. # That's ok, since the offending, partial commit, should be classified # as "too recent", and deferred. # FIXME: so, to implement the above, rsync-exclude any in-tree write-lock files. # FIXME: Document that user-map file is required only on the first iteration ME=$(basename "$0") die() { echo >&2 "$ME: $@"; exit 2; } # Return 0 if the mtime of $FILE is at least $N_MINUTES in the past. timestamp_old_enough() { case $# in 2) ;; *) die 'Usage: timestamp_old_enough file n_minutes';; esac case $2 in [0-9]*) ;; *) die 'Usage: timestamp_old_enough file n_minutes';; esac file=$1 n=$2 old_enough=$(find "$file" -mmin +$n) case $old_enough in '') return 1;; *) return 0;; esac } move_if_change() { case $# in 2) ;; *) eval echo >&2 'Usage: move_if_change file_1 file_2'; exit 2;; esac if [ -r $2 ] ; then if cmp $1 $2 > /dev/null ; then : # echo $2 is unchanged rm $1 else mv $1 $2 fi else mv $1 $2 fi } usage () { echo "\ Usage: $ME \\ --rsync-url=example.org::dir/... \\ --module=module_name \\ --git-dir=DIR \\ --state-dir=DIR \\ [OPTION...] Synchronize a git repository mirror from a cvs repository. Options: --rsync-url=RSYNC_URL Specify an rsync URL, e.g., example.org::repo. You'd rsync this URL to get the entire CVS repository, often including a top-level CVSROOT/ directory. --exclude=SPC_SEPARATED_LIST Specify names to exclude. --module=CVS_MODULE Specify the CVS module name to mirror into a git repository. --state-dir=DIR Specify a directory, DIR, in which to store sufficient state so that subsequent runs are efficient. Things we store in there: rsync'd CVS repo, the user-map file. FIXME: currently we create DIR if it doesn't exist, but only using "mkdir", not "mkdir -p". --git-dir=DIR Specify a directory, DIR, in which the resulting .git directory resides (or will reside, if it doesn't yet exist). FIXME: currently we create DIR if it doesn't exist, but only using "mkdir", not "mkdir -p". --key-file=FILE Specify a file (usually ending in ,v) in the desired repo. FIXME: give an example. --user-map=FILE Specify the mapping of user names in CVS logs to the (User Name, ) pairs git uses. Each line must look like \"username=User Name \". Specify this file so that git tools can display the real name and email address of each change-set committer. --help display this help and exit --version output version information and exit Exit status: 0 rsync pulled in _no_ changes 1 rsync *did* pull in changes 2 abnormal (error) termination EXAMPLE m=device-mapper url=sourceware.org::dm-cvs f=dmsetup/dmsetup.c,v mirror-cvs-to-git \\ --rsync-url=\$url \\ --module=\$m \\ --git-dir=/work/co/git-repo/\$m \\ --state-dir=/work/co/git-repo/.state/\$m \\ --key-file=\$f \\ --user-map=\$HOME/tmp/misc/git-user-map/\$m m=LVM2 url=sourceware.org::lvm2-cvs f=tools/lvm.c,v m=emacs url=cvs.sv.gnu.org::sources/emacs f=src/emacs.c,v m=gnulib url=cvs.sv.gnu.org::sources/gnulib f=doc/gnulib.texi,v Report bugs to ." } Exit () { (exit $1); exit $1 } rsync_url= module= exclude= git_dir= state_dir= user_map= verbose= { while test $# -gt 0; do case "$1" in --module ) shift if test $# = 0; then die "missing argument for --module" fi module="$1" shift ;; --module=* ) module=`echo "X$1" | sed -e 's/^X--module=//'` shift ;; --exclude ) shift if test $# = 0; then die "missing argument for --exclude" fi exclude="$1" shift ;; --exclude=* ) exclude=`echo "X$1" | sed -e 's/^X--exclude=//'` shift ;; --git-dir ) shift if test $# = 0; then die "missing argument for --git-dir" fi git_dir="$1" shift ;; --git-dir=* ) git_dir=`echo "X$1" | sed -e 's/^X--git-dir=//'` shift ;; --key-file ) shift if test $# = 0; then die "missing argument for --key-file" fi key_file="$1" shift ;; --key-file=* ) key_file=`echo "X$1" | sed -e 's/^X--key-file=//'` shift ;; --user-map ) shift if test $# = 0; then die "missing argument for --user-map" fi user_map="$1" shift ;; --user-map=* ) user_map=`echo "X$1" | sed -e 's/^X--user-map=//'` shift ;; --state-dir ) shift if test $# = 0; then die "missing argument for --state-dir" fi state_dir="$1" shift ;; --state-dir=* ) state_dir=`echo "X$1" | sed -e 's/^X--state-dir=//'` shift ;; --rsync-url ) shift if test $# = 0; then die "missing argument for --rsync-url" fi rsync_url="$1" shift ;; --rsync-url=* ) rsync_url=`echo "X$1" | sed -e 's/^X--rsync-url=//'` shift ;; --verbose | --verbos | --verbo | --verb ) verbose=-v shift ;; --help | --hel | --he | --h ) usage Exit $? ;; --version | --versio | --versi | --vers ) echo "$ME version $VERSION" Exit $? ;; -- ) # Stop option processing shift break ;; -* ) echo "$ME: unknown option $1" 1>&2 echo "Try '$ME --help' for more information." 1>&2 Exit 1 ;; * ) break ;; esac done } ok=no test -n "$rsync_url" && test -n "$module" && test -n "$git_dir" && test -n "$key_file" && test -n "$state_dir" && ok=yes test $ok = yes || die "required argument not specified" ############################################################# test -d "$git_dir" \ || { mkdir "$git_dir" || die "cannot create git dir, \"$git_dir\""; } test -d "$state_dir" \ || { mkdir "$state_dir" || die "cannot create state dir, \"$state_dir\""; } if test x"$user_map" = x; then test -f "$state_dir/user-map" \ && user_map="$state_dir/user-map" \ || user_map=/dev/null else cat "$user_map" > $state_dir/user-map \ || die "can't read user-map file: $user_map" fi user_map=$state_dir/user-map cvs_repo=$state_dir/cvsrepo # If this succeeds, then the key-file is valid t="$rsync_url/$module/$key_file" rsync "$t" > /dev/null \ || die "invalid key-file? failed to rsync $t" excl_opt= for i in $(echo $exclude); do excl_opt="$excl_opt --exclude $module/$i/" excl_opt="$excl_opt --exclude '$module/$i/**'" done rsync -az $verbose \ --delete \ --delete-excluded \ $excl_opt \ --include "$module/" \ --include "$module/**" \ --exclude '*' \ $rsync_url/ $cvs_repo/$module # Handle two different repository layouts: # - savannah-style, where there's an extra layer of hierarchy, e.g., # gnulib/gnulib/ FIXME: finish this... new_repo=$cvs_repo/$module if test -d $new_repo && test -d $new_repo/$module; then cvs_repo=$new_repo rm -rf $cvs_repo/CVSROOT fi cvs -d $cvs_repo init rsync_change_timestamp=$state_dir/rsync-change-timestamp # See if "find" run exposes changes in name, mtime, or mode bits. rsync_pulled_changes=yes f=$state_dir/find (cd $cvs_repo/$module && find . -type f -printf '%m %T@ %P\0') > $f.t test -f $f && cmp $f $f.t > /dev/null \ && rsync_pulled_changes=no move_if_change $f.t $f n_minutes=10 # If the previous "git cvsimport" run was too soon after the rsync that # changed the CVS repository, then run it again, even if the latest # rsync pulled in no changes. need_git_cvsimport=yes if test $rsync_pulled_changes = yes; then touch $rsync_change_timestamp else if test -f $rsync_change_timestamp; then if timestamp_old_enough $rsync_change_timestamp $n_minutes; then # If there have been no changes for $n_minutes, remove the timestamp # file so that subsequent no-rsync-change iterations won't bother # to run "git cvsimport". However, this one must still run it. rm $rsync_change_timestamp fi else need_git_cvsimport=no fi fi test $need_git_cvsimport = no && Exit 0 # Be sure to use cvsps-2.2b1, not 2.1 export PATH=/home/meyering/bin:$PATH # Set HOME, so cvsps leaves its .cvspsrc file here, not in ~/. export HOME=$state_dir rm -rf $HOME/.cvsps git cvsimport \ $verbose \ -A $user_map \ -d $cvs_repo \ -C $git_dir \ -p -z,120 \ -o master \ -k -i \ $module # Clean up (needed because git cvsimport's -i option doesn't work). # Remove everything in $git_dir except .git/. find $git_dir -mindepth 1 -maxdepth 1 -name '.git' -prune -o -print0 \ | xargs -0 rm -rf Exit 1 ## Local Variables: ## eval: (add-hook 'write-file-hooks 'time-stamp) ## time-stamp-start: "VERSION='" ## time-stamp-format: "%:y-%02m-%02d %02H:%02M" ## time-stamp-time-zone: "UTC" ## time-stamp-end: "' # UTC" ## End: --=-=-= Content-Type: text/plain Content-length: 54 The crontab entry runs this command: "mirror-sw gdb" --=-=-=--