Fri, 17 May 2013 10:06:56 -0700
Merge
ohrstrom@538 | 1 | #!/bin/sh |
ohrstrom@538 | 2 | |
ohrstrom@538 | 3 | # |
ohrstrom@538 | 4 | # Copyright (c) 2009, 2012, Oracle and/or its affiliates. All rights reserved. |
ohrstrom@538 | 5 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
ohrstrom@538 | 6 | # |
ohrstrom@538 | 7 | # This code is free software; you can redistribute it and/or modify it |
ohrstrom@538 | 8 | # under the terms of the GNU General Public License version 2 only, as |
ohrstrom@538 | 9 | # published by the Free Software Foundation. |
ohrstrom@538 | 10 | # |
ohrstrom@538 | 11 | # This code is distributed in the hope that it will be useful, but WITHOUT |
ohrstrom@538 | 12 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
ohrstrom@538 | 13 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
ohrstrom@538 | 14 | # version 2 for more details (a copy is included in the LICENSE file that |
ohrstrom@538 | 15 | # accompanied this code). |
ohrstrom@538 | 16 | # |
ohrstrom@538 | 17 | # You should have received a copy of the GNU General Public License version |
ohrstrom@538 | 18 | # 2 along with this work; if not, write to the Free Software Foundation, |
ohrstrom@538 | 19 | # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
ohrstrom@538 | 20 | # |
ohrstrom@538 | 21 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
ohrstrom@538 | 22 | # or visit www.oracle.com if you need additional information or have any |
ohrstrom@538 | 23 | # questions. |
ohrstrom@538 | 24 | # |
ohrstrom@538 | 25 | |
ohrstrom@538 | 26 | # Shell script for a fast parallel forest command |
ohrstrom@538 | 27 | command="$1" |
ohrstrom@538 | 28 | pull_extra_base="$2" |
ohrstrom@538 | 29 | |
ohrstrom@538 | 30 | # Python always buffers stdout significantly, thus we will not see any output from hg clone jdk, |
ohrstrom@538 | 31 | # until a lot of time has passed! By passing -u to python, we get incremental updates |
ohrstrom@538 | 32 | # on stdout. Much nicer. |
mduigou@682 | 33 | whichhg="`which hg 2> /dev/null | grep -v '^no hg in'`" |
ohrstrom@538 | 34 | |
ohrstrom@538 | 35 | if [ "${whichhg}" = "" ] ; then |
ohrstrom@538 | 36 | echo Cannot find hg! |
ohrstrom@538 | 37 | exit 1 |
ohrstrom@538 | 38 | fi |
ohrstrom@538 | 39 | |
ohrstrom@538 | 40 | if [ "" = "$command" ] ; then |
ohrstrom@538 | 41 | echo No command to hg supplied! |
ohrstrom@538 | 42 | exit 1 |
ohrstrom@538 | 43 | fi |
ohrstrom@538 | 44 | |
ohrstrom@538 | 45 | has_hash_bang="`head -n 1 "${whichhg}" | cut -b 1-2`" |
ohrstrom@538 | 46 | python="" |
ohrstrom@538 | 47 | bpython="" |
ohrstrom@538 | 48 | |
ohrstrom@538 | 49 | if [ "#!" = "$has_hash_bang" ] ; then |
ohrstrom@538 | 50 | python="`head -n 1 ${whichhg} | cut -b 3-`" |
ohrstrom@538 | 51 | bpython="`basename "$python"`" |
ohrstrom@538 | 52 | fi |
ohrstrom@538 | 53 | |
mduigou@682 | 54 | if [ -x "$python" -a ! -d "$python" -a "`${python} -V 2>&1 | cut -f 1 -d ' '`" = "Python" ] ; then |
ohrstrom@538 | 55 | hg="${python} -u ${whichhg}" |
ohrstrom@538 | 56 | else |
ohrstrom@538 | 57 | echo Cannot find python from hg launcher. Running plain hg, which probably has buffered stdout. |
ohrstrom@538 | 58 | hg="hg" |
ohrstrom@538 | 59 | fi |
ohrstrom@538 | 60 | |
ohrstrom@538 | 61 | # Clean out the temporary directory that stores the pid files. |
ohrstrom@538 | 62 | tmp=/tmp/forest.$$ |
ohrstrom@538 | 63 | rm -f -r ${tmp} |
ohrstrom@538 | 64 | mkdir -p ${tmp} |
ohrstrom@538 | 65 | |
ohrstrom@538 | 66 | safe_interrupt () { |
chegar@615 | 67 | if [ -d ${tmp} ]; then |
chegar@615 | 68 | if [ "`ls ${tmp}/*.pid`" != "" ]; then |
chegar@615 | 69 | echo "Waiting for processes ( `cat ${tmp}/*.pid | tr '\n' ' '`) to terminate nicely!" |
ohrstrom@538 | 70 | sleep 1 |
ohrstrom@538 | 71 | # Pipe stderr to dev/null to silence kill, that complains when trying to kill |
ohrstrom@538 | 72 | # a subprocess that has already exited. |
chegar@615 | 73 | kill -TERM `cat ${tmp}/*.pid | tr '\n' ' '` 2> /dev/null |
chegar@615 | 74 | wait |
chegar@615 | 75 | echo Interrupt complete! |
chegar@615 | 76 | fi |
ohrstrom@538 | 77 | fi |
ohrstrom@538 | 78 | rm -f -r ${tmp} |
ohrstrom@538 | 79 | exit 1 |
ohrstrom@538 | 80 | } |
ohrstrom@538 | 81 | |
ohrstrom@538 | 82 | nice_exit () { |
chegar@615 | 83 | if [ -d ${tmp} ]; then |
chegar@615 | 84 | if [ "`ls ${tmp}`" != "" ]; then |
chegar@615 | 85 | wait |
chegar@615 | 86 | fi |
ohrstrom@538 | 87 | fi |
ohrstrom@538 | 88 | rm -f -r ${tmp} |
ohrstrom@538 | 89 | } |
ohrstrom@538 | 90 | |
ohrstrom@538 | 91 | trap 'safe_interrupt' INT QUIT |
ohrstrom@538 | 92 | trap 'nice_exit' EXIT |
chegar@615 | 93 | |
ohrstrom@538 | 94 | # Only look in specific locations for possible forests (avoids long searches) |
ohrstrom@538 | 95 | pull_default="" |
ohrstrom@538 | 96 | repos="" |
ohrstrom@538 | 97 | repos_extra="" |
ohrstrom@538 | 98 | if [ "${command}" = "clone" -o "${command}" = "fclone" ] ; then |
jjg@645 | 99 | subrepos="corba jaxp jaxws langtools jdk hotspot nashorn" |
ohrstrom@538 | 100 | if [ -f .hg/hgrc ] ; then |
ohrstrom@538 | 101 | pull_default=`hg paths default` |
ohrstrom@538 | 102 | if [ "${pull_default}" = "" ] ; then |
ohrstrom@538 | 103 | echo "ERROR: Need initial clone with 'hg paths default' defined" |
ohrstrom@538 | 104 | exit 1 |
ohrstrom@538 | 105 | fi |
ohrstrom@538 | 106 | fi |
ohrstrom@538 | 107 | if [ "${pull_default}" = "" ] ; then |
ohrstrom@538 | 108 | echo "ERROR: Need initial repository to use this script" |
ohrstrom@538 | 109 | exit 1 |
ohrstrom@538 | 110 | fi |
ohrstrom@538 | 111 | for i in ${subrepos} ; do |
ohrstrom@538 | 112 | if [ ! -f ${i}/.hg/hgrc ] ; then |
ohrstrom@538 | 113 | repos="${repos} ${i}" |
ohrstrom@538 | 114 | fi |
ohrstrom@538 | 115 | done |
ohrstrom@538 | 116 | if [ "${pull_extra_base}" != "" ] ; then |
ohrstrom@538 | 117 | subrepos_extra="jdk/src/closed jdk/make/closed jdk/test/closed hotspot/make/closed hotspot/src/closed hotspot/test/closed deploy install sponsors pubs" |
ohrstrom@538 | 118 | pull_default_tail=`echo ${pull_default} | sed -e 's@^.*://[^/]*/\(.*\)@\1@'` |
ohrstrom@538 | 119 | pull_extra="${pull_extra_base}/${pull_default_tail}" |
ohrstrom@538 | 120 | for i in ${subrepos_extra} ; do |
ohrstrom@538 | 121 | if [ ! -f ${i}/.hg/hgrc ] ; then |
ohrstrom@538 | 122 | repos_extra="${repos_extra} ${i}" |
ohrstrom@538 | 123 | fi |
ohrstrom@538 | 124 | done |
ohrstrom@538 | 125 | fi |
ohrstrom@538 | 126 | at_a_time=2 |
ohrstrom@538 | 127 | # Any repos to deal with? |
ohrstrom@538 | 128 | if [ "${repos}" = "" -a "${repos_extra}" = "" ] ; then |
ohrstrom@538 | 129 | exit |
ohrstrom@538 | 130 | fi |
ohrstrom@538 | 131 | else |
ohrstrom@538 | 132 | hgdirs=`ls -d ./.hg ./*/.hg ./*/*/.hg ./*/*/*/.hg ./*/*/*/*/.hg 2>/dev/null` |
ohrstrom@538 | 133 | # Derive repository names from the .hg directory locations |
ohrstrom@538 | 134 | for i in ${hgdirs} ; do |
ohrstrom@538 | 135 | repos="${repos} `echo ${i} | sed -e 's@/.hg$@@'`" |
ohrstrom@538 | 136 | done |
ohrstrom@538 | 137 | for i in ${repos} ; do |
ohrstrom@538 | 138 | if [ -h ${i}/.hg/store/lock -o -f ${i}/.hg/store/lock ] ; then |
ohrstrom@538 | 139 | locked="${i} ${locked}" |
ohrstrom@538 | 140 | fi |
ohrstrom@538 | 141 | done |
ohrstrom@538 | 142 | at_a_time=8 |
ohrstrom@538 | 143 | # Any repos to deal with? |
ohrstrom@538 | 144 | if [ "${repos}" = "" ] ; then |
ohrstrom@538 | 145 | echo "No repositories to process." |
ohrstrom@538 | 146 | exit |
ohrstrom@538 | 147 | fi |
ohrstrom@538 | 148 | if [ "${locked}" != "" ] ; then |
ohrstrom@538 | 149 | echo "These repositories are locked: ${locked}" |
ohrstrom@538 | 150 | exit |
ohrstrom@538 | 151 | fi |
ohrstrom@538 | 152 | fi |
ohrstrom@538 | 153 | |
ohrstrom@538 | 154 | # Echo out what repositories we do a command on. |
ohrstrom@538 | 155 | echo "# Repositories: ${repos} ${repos_extra}" |
ohrstrom@538 | 156 | echo |
ohrstrom@538 | 157 | |
ohrstrom@538 | 158 | # Run the supplied command on all repos in parallel. |
ohrstrom@538 | 159 | n=0 |
ohrstrom@538 | 160 | for i in ${repos} ${repos_extra} ; do |
ohrstrom@538 | 161 | n=`expr ${n} '+' 1` |
ohrstrom@538 | 162 | repopidfile=`echo ${i} | sed -e 's@./@@' -e 's@/@_@g'` |
ohrstrom@538 | 163 | reponame=`echo ${i} | sed -e :a -e 's/^.\{1,20\}$/ &/;ta'` |
ohrstrom@538 | 164 | pull_base="${pull_default}" |
ohrstrom@538 | 165 | for j in $repos_extra ; do |
ohrstrom@538 | 166 | if [ "$i" = "$j" ] ; then |
ohrstrom@538 | 167 | pull_base="${pull_extra}" |
ohrstrom@538 | 168 | fi |
ohrstrom@538 | 169 | done |
ohrstrom@538 | 170 | ( |
ohrstrom@538 | 171 | ( |
ohrstrom@538 | 172 | if [ "${command}" = "clone" -o "${command}" = "fclone" ] ; then |
ohrstrom@538 | 173 | pull_newrepo="`echo ${pull_base}/${i} | sed -e 's@\([^:]/\)//*@\1@g'`" |
ohrstrom@538 | 174 | echo ${hg} clone ${pull_newrepo} ${i} |
chegar@615 | 175 | path="`dirname ${i}`" |
chegar@615 | 176 | if [ "${path}" != "." ] ; then |
chegar@615 | 177 | times=0 |
chegar@615 | 178 | while [ ! -d "${path}" ] ## nested repo, ensure containing dir exists |
chegar@615 | 179 | do |
chegar@615 | 180 | times=`expr ${times} '+' 1` |
chegar@615 | 181 | if [ `expr ${times} '%' 10` -eq 0 ] ; then |
chegar@615 | 182 | echo ${path} still not created, waiting... |
chegar@615 | 183 | fi |
chegar@615 | 184 | sleep 5 |
chegar@615 | 185 | done |
chegar@615 | 186 | fi |
chegar@615 | 187 | (${hg} clone ${pull_newrepo} ${i}; echo "$?" > ${tmp}/${repopidfile}.pid.rc )& |
ohrstrom@538 | 188 | else |
ohrstrom@538 | 189 | echo "cd ${i} && ${hg} $*" |
chegar@615 | 190 | cd ${i} && (${hg} "$@"; echo "$?" > ${tmp}/${repopidfile}.pid.rc )& |
chegar@615 | 191 | fi |
ohrstrom@538 | 192 | echo $! > ${tmp}/${repopidfile}.pid |
ohrstrom@538 | 193 | ) 2>&1 | sed -e "s@^@${reponame}: @") & |
chegar@615 | 194 | |
ohrstrom@538 | 195 | if [ `expr ${n} '%' ${at_a_time}` -eq 0 ] ; then |
ohrstrom@538 | 196 | sleep 2 |
ohrstrom@538 | 197 | echo Waiting 5 secs before spawning next background command. |
ohrstrom@538 | 198 | sleep 3 |
ohrstrom@538 | 199 | fi |
ohrstrom@538 | 200 | done |
ohrstrom@538 | 201 | # Wait for all hg commands to complete |
ohrstrom@538 | 202 | wait |
ohrstrom@538 | 203 | |
chegar@615 | 204 | # Terminate with exit 0 only if all subprocesses were successful |
chegar@615 | 205 | ec=0 |
chegar@615 | 206 | if [ -d ${tmp} ]; then |
chegar@615 | 207 | for rc in ${tmp}/*.pid.rc ; do |
chegar@615 | 208 | exit_code=`cat ${rc} | tr -d ' \n\r'` |
chegar@615 | 209 | if [ "${exit_code}" != "0" ] ; then |
chegar@615 | 210 | echo "WARNING: ${rc} exited abnormally." |
chegar@615 | 211 | ec=1 |
chegar@615 | 212 | fi |
chegar@615 | 213 | done |
chegar@615 | 214 | fi |
chegar@615 | 215 | exit ${ec} |