Tue, 06 May 2014 13:24:51 -0700
8042417: hgforest: allow local clone of extra repos
Reviewed-by: chegar, erikj
mduigou@1142 | 1 | #!/bin/sh |
ohrstrom@538 | 2 | |
ohrstrom@538 | 3 | # |
mduigou@845 | 4 | # Copyright (c) 2009, 2013, Oracle and/or its affiliates. All rights reserved. |
ohrstrom@538 | 5 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
ohrstrom@538 | 6 | # |
ohrstrom@538 | 7 | # This code is free software; you can redistribute it and/or modify it |
ohrstrom@538 | 8 | # under the terms of the GNU General Public License version 2 only, as |
ohrstrom@538 | 9 | # published by the Free Software Foundation. |
ohrstrom@538 | 10 | # |
ohrstrom@538 | 11 | # This code is distributed in the hope that it will be useful, but WITHOUT |
ohrstrom@538 | 12 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
ohrstrom@538 | 13 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
ohrstrom@538 | 14 | # version 2 for more details (a copy is included in the LICENSE file that |
ohrstrom@538 | 15 | # accompanied this code). |
ohrstrom@538 | 16 | # |
ohrstrom@538 | 17 | # You should have received a copy of the GNU General Public License version |
ohrstrom@538 | 18 | # 2 along with this work; if not, write to the Free Software Foundation, |
ohrstrom@538 | 19 | # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
ohrstrom@538 | 20 | # |
ohrstrom@538 | 21 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
ohrstrom@538 | 22 | # or visit www.oracle.com if you need additional information or have any |
ohrstrom@538 | 23 | # questions. |
ohrstrom@538 | 24 | # |
ohrstrom@538 | 25 | |
ohrstrom@538 | 26 | # Shell script for a fast parallel forest command |
ohrstrom@538 | 27 | |
mduigou@1141 | 28 | global_opts="" |
mduigou@1141 | 29 | status_output="/dev/stdout" |
mduigou@1141 | 30 | qflag="false" |
mduigou@1141 | 31 | vflag="false" |
chegar@1143 | 32 | sflag="false" |
mduigou@1141 | 33 | while [ $# -gt 0 ] |
mduigou@1141 | 34 | do |
mduigou@1141 | 35 | case $1 in |
mduigou@1141 | 36 | -q | --quiet ) |
mduigou@1141 | 37 | qflag="true" |
mduigou@1141 | 38 | global_opts="${global_opts} -q" |
mduigou@1141 | 39 | status_output="/dev/null" |
mduigou@1141 | 40 | ;; |
mduigou@1141 | 41 | |
mduigou@1141 | 42 | -v | --verbose ) |
mduigou@1141 | 43 | vflag="true" |
mduigou@1141 | 44 | global_opts="${global_opts} -v" |
mduigou@1141 | 45 | ;; |
mduigou@1141 | 46 | |
chegar@1143 | 47 | -s | --sequential ) |
chegar@1143 | 48 | sflag="true" |
chegar@1143 | 49 | ;; |
chegar@1143 | 50 | |
mduigou@1141 | 51 | '--' ) # no more options |
mduigou@1141 | 52 | shift; break |
mduigou@1141 | 53 | ;; |
mduigou@1141 | 54 | |
mduigou@1141 | 55 | -*) # bad option |
mduigou@1141 | 56 | usage |
mduigou@1141 | 57 | ;; |
mduigou@1141 | 58 | |
mduigou@1141 | 59 | * ) # non option |
mduigou@1141 | 60 | break |
mduigou@1141 | 61 | ;; |
mduigou@1141 | 62 | esac |
mduigou@1141 | 63 | shift |
mduigou@1141 | 64 | done |
mduigou@1141 | 65 | |
mduigou@1141 | 66 | |
mduigou@1141 | 67 | command="$1"; shift |
mduigou@1142 | 68 | command_args="$@" |
mduigou@1141 | 69 | |
mduigou@1141 | 70 | usage() { |
chegar@1143 | 71 | echo "usage: $0 [-q|--quiet] [-v|--verbose] [-s|--sequential] [--] <command> [commands...]" > ${status_output} |
mduigou@1141 | 72 | exit 1 |
mduigou@1141 | 73 | } |
mduigou@1141 | 74 | |
mduigou@1141 | 75 | if [ "x" = "x$command" ] ; then |
mduigou@1141 | 76 | echo "ERROR: No command to hg supplied!" |
mduigou@1141 | 77 | usage |
ohrstrom@538 | 78 | fi |
ohrstrom@538 | 79 | |
mduigou@1144 | 80 | # Check if we can use fifos for monitoring sub-process completion. |
mduigou@1144 | 81 | on_windows=`uname -s | egrep -ic -e 'cygwin|msys'` |
mduigou@1144 | 82 | if [ ${on_windows} = "1" ]; then |
mduigou@1144 | 83 | # cygwin has (2014-04-18) broken (single writer only) FIFOs |
mduigou@1144 | 84 | # msys has (2014-04-18) no FIFOs. |
mduigou@1144 | 85 | have_fifos="false" |
mduigou@1144 | 86 | else |
mduigou@1144 | 87 | have_fifos="true" |
mduigou@1144 | 88 | fi |
mduigou@1144 | 89 | |
ohrstrom@538 | 90 | # Clean out the temporary directory that stores the pid files. |
ohrstrom@538 | 91 | tmp=/tmp/forest.$$ |
ohrstrom@538 | 92 | rm -f -r ${tmp} |
ohrstrom@538 | 93 | mkdir -p ${tmp} |
ohrstrom@538 | 94 | |
ohrstrom@538 | 95 | safe_interrupt () { |
chegar@615 | 96 | if [ -d ${tmp} ]; then |
chegar@615 | 97 | if [ "`ls ${tmp}/*.pid`" != "" ]; then |
mduigou@1141 | 98 | echo "Waiting for processes ( `cat ${tmp}/*.pid | tr '\n' ' '`) to terminate nicely!" > ${status_output} |
ohrstrom@538 | 99 | sleep 1 |
ohrstrom@538 | 100 | # Pipe stderr to dev/null to silence kill, that complains when trying to kill |
ohrstrom@538 | 101 | # a subprocess that has already exited. |
chegar@615 | 102 | kill -TERM `cat ${tmp}/*.pid | tr '\n' ' '` 2> /dev/null |
chegar@615 | 103 | wait |
mduigou@1141 | 104 | echo "Interrupt complete!" > ${status_output} |
chegar@615 | 105 | fi |
mduigou@1141 | 106 | rm -f -r ${tmp} |
ohrstrom@538 | 107 | fi |
mduigou@1141 | 108 | exit 130 |
ohrstrom@538 | 109 | } |
ohrstrom@538 | 110 | |
ohrstrom@538 | 111 | nice_exit () { |
chegar@615 | 112 | if [ -d ${tmp} ]; then |
chegar@615 | 113 | if [ "`ls ${tmp}`" != "" ]; then |
chegar@615 | 114 | wait |
chegar@615 | 115 | fi |
mduigou@1141 | 116 | rm -f -r ${tmp} |
ohrstrom@538 | 117 | fi |
ohrstrom@538 | 118 | } |
ohrstrom@538 | 119 | |
ohrstrom@538 | 120 | trap 'safe_interrupt' INT QUIT |
ohrstrom@538 | 121 | trap 'nice_exit' EXIT |
chegar@615 | 122 | |
mduigou@1141 | 123 | subrepos="corba jaxp jaxws langtools jdk hotspot nashorn" |
mduigou@1141 | 124 | subrepos_extra="jdk/src/closed jdk/make/closed jdk/test/closed hotspot/make/closed hotspot/src/closed hotspot/test/closed deploy install sponsors pubs" |
mduigou@1141 | 125 | |
ohrstrom@538 | 126 | # Only look in specific locations for possible forests (avoids long searches) |
ohrstrom@538 | 127 | pull_default="" |
ohrstrom@538 | 128 | repos="" |
ohrstrom@538 | 129 | repos_extra="" |
mduigou@1142 | 130 | if [ "${command}" = "clone" -o "${command}" = "fclone" -o "${command}" = "tclone" ] ; then |
mduigou@1142 | 131 | if [ ! -f .hg/hgrc ] ; then |
mduigou@1141 | 132 | echo "ERROR: Need initial repository to use this script" > ${status_output} |
ohrstrom@538 | 133 | exit 1 |
ohrstrom@538 | 134 | fi |
mduigou@1142 | 135 | |
mduigou@1142 | 136 | pull_default=`hg paths default` |
mduigou@1142 | 137 | if [ "${pull_default}" = "" ] ; then |
mduigou@1142 | 138 | echo "ERROR: Need initial clone with 'hg paths default' defined" > ${status_output} |
mduigou@1142 | 139 | exit 1 |
mduigou@1142 | 140 | fi |
mduigou@1142 | 141 | |
ohrstrom@538 | 142 | for i in ${subrepos} ; do |
ohrstrom@538 | 143 | if [ ! -f ${i}/.hg/hgrc ] ; then |
ohrstrom@538 | 144 | repos="${repos} ${i}" |
ohrstrom@538 | 145 | fi |
ohrstrom@538 | 146 | done |
mduigou@1145 | 147 | |
mduigou@1145 | 148 | pull_default_tail=`echo ${pull_default} | sed -e 's@^.*://[^/]*/\(.*\)@\1@'` |
mduigou@1145 | 149 | |
mduigou@1142 | 150 | if [ "${command_args}" != "" ] ; then |
mduigou@1142 | 151 | if [ "x${pull_default}" = "x${pull_default_tail}" ] ; then |
mduigou@1142 | 152 | echo "ERROR: Need initial clone from non-local source" > ${status_output} |
mduigou@1142 | 153 | exit 1 |
mduigou@1142 | 154 | fi |
mduigou@1142 | 155 | pull_extra="${command_args}/${pull_default_tail}" |
ohrstrom@538 | 156 | for i in ${subrepos_extra} ; do |
ohrstrom@538 | 157 | if [ ! -f ${i}/.hg/hgrc ] ; then |
ohrstrom@538 | 158 | repos_extra="${repos_extra} ${i}" |
ohrstrom@538 | 159 | fi |
ohrstrom@538 | 160 | done |
mduigou@1145 | 161 | else |
mduigou@1145 | 162 | if [ "x${pull_default}" = "x${pull_default_tail}" ] ; then |
mduigou@1145 | 163 | # local source repo. Copy the extras ones that exist there. |
mduigou@1145 | 164 | for i in ${subrepos_extra} ; do |
mduigou@1145 | 165 | if [ -f ${pull_default}/${i}/.hg/hgrc -a ! -f ${i}/.hg/hgrc ] ; then |
mduigou@1145 | 166 | # sub-repo there in source but not here |
mduigou@1145 | 167 | repos_extra="${repos_extra} ${i}" |
mduigou@1145 | 168 | fi |
mduigou@1145 | 169 | done |
mduigou@1145 | 170 | fi |
ohrstrom@538 | 171 | fi |
ohrstrom@538 | 172 | at_a_time=2 |
ohrstrom@538 | 173 | # Any repos to deal with? |
ohrstrom@538 | 174 | if [ "${repos}" = "" -a "${repos_extra}" = "" ] ; then |
mduigou@1141 | 175 | echo "No repositories to process." > ${status_output} |
ohrstrom@538 | 176 | exit |
ohrstrom@538 | 177 | fi |
ohrstrom@538 | 178 | else |
mduigou@1141 | 179 | for i in . ${subrepos} ${subrepos_extra} ; do |
mduigou@1141 | 180 | if [ -d ${i}/.hg ] ; then |
mduigou@1141 | 181 | repos="${repos} ${i}" |
mduigou@1141 | 182 | fi |
ohrstrom@538 | 183 | done |
mduigou@1141 | 184 | |
mduigou@1141 | 185 | # Any repos to deal with? |
mduigou@1141 | 186 | if [ "${repos}" = "" ] ; then |
mduigou@1141 | 187 | echo "No repositories to process." > ${status_output} |
mduigou@1141 | 188 | exit |
mduigou@1141 | 189 | fi |
mduigou@1141 | 190 | |
mduigou@1141 | 191 | # any of the repos locked? |
ohrstrom@538 | 192 | for i in ${repos} ; do |
ohrstrom@538 | 193 | if [ -h ${i}/.hg/store/lock -o -f ${i}/.hg/store/lock ] ; then |
ohrstrom@538 | 194 | locked="${i} ${locked}" |
ohrstrom@538 | 195 | fi |
ohrstrom@538 | 196 | done |
mduigou@1141 | 197 | if [ "${locked}" != "" ] ; then |
mduigou@1141 | 198 | echo "ERROR: These repositories are locked: ${locked}" > ${status_output} |
mduigou@1141 | 199 | exit 1 |
mduigou@1141 | 200 | fi |
ohrstrom@538 | 201 | at_a_time=8 |
ohrstrom@538 | 202 | fi |
ohrstrom@538 | 203 | |
ohrstrom@538 | 204 | # Echo out what repositories we do a command on. |
mduigou@1141 | 205 | echo "# Repositories: ${repos} ${repos_extra}" > ${status_output} |
ohrstrom@538 | 206 | |
mduigou@1141 | 207 | if [ "${command}" = "serve" ] ; then |
mduigou@1141 | 208 | # "serve" is run for all the repos. |
mduigou@1141 | 209 | ( |
mduigou@1141 | 210 | ( |
mduigou@1141 | 211 | ( |
mduigou@1141 | 212 | echo "[web]" |
mduigou@1141 | 213 | echo "description = $(basename $(pwd))" |
mduigou@1141 | 214 | echo "allow_push = *" |
mduigou@1141 | 215 | echo "push_ssl = False" |
mduigou@1141 | 216 | |
mduigou@1141 | 217 | echo "[paths]" |
mduigou@1141 | 218 | for i in ${repos} ${repos_extra} ; do |
mduigou@1141 | 219 | if [ "${i}" != "." ] ; then |
mduigou@1141 | 220 | echo "/$(basename $(pwd))/${i} = ${i}" |
mduigou@1141 | 221 | else |
mduigou@1141 | 222 | echo "/$(basename $(pwd)) = $(pwd)" |
mduigou@1141 | 223 | fi |
mduigou@1141 | 224 | done |
mduigou@1141 | 225 | ) > ${tmp}/serve.web-conf |
mduigou@1141 | 226 | |
mduigou@1141 | 227 | echo "serving root repo $(basename $(pwd))" |
mduigou@1141 | 228 | |
mduigou@1141 | 229 | (PYTHONUNBUFFERED=true hg${global_opts} serve -A ${status_output} -E ${status_output} --pid-file ${tmp}/serve.pid --web-conf ${tmp}/serve.web-conf; echo "$?" > ${tmp}/serve.pid.rc ) 2>&1 & |
mduigou@1141 | 230 | ) 2>&1 | sed -e "s@^@serve: @" > ${status_output} |
mduigou@1141 | 231 | ) & |
mduigou@1141 | 232 | else |
mduigou@1141 | 233 | # Run the supplied command on all repos in parallel. |
mduigou@1144 | 234 | |
mduigou@1144 | 235 | # n is the number of subprocess started or which might still be running. |
mduigou@1141 | 236 | n=0 |
mduigou@1144 | 237 | if [ $have_fifos = "true" ]; then |
mduigou@1144 | 238 | # if we have fifos use them to detect command completion. |
mduigou@1144 | 239 | mkfifo ${tmp}/fifo |
mduigou@1144 | 240 | exec 3<>${tmp}/fifo |
mduigou@1144 | 241 | if [ "${sflag}" = "true" ] ; then |
mduigou@1144 | 242 | # force sequential |
mduigou@1144 | 243 | at_a_time=1 |
mduigou@1144 | 244 | fi |
mduigou@1144 | 245 | fi |
mduigou@1144 | 246 | |
mduigou@1141 | 247 | for i in ${repos} ${repos_extra} ; do |
mduigou@1141 | 248 | n=`expr ${n} '+' 1` |
mduigou@1141 | 249 | repopidfile=`echo ${i} | sed -e 's@./@@' -e 's@/@_@g'` |
mduigou@1141 | 250 | reponame=`echo ${i} | sed -e :a -e 's/^.\{1,20\}$/ &/;ta'` |
mduigou@1141 | 251 | pull_base="${pull_default}" |
mduigou@1141 | 252 | for j in $repos_extra ; do |
ohrstrom@538 | 253 | if [ "$i" = "$j" ] ; then |
ohrstrom@538 | 254 | pull_base="${pull_extra}" |
ohrstrom@538 | 255 | fi |
mduigou@1141 | 256 | done |
mduigou@1144 | 257 | pull_base="`echo ${pull_base} | sed -e 's@[/]*$@@'`" |
mduigou@1141 | 258 | ( |
mduigou@1141 | 259 | ( |
mduigou@1142 | 260 | if [ "${command}" = "clone" -o "${command}" = "fclone" -o "${command}" = "tclone" ] ; then |
mduigou@1144 | 261 | pull_newrepo="${pull_base}/${i}" |
mduigou@1141 | 262 | path="`dirname ${i}`" |
mduigou@1141 | 263 | if [ "${path}" != "." ] ; then |
mduigou@1141 | 264 | times=0 |
mduigou@1141 | 265 | while [ ! -d "${path}" ] ## nested repo, ensure containing dir exists |
mduigou@1141 | 266 | do |
mduigou@1141 | 267 | times=`expr ${times} '+' 1` |
mduigou@1141 | 268 | if [ `expr ${times} '%' 10` -eq 0 ] ; then |
mduigou@1141 | 269 | echo "${path} still not created, waiting..." > ${status_output} |
mduigou@1141 | 270 | fi |
mduigou@1141 | 271 | sleep 5 |
mduigou@1141 | 272 | done |
mduigou@1141 | 273 | fi |
mduigou@1144 | 274 | echo "hg${global_opts} clone ${pull_newrepo} ${i}" > ${status_output} |
mduigou@1141 | 275 | (PYTHONUNBUFFERED=true hg${global_opts} clone ${pull_newrepo} ${i}; echo "$?" > ${tmp}/${repopidfile}.pid.rc ) 2>&1 & |
mduigou@1141 | 276 | else |
mduigou@1142 | 277 | echo "cd ${i} && hg${global_opts} ${command} ${command_args}" > ${status_output} |
mduigou@1142 | 278 | cd ${i} && (PYTHONUNBUFFERED=true hg${global_opts} ${command} ${command_args}; echo "$?" > ${tmp}/${repopidfile}.pid.rc ) 2>&1 & |
mduigou@1141 | 279 | fi |
mduigou@1141 | 280 | |
mduigou@1141 | 281 | echo $! > ${tmp}/${repopidfile}.pid |
mduigou@1141 | 282 | ) 2>&1 | sed -e "s@^@${reponame}: @" > ${status_output} |
mduigou@1144 | 283 | if [ $have_fifos = "true" ]; then |
mduigou@1144 | 284 | echo "${reponame}" >&3 |
mduigou@1144 | 285 | fi |
mduigou@1141 | 286 | ) & |
mduigou@1141 | 287 | |
mduigou@1144 | 288 | if [ $have_fifos = "true" ]; then |
mduigou@1144 | 289 | # check on count of running subprocesses and possibly wait for completion |
mduigou@1144 | 290 | if [ ${at_a_time} -lt ${n} ] ; then |
mduigou@1144 | 291 | # read will block until there are completed subprocesses |
mduigou@1144 | 292 | while read repo_done; do |
mduigou@1144 | 293 | n=`expr ${n} '-' 1` |
mduigou@1144 | 294 | if [ ${n} -lt ${at_a_time} ] ; then |
mduigou@1144 | 295 | # we should start more subprocesses |
mduigou@1144 | 296 | break; |
mduigou@1144 | 297 | fi |
mduigou@1144 | 298 | done <&3 |
mduigou@1144 | 299 | fi |
mduigou@1144 | 300 | else |
mduigou@1144 | 301 | if [ "${sflag}" = "false" ] ; then |
mduigou@1144 | 302 | # Compare completions to starts |
mduigou@1144 | 303 | completed="`(ls -1 ${tmp}/*.pid.rc 2> /dev/null | wc -l) || echo 0`" |
mduigou@1144 | 304 | while [ ${at_a_time} -lt `expr ${n} '-' ${completed}` ] ; do |
mduigou@1144 | 305 | # sleep a short time to give time for something to complete |
mduigou@1144 | 306 | sleep 1 |
mduigou@1144 | 307 | completed="`(ls -1 ${tmp}/*.pid.rc 2> /dev/null | wc -l) || echo 0`" |
mduigou@1144 | 308 | done |
mduigou@1144 | 309 | else |
mduigou@1144 | 310 | # complete this task before starting another. |
chegar@1143 | 311 | wait |
mduigou@1144 | 312 | fi |
chegar@1143 | 313 | fi |
ohrstrom@538 | 314 | done |
mduigou@1141 | 315 | fi |
chegar@615 | 316 | |
mduigou@1144 | 317 | # Wait for all subprocesses to complete |
ohrstrom@538 | 318 | wait |
ohrstrom@538 | 319 | |
chegar@615 | 320 | # Terminate with exit 0 only if all subprocesses were successful |
chegar@615 | 321 | ec=0 |
chegar@615 | 322 | if [ -d ${tmp} ]; then |
chegar@615 | 323 | for rc in ${tmp}/*.pid.rc ; do |
chegar@615 | 324 | exit_code=`cat ${rc} | tr -d ' \n\r'` |
chegar@615 | 325 | if [ "${exit_code}" != "0" ] ; then |
mduigou@1141 | 326 | repo="`echo ${rc} | sed -e s@^${tmp}@@ -e 's@/*\([^/]*\)\.pid\.rc$@\1@' -e 's@_@/@g'`" |
mduigou@1144 | 327 | echo "WARNING: ${repo} exited abnormally ($exit_code)" > ${status_output} |
chegar@615 | 328 | ec=1 |
chegar@615 | 329 | fi |
chegar@615 | 330 | done |
chegar@615 | 331 | fi |
chegar@615 | 332 | exit ${ec} |