Tue, 13 May 2014 08:42:44 -0700
8042810: hgforest: some shells run read in sub-shell and can't use fifo
Reviewed-by: chegar, erikj
common/bin/hgforest.sh | file | annotate | diff | comparison | revisions |
1.1 --- a/common/bin/hgforest.sh Tue Jul 08 11:21:43 2014 -0700 1.2 +++ b/common/bin/hgforest.sh Tue May 13 08:42:44 2014 -0700 1.3 @@ -1,5 +1,4 @@ 1.4 #!/bin/sh 1.5 - 1.6 # 1.7 # Copyright (c) 2009, 2013, Oracle and/or its affiliates. All rights reserved. 1.8 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 1.9 @@ -23,25 +22,40 @@ 1.10 # questions. 1.11 # 1.12 1.13 -# Shell script for a fast parallel forest command 1.14 +# Shell script for a fast parallel forest/trees command 1.15 1.16 -global_opts="" 1.17 -status_output="/dev/stdout" 1.18 -qflag="false" 1.19 -vflag="false" 1.20 -sflag="false" 1.21 +usage() { 1.22 + echo "usage: $0 [-h|--help] [-q|--quiet] [-v|--verbose] [-s|--sequential] [--] <command> [commands...]" > ${status_output} 1.23 + echo "Environment variables which modify behaviour:" 1.24 + echo " HGFOREST_QUIET : (boolean) If 'true' then standard output is redirected to /dev/null" 1.25 + echo " HGFOREST_VERBOSE : (boolean) If 'true' then Mercurial asked to produce verbose output" 1.26 + echo " HGFOREST_SEQUENTIAL : (boolean) If 'true' then repos are processed sequentially. Disables concurrency" 1.27 + echo " HGFOREST_GLOBALOPTS : (string, must begin with space) Additional Mercurial global options" 1.28 + echo " HGFOREST_REDIRECT : (file path) Redirect standard output to specified file" 1.29 + echo " HGFOREST_FIFOS : (boolean) Default behaviour for FIFO detection. Does not override FIFOs disabled" 1.30 + echo " HGFOREST_CONCURRENCY: (positive integer) Number of repos to process concurrently" 1.31 + echo " HGFOREST_DEBUG : (boolean) If 'true' then temp files are retained" 1.32 + exit 1 1.33 +} 1.34 + 1.35 +global_opts="${HGFOREST_GLOBALOPTS:-}" 1.36 +status_output="${HGFOREST_REDIRECT:-/dev/stdout}" 1.37 +qflag="${HGFOREST_QUIET:-false}" 1.38 +vflag="${HGFOREST_VERBOSE:-false}" 1.39 +sflag="${HGFOREST_SEQUENTIAL:-false}" 1.40 while [ $# -gt 0 ] 1.41 do 1.42 case $1 in 1.43 + -h | --help ) 1.44 + usage 1.45 + ;; 1.46 + 1.47 -q | --quiet ) 1.48 qflag="true" 1.49 - global_opts="${global_opts} -q" 1.50 - status_output="/dev/null" 1.51 ;; 1.52 1.53 -v | --verbose ) 1.54 vflag="true" 1.55 - global_opts="${global_opts} -v" 1.56 ;; 1.57 1.58 -s | --sequential ) 1.59 @@ -63,39 +77,60 @@ 1.60 shift 1.61 done 1.62 1.63 +# silence standard output? 1.64 +if [ ${qflag} = "true" ] ; then 1.65 + global_opts="${global_opts} -q" 1.66 + status_output="/dev/null" 1.67 +fi 1.68 1.69 -command="$1"; shift 1.70 -command_args="$@" 1.71 +# verbose output? 1.72 +if [ ${vflag} = "true" ] ; then 1.73 + global_opts="${global_opts} -v" 1.74 +fi 1.75 1.76 -usage() { 1.77 - echo "usage: $0 [-q|--quiet] [-v|--verbose] [-s|--sequential] [--] <command> [commands...]" > ${status_output} 1.78 - exit 1 1.79 -} 1.80 - 1.81 -if [ "x" = "x$command" ] ; then 1.82 +# Make sure we have a command. 1.83 +if [ $# -lt 1 -o -z "${1:-}" ] ; then 1.84 echo "ERROR: No command to hg supplied!" 1.85 usage 1.86 fi 1.87 1.88 -# Check if we can use fifos for monitoring sub-process completion. 1.89 -on_windows=`uname -s | egrep -ic -e 'cygwin|msys'` 1.90 -if [ ${on_windows} = "1" ]; then 1.91 - # cygwin has (2014-04-18) broken (single writer only) FIFOs 1.92 - # msys has (2014-04-18) no FIFOs. 1.93 - have_fifos="false" 1.94 -else 1.95 - have_fifos="true" 1.96 -fi 1.97 +command="$1"; shift 1.98 +command_args="${@:-}" 1.99 1.100 # Clean out the temporary directory that stores the pid files. 1.101 tmp=/tmp/forest.$$ 1.102 rm -f -r ${tmp} 1.103 mkdir -p ${tmp} 1.104 1.105 + 1.106 +if [ "${HGFOREST_DEBUG:-false}" = "true" ] ; then 1.107 + echo "DEBUG: temp files are in: ${tmp}" 1.108 +fi 1.109 + 1.110 +# Check if we can use fifos for monitoring sub-process completion. 1.111 +echo "1" > ${tmp}/read 1.112 +while_subshell=1 1.113 +while read line; do 1.114 + while_subshell=0 1.115 + break; 1.116 +done < ${tmp}/read 1.117 +rm ${tmp}/read 1.118 + 1.119 +on_windows=`uname -s | egrep -ic -e 'cygwin|msys'` 1.120 + 1.121 +if [ ${while_subshell} = "1" -o ${on_windows} = "1" ]; then 1.122 + # cygwin has (2014-04-18) broken (single writer only) FIFOs 1.123 + # msys has (2014-04-18) no FIFOs. 1.124 + # older shells create a sub-shell for redirect to while 1.125 + have_fifos="false" 1.126 +else 1.127 + have_fifos="${HGFOREST_FIFOS:-true}" 1.128 +fi 1.129 + 1.130 safe_interrupt () { 1.131 if [ -d ${tmp} ]; then 1.132 if [ "`ls ${tmp}/*.pid`" != "" ]; then 1.133 - echo "Waiting for processes ( `cat ${tmp}/*.pid | tr '\n' ' '`) to terminate nicely!" > ${status_output} 1.134 + echo "Waiting for processes ( `cat ${tmp}/.*.pid ${tmp}/*.pid 2> /dev/null | tr '\n' ' '`) to terminate nicely!" > ${status_output} 1.135 sleep 1 1.136 # Pipe stderr to dev/null to silence kill, that complains when trying to kill 1.137 # a subprocess that has already exited. 1.138 @@ -110,10 +145,12 @@ 1.139 1.140 nice_exit () { 1.141 if [ -d ${tmp} ]; then 1.142 - if [ "`ls ${tmp}`" != "" ]; then 1.143 + if [ "`ls -A ${tmp} 2> /dev/null`" != "" ]; then 1.144 wait 1.145 fi 1.146 - rm -f -r ${tmp} 1.147 + if [ "${HGFOREST_DEBUG:-false}" != "true" ] ; then 1.148 + rm -f -r ${tmp} 1.149 + fi 1.150 fi 1.151 } 1.152 1.153 @@ -128,17 +165,20 @@ 1.154 repos="" 1.155 repos_extra="" 1.156 if [ "${command}" = "clone" -o "${command}" = "fclone" -o "${command}" = "tclone" ] ; then 1.157 + # we must be a clone 1.158 if [ ! -f .hg/hgrc ] ; then 1.159 echo "ERROR: Need initial repository to use this script" > ${status_output} 1.160 exit 1 1.161 fi 1.162 1.163 + # the clone must know where it came from (have a default pull path). 1.164 pull_default=`hg paths default` 1.165 if [ "${pull_default}" = "" ] ; then 1.166 echo "ERROR: Need initial clone with 'hg paths default' defined" > ${status_output} 1.167 exit 1 1.168 fi 1.169 1.170 + # determine which sub repos need to be cloned. 1.171 for i in ${subrepos} ; do 1.172 if [ ! -f ${i}/.hg/hgrc ] ; then 1.173 repos="${repos} ${i}" 1.174 @@ -147,12 +187,15 @@ 1.175 1.176 pull_default_tail=`echo ${pull_default} | sed -e 's@^.*://[^/]*/\(.*\)@\1@'` 1.177 1.178 - if [ "${command_args}" != "" ] ; then 1.179 + if [ -n "${command_args}" ] ; then 1.180 + # if there is an "extra sources" path then reparent "extra" repos to that path 1.181 if [ "x${pull_default}" = "x${pull_default_tail}" ] ; then 1.182 echo "ERROR: Need initial clone from non-local source" > ${status_output} 1.183 exit 1 1.184 fi 1.185 pull_extra="${command_args}/${pull_default_tail}" 1.186 + 1.187 + # determine which extra subrepos need to be cloned. 1.188 for i in ${subrepos_extra} ; do 1.189 if [ ! -f ${i}/.hg/hgrc ] ; then 1.190 repos_extra="${repos_extra} ${i}" 1.191 @@ -160,7 +203,7 @@ 1.192 done 1.193 else 1.194 if [ "x${pull_default}" = "x${pull_default_tail}" ] ; then 1.195 - # local source repo. Copy the extras ones that exist there. 1.196 + # local source repo. Clone the "extra" subrepos that exist there. 1.197 for i in ${subrepos_extra} ; do 1.198 if [ -f ${pull_default}/${i}/.hg/hgrc -a ! -f ${i}/.hg/hgrc ] ; then 1.199 # sub-repo there in source but not here 1.200 @@ -169,13 +212,17 @@ 1.201 done 1.202 fi 1.203 fi 1.204 - at_a_time=2 1.205 + 1.206 # Any repos to deal with? 1.207 if [ "${repos}" = "" -a "${repos_extra}" = "" ] ; then 1.208 echo "No repositories to process." > ${status_output} 1.209 exit 1.210 fi 1.211 + 1.212 + # Repos to process concurrently. Clone does better with low concurrency. 1.213 + at_a_time="${HGFOREST_CONCURRENCY:-2}" 1.214 else 1.215 + # Process command for all of the present repos 1.216 for i in . ${subrepos} ${subrepos_extra} ; do 1.217 if [ -d ${i}/.hg ] ; then 1.218 repos="${repos} ${i}" 1.219 @@ -189,6 +236,7 @@ 1.220 fi 1.221 1.222 # any of the repos locked? 1.223 + locked="" 1.224 for i in ${repos} ; do 1.225 if [ -h ${i}/.hg/store/lock -o -f ${i}/.hg/store/lock ] ; then 1.226 locked="${i} ${locked}" 1.227 @@ -198,34 +246,39 @@ 1.228 echo "ERROR: These repositories are locked: ${locked}" > ${status_output} 1.229 exit 1 1.230 fi 1.231 - at_a_time=8 1.232 + 1.233 + # Repos to process concurrently. 1.234 + at_a_time="${HGFOREST_CONCURRENCY:-8}" 1.235 fi 1.236 1.237 # Echo out what repositories we do a command on. 1.238 echo "# Repositories: ${repos} ${repos_extra}" > ${status_output} 1.239 1.240 if [ "${command}" = "serve" ] ; then 1.241 - # "serve" is run for all the repos. 1.242 + # "serve" is run for all the repos as one command. 1.243 ( 1.244 ( 1.245 + cwd=`pwd` 1.246 + serving=`basename ${cwd}` 1.247 ( 1.248 echo "[web]" 1.249 - echo "description = $(basename $(pwd))" 1.250 + echo "description = ${serving}" 1.251 echo "allow_push = *" 1.252 echo "push_ssl = False" 1.253 1.254 echo "[paths]" 1.255 - for i in ${repos} ${repos_extra} ; do 1.256 + for i in ${repos} ; do 1.257 if [ "${i}" != "." ] ; then 1.258 - echo "/$(basename $(pwd))/${i} = ${i}" 1.259 + echo "/${serving}/${i} = ${i}" 1.260 else 1.261 - echo "/$(basename $(pwd)) = $(pwd)" 1.262 + echo "/${serving} = ${cwd}" 1.263 fi 1.264 done 1.265 ) > ${tmp}/serve.web-conf 1.266 1.267 - echo "serving root repo $(basename $(pwd))" 1.268 + echo "serving root repo ${serving}" > ${status_output} 1.269 1.270 + echo "hg${global_opts} serve" > ${status_output} 1.271 (PYTHONUNBUFFERED=true hg${global_opts} serve -A ${status_output} -E ${status_output} --pid-file ${tmp}/serve.pid --web-conf ${tmp}/serve.web-conf; echo "$?" > ${tmp}/serve.pid.rc ) 2>&1 & 1.272 ) 2>&1 | sed -e "s@^@serve: @" > ${status_output} 1.273 ) & 1.274 @@ -234,81 +287,93 @@ 1.275 1.276 # n is the number of subprocess started or which might still be running. 1.277 n=0 1.278 - if [ $have_fifos = "true" ]; then 1.279 + if [ ${have_fifos} = "true" ]; then 1.280 # if we have fifos use them to detect command completion. 1.281 mkfifo ${tmp}/fifo 1.282 exec 3<>${tmp}/fifo 1.283 - if [ "${sflag}" = "true" ] ; then 1.284 - # force sequential 1.285 - at_a_time=1 1.286 - fi 1.287 fi 1.288 1.289 + # iterate over all of the subrepos. 1.290 for i in ${repos} ${repos_extra} ; do 1.291 n=`expr ${n} '+' 1` 1.292 repopidfile=`echo ${i} | sed -e 's@./@@' -e 's@/@_@g'` 1.293 reponame=`echo ${i} | sed -e :a -e 's/^.\{1,20\}$/ &/;ta'` 1.294 pull_base="${pull_default}" 1.295 - for j in $repos_extra ; do 1.296 - if [ "$i" = "$j" ] ; then 1.297 - pull_base="${pull_extra}" 1.298 + 1.299 + # regular repo or "extra" repo? 1.300 + for j in ${repos_extra} ; do 1.301 + if [ "${i}" = "${j}" ] ; then 1.302 + # it's an "extra" 1.303 + pull_base="${pull_extra}" 1.304 fi 1.305 done 1.306 + 1.307 + # remove trailing slash 1.308 pull_base="`echo ${pull_base} | sed -e 's@[/]*$@@'`" 1.309 + 1.310 + # execute the command on the subrepo 1.311 ( 1.312 ( 1.313 if [ "${command}" = "clone" -o "${command}" = "fclone" -o "${command}" = "tclone" ] ; then 1.314 - pull_newrepo="${pull_base}/${i}" 1.315 - path="`dirname ${i}`" 1.316 - if [ "${path}" != "." ] ; then 1.317 + # some form of clone 1.318 + clone_newrepo="${pull_base}/${i}" 1.319 + parent_path="`dirname ${i}`" 1.320 + if [ "${parent_path}" != "." ] ; then 1.321 times=0 1.322 - while [ ! -d "${path}" ] ## nested repo, ensure containing dir exists 1.323 - do 1.324 - times=`expr ${times} '+' 1` 1.325 + while [ ! -d "${parent_path}" ] ; do ## nested repo, ensure containing dir exists 1.326 + if [ "${sflag}" = "true" ] ; then 1.327 + # Missing parent is fatal during sequential operation. 1.328 + echo "ERROR: Missing parent path: ${parent_path}" > ${status_output} 1.329 + exit 1 1.330 + fi 1.331 + times=`expr ${times} '+' 1)` 1.332 if [ `expr ${times} '%' 10` -eq 0 ] ; then 1.333 - echo "${path} still not created, waiting..." > ${status_output} 1.334 + echo "${parent_path} still not created, waiting..." > ${status_output} 1.335 fi 1.336 sleep 5 1.337 done 1.338 fi 1.339 - echo "hg${global_opts} clone ${pull_newrepo} ${i}" > ${status_output} 1.340 - (PYTHONUNBUFFERED=true hg${global_opts} clone ${pull_newrepo} ${i}; echo "$?" > ${tmp}/${repopidfile}.pid.rc ) 2>&1 & 1.341 + # run the clone command. 1.342 + echo "hg${global_opts} clone ${clone_newrepo} ${i}" > ${status_output} 1.343 + (PYTHONUNBUFFERED=true hg${global_opts} clone ${clone_newrepo} ${i}; echo "$?" > ${tmp}/${repopidfile}.pid.rc ) 2>&1 & 1.344 else 1.345 + # run the command. 1.346 echo "cd ${i} && hg${global_opts} ${command} ${command_args}" > ${status_output} 1.347 cd ${i} && (PYTHONUNBUFFERED=true hg${global_opts} ${command} ${command_args}; echo "$?" > ${tmp}/${repopidfile}.pid.rc ) 2>&1 & 1.348 fi 1.349 1.350 echo $! > ${tmp}/${repopidfile}.pid 1.351 ) 2>&1 | sed -e "s@^@${reponame}: @" > ${status_output} 1.352 - if [ $have_fifos = "true" ]; then 1.353 - echo "${reponame}" >&3 1.354 + # tell the fifo waiter that this subprocess is done. 1.355 + if [ ${have_fifos} = "true" ]; then 1.356 + echo "${i}" >&3 1.357 fi 1.358 ) & 1.359 1.360 - if [ $have_fifos = "true" ]; then 1.361 - # check on count of running subprocesses and possibly wait for completion 1.362 - if [ ${at_a_time} -lt ${n} ] ; then 1.363 - # read will block until there are completed subprocesses 1.364 - while read repo_done; do 1.365 - n=`expr ${n} '-' 1` 1.366 - if [ ${n} -lt ${at_a_time} ] ; then 1.367 - # we should start more subprocesses 1.368 - break; 1.369 - fi 1.370 - done <&3 1.371 - fi 1.372 + if [ "${sflag}" = "true" ] ; then 1.373 + # complete this task before starting another. 1.374 + wait 1.375 else 1.376 - if [ "${sflag}" = "false" ] ; then 1.377 + if [ "${have_fifos}" = "true" ]; then 1.378 + # check on count of running subprocesses and possibly wait for completion 1.379 + if [ ${n} -ge ${at_a_time} ] ; then 1.380 + # read will block until there are completed subprocesses 1.381 + while read repo_done; do 1.382 + n=`expr ${n} '-' 1` 1.383 + if [ ${n} -lt ${at_a_time} ] ; then 1.384 + # we should start more subprocesses 1.385 + break; 1.386 + fi 1.387 + done <&3 1.388 + fi 1.389 + else 1.390 # Compare completions to starts 1.391 - completed="`(ls -1 ${tmp}/*.pid.rc 2> /dev/null | wc -l) || echo 0`" 1.392 - while [ ${at_a_time} -lt `expr ${n} '-' ${completed}` ] ; do 1.393 + completed="`(ls -a1 ${tmp}/*.pid.rc 2> /dev/null | wc -l) || echo 0`" 1.394 + while [ `expr ${n} '-' ${completed}` -ge ${at_a_time} ] ; do 1.395 # sleep a short time to give time for something to complete 1.396 sleep 1 1.397 - completed="`(ls -1 ${tmp}/*.pid.rc 2> /dev/null | wc -l) || echo 0`" 1.398 + completed="`(ls -a1 ${tmp}/*.pid.rc 2> /dev/null | wc -l) || echo 0`" 1.399 done 1.400 - else 1.401 - # complete this task before starting another. 1.402 - wait 1.403 fi 1.404 fi 1.405 done 1.406 @@ -320,11 +385,12 @@ 1.407 # Terminate with exit 0 only if all subprocesses were successful 1.408 ec=0 1.409 if [ -d ${tmp} ]; then 1.410 - for rc in ${tmp}/*.pid.rc ; do 1.411 + rcfiles="`(ls -a ${tmp}/*.pid.rc 2> /dev/null) || echo ''`" 1.412 + for rc in ${rcfiles} ; do 1.413 exit_code=`cat ${rc} | tr -d ' \n\r'` 1.414 if [ "${exit_code}" != "0" ] ; then 1.415 - repo="`echo ${rc} | sed -e s@^${tmp}@@ -e 's@/*\([^/]*\)\.pid\.rc$@\1@' -e 's@_@/@g'`" 1.416 - echo "WARNING: ${repo} exited abnormally ($exit_code)" > ${status_output} 1.417 + repo="`echo ${rc} | sed -e 's@^'${tmp}'@@' -e 's@/*\([^/]*\)\.pid\.rc$@\1@' -e 's@_@/@g'`" 1.418 + echo "WARNING: ${repo} exited abnormally (${exit_code})" > ${status_output} 1.419 ec=1 1.420 fi 1.421 done