Tue, 06 May 2014 13:57:30 -0700
8030681: add "serve" command and --quiet and --verbose options to hgforest
Reviewed-by: ihse
mduigou@1141 | 1 | #!/bin/bash |
ohrstrom@538 | 2 | |
ohrstrom@538 | 3 | # |
mduigou@845 | 4 | # Copyright (c) 2009, 2013, Oracle and/or its affiliates. All rights reserved. |
ohrstrom@538 | 5 | # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
ohrstrom@538 | 6 | # |
ohrstrom@538 | 7 | # This code is free software; you can redistribute it and/or modify it |
ohrstrom@538 | 8 | # under the terms of the GNU General Public License version 2 only, as |
ohrstrom@538 | 9 | # published by the Free Software Foundation. |
ohrstrom@538 | 10 | # |
ohrstrom@538 | 11 | # This code is distributed in the hope that it will be useful, but WITHOUT |
ohrstrom@538 | 12 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
ohrstrom@538 | 13 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
ohrstrom@538 | 14 | # version 2 for more details (a copy is included in the LICENSE file that |
ohrstrom@538 | 15 | # accompanied this code). |
ohrstrom@538 | 16 | # |
ohrstrom@538 | 17 | # You should have received a copy of the GNU General Public License version |
ohrstrom@538 | 18 | # 2 along with this work; if not, write to the Free Software Foundation, |
ohrstrom@538 | 19 | # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
ohrstrom@538 | 20 | # |
ohrstrom@538 | 21 | # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
ohrstrom@538 | 22 | # or visit www.oracle.com if you need additional information or have any |
ohrstrom@538 | 23 | # questions. |
ohrstrom@538 | 24 | # |
ohrstrom@538 | 25 | |
ohrstrom@538 | 26 | # Shell script for a fast parallel forest command |
ohrstrom@538 | 27 | |
mduigou@1141 | 28 | global_opts="" |
mduigou@1141 | 29 | status_output="/dev/stdout" |
mduigou@1141 | 30 | qflag="false" |
mduigou@1141 | 31 | vflag="false" |
mduigou@1141 | 32 | while [ $# -gt 0 ] |
mduigou@1141 | 33 | do |
mduigou@1141 | 34 | case $1 in |
mduigou@1141 | 35 | -q | --quiet ) |
mduigou@1141 | 36 | qflag="true" |
mduigou@1141 | 37 | global_opts="${global_opts} -q" |
mduigou@1141 | 38 | status_output="/dev/null" |
mduigou@1141 | 39 | ;; |
mduigou@1141 | 40 | |
mduigou@1141 | 41 | -v | --verbose ) |
mduigou@1141 | 42 | vflag="true" |
mduigou@1141 | 43 | global_opts="${global_opts} -v" |
mduigou@1141 | 44 | ;; |
mduigou@1141 | 45 | |
mduigou@1141 | 46 | '--' ) # no more options |
mduigou@1141 | 47 | shift; break |
mduigou@1141 | 48 | ;; |
mduigou@1141 | 49 | |
mduigou@1141 | 50 | -*) # bad option |
mduigou@1141 | 51 | usage |
mduigou@1141 | 52 | ;; |
mduigou@1141 | 53 | |
mduigou@1141 | 54 | * ) # non option |
mduigou@1141 | 55 | break |
mduigou@1141 | 56 | ;; |
mduigou@1141 | 57 | esac |
mduigou@1141 | 58 | shift |
mduigou@1141 | 59 | done |
mduigou@1141 | 60 | |
mduigou@1141 | 61 | |
mduigou@1141 | 62 | command="$1"; shift |
mduigou@1141 | 63 | repo_base="$@" |
mduigou@1141 | 64 | |
mduigou@1141 | 65 | usage() { |
mduigou@1141 | 66 | echo "usage: $0 [-q|--quiet] [-v|--verbose] [--] <command> [repo_base_path]" > ${status_output} |
mduigou@1141 | 67 | exit 1 |
mduigou@1141 | 68 | } |
mduigou@1141 | 69 | |
mduigou@1141 | 70 | |
mduigou@1141 | 71 | if [ "x" = "x$command" ] ; then |
mduigou@1141 | 72 | echo "ERROR: No command to hg supplied!" |
mduigou@1141 | 73 | usage |
ohrstrom@538 | 74 | fi |
ohrstrom@538 | 75 | |
ohrstrom@538 | 76 | # Clean out the temporary directory that stores the pid files. |
ohrstrom@538 | 77 | tmp=/tmp/forest.$$ |
ohrstrom@538 | 78 | rm -f -r ${tmp} |
ohrstrom@538 | 79 | mkdir -p ${tmp} |
ohrstrom@538 | 80 | |
ohrstrom@538 | 81 | safe_interrupt () { |
chegar@615 | 82 | if [ -d ${tmp} ]; then |
chegar@615 | 83 | if [ "`ls ${tmp}/*.pid`" != "" ]; then |
mduigou@1141 | 84 | echo "Waiting for processes ( `cat ${tmp}/*.pid | tr '\n' ' '`) to terminate nicely!" > ${status_output} |
ohrstrom@538 | 85 | sleep 1 |
ohrstrom@538 | 86 | # Pipe stderr to dev/null to silence kill, that complains when trying to kill |
ohrstrom@538 | 87 | # a subprocess that has already exited. |
chegar@615 | 88 | kill -TERM `cat ${tmp}/*.pid | tr '\n' ' '` 2> /dev/null |
chegar@615 | 89 | wait |
mduigou@1141 | 90 | echo "Interrupt complete!" > ${status_output} |
chegar@615 | 91 | fi |
mduigou@1141 | 92 | rm -f -r ${tmp} |
ohrstrom@538 | 93 | fi |
mduigou@1141 | 94 | exit 130 |
ohrstrom@538 | 95 | } |
ohrstrom@538 | 96 | |
ohrstrom@538 | 97 | nice_exit () { |
chegar@615 | 98 | if [ -d ${tmp} ]; then |
chegar@615 | 99 | if [ "`ls ${tmp}`" != "" ]; then |
chegar@615 | 100 | wait |
chegar@615 | 101 | fi |
mduigou@1141 | 102 | rm -f -r ${tmp} |
ohrstrom@538 | 103 | fi |
ohrstrom@538 | 104 | } |
ohrstrom@538 | 105 | |
ohrstrom@538 | 106 | trap 'safe_interrupt' INT QUIT |
ohrstrom@538 | 107 | trap 'nice_exit' EXIT |
chegar@615 | 108 | |
mduigou@1141 | 109 | subrepos="corba jaxp jaxws langtools jdk hotspot nashorn" |
mduigou@1141 | 110 | subrepos_extra="jdk/src/closed jdk/make/closed jdk/test/closed hotspot/make/closed hotspot/src/closed hotspot/test/closed deploy install sponsors pubs" |
mduigou@1141 | 111 | |
ohrstrom@538 | 112 | # Only look in specific locations for possible forests (avoids long searches) |
ohrstrom@538 | 113 | pull_default="" |
ohrstrom@538 | 114 | repos="" |
ohrstrom@538 | 115 | repos_extra="" |
ohrstrom@538 | 116 | if [ "${command}" = "clone" -o "${command}" = "fclone" ] ; then |
ohrstrom@538 | 117 | if [ -f .hg/hgrc ] ; then |
ohrstrom@538 | 118 | pull_default=`hg paths default` |
ohrstrom@538 | 119 | if [ "${pull_default}" = "" ] ; then |
mduigou@1141 | 120 | echo "ERROR: Need initial clone with 'hg paths default' defined" > ${status_output} |
ohrstrom@538 | 121 | exit 1 |
ohrstrom@538 | 122 | fi |
ohrstrom@538 | 123 | fi |
ohrstrom@538 | 124 | if [ "${pull_default}" = "" ] ; then |
mduigou@1141 | 125 | echo "ERROR: Need initial repository to use this script" > ${status_output} |
ohrstrom@538 | 126 | exit 1 |
ohrstrom@538 | 127 | fi |
ohrstrom@538 | 128 | for i in ${subrepos} ; do |
ohrstrom@538 | 129 | if [ ! -f ${i}/.hg/hgrc ] ; then |
ohrstrom@538 | 130 | repos="${repos} ${i}" |
ohrstrom@538 | 131 | fi |
ohrstrom@538 | 132 | done |
mduigou@1141 | 133 | if [ "${repo_base}" != "" ] ; then |
ohrstrom@538 | 134 | pull_default_tail=`echo ${pull_default} | sed -e 's@^.*://[^/]*/\(.*\)@\1@'` |
mduigou@1141 | 135 | pull_extra="${repo_base}/${pull_default_tail}" |
ohrstrom@538 | 136 | for i in ${subrepos_extra} ; do |
ohrstrom@538 | 137 | if [ ! -f ${i}/.hg/hgrc ] ; then |
ohrstrom@538 | 138 | repos_extra="${repos_extra} ${i}" |
ohrstrom@538 | 139 | fi |
ohrstrom@538 | 140 | done |
ohrstrom@538 | 141 | fi |
ohrstrom@538 | 142 | at_a_time=2 |
ohrstrom@538 | 143 | # Any repos to deal with? |
ohrstrom@538 | 144 | if [ "${repos}" = "" -a "${repos_extra}" = "" ] ; then |
mduigou@1141 | 145 | echo "No repositories to process." > ${status_output} |
ohrstrom@538 | 146 | exit |
ohrstrom@538 | 147 | fi |
ohrstrom@538 | 148 | else |
mduigou@1141 | 149 | for i in . ${subrepos} ${subrepos_extra} ; do |
mduigou@1141 | 150 | if [ -d ${i}/.hg ] ; then |
mduigou@1141 | 151 | repos="${repos} ${i}" |
mduigou@1141 | 152 | fi |
ohrstrom@538 | 153 | done |
mduigou@1141 | 154 | |
mduigou@1141 | 155 | # Any repos to deal with? |
mduigou@1141 | 156 | if [ "${repos}" = "" ] ; then |
mduigou@1141 | 157 | echo "No repositories to process." > ${status_output} |
mduigou@1141 | 158 | exit |
mduigou@1141 | 159 | fi |
mduigou@1141 | 160 | |
mduigou@1141 | 161 | # any of the repos locked? |
ohrstrom@538 | 162 | for i in ${repos} ; do |
ohrstrom@538 | 163 | if [ -h ${i}/.hg/store/lock -o -f ${i}/.hg/store/lock ] ; then |
ohrstrom@538 | 164 | locked="${i} ${locked}" |
ohrstrom@538 | 165 | fi |
ohrstrom@538 | 166 | done |
mduigou@1141 | 167 | if [ "${locked}" != "" ] ; then |
mduigou@1141 | 168 | echo "ERROR: These repositories are locked: ${locked}" > ${status_output} |
mduigou@1141 | 169 | exit 1 |
mduigou@1141 | 170 | fi |
ohrstrom@538 | 171 | at_a_time=8 |
ohrstrom@538 | 172 | fi |
ohrstrom@538 | 173 | |
ohrstrom@538 | 174 | # Echo out what repositories we do a command on. |
mduigou@1141 | 175 | echo "# Repositories: ${repos} ${repos_extra}" > ${status_output} |
ohrstrom@538 | 176 | |
mduigou@1141 | 177 | if [ "${command}" = "serve" ] ; then |
mduigou@1141 | 178 | # "serve" is run for all the repos. |
mduigou@1141 | 179 | ( |
mduigou@1141 | 180 | ( |
mduigou@1141 | 181 | ( |
mduigou@1141 | 182 | echo "[web]" |
mduigou@1141 | 183 | echo "description = $(basename $(pwd))" |
mduigou@1141 | 184 | echo "allow_push = *" |
mduigou@1141 | 185 | echo "push_ssl = False" |
mduigou@1141 | 186 | |
mduigou@1141 | 187 | echo "[paths]" |
mduigou@1141 | 188 | for i in ${repos} ${repos_extra} ; do |
mduigou@1141 | 189 | if [ "${i}" != "." ] ; then |
mduigou@1141 | 190 | echo "/$(basename $(pwd))/${i} = ${i}" |
mduigou@1141 | 191 | else |
mduigou@1141 | 192 | echo "/$(basename $(pwd)) = $(pwd)" |
mduigou@1141 | 193 | fi |
mduigou@1141 | 194 | done |
mduigou@1141 | 195 | ) > ${tmp}/serve.web-conf |
mduigou@1141 | 196 | |
mduigou@1141 | 197 | echo "serving root repo $(basename $(pwd))" |
mduigou@1141 | 198 | |
mduigou@1141 | 199 | (PYTHONUNBUFFERED=true hg${global_opts} serve -A ${status_output} -E ${status_output} --pid-file ${tmp}/serve.pid --web-conf ${tmp}/serve.web-conf; echo "$?" > ${tmp}/serve.pid.rc ) 2>&1 & |
mduigou@1141 | 200 | ) 2>&1 | sed -e "s@^@serve: @" > ${status_output} |
mduigou@1141 | 201 | ) & |
mduigou@1141 | 202 | else |
mduigou@1141 | 203 | # Run the supplied command on all repos in parallel. |
mduigou@1141 | 204 | n=0 |
mduigou@1141 | 205 | for i in ${repos} ${repos_extra} ; do |
mduigou@1141 | 206 | n=`expr ${n} '+' 1` |
mduigou@1141 | 207 | repopidfile=`echo ${i} | sed -e 's@./@@' -e 's@/@_@g'` |
mduigou@1141 | 208 | reponame=`echo ${i} | sed -e :a -e 's/^.\{1,20\}$/ &/;ta'` |
mduigou@1141 | 209 | pull_base="${pull_default}" |
mduigou@1141 | 210 | for j in $repos_extra ; do |
ohrstrom@538 | 211 | if [ "$i" = "$j" ] ; then |
ohrstrom@538 | 212 | pull_base="${pull_extra}" |
ohrstrom@538 | 213 | fi |
mduigou@1141 | 214 | done |
mduigou@1141 | 215 | ( |
mduigou@1141 | 216 | ( |
mduigou@1141 | 217 | if [ "${command}" = "clone" -o "${command}" = "fclone" ] ; then |
mduigou@1141 | 218 | pull_newrepo="`echo ${pull_base}/${i} | sed -e 's@\([^:]/\)//*@\1@g'`" |
mduigou@1141 | 219 | echo "hg clone ${pull_newrepo} ${i}" > ${status_output} |
mduigou@1141 | 220 | path="`dirname ${i}`" |
mduigou@1141 | 221 | if [ "${path}" != "." ] ; then |
mduigou@1141 | 222 | times=0 |
mduigou@1141 | 223 | while [ ! -d "${path}" ] ## nested repo, ensure containing dir exists |
mduigou@1141 | 224 | do |
mduigou@1141 | 225 | times=`expr ${times} '+' 1` |
mduigou@1141 | 226 | if [ `expr ${times} '%' 10` -eq 0 ] ; then |
mduigou@1141 | 227 | echo "${path} still not created, waiting..." > ${status_output} |
mduigou@1141 | 228 | fi |
mduigou@1141 | 229 | sleep 5 |
mduigou@1141 | 230 | done |
mduigou@1141 | 231 | fi |
mduigou@1141 | 232 | (PYTHONUNBUFFERED=true hg${global_opts} clone ${pull_newrepo} ${i}; echo "$?" > ${tmp}/${repopidfile}.pid.rc ) 2>&1 & |
mduigou@1141 | 233 | else |
mduigou@1141 | 234 | echo "cd ${i} && hg${global_opts} ${command} ${repo_base}" > ${status_output} |
mduigou@1141 | 235 | cd ${i} && (PYTHONUNBUFFERED=true hg${global_opts} ${command} ${command_repo}; echo "$?" > ${tmp}/${repopidfile}.pid.rc ) 2>&1 & |
mduigou@1141 | 236 | fi |
mduigou@1141 | 237 | |
mduigou@1141 | 238 | echo $! > ${tmp}/${repopidfile}.pid |
mduigou@1141 | 239 | ) 2>&1 | sed -e "s@^@${reponame}: @" > ${status_output} |
mduigou@1141 | 240 | ) & |
mduigou@1141 | 241 | |
mduigou@1141 | 242 | if [ `expr ${n} '%' ${at_a_time}` -eq 0 ] ; then |
mduigou@1141 | 243 | sleep 2 |
mduigou@1141 | 244 | echo "Waiting 5 secs before spawning next background command." > ${status_output} |
mduigou@1141 | 245 | sleep 3 |
mduigou@1141 | 246 | fi |
ohrstrom@538 | 247 | done |
mduigou@1141 | 248 | fi |
chegar@615 | 249 | |
ohrstrom@538 | 250 | # Wait for all hg commands to complete |
ohrstrom@538 | 251 | wait |
ohrstrom@538 | 252 | |
chegar@615 | 253 | # Terminate with exit 0 only if all subprocesses were successful |
chegar@615 | 254 | ec=0 |
chegar@615 | 255 | if [ -d ${tmp} ]; then |
chegar@615 | 256 | for rc in ${tmp}/*.pid.rc ; do |
chegar@615 | 257 | exit_code=`cat ${rc} | tr -d ' \n\r'` |
chegar@615 | 258 | if [ "${exit_code}" != "0" ] ; then |
mduigou@1141 | 259 | repo="`echo ${rc} | sed -e s@^${tmp}@@ -e 's@/*\([^/]*\)\.pid\.rc$@\1@' -e 's@_@/@g'`" |
mduigou@1141 | 260 | echo "WARNING: ${repo} exited abnormally." > ${status_output} |
chegar@615 | 261 | ec=1 |
chegar@615 | 262 | fi |
chegar@615 | 263 | done |
chegar@615 | 264 | fi |
chegar@615 | 265 | exit ${ec} |