21 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
20 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
22 # or visit www.oracle.com if you need additional information or have any |
21 # or visit www.oracle.com if you need additional information or have any |
23 # questions. |
22 # questions. |
24 # |
23 # |
25 |
24 |
26 # Shell script for a fast parallel forest command |
25 # Shell script for a fast parallel forest/trees command |
27 |
26 |
28 global_opts="" |
27 usage() { |
29 status_output="/dev/stdout" |
28 echo "usage: $0 [-h|--help] [-q|--quiet] [-v|--verbose] [-s|--sequential] [--] <command> [commands...]" > ${status_output} |
30 qflag="false" |
29 echo "Environment variables which modify behaviour:" |
31 vflag="false" |
30 echo " HGFOREST_QUIET : (boolean) If 'true' then standard output is redirected to /dev/null" |
32 sflag="false" |
31 echo " HGFOREST_VERBOSE : (boolean) If 'true' then Mercurial asked to produce verbose output" |
|
32 echo " HGFOREST_SEQUENTIAL : (boolean) If 'true' then repos are processed sequentially. Disables concurrency" |
|
33 echo " HGFOREST_GLOBALOPTS : (string, must begin with space) Additional Mercurial global options" |
|
34 echo " HGFOREST_REDIRECT : (file path) Redirect standard output to specified file" |
|
35 echo " HGFOREST_FIFOS : (boolean) Default behaviour for FIFO detection. Does not override FIFOs disabled" |
|
36 echo " HGFOREST_CONCURRENCY: (positive integer) Number of repos to process concurrently" |
|
37 echo " HGFOREST_DEBUG : (boolean) If 'true' then temp files are retained" |
|
38 exit 1 |
|
39 } |
|
40 |
|
41 global_opts="${HGFOREST_GLOBALOPTS:-}" |
|
42 status_output="${HGFOREST_REDIRECT:-/dev/stdout}" |
|
43 qflag="${HGFOREST_QUIET:-false}" |
|
44 vflag="${HGFOREST_VERBOSE:-false}" |
|
45 sflag="${HGFOREST_SEQUENTIAL:-false}" |
33 while [ $# -gt 0 ] |
46 while [ $# -gt 0 ] |
34 do |
47 do |
35 case $1 in |
48 case $1 in |
|
49 -h | --help ) |
|
50 usage |
|
51 ;; |
|
52 |
36 -q | --quiet ) |
53 -q | --quiet ) |
37 qflag="true" |
54 qflag="true" |
38 global_opts="${global_opts} -q" |
|
39 status_output="/dev/null" |
|
40 ;; |
55 ;; |
41 |
56 |
42 -v | --verbose ) |
57 -v | --verbose ) |
43 vflag="true" |
58 vflag="true" |
44 global_opts="${global_opts} -v" |
|
45 ;; |
59 ;; |
46 |
60 |
47 -s | --sequential ) |
61 -s | --sequential ) |
48 sflag="true" |
62 sflag="true" |
49 ;; |
63 ;; |
61 ;; |
75 ;; |
62 esac |
76 esac |
63 shift |
77 shift |
64 done |
78 done |
65 |
79 |
66 |
80 # silence standard output? |
67 command="$1"; shift |
81 if [ ${qflag} = "true" ] ; then |
68 command_args="$@" |
82 global_opts="${global_opts} -q" |
69 |
83 status_output="/dev/null" |
70 usage() { |
84 fi |
71 echo "usage: $0 [-q|--quiet] [-v|--verbose] [-s|--sequential] [--] <command> [commands...]" > ${status_output} |
85 |
72 exit 1 |
86 # verbose output? |
73 } |
87 if [ ${vflag} = "true" ] ; then |
74 |
88 global_opts="${global_opts} -v" |
75 if [ "x" = "x$command" ] ; then |
89 fi |
|
90 |
|
91 # Make sure we have a command. |
|
92 if [ $# -lt 1 -o -z "${1:-}" ] ; then |
76 echo "ERROR: No command to hg supplied!" |
93 echo "ERROR: No command to hg supplied!" |
77 usage |
94 usage |
78 fi |
95 fi |
79 |
96 |
80 # Check if we can use fifos for monitoring sub-process completion. |
97 command="$1"; shift |
81 on_windows=`uname -s | egrep -ic -e 'cygwin|msys'` |
98 command_args="${@:-}" |
82 if [ ${on_windows} = "1" ]; then |
|
83 # cygwin has (2014-04-18) broken (single writer only) FIFOs |
|
84 # msys has (2014-04-18) no FIFOs. |
|
85 have_fifos="false" |
|
86 else |
|
87 have_fifos="true" |
|
88 fi |
|
89 |
99 |
90 # Clean out the temporary directory that stores the pid files. |
100 # Clean out the temporary directory that stores the pid files. |
91 tmp=/tmp/forest.$$ |
101 tmp=/tmp/forest.$$ |
92 rm -f -r ${tmp} |
102 rm -f -r ${tmp} |
93 mkdir -p ${tmp} |
103 mkdir -p ${tmp} |
94 |
104 |
|
105 |
|
106 if [ "${HGFOREST_DEBUG:-false}" = "true" ] ; then |
|
107 echo "DEBUG: temp files are in: ${tmp}" |
|
108 fi |
|
109 |
|
110 # Check if we can use fifos for monitoring sub-process completion. |
|
111 echo "1" > ${tmp}/read |
|
112 while_subshell=1 |
|
113 while read line; do |
|
114 while_subshell=0 |
|
115 break; |
|
116 done < ${tmp}/read |
|
117 rm ${tmp}/read |
|
118 |
|
119 on_windows=`uname -s | egrep -ic -e 'cygwin|msys'` |
|
120 |
|
121 if [ ${while_subshell} = "1" -o ${on_windows} = "1" ]; then |
|
122 # cygwin has (2014-04-18) broken (single writer only) FIFOs |
|
123 # msys has (2014-04-18) no FIFOs. |
|
124 # older shells create a sub-shell for redirect to while |
|
125 have_fifos="false" |
|
126 else |
|
127 have_fifos="${HGFOREST_FIFOS:-true}" |
|
128 fi |
|
129 |
95 safe_interrupt () { |
130 safe_interrupt () { |
96 if [ -d ${tmp} ]; then |
131 if [ -d ${tmp} ]; then |
97 if [ "`ls ${tmp}/*.pid`" != "" ]; then |
132 if [ "`ls ${tmp}/*.pid`" != "" ]; then |
98 echo "Waiting for processes ( `cat ${tmp}/*.pid | tr '\n' ' '`) to terminate nicely!" > ${status_output} |
133 echo "Waiting for processes ( `cat ${tmp}/.*.pid ${tmp}/*.pid 2> /dev/null | tr '\n' ' '`) to terminate nicely!" > ${status_output} |
99 sleep 1 |
134 sleep 1 |
100 # Pipe stderr to dev/null to silence kill, that complains when trying to kill |
135 # Pipe stderr to dev/null to silence kill, that complains when trying to kill |
101 # a subprocess that has already exited. |
136 # a subprocess that has already exited. |
102 kill -TERM `cat ${tmp}/*.pid | tr '\n' ' '` 2> /dev/null |
137 kill -TERM `cat ${tmp}/*.pid | tr '\n' ' '` 2> /dev/null |
103 wait |
138 wait |
126 # Only look in specific locations for possible forests (avoids long searches) |
163 # Only look in specific locations for possible forests (avoids long searches) |
127 pull_default="" |
164 pull_default="" |
128 repos="" |
165 repos="" |
129 repos_extra="" |
166 repos_extra="" |
130 if [ "${command}" = "clone" -o "${command}" = "fclone" -o "${command}" = "tclone" ] ; then |
167 if [ "${command}" = "clone" -o "${command}" = "fclone" -o "${command}" = "tclone" ] ; then |
|
168 # we must be a clone |
131 if [ ! -f .hg/hgrc ] ; then |
169 if [ ! -f .hg/hgrc ] ; then |
132 echo "ERROR: Need initial repository to use this script" > ${status_output} |
170 echo "ERROR: Need initial repository to use this script" > ${status_output} |
133 exit 1 |
171 exit 1 |
134 fi |
172 fi |
135 |
173 |
|
174 # the clone must know where it came from (have a default pull path). |
136 pull_default=`hg paths default` |
175 pull_default=`hg paths default` |
137 if [ "${pull_default}" = "" ] ; then |
176 if [ "${pull_default}" = "" ] ; then |
138 echo "ERROR: Need initial clone with 'hg paths default' defined" > ${status_output} |
177 echo "ERROR: Need initial clone with 'hg paths default' defined" > ${status_output} |
139 exit 1 |
178 exit 1 |
140 fi |
179 fi |
141 |
180 |
|
181 # determine which sub repos need to be cloned. |
142 for i in ${subrepos} ; do |
182 for i in ${subrepos} ; do |
143 if [ ! -f ${i}/.hg/hgrc ] ; then |
183 if [ ! -f ${i}/.hg/hgrc ] ; then |
144 repos="${repos} ${i}" |
184 repos="${repos} ${i}" |
145 fi |
185 fi |
146 done |
186 done |
147 |
187 |
148 pull_default_tail=`echo ${pull_default} | sed -e 's@^.*://[^/]*/\(.*\)@\1@'` |
188 pull_default_tail=`echo ${pull_default} | sed -e 's@^.*://[^/]*/\(.*\)@\1@'` |
149 |
189 |
150 if [ "${command_args}" != "" ] ; then |
190 if [ -n "${command_args}" ] ; then |
|
191 # if there is an "extra sources" path then reparent "extra" repos to that path |
151 if [ "x${pull_default}" = "x${pull_default_tail}" ] ; then |
192 if [ "x${pull_default}" = "x${pull_default_tail}" ] ; then |
152 echo "ERROR: Need initial clone from non-local source" > ${status_output} |
193 echo "ERROR: Need initial clone from non-local source" > ${status_output} |
153 exit 1 |
194 exit 1 |
154 fi |
195 fi |
155 pull_extra="${command_args}/${pull_default_tail}" |
196 pull_extra="${command_args}/${pull_default_tail}" |
|
197 |
|
198 # determine which extra subrepos need to be cloned. |
156 for i in ${subrepos_extra} ; do |
199 for i in ${subrepos_extra} ; do |
157 if [ ! -f ${i}/.hg/hgrc ] ; then |
200 if [ ! -f ${i}/.hg/hgrc ] ; then |
158 repos_extra="${repos_extra} ${i}" |
201 repos_extra="${repos_extra} ${i}" |
159 fi |
202 fi |
160 done |
203 done |
161 else |
204 else |
162 if [ "x${pull_default}" = "x${pull_default_tail}" ] ; then |
205 if [ "x${pull_default}" = "x${pull_default_tail}" ] ; then |
163 # local source repo. Copy the extras ones that exist there. |
206 # local source repo. Clone the "extra" subrepos that exist there. |
164 for i in ${subrepos_extra} ; do |
207 for i in ${subrepos_extra} ; do |
165 if [ -f ${pull_default}/${i}/.hg/hgrc -a ! -f ${i}/.hg/hgrc ] ; then |
208 if [ -f ${pull_default}/${i}/.hg/hgrc -a ! -f ${i}/.hg/hgrc ] ; then |
166 # sub-repo there in source but not here |
209 # sub-repo there in source but not here |
167 repos_extra="${repos_extra} ${i}" |
210 repos_extra="${repos_extra} ${i}" |
168 fi |
211 fi |
169 done |
212 done |
170 fi |
213 fi |
171 fi |
214 fi |
172 at_a_time=2 |
215 |
173 # Any repos to deal with? |
216 # Any repos to deal with? |
174 if [ "${repos}" = "" -a "${repos_extra}" = "" ] ; then |
217 if [ "${repos}" = "" -a "${repos_extra}" = "" ] ; then |
175 echo "No repositories to process." > ${status_output} |
218 echo "No repositories to process." > ${status_output} |
176 exit |
219 exit |
177 fi |
220 fi |
|
221 |
|
222 # Repos to process concurrently. Clone does better with low concurrency. |
|
223 at_a_time="${HGFOREST_CONCURRENCY:-2}" |
178 else |
224 else |
|
225 # Process command for all of the present repos |
179 for i in . ${subrepos} ${subrepos_extra} ; do |
226 for i in . ${subrepos} ${subrepos_extra} ; do |
180 if [ -d ${i}/.hg ] ; then |
227 if [ -d ${i}/.hg ] ; then |
181 repos="${repos} ${i}" |
228 repos="${repos} ${i}" |
182 fi |
229 fi |
183 done |
230 done |
187 echo "No repositories to process." > ${status_output} |
234 echo "No repositories to process." > ${status_output} |
188 exit |
235 exit |
189 fi |
236 fi |
190 |
237 |
191 # any of the repos locked? |
238 # any of the repos locked? |
|
239 locked="" |
192 for i in ${repos} ; do |
240 for i in ${repos} ; do |
193 if [ -h ${i}/.hg/store/lock -o -f ${i}/.hg/store/lock ] ; then |
241 if [ -h ${i}/.hg/store/lock -o -f ${i}/.hg/store/lock ] ; then |
194 locked="${i} ${locked}" |
242 locked="${i} ${locked}" |
195 fi |
243 fi |
196 done |
244 done |
197 if [ "${locked}" != "" ] ; then |
245 if [ "${locked}" != "" ] ; then |
198 echo "ERROR: These repositories are locked: ${locked}" > ${status_output} |
246 echo "ERROR: These repositories are locked: ${locked}" > ${status_output} |
199 exit 1 |
247 exit 1 |
200 fi |
248 fi |
201 at_a_time=8 |
249 |
|
250 # Repos to process concurrently. |
|
251 at_a_time="${HGFOREST_CONCURRENCY:-8}" |
202 fi |
252 fi |
203 |
253 |
204 # Echo out what repositories we do a command on. |
254 # Echo out what repositories we do a command on. |
205 echo "# Repositories: ${repos} ${repos_extra}" > ${status_output} |
255 echo "# Repositories: ${repos} ${repos_extra}" > ${status_output} |
206 |
256 |
207 if [ "${command}" = "serve" ] ; then |
257 if [ "${command}" = "serve" ] ; then |
208 # "serve" is run for all the repos. |
258 # "serve" is run for all the repos as one command. |
209 ( |
259 ( |
210 ( |
260 ( |
|
261 cwd=`pwd` |
|
262 serving=`basename ${cwd}` |
211 ( |
263 ( |
212 echo "[web]" |
264 echo "[web]" |
213 echo "description = $(basename $(pwd))" |
265 echo "description = ${serving}" |
214 echo "allow_push = *" |
266 echo "allow_push = *" |
215 echo "push_ssl = False" |
267 echo "push_ssl = False" |
216 |
268 |
217 echo "[paths]" |
269 echo "[paths]" |
218 for i in ${repos} ${repos_extra} ; do |
270 for i in ${repos} ; do |
219 if [ "${i}" != "." ] ; then |
271 if [ "${i}" != "." ] ; then |
220 echo "/$(basename $(pwd))/${i} = ${i}" |
272 echo "/${serving}/${i} = ${i}" |
221 else |
273 else |
222 echo "/$(basename $(pwd)) = $(pwd)" |
274 echo "/${serving} = ${cwd}" |
223 fi |
275 fi |
224 done |
276 done |
225 ) > ${tmp}/serve.web-conf |
277 ) > ${tmp}/serve.web-conf |
226 |
278 |
227 echo "serving root repo $(basename $(pwd))" |
279 echo "serving root repo ${serving}" > ${status_output} |
228 |
280 |
|
281 echo "hg${global_opts} serve" > ${status_output} |
229 (PYTHONUNBUFFERED=true hg${global_opts} serve -A ${status_output} -E ${status_output} --pid-file ${tmp}/serve.pid --web-conf ${tmp}/serve.web-conf; echo "$?" > ${tmp}/serve.pid.rc ) 2>&1 & |
282 (PYTHONUNBUFFERED=true hg${global_opts} serve -A ${status_output} -E ${status_output} --pid-file ${tmp}/serve.pid --web-conf ${tmp}/serve.web-conf; echo "$?" > ${tmp}/serve.pid.rc ) 2>&1 & |
230 ) 2>&1 | sed -e "s@^@serve: @" > ${status_output} |
283 ) 2>&1 | sed -e "s@^@serve: @" > ${status_output} |
231 ) & |
284 ) & |
232 else |
285 else |
233 # Run the supplied command on all repos in parallel. |
286 # Run the supplied command on all repos in parallel. |
234 |
287 |
235 # n is the number of subprocess started or which might still be running. |
288 # n is the number of subprocess started or which might still be running. |
236 n=0 |
289 n=0 |
237 if [ $have_fifos = "true" ]; then |
290 if [ ${have_fifos} = "true" ]; then |
238 # if we have fifos use them to detect command completion. |
291 # if we have fifos use them to detect command completion. |
239 mkfifo ${tmp}/fifo |
292 mkfifo ${tmp}/fifo |
240 exec 3<>${tmp}/fifo |
293 exec 3<>${tmp}/fifo |
241 if [ "${sflag}" = "true" ] ; then |
294 fi |
242 # force sequential |
295 |
243 at_a_time=1 |
296 # iterate over all of the subrepos. |
244 fi |
|
245 fi |
|
246 |
|
247 for i in ${repos} ${repos_extra} ; do |
297 for i in ${repos} ${repos_extra} ; do |
248 n=`expr ${n} '+' 1` |
298 n=`expr ${n} '+' 1` |
249 repopidfile=`echo ${i} | sed -e 's@./@@' -e 's@/@_@g'` |
299 repopidfile=`echo ${i} | sed -e 's@./@@' -e 's@/@_@g'` |
250 reponame=`echo ${i} | sed -e :a -e 's/^.\{1,20\}$/ &/;ta'` |
300 reponame=`echo ${i} | sed -e :a -e 's/^.\{1,20\}$/ &/;ta'` |
251 pull_base="${pull_default}" |
301 pull_base="${pull_default}" |
252 for j in $repos_extra ; do |
302 |
253 if [ "$i" = "$j" ] ; then |
303 # regular repo or "extra" repo? |
254 pull_base="${pull_extra}" |
304 for j in ${repos_extra} ; do |
|
305 if [ "${i}" = "${j}" ] ; then |
|
306 # it's an "extra" |
|
307 pull_base="${pull_extra}" |
255 fi |
308 fi |
256 done |
309 done |
|
310 |
|
311 # remove trailing slash |
257 pull_base="`echo ${pull_base} | sed -e 's@[/]*$@@'`" |
312 pull_base="`echo ${pull_base} | sed -e 's@[/]*$@@'`" |
|
313 |
|
314 # execute the command on the subrepo |
258 ( |
315 ( |
259 ( |
316 ( |
260 if [ "${command}" = "clone" -o "${command}" = "fclone" -o "${command}" = "tclone" ] ; then |
317 if [ "${command}" = "clone" -o "${command}" = "fclone" -o "${command}" = "tclone" ] ; then |
261 pull_newrepo="${pull_base}/${i}" |
318 # some form of clone |
262 path="`dirname ${i}`" |
319 clone_newrepo="${pull_base}/${i}" |
263 if [ "${path}" != "." ] ; then |
320 parent_path="`dirname ${i}`" |
|
321 if [ "${parent_path}" != "." ] ; then |
264 times=0 |
322 times=0 |
265 while [ ! -d "${path}" ] ## nested repo, ensure containing dir exists |
323 while [ ! -d "${parent_path}" ] ; do ## nested repo, ensure containing dir exists |
266 do |
324 if [ "${sflag}" = "true" ] ; then |
267 times=`expr ${times} '+' 1` |
325 # Missing parent is fatal during sequential operation. |
|
326 echo "ERROR: Missing parent path: ${parent_path}" > ${status_output} |
|
327 exit 1 |
|
328 fi |
|
329 times=`expr ${times} '+' 1)` |
268 if [ `expr ${times} '%' 10` -eq 0 ] ; then |
330 if [ `expr ${times} '%' 10` -eq 0 ] ; then |
269 echo "${path} still not created, waiting..." > ${status_output} |
331 echo "${parent_path} still not created, waiting..." > ${status_output} |
270 fi |
332 fi |
271 sleep 5 |
333 sleep 5 |
272 done |
334 done |
273 fi |
335 fi |
274 echo "hg${global_opts} clone ${pull_newrepo} ${i}" > ${status_output} |
336 # run the clone command. |
275 (PYTHONUNBUFFERED=true hg${global_opts} clone ${pull_newrepo} ${i}; echo "$?" > ${tmp}/${repopidfile}.pid.rc ) 2>&1 & |
337 echo "hg${global_opts} clone ${clone_newrepo} ${i}" > ${status_output} |
|
338 (PYTHONUNBUFFERED=true hg${global_opts} clone ${clone_newrepo} ${i}; echo "$?" > ${tmp}/${repopidfile}.pid.rc ) 2>&1 & |
276 else |
339 else |
|
340 # run the command. |
277 echo "cd ${i} && hg${global_opts} ${command} ${command_args}" > ${status_output} |
341 echo "cd ${i} && hg${global_opts} ${command} ${command_args}" > ${status_output} |
278 cd ${i} && (PYTHONUNBUFFERED=true hg${global_opts} ${command} ${command_args}; echo "$?" > ${tmp}/${repopidfile}.pid.rc ) 2>&1 & |
342 cd ${i} && (PYTHONUNBUFFERED=true hg${global_opts} ${command} ${command_args}; echo "$?" > ${tmp}/${repopidfile}.pid.rc ) 2>&1 & |
279 fi |
343 fi |
280 |
344 |
281 echo $! > ${tmp}/${repopidfile}.pid |
345 echo $! > ${tmp}/${repopidfile}.pid |
282 ) 2>&1 | sed -e "s@^@${reponame}: @" > ${status_output} |
346 ) 2>&1 | sed -e "s@^@${reponame}: @" > ${status_output} |
283 if [ $have_fifos = "true" ]; then |
347 # tell the fifo waiter that this subprocess is done. |
284 echo "${reponame}" >&3 |
348 if [ ${have_fifos} = "true" ]; then |
|
349 echo "${i}" >&3 |
285 fi |
350 fi |
286 ) & |
351 ) & |
287 |
352 |
288 if [ $have_fifos = "true" ]; then |
353 if [ "${sflag}" = "true" ] ; then |
289 # check on count of running subprocesses and possibly wait for completion |
354 # complete this task before starting another. |
290 if [ ${at_a_time} -lt ${n} ] ; then |
355 wait |
291 # read will block until there are completed subprocesses |
|
292 while read repo_done; do |
|
293 n=`expr ${n} '-' 1` |
|
294 if [ ${n} -lt ${at_a_time} ] ; then |
|
295 # we should start more subprocesses |
|
296 break; |
|
297 fi |
|
298 done <&3 |
|
299 fi |
|
300 else |
356 else |
301 if [ "${sflag}" = "false" ] ; then |
357 if [ "${have_fifos}" = "true" ]; then |
|
358 # check on count of running subprocesses and possibly wait for completion |
|
359 if [ ${n} -ge ${at_a_time} ] ; then |
|
360 # read will block until there are completed subprocesses |
|
361 while read repo_done; do |
|
362 n=`expr ${n} '-' 1` |
|
363 if [ ${n} -lt ${at_a_time} ] ; then |
|
364 # we should start more subprocesses |
|
365 break; |
|
366 fi |
|
367 done <&3 |
|
368 fi |
|
369 else |
302 # Compare completions to starts |
370 # Compare completions to starts |
303 completed="`(ls -1 ${tmp}/*.pid.rc 2> /dev/null | wc -l) || echo 0`" |
371 completed="`(ls -a1 ${tmp}/*.pid.rc 2> /dev/null | wc -l) || echo 0`" |
304 while [ ${at_a_time} -lt `expr ${n} '-' ${completed}` ] ; do |
372 while [ `expr ${n} '-' ${completed}` -ge ${at_a_time} ] ; do |
305 # sleep a short time to give time for something to complete |
373 # sleep a short time to give time for something to complete |
306 sleep 1 |
374 sleep 1 |
307 completed="`(ls -1 ${tmp}/*.pid.rc 2> /dev/null | wc -l) || echo 0`" |
375 completed="`(ls -a1 ${tmp}/*.pid.rc 2> /dev/null | wc -l) || echo 0`" |
308 done |
376 done |
309 else |
|
310 # complete this task before starting another. |
|
311 wait |
|
312 fi |
377 fi |
313 fi |
378 fi |
314 done |
379 done |
315 fi |
380 fi |
316 |
381 |