22 # or visit www.oracle.com if you need additional information or have any |
22 # or visit www.oracle.com if you need additional information or have any |
23 # questions. |
23 # questions. |
24 # |
24 # |
25 |
25 |
26 # Shell script for a fast parallel forest command |
26 # Shell script for a fast parallel forest command |
27 command="$1" |
27 |
28 pull_extra_base="$2" |
28 global_opts="" |
29 |
29 status_output="/dev/stdout" |
30 if [ "" = "$command" ] ; then |
30 qflag="false" |
31 echo No command to hg supplied! |
31 vflag="false" |
32 exit 1 |
32 while [ $# -gt 0 ] |
|
33 do |
|
34 case $1 in |
|
35 -q | --quiet ) |
|
36 qflag="true" |
|
37 global_opts="${global_opts} -q" |
|
38 status_output="/dev/null" |
|
39 ;; |
|
40 |
|
41 -v | --verbose ) |
|
42 vflag="true" |
|
43 global_opts="${global_opts} -v" |
|
44 ;; |
|
45 |
|
46 '--' ) # no more options |
|
47 shift; break |
|
48 ;; |
|
49 |
|
50 -*) # bad option |
|
51 usage |
|
52 ;; |
|
53 |
|
54 * ) # non option |
|
55 break |
|
56 ;; |
|
57 esac |
|
58 shift |
|
59 done |
|
60 |
|
61 |
|
62 command="$1"; shift |
|
63 repo_base="$@" |
|
64 |
|
65 usage() { |
|
66 echo "usage: $0 [-q|--quiet] [-v|--verbose] [--] <command> [repo_base_path]" > ${status_output} |
|
67 exit 1 |
|
68 } |
|
69 |
|
70 |
|
71 if [ "x" = "x$command" ] ; then |
|
72 echo "ERROR: No command to hg supplied!" |
|
73 usage |
33 fi |
74 fi |
34 |
75 |
35 # Clean out the temporary directory that stores the pid files. |
76 # Clean out the temporary directory that stores the pid files. |
36 tmp=/tmp/forest.$$ |
77 tmp=/tmp/forest.$$ |
37 rm -f -r ${tmp} |
78 rm -f -r ${tmp} |
38 mkdir -p ${tmp} |
79 mkdir -p ${tmp} |
39 |
80 |
40 safe_interrupt () { |
81 safe_interrupt () { |
41 if [ -d ${tmp} ]; then |
82 if [ -d ${tmp} ]; then |
42 if [ "`ls ${tmp}/*.pid`" != "" ]; then |
83 if [ "`ls ${tmp}/*.pid`" != "" ]; then |
43 echo "Waiting for processes ( `cat ${tmp}/*.pid | tr '\n' ' '`) to terminate nicely!" |
84 echo "Waiting for processes ( `cat ${tmp}/*.pid | tr '\n' ' '`) to terminate nicely!" > ${status_output} |
44 sleep 1 |
85 sleep 1 |
45 # Pipe stderr to dev/null to silence kill, that complains when trying to kill |
86 # Pipe stderr to dev/null to silence kill, that complains when trying to kill |
46 # a subprocess that has already exited. |
87 # a subprocess that has already exited. |
47 kill -TERM `cat ${tmp}/*.pid | tr '\n' ' '` 2> /dev/null |
88 kill -TERM `cat ${tmp}/*.pid | tr '\n' ' '` 2> /dev/null |
48 wait |
89 wait |
49 echo Interrupt complete! |
90 echo "Interrupt complete!" > ${status_output} |
50 fi |
91 fi |
51 fi |
92 rm -f -r ${tmp} |
52 rm -f -r ${tmp} |
93 fi |
53 exit 1 |
94 exit 130 |
54 } |
95 } |
55 |
96 |
56 nice_exit () { |
97 nice_exit () { |
57 if [ -d ${tmp} ]; then |
98 if [ -d ${tmp} ]; then |
58 if [ "`ls ${tmp}`" != "" ]; then |
99 if [ "`ls ${tmp}`" != "" ]; then |
59 wait |
100 wait |
60 fi |
101 fi |
61 fi |
102 rm -f -r ${tmp} |
62 rm -f -r ${tmp} |
103 fi |
63 } |
104 } |
64 |
105 |
65 trap 'safe_interrupt' INT QUIT |
106 trap 'safe_interrupt' INT QUIT |
66 trap 'nice_exit' EXIT |
107 trap 'nice_exit' EXIT |
|
108 |
|
109 subrepos="corba jaxp jaxws langtools jdk hotspot nashorn" |
|
110 subrepos_extra="jdk/src/closed jdk/make/closed jdk/test/closed hotspot/make/closed hotspot/src/closed hotspot/test/closed deploy install sponsors pubs" |
67 |
111 |
68 # Only look in specific locations for possible forests (avoids long searches) |
112 # Only look in specific locations for possible forests (avoids long searches) |
69 pull_default="" |
113 pull_default="" |
70 repos="" |
114 repos="" |
71 repos_extra="" |
115 repos_extra="" |
72 if [ "${command}" = "clone" -o "${command}" = "fclone" ] ; then |
116 if [ "${command}" = "clone" -o "${command}" = "fclone" ] ; then |
73 subrepos="corba jaxp jaxws langtools jdk hotspot nashorn" |
|
74 if [ -f .hg/hgrc ] ; then |
117 if [ -f .hg/hgrc ] ; then |
75 pull_default=`hg paths default` |
118 pull_default=`hg paths default` |
76 if [ "${pull_default}" = "" ] ; then |
119 if [ "${pull_default}" = "" ] ; then |
77 echo "ERROR: Need initial clone with 'hg paths default' defined" |
120 echo "ERROR: Need initial clone with 'hg paths default' defined" > ${status_output} |
78 exit 1 |
121 exit 1 |
79 fi |
122 fi |
80 fi |
123 fi |
81 if [ "${pull_default}" = "" ] ; then |
124 if [ "${pull_default}" = "" ] ; then |
82 echo "ERROR: Need initial repository to use this script" |
125 echo "ERROR: Need initial repository to use this script" > ${status_output} |
83 exit 1 |
126 exit 1 |
84 fi |
127 fi |
85 for i in ${subrepos} ; do |
128 for i in ${subrepos} ; do |
86 if [ ! -f ${i}/.hg/hgrc ] ; then |
129 if [ ! -f ${i}/.hg/hgrc ] ; then |
87 repos="${repos} ${i}" |
130 repos="${repos} ${i}" |
88 fi |
131 fi |
89 done |
132 done |
90 if [ "${pull_extra_base}" != "" ] ; then |
133 if [ "${repo_base}" != "" ] ; then |
91 subrepos_extra="jdk/src/closed jdk/make/closed jdk/test/closed hotspot/make/closed hotspot/src/closed hotspot/test/closed deploy install sponsors pubs" |
|
92 pull_default_tail=`echo ${pull_default} | sed -e 's@^.*://[^/]*/\(.*\)@\1@'` |
134 pull_default_tail=`echo ${pull_default} | sed -e 's@^.*://[^/]*/\(.*\)@\1@'` |
93 pull_extra="${pull_extra_base}/${pull_default_tail}" |
135 pull_extra="${repo_base}/${pull_default_tail}" |
94 for i in ${subrepos_extra} ; do |
136 for i in ${subrepos_extra} ; do |
95 if [ ! -f ${i}/.hg/hgrc ] ; then |
137 if [ ! -f ${i}/.hg/hgrc ] ; then |
96 repos_extra="${repos_extra} ${i}" |
138 repos_extra="${repos_extra} ${i}" |
97 fi |
139 fi |
98 done |
140 done |
99 fi |
141 fi |
100 at_a_time=2 |
142 at_a_time=2 |
101 # Any repos to deal with? |
143 # Any repos to deal with? |
102 if [ "${repos}" = "" -a "${repos_extra}" = "" ] ; then |
144 if [ "${repos}" = "" -a "${repos_extra}" = "" ] ; then |
|
145 echo "No repositories to process." > ${status_output} |
103 exit |
146 exit |
104 fi |
147 fi |
105 else |
148 else |
106 hgdirs=`ls -d ./.hg ./*/.hg ./*/*/.hg ./*/*/*/.hg ./*/*/*/*/.hg 2>/dev/null` |
149 for i in . ${subrepos} ${subrepos_extra} ; do |
107 # Derive repository names from the .hg directory locations |
150 if [ -d ${i}/.hg ] ; then |
108 for i in ${hgdirs} ; do |
151 repos="${repos} ${i}" |
109 repos="${repos} `echo ${i} | sed -e 's@/.hg$@@'`" |
152 fi |
110 done |
153 done |
|
154 |
|
155 # Any repos to deal with? |
|
156 if [ "${repos}" = "" ] ; then |
|
157 echo "No repositories to process." > ${status_output} |
|
158 exit |
|
159 fi |
|
160 |
|
161 # any of the repos locked? |
111 for i in ${repos} ; do |
162 for i in ${repos} ; do |
112 if [ -h ${i}/.hg/store/lock -o -f ${i}/.hg/store/lock ] ; then |
163 if [ -h ${i}/.hg/store/lock -o -f ${i}/.hg/store/lock ] ; then |
113 locked="${i} ${locked}" |
164 locked="${i} ${locked}" |
114 fi |
165 fi |
115 done |
166 done |
|
167 if [ "${locked}" != "" ] ; then |
|
168 echo "ERROR: These repositories are locked: ${locked}" > ${status_output} |
|
169 exit 1 |
|
170 fi |
116 at_a_time=8 |
171 at_a_time=8 |
117 # Any repos to deal with? |
|
118 if [ "${repos}" = "" ] ; then |
|
119 echo "No repositories to process." |
|
120 exit |
|
121 fi |
|
122 if [ "${locked}" != "" ] ; then |
|
123 echo "These repositories are locked: ${locked}" |
|
124 exit |
|
125 fi |
|
126 fi |
172 fi |
127 |
173 |
128 # Echo out what repositories we do a command on. |
174 # Echo out what repositories we do a command on. |
129 echo "# Repositories: ${repos} ${repos_extra}" |
175 echo "# Repositories: ${repos} ${repos_extra}" > ${status_output} |
130 echo |
176 |
131 |
177 if [ "${command}" = "serve" ] ; then |
132 # Run the supplied command on all repos in parallel. |
178 # "serve" is run for all the repos. |
133 n=0 |
179 ( |
134 for i in ${repos} ${repos_extra} ; do |
180 ( |
135 n=`expr ${n} '+' 1` |
181 ( |
136 repopidfile=`echo ${i} | sed -e 's@./@@' -e 's@/@_@g'` |
182 echo "[web]" |
137 reponame=`echo ${i} | sed -e :a -e 's/^.\{1,20\}$/ &/;ta'` |
183 echo "description = $(basename $(pwd))" |
138 pull_base="${pull_default}" |
184 echo "allow_push = *" |
139 for j in $repos_extra ; do |
185 echo "push_ssl = False" |
|
186 |
|
187 echo "[paths]" |
|
188 for i in ${repos} ${repos_extra} ; do |
|
189 if [ "${i}" != "." ] ; then |
|
190 echo "/$(basename $(pwd))/${i} = ${i}" |
|
191 else |
|
192 echo "/$(basename $(pwd)) = $(pwd)" |
|
193 fi |
|
194 done |
|
195 ) > ${tmp}/serve.web-conf |
|
196 |
|
197 echo "serving root repo $(basename $(pwd))" |
|
198 |
|
199 (PYTHONUNBUFFERED=true hg${global_opts} serve -A ${status_output} -E ${status_output} --pid-file ${tmp}/serve.pid --web-conf ${tmp}/serve.web-conf; echo "$?" > ${tmp}/serve.pid.rc ) 2>&1 & |
|
200 ) 2>&1 | sed -e "s@^@serve: @" > ${status_output} |
|
201 ) & |
|
202 else |
|
203 # Run the supplied command on all repos in parallel. |
|
204 n=0 |
|
205 for i in ${repos} ${repos_extra} ; do |
|
206 n=`expr ${n} '+' 1` |
|
207 repopidfile=`echo ${i} | sed -e 's@./@@' -e 's@/@_@g'` |
|
208 reponame=`echo ${i} | sed -e :a -e 's/^.\{1,20\}$/ &/;ta'` |
|
209 pull_base="${pull_default}" |
|
210 for j in $repos_extra ; do |
140 if [ "$i" = "$j" ] ; then |
211 if [ "$i" = "$j" ] ; then |
141 pull_base="${pull_extra}" |
212 pull_base="${pull_extra}" |
142 fi |
213 fi |
143 done |
214 done |
144 ( |
|
145 ( |
215 ( |
146 if [ "${command}" = "clone" -o "${command}" = "fclone" ] ; then |
216 ( |
147 pull_newrepo="`echo ${pull_base}/${i} | sed -e 's@\([^:]/\)//*@\1@g'`" |
217 if [ "${command}" = "clone" -o "${command}" = "fclone" ] ; then |
148 echo hg clone ${pull_newrepo} ${i} |
218 pull_newrepo="`echo ${pull_base}/${i} | sed -e 's@\([^:]/\)//*@\1@g'`" |
149 path="`dirname ${i}`" |
219 echo "hg clone ${pull_newrepo} ${i}" > ${status_output} |
150 if [ "${path}" != "." ] ; then |
220 path="`dirname ${i}`" |
151 times=0 |
221 if [ "${path}" != "." ] ; then |
152 while [ ! -d "${path}" ] ## nested repo, ensure containing dir exists |
222 times=0 |
153 do |
223 while [ ! -d "${path}" ] ## nested repo, ensure containing dir exists |
154 times=`expr ${times} '+' 1` |
224 do |
155 if [ `expr ${times} '%' 10` -eq 0 ] ; then |
225 times=`expr ${times} '+' 1` |
156 echo ${path} still not created, waiting... |
226 if [ `expr ${times} '%' 10` -eq 0 ] ; then |
157 fi |
227 echo "${path} still not created, waiting..." > ${status_output} |
158 sleep 5 |
228 fi |
159 done |
229 sleep 5 |
|
230 done |
|
231 fi |
|
232 (PYTHONUNBUFFERED=true hg${global_opts} clone ${pull_newrepo} ${i}; echo "$?" > ${tmp}/${repopidfile}.pid.rc ) 2>&1 & |
|
233 else |
|
234 echo "cd ${i} && hg${global_opts} ${command} ${repo_base}" > ${status_output} |
|
235 cd ${i} && (PYTHONUNBUFFERED=true hg${global_opts} ${command} ${command_repo}; echo "$?" > ${tmp}/${repopidfile}.pid.rc ) 2>&1 & |
160 fi |
236 fi |
161 (PYTHONUNBUFFERED=true hg clone ${pull_newrepo} ${i}; echo "$?" > ${tmp}/${repopidfile}.pid.rc )& |
237 |
162 else |
238 echo $! > ${tmp}/${repopidfile}.pid |
163 echo "cd ${i} && hg $*" |
239 ) 2>&1 | sed -e "s@^@${reponame}: @" > ${status_output} |
164 cd ${i} && (PYTHONUNBUFFERED=true hg "$@"; echo "$?" > ${tmp}/${repopidfile}.pid.rc )& |
240 ) & |
165 fi |
241 |
166 echo $! > ${tmp}/${repopidfile}.pid |
242 if [ `expr ${n} '%' ${at_a_time}` -eq 0 ] ; then |
167 ) 2>&1 | sed -e "s@^@${reponame}: @") & |
243 sleep 2 |
168 |
244 echo "Waiting 5 secs before spawning next background command." > ${status_output} |
169 if [ `expr ${n} '%' ${at_a_time}` -eq 0 ] ; then |
245 sleep 3 |
170 sleep 2 |
246 fi |
171 echo Waiting 5 secs before spawning next background command. |
247 done |
172 sleep 3 |
248 fi |
173 fi |
249 |
174 done |
|
175 # Wait for all hg commands to complete |
250 # Wait for all hg commands to complete |
176 wait |
251 wait |
177 |
252 |
178 # Terminate with exit 0 only if all subprocesses were successful |
253 # Terminate with exit 0 only if all subprocesses were successful |
179 ec=0 |
254 ec=0 |
180 if [ -d ${tmp} ]; then |
255 if [ -d ${tmp} ]; then |
181 for rc in ${tmp}/*.pid.rc ; do |
256 for rc in ${tmp}/*.pid.rc ; do |
182 exit_code=`cat ${rc} | tr -d ' \n\r'` |
257 exit_code=`cat ${rc} | tr -d ' \n\r'` |
183 if [ "${exit_code}" != "0" ] ; then |
258 if [ "${exit_code}" != "0" ] ; then |
184 echo "WARNING: ${rc} exited abnormally." |
259 repo="`echo ${rc} | sed -e s@^${tmp}@@ -e 's@/*\([^/]*\)\.pid\.rc$@\1@' -e 's@_@/@g'`" |
|
260 echo "WARNING: ${repo} exited abnormally." > ${status_output} |
185 ec=1 |
261 ec=1 |
186 fi |
262 fi |
187 done |
263 done |
188 fi |
264 fi |
189 exit ${ec} |
265 exit ${ec} |