Mercurial > jdk8-mips64-public / changeset
changeset
8004352: build-infra: Limit JOBS on large machines
Mon, 04 Mar 2013 16:45:07 +0100
- author
- erikj
- date
- Mon, 04 Mar 2013 16:45:07 +0100
- changeset 635
- 907a926d3c96
- parent 634
- 0adf9c626bb1
- child 636
- cd7f2c7e2a0e
- child 637
- 52741ab7c601
8004352: build-infra: Limit JOBS on large machines
Reviewed-by: mduigou
1.1 --- a/common/autoconf/build-performance.m4 Thu Feb 28 20:29:19 2013 -0800 1.2 +++ b/common/autoconf/build-performance.m4 Mon Mar 04 16:45:07 2013 +0100 1.3 @@ -47,10 +47,6 @@ 1.4 FOUND_CORES=yes 1.5 fi 1.6 1.7 - # For c/c++ code we run twice as many concurrent build 1.8 - # jobs than we have cores, otherwise we will stall on io. 1.9 - CONCURRENT_BUILD_JOBS=`expr $NUM_CORES \* 2` 1.10 - 1.11 if test "x$FOUND_CORES" = xyes; then 1.12 AC_MSG_RESULT([$NUM_CORES]) 1.13 else 1.14 @@ -98,32 +94,62 @@ 1.15 1.16 AC_DEFUN_ONCE([BPERF_SETUP_BUILD_CORES], 1.17 [ 1.18 -# How many cores do we have on this build system? 1.19 -AC_ARG_WITH(num-cores, [AS_HELP_STRING([--with-num-cores], 1.20 + # How many cores do we have on this build system? 1.21 + AC_ARG_WITH(num-cores, [AS_HELP_STRING([--with-num-cores], 1.22 [number of cores in the build system, e.g. --with-num-cores=8 @<:@probed@:>@])]) 1.23 -if test "x$with_num_cores" = x; then 1.24 + if test "x$with_num_cores" = x; then 1.25 # The number of cores were not specified, try to probe them. 1.26 BPERF_CHECK_CORES 1.27 -else 1.28 + else 1.29 NUM_CORES=$with_num_cores 1.30 - CONCURRENT_BUILD_JOBS=`expr $NUM_CORES \* 2` 1.31 -fi 1.32 -AC_SUBST(NUM_CORES) 1.33 -AC_SUBST(CONCURRENT_BUILD_JOBS) 1.34 + fi 1.35 + AC_SUBST(NUM_CORES) 1.36 ]) 1.37 1.38 AC_DEFUN_ONCE([BPERF_SETUP_BUILD_MEMORY], 1.39 [ 1.40 -# How much memory do we have on this build system? 1.41 -AC_ARG_WITH(memory-size, [AS_HELP_STRING([--with-memory-size], 1.42 + # How much memory do we have on this build system? 1.43 + AC_ARG_WITH(memory-size, [AS_HELP_STRING([--with-memory-size], 1.44 [memory (in MB) available in the build system, e.g. --with-memory-size=1024 @<:@probed@:>@])]) 1.45 -if test "x$with_memory_size" = x; then 1.46 + if test "x$with_memory_size" = x; then 1.47 # The memory size was not specified, try to probe it. 1.48 BPERF_CHECK_MEMORY_SIZE 1.49 -else 1.50 + else 1.51 MEMORY_SIZE=$with_memory_size 1.52 -fi 1.53 -AC_SUBST(MEMORY_SIZE) 1.54 + fi 1.55 + AC_SUBST(MEMORY_SIZE) 1.56 +]) 1.57 + 1.58 +AC_DEFUN_ONCE([BPERF_SETUP_BUILD_JOBS], 1.59 +[ 1.60 + # Provide a decent default number of parallel jobs for make depending on 1.61 + # number of cores, amount of memory and machine architecture. 1.62 + AC_ARG_WITH(jobs, [AS_HELP_STRING([--with-jobs], 1.63 + [number of parallel jobs to let make run @<:@calculated based on cores and memory@:>@])]) 1.64 + if test "x$with_jobs" = x; then 1.65 + # Number of jobs was not specified, calculate. 1.66 + AC_MSG_CHECKING([for appropriate number of jobs to run in parallel]) 1.67 + # Approximate memory in GB, rounding up a bit. 1.68 + memory_gb=`expr $MEMORY_SIZE / 1100` 1.69 + # Pick the lowest of memory in gb and number of cores. 1.70 + if test "$memory_gb" -lt "$NUM_CORES"; then 1.71 + JOBS="$memory_gb" 1.72 + else 1.73 + JOBS="$NUM_CORES" 1.74 + # On bigger machines, leave some room for other processes to run 1.75 + if test "$JOBS" -gt "4"; then 1.76 + JOBS=`expr $JOBS '*' 90 / 100` 1.77 + fi 1.78 + fi 1.79 + # Cap number of jobs to 16 1.80 + if test "$JOBS" -gt "16"; then 1.81 + JOBS=16 1.82 + fi 1.83 + AC_MSG_RESULT([$JOBS]) 1.84 + else 1.85 + JOBS=$with_jobs 1.86 + fi 1.87 + AC_SUBST(JOBS) 1.88 ]) 1.89 1.90 AC_DEFUN([BPERF_SETUP_CCACHE],
2.1 --- a/common/autoconf/configure.ac Thu Feb 28 20:29:19 2013 -0800 2.2 +++ b/common/autoconf/configure.ac Mon Mar 04 16:45:07 2013 +0100 2.3 @@ -204,6 +204,7 @@ 2.4 2.5 BPERF_SETUP_BUILD_CORES 2.6 BPERF_SETUP_BUILD_MEMORY 2.7 +BPERF_SETUP_BUILD_JOBS 2.8 2.9 # Setup smart javac (after cores and memory have been setup) 2.10 BPERF_SETUP_SMART_JAVAC
3.1 --- a/common/autoconf/generated-configure.sh Thu Feb 28 20:29:19 2013 -0800 3.2 +++ b/common/autoconf/generated-configure.sh Mon Mar 04 16:45:07 2013 +0100 3.3 @@ -601,8 +601,8 @@ 3.4 ENABLE_SJAVAC 3.5 SJAVAC_SERVER_CORES 3.6 SJAVAC_SERVER_JAVA 3.7 +JOBS 3.8 MEMORY_SIZE 3.9 -CONCURRENT_BUILD_JOBS 3.10 NUM_CORES 3.11 SALIB_NAME 3.12 HOTSPOT_MAKE_ARGS 3.13 @@ -1002,6 +1002,7 @@ 3.14 with_stdc__lib 3.15 with_num_cores 3.16 with_memory_size 3.17 +with_jobs 3.18 with_sjavac_server_java 3.19 with_sjavac_server_cores 3.20 enable_sjavac 3.21 @@ -1760,6 +1761,8 @@ 3.22 --with-num-cores=8 [probed] 3.23 --with-memory-size memory (in MB) available in the build system, e.g. 3.24 --with-memory-size=1024 [probed] 3.25 + --with-jobs number of parallel jobs to let make run [calculated 3.26 + based on cores and memory] 3.27 --with-sjavac-server-java 3.28 use this java binary for running the sjavac 3.29 background server [Boot JDK java] 3.30 @@ -3329,6 +3332,8 @@ 3.31 3.32 3.33 3.34 + 3.35 + 3.36 # 3.37 # Copyright (c) 2011, 2012, Oracle and/or its affiliates. All rights reserved. 3.38 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 3.39 @@ -3724,7 +3729,7 @@ 3.40 #CUSTOM_AUTOCONF_INCLUDE 3.41 3.42 # Do not change or remove the following line, it is needed for consistency checks: 3.43 -DATE_WHEN_GENERATED=1361452590 3.44 +DATE_WHEN_GENERATED=1362411827 3.45 3.46 ############################################################################### 3.47 # 3.48 @@ -31225,14 +31230,14 @@ 3.49 ############################################################################### 3.50 3.51 3.52 -# How many cores do we have on this build system? 3.53 + # How many cores do we have on this build system? 3.54 3.55 # Check whether --with-num-cores was given. 3.56 if test "${with_num_cores+set}" = set; then : 3.57 withval=$with_num_cores; 3.58 fi 3.59 3.60 -if test "x$with_num_cores" = x; then 3.61 + if test "x$with_num_cores" = x; then 3.62 # The number of cores were not specified, try to probe them. 3.63 3.64 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for number of cores" >&5 3.65 @@ -31258,10 +31263,6 @@ 3.66 FOUND_CORES=yes 3.67 fi 3.68 3.69 - # For c/c++ code we run twice as many concurrent build 3.70 - # jobs than we have cores, otherwise we will stall on io. 3.71 - CONCURRENT_BUILD_JOBS=`expr $NUM_CORES \* 2` 3.72 - 3.73 if test "x$FOUND_CORES" = xyes; then 3.74 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $NUM_CORES" >&5 3.75 $as_echo "$NUM_CORES" >&6; } 3.76 @@ -31273,22 +31274,20 @@ 3.77 fi 3.78 3.79 3.80 -else 3.81 + else 3.82 NUM_CORES=$with_num_cores 3.83 - CONCURRENT_BUILD_JOBS=`expr $NUM_CORES \* 2` 3.84 -fi 3.85 - 3.86 - 3.87 - 3.88 - 3.89 -# How much memory do we have on this build system? 3.90 + fi 3.91 + 3.92 + 3.93 + 3.94 + # How much memory do we have on this build system? 3.95 3.96 # Check whether --with-memory-size was given. 3.97 if test "${with_memory_size+set}" = set; then : 3.98 withval=$with_memory_size; 3.99 fi 3.100 3.101 -if test "x$with_memory_size" = x; then 3.102 + if test "x$with_memory_size" = x; then 3.103 # The memory size was not specified, try to probe it. 3.104 3.105 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for memory size" >&5 3.106 @@ -31328,9 +31327,45 @@ 3.107 $as_echo "$as_me: WARNING: This might seriously impact build performance!" >&2;} 3.108 fi 3.109 3.110 -else 3.111 + else 3.112 MEMORY_SIZE=$with_memory_size 3.113 -fi 3.114 + fi 3.115 + 3.116 + 3.117 + 3.118 + # Provide a decent default number of parallel jobs for make depending on 3.119 + # number of cores, amount of memory and machine architecture. 3.120 + 3.121 +# Check whether --with-jobs was given. 3.122 +if test "${with_jobs+set}" = set; then : 3.123 + withval=$with_jobs; 3.124 +fi 3.125 + 3.126 + if test "x$with_jobs" = x; then 3.127 + # Number of jobs was not specified, calculate. 3.128 + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for appropriate number of jobs to run in parallel" >&5 3.129 +$as_echo_n "checking for appropriate number of jobs to run in parallel... " >&6; } 3.130 + # Approximate memory in GB, rounding up a bit. 3.131 + memory_gb=`expr $MEMORY_SIZE / 1100` 3.132 + # Pick the lowest of memory in gb and number of cores. 3.133 + if test "$memory_gb" -lt "$NUM_CORES"; then 3.134 + JOBS="$memory_gb" 3.135 + else 3.136 + JOBS="$NUM_CORES" 3.137 + # On bigger machines, leave some room for other processes to run 3.138 + if test "$JOBS" -gt "4"; then 3.139 + JOBS=`expr $JOBS '*' 90 / 100` 3.140 + fi 3.141 + fi 3.142 + # Cap number of jobs to 16 3.143 + if test "$JOBS" -gt "16"; then 3.144 + JOBS=16 3.145 + fi 3.146 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JOBS" >&5 3.147 +$as_echo "$JOBS" >&6; } 3.148 + else 3.149 + JOBS=$with_jobs 3.150 + fi 3.151 3.152 3.153 3.154 @@ -33143,7 +33178,7 @@ 3.155 3.156 printf "\n" 3.157 printf "Build performance summary:\n" 3.158 -printf "* Cores to use: $NUM_CORES\n" 3.159 +printf "* Cores to use: $JOBS\n" 3.160 printf "* Memory limit: $MEMORY_SIZE MB\n" 3.161 printf "* ccache status: $CCACHE_STATUS\n" 3.162 printf "\n"
4.1 --- a/common/autoconf/help.m4 Thu Feb 28 20:29:19 2013 -0800 4.2 +++ b/common/autoconf/help.m4 Mon Mar 04 16:45:07 2013 +0100 4.3 @@ -174,7 +174,7 @@ 4.4 4.5 printf "\n" 4.6 printf "Build performance summary:\n" 4.7 -printf "* Cores to use: $NUM_CORES\n" 4.8 +printf "* Cores to use: $JOBS\n" 4.9 printf "* Memory limit: $MEMORY_SIZE MB\n" 4.10 printf "* ccache status: $CCACHE_STATUS\n" 4.11 printf "\n"
5.1 --- a/common/autoconf/hotspot-spec.gmk.in Thu Feb 28 20:29:19 2013 -0800 5.2 +++ b/common/autoconf/hotspot-spec.gmk.in Mon Mar 04 16:45:07 2013 +0100 5.3 @@ -80,7 +80,7 @@ 5.4 5.5 HOTSPOT_MAKE_ARGS:=@HOTSPOT_MAKE_ARGS@ @STATIC_CXX_SETTING@ 5.6 # This is used from the libjvm build for C/C++ code. 5.7 -HOTSPOT_BUILD_JOBS:=@CONCURRENT_BUILD_JOBS@ 5.8 +HOTSPOT_BUILD_JOBS:=$(JOBS) 5.9 # Control wether Hotspot runs Queens test after building 5.10 TEST_IN_BUILD=@TEST_IN_BUILD@ 5.11
6.1 --- a/common/autoconf/spec.gmk.in Thu Feb 28 20:29:19 2013 -0800 6.2 +++ b/common/autoconf/spec.gmk.in Mon Mar 04 16:45:07 2013 +0100 6.3 @@ -260,6 +260,9 @@ 6.4 # the sjavac server log files. 6.5 SJAVAC_SERVER_DIR:=@SJAVAC_SERVER_DIR@ 6.6 6.7 +# Number of parallel jobs to use for compilation 6.8 +JOBS?=@JOBS@ 6.9 + 6.10 # The OpenJDK makefiles should be changed to using the standard 6.11 # configure output ..._CFLAGS and ..._LIBS. In the meantime we 6.12 # extract the information here.
7.1 --- a/common/makefiles/JavaCompilation.gmk Thu Feb 28 20:29:19 2013 -0800 7.2 +++ b/common/makefiles/JavaCompilation.gmk Mon Mar 04 16:45:07 2013 +0100 7.3 @@ -501,7 +501,7 @@ 7.4 $(ECHO) Compiling $1 7.5 ($$($1_JVM) $$($1_SJAVAC) \ 7.6 $$($1_REMOTE) \ 7.7 - -j $(NUM_CORES) \ 7.8 + -j $(JOBS) \ 7.9 --permit-unidentified-artifacts \ 7.10 --permit-sources-without-package \ 7.11 --compare-found-sources $$($1_BIN)/_the.batch.tmp \
8.1 --- a/common/makefiles/Main.gmk Thu Feb 28 20:29:19 2013 -0800 8.2 +++ b/common/makefiles/Main.gmk Mon Mar 04 16:45:07 2013 +0100 8.3 @@ -58,9 +58,6 @@ 8.4 8.5 # Setup number of jobs to use. -jN is unfortunately not available for us to parse from the command line, 8.6 # hence this workaround. 8.7 -ifeq ($(JOBS),) 8.8 - JOBS=$(NUM_CORES) 8.9 -endif 8.10 MAKE_ARGS:=$(MAKE_ARGS) -j$(JOBS) 8.11 8.12 ### Main targets
