1.1 --- a/test/script/basic/run-octane.js Fri May 24 23:27:52 2013 +0530
1.2 +++ b/test/script/basic/run-octane.js Mon May 27 13:11:13 2013 +0200
1.3 @@ -26,36 +26,20 @@
1.4 */
1.5
1.6 var tests = [
1.7 - "box2d.js",
1.8 - "code-load.js",
1.9 - "crypto.js",
1.10 - "deltablue.js",
1.11 - "earley-boyer.js",
1.12 - "gbemu.js",
1.13 - "mandreel.js",
1.14 - "navier-stokes.js",
1.15 - "pdfjs.js",
1.16 - "raytrace.js",
1.17 - "regexp.js",
1.18 - "richards.js",
1.19 - "splay.js"
1.20 + {file:"box2d",suite:"Box2DBenchmark"},
1.21 + {file:"code-load",suite:"CodeLoad"},
1.22 + {file:"crypto",suite:"Crypto"},
1.23 + {file:"deltablue",suite:"DeltaBlue"},
1.24 + {file:"earley-boyer", suite:"EarleyBoyer"},
1.25 + {file:"gbemu", suite:"GameboyBenchmark"},
1.26 + {file:"mandreel", suite:"MandreelBenchmark"},
1.27 + {file:"navier-stokes", suite:"NavierStokes"},
1.28 + {file:"pdfjs", suite:"PdfJS"},
1.29 + {file:"raytrace", suite:"RayTrace"},
1.30 + {file:"regexp", suite:"RegExpSuite"},
1.31 + {file:"richards", suite:"Richards"},
1.32 + {file:"splay", suite:"Splay"}
1.33 ];
1.34 -
1.35 -// hack, teardown breaks things defined in the global space, making it impossible
1.36 -// to do multiple consecutive benchmark runs with the same harness. I think it's a bug
1.37 -// that the setup and teardown aren't each others constructor and destructor but rather
1.38 -// that the benchmarks rely on partial global state. For shame, Octane!
1.39 -var ignoreTeardown = [
1.40 - { name: "box2d.js" },
1.41 - { name: "gbemu.js" },
1.42 -];
1.43 -
1.44 -
1.45 -//TODO mandreel can be compiled as a test, but not run multiple times unless modified to not have global state
1.46 -var compileOnly = {
1.47 - "mandreel.js" : true
1.48 -};
1.49 -
1.50 var dir = (typeof(__DIR__) == 'undefined') ? "test/script/basic/" : __DIR__;
1.51
1.52 // TODO: why is this path hard coded when it's defined in project properties?
1.53 @@ -71,110 +55,96 @@
1.54 }
1.55
1.56 function should_compile_only(name) {
1.57 - return (typeof compile_only !== 'undefined') || compileOnly[name] === true;
1.58 + return (typeof compile_only !== 'undefined')
1.59 }
1.60
1.61 function run_one_benchmark(arg, iters) {
1.62 -
1.63 var file_name;
1.64 - var file = arg.split('/');
1.65 - if (file.length == 1) {
1.66 - file = arg.split('\\');
1.67 - }
1.68 -
1.69 - //trim off trailing path separators
1.70 - while (file[file.length - 1].indexOf(".js") == -1) {
1.71 - file.pop();
1.72 - }
1.73 - file_name = file[file.length - 1];
1.74 -
1.75 + var file = (arg.file + ".js").split('/');
1.76 +
1.77 + file_name = path + file[file.length - 1];
1.78 +
1.79 var compile_and_return = should_compile_only(file_name);
1.80 if (compile_and_return) {
1.81 if (typeof compile_only === 'undefined') { //for a run, skip compile onlies, don't even compile them
1.82 return;
1.83 }
1.84 - print("Compiling... " + file_name);
1.85 }
1.86 -
1.87 - load(path + 'base.js');
1.88 - load(arg);
1.89 +
1.90 + print_verbose("Loading... " + file_name);
1.91 + load(file_name);
1.92
1.93 if (compile_and_return) {
1.94 - print("Compiled OK: " + file_name);
1.95 - print("");
1.96 + print_always("Compiled OK: " + arg.file);
1.97 return;
1.98 }
1.99
1.100 var success = true;
1.101 - var hiscore = 0;
1.102 - var loscore = 10e8;
1.103 var current_name;
1.104
1.105 - function PrintResult(name, result) {
1.106 - current_name = name;
1.107 - }
1.108 -
1.109 - function PrintError(name, error) {
1.110 - current_name = name;
1.111 - PrintResult(name, error);
1.112 - success = false;
1.113 - }
1.114 -
1.115 - function PrintScore(score) {
1.116 - if (success) {
1.117 - if (+score >= hiscore) {
1.118 - hiscore = +score;
1.119 - }
1.120 - if (+score <= loscore) {
1.121 - loscore = +score;
1.122 - }
1.123 - }
1.124 -
1.125 - if (verbose) {
1.126 - print("Score: " + score);
1.127 - }
1.128 - }
1.129 -
1.130 if (iters == undefined) {
1.131 iters = numberOfIterations;
1.132 } else {
1.133 numberOfIterations = iters;
1.134 }
1.135 +
1.136 + var benchmarks = eval(arg.suite + ".benchmarks");
1.137 + for (var x = 0; x < benchmarks.length ; x++) {
1.138 + benchmarks[x].Setup();
1.139 + }
1.140 + print_verbose("Running '" + arg.file + "' for " + iters + " iterations of no less than " + min_time + " seconds (" + runtime + ")");
1.141 +
1.142 + var scores = [];
1.143 +
1.144 + var min_time_ms = min_time * 1000;
1.145 + var len = benchmarks.length;
1.146 +
1.147 + for (var it = 0; it < iters + 1; it++) {
1.148 + //every iteration must take a minimum of 10 secs
1.149 + var ops = 0;
1.150 + var elapsed = 0;
1.151 + var start = new Date;
1.152 + do {
1.153 + for (var i = 0; i < len; i++) {
1.154 + benchmarks[i].run();
1.155 + }
1.156 + ops += len;
1.157 + elapsed = new Date - start;
1.158 + } while (elapsed < min_time * 1000);
1.159 +
1.160 + var score = ops / elapsed * 1000 * 60;
1.161 + scores.push(score);
1.162 + var name = it == 0 ? "warmup" : "iteration " + it;
1.163 + print_verbose("[" + arg.file + "] " + name + " finished " + score.toFixed(0) + " ops/minute");
1.164 + }
1.165 + for (var x = 0; x < benchmarks.length ; x++) {
1.166 + benchmarks[x].TearDown();
1.167 + }
1.168
1.169 - print(runtime + ": running " + file_name + "...");
1.170 + var min_score = 1e9;
1.171 + var max_score = 0;
1.172 + var mean_score = 0;
1.173 + for (var x = 1; x < iters + 1 ; x++) {
1.174 + mean_score += scores[x];
1.175 + min_score = Math.min(min_score, scores[x]);
1.176 + max_score = Math.max(max_score, scores[x]);
1.177 + }
1.178 + mean_score /= iters;
1.179 + var res = "[" + arg.file + "] " + mean_score.toFixed(0);
1.180 + if (verbose) {
1.181 + res += " ops/minute (" + min_score.toFixed(0) + "-" + max_score.toFixed(0) + "), warmup=" + scores[0].toFixed(0);
1.182 + }
1.183 + print_always(res);
1.184 +}
1.185
1.186 - for (var i = 0; i < numberOfIterations; i++) {
1.187 - var callbacks =
1.188 - { NotifyResult: PrintResult,
1.189 - NotifyError: PrintError,
1.190 - NotifyScore: PrintScore };
1.191 +function print_always(x) {
1.192 + print(x);
1.193 +}
1.194
1.195 - for (j in ignoreTeardown) {
1.196 - var ignore = ignoreTeardown[j];
1.197 - if (endsWith(arg, ignore.name)) {
1.198 - var teardownOverride = ignore.teardown;
1.199 - if (!teardownOverride) {
1.200 - teardownOverride = function() {};
1.201 - }
1.202 -
1.203 - for (k in BenchmarkSuite.suites) {
1.204 - var benchmarks = BenchmarkSuite.suites[k].benchmarks;
1.205 - for (l in benchmarks) {
1.206 - benchmarks[l].TearDown = teardownOverride;
1.207 - }
1.208 - }
1.209 - break;
1.210 - }
1.211 - }
1.212 -
1.213 - BenchmarkSuite.RunSuites(callbacks);
1.214 +function print_verbose(x) {
1.215 + if (verbose) {
1.216 + print(x);
1.217 }
1.218 -
1.219 - var start = "Score: ";
1.220 - if (runtime != "") {
1.221 - start = runtime + ": ";
1.222 - }
1.223 - print(start + current_name + ' (version ' + BenchmarkSuite.version + '): ' + loscore + '-' + hiscore);
1.224 }
1.225
1.226 function run_suite(tests, iters) {
1.227 @@ -211,6 +181,7 @@
1.228
1.229 var tests_found = [];
1.230 var iters = undefined;
1.231 +var min_time = 5;
1.232
1.233 for (var i = 0; i < args.length; i++) {
1.234 arg = args[i];
1.235 @@ -220,21 +191,41 @@
1.236 runtime = args[++i];
1.237 } else if (arg == "--verbose") {
1.238 verbose = true;
1.239 + } else if (arg == "--min-time") {
1.240 + min_time = +args[++i];
1.241 } else if (arg == "") {
1.242 continue; //skip
1.243 } else {
1.244 - tests_found.push(arg);
1.245 + var found = false;
1.246 + for (j in tests) {
1.247 + if (tests[j].file === arg) {
1.248 + tests_found.push(tests[j]);
1.249 + found = true;
1.250 + break;
1.251 + }
1.252 + }
1.253 + if (!found) {
1.254 + var str = "unknown test name: '" + arg + "' -- valid names are: ";
1.255 + for (j in tests) {
1.256 + if (j != 0) {
1.257 + str += ", ";
1.258 + }
1.259 + str += "'" + tests[j].file + "'";
1.260 + }
1.261 + throw str;
1.262 + }
1.263 }
1.264 }
1.265
1.266 if (tests_found.length == 0) {
1.267 for (i in tests) {
1.268 - tests_found.push(path + tests[i]);
1.269 + tests_found.push(tests[i]);
1.270 }
1.271 }
1.272
1.273 tests_found.sort();
1.274
1.275 +load(path + 'base.js');
1.276 run_suite(tests_found, iters);
1.277
1.278
