Thu, 07 Apr 2011 09:53:20 -0700
7009266: G1: assert(obj->is_oop_or_null(true )) failed: Error
Summary: A referent object that is only weakly reachable at the start of concurrent marking but is re-attached to the strongly reachable object graph during marking may not be marked as live. This can cause the reference object to be processed prematurely and leave dangling pointers to the referent object. Implement a read barrier for the java.lang.ref.Reference::referent field by intrinsifying the Reference.get() method, and intercepting accesses though JNI, reflection, and Unsafe, so that when a non-null referent object is read it is also logged in an SATB buffer.
Reviewed-by: kvn, iveresov, never, tonyp, dholmes
1 /*
2 * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
25 #include "precompiled.hpp"
26 #include "compiler/compileLog.hpp"
27 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
28 #include "gc_implementation/g1/heapRegion.hpp"
29 #include "gc_interface/collectedHeap.hpp"
30 #include "memory/barrierSet.hpp"
31 #include "memory/cardTableModRefBS.hpp"
32 #include "opto/addnode.hpp"
33 #include "opto/graphKit.hpp"
34 #include "opto/idealKit.hpp"
35 #include "opto/locknode.hpp"
36 #include "opto/machnode.hpp"
37 #include "opto/parse.hpp"
38 #include "opto/rootnode.hpp"
39 #include "opto/runtime.hpp"
40 #include "runtime/deoptimization.hpp"
41 #include "runtime/sharedRuntime.hpp"
43 //----------------------------GraphKit-----------------------------------------
44 // Main utility constructor.
45 GraphKit::GraphKit(JVMState* jvms)
46 : Phase(Phase::Parser),
47 _env(C->env()),
48 _gvn(*C->initial_gvn())
49 {
50 _exceptions = jvms->map()->next_exception();
51 if (_exceptions != NULL) jvms->map()->set_next_exception(NULL);
52 set_jvms(jvms);
53 }
55 // Private constructor for parser.
56 GraphKit::GraphKit()
57 : Phase(Phase::Parser),
58 _env(C->env()),
59 _gvn(*C->initial_gvn())
60 {
61 _exceptions = NULL;
62 set_map(NULL);
63 debug_only(_sp = -99);
64 debug_only(set_bci(-99));
65 }
69 //---------------------------clean_stack---------------------------------------
70 // Clear away rubbish from the stack area of the JVM state.
71 // This destroys any arguments that may be waiting on the stack.
72 void GraphKit::clean_stack(int from_sp) {
73 SafePointNode* map = this->map();
74 JVMState* jvms = this->jvms();
75 int stk_size = jvms->stk_size();
76 int stkoff = jvms->stkoff();
77 Node* top = this->top();
78 for (int i = from_sp; i < stk_size; i++) {
79 if (map->in(stkoff + i) != top) {
80 map->set_req(stkoff + i, top);
81 }
82 }
83 }
86 //--------------------------------sync_jvms-----------------------------------
87 // Make sure our current jvms agrees with our parse state.
88 JVMState* GraphKit::sync_jvms() const {
89 JVMState* jvms = this->jvms();
90 jvms->set_bci(bci()); // Record the new bci in the JVMState
91 jvms->set_sp(sp()); // Record the new sp in the JVMState
92 assert(jvms_in_sync(), "jvms is now in sync");
93 return jvms;
94 }
96 #ifdef ASSERT
97 bool GraphKit::jvms_in_sync() const {
98 Parse* parse = is_Parse();
99 if (parse == NULL) {
100 if (bci() != jvms()->bci()) return false;
101 if (sp() != (int)jvms()->sp()) return false;
102 return true;
103 }
104 if (jvms()->method() != parse->method()) return false;
105 if (jvms()->bci() != parse->bci()) return false;
106 int jvms_sp = jvms()->sp();
107 if (jvms_sp != parse->sp()) return false;
108 int jvms_depth = jvms()->depth();
109 if (jvms_depth != parse->depth()) return false;
110 return true;
111 }
113 // Local helper checks for special internal merge points
114 // used to accumulate and merge exception states.
115 // They are marked by the region's in(0) edge being the map itself.
116 // Such merge points must never "escape" into the parser at large,
117 // until they have been handed to gvn.transform.
118 static bool is_hidden_merge(Node* reg) {
119 if (reg == NULL) return false;
120 if (reg->is_Phi()) {
121 reg = reg->in(0);
122 if (reg == NULL) return false;
123 }
124 return reg->is_Region() && reg->in(0) != NULL && reg->in(0)->is_Root();
125 }
127 void GraphKit::verify_map() const {
128 if (map() == NULL) return; // null map is OK
129 assert(map()->req() <= jvms()->endoff(), "no extra garbage on map");
130 assert(!map()->has_exceptions(), "call add_exception_states_from 1st");
131 assert(!is_hidden_merge(control()), "call use_exception_state, not set_map");
132 }
134 void GraphKit::verify_exception_state(SafePointNode* ex_map) {
135 assert(ex_map->next_exception() == NULL, "not already part of a chain");
136 assert(has_saved_ex_oop(ex_map), "every exception state has an ex_oop");
137 }
138 #endif
140 //---------------------------stop_and_kill_map---------------------------------
141 // Set _map to NULL, signalling a stop to further bytecode execution.
142 // First smash the current map's control to a constant, to mark it dead.
143 void GraphKit::stop_and_kill_map() {
144 SafePointNode* dead_map = stop();
145 if (dead_map != NULL) {
146 dead_map->disconnect_inputs(NULL); // Mark the map as killed.
147 assert(dead_map->is_killed(), "must be so marked");
148 }
149 }
152 //--------------------------------stopped--------------------------------------
153 // Tell if _map is NULL, or control is top.
154 bool GraphKit::stopped() {
155 if (map() == NULL) return true;
156 else if (control() == top()) return true;
157 else return false;
158 }
161 //-----------------------------has_ex_handler----------------------------------
162 // Tell if this method or any caller method has exception handlers.
163 bool GraphKit::has_ex_handler() {
164 for (JVMState* jvmsp = jvms(); jvmsp != NULL; jvmsp = jvmsp->caller()) {
165 if (jvmsp->has_method() && jvmsp->method()->has_exception_handlers()) {
166 return true;
167 }
168 }
169 return false;
170 }
172 //------------------------------save_ex_oop------------------------------------
173 // Save an exception without blowing stack contents or other JVM state.
174 void GraphKit::set_saved_ex_oop(SafePointNode* ex_map, Node* ex_oop) {
175 assert(!has_saved_ex_oop(ex_map), "clear ex-oop before setting again");
176 ex_map->add_req(ex_oop);
177 debug_only(verify_exception_state(ex_map));
178 }
180 inline static Node* common_saved_ex_oop(SafePointNode* ex_map, bool clear_it) {
181 assert(GraphKit::has_saved_ex_oop(ex_map), "ex_oop must be there");
182 Node* ex_oop = ex_map->in(ex_map->req()-1);
183 if (clear_it) ex_map->del_req(ex_map->req()-1);
184 return ex_oop;
185 }
187 //-----------------------------saved_ex_oop------------------------------------
188 // Recover a saved exception from its map.
189 Node* GraphKit::saved_ex_oop(SafePointNode* ex_map) {
190 return common_saved_ex_oop(ex_map, false);
191 }
193 //--------------------------clear_saved_ex_oop---------------------------------
194 // Erase a previously saved exception from its map.
195 Node* GraphKit::clear_saved_ex_oop(SafePointNode* ex_map) {
196 return common_saved_ex_oop(ex_map, true);
197 }
199 #ifdef ASSERT
200 //---------------------------has_saved_ex_oop----------------------------------
201 // Erase a previously saved exception from its map.
202 bool GraphKit::has_saved_ex_oop(SafePointNode* ex_map) {
203 return ex_map->req() == ex_map->jvms()->endoff()+1;
204 }
205 #endif
207 //-------------------------make_exception_state--------------------------------
208 // Turn the current JVM state into an exception state, appending the ex_oop.
209 SafePointNode* GraphKit::make_exception_state(Node* ex_oop) {
210 sync_jvms();
211 SafePointNode* ex_map = stop(); // do not manipulate this map any more
212 set_saved_ex_oop(ex_map, ex_oop);
213 return ex_map;
214 }
217 //--------------------------add_exception_state--------------------------------
218 // Add an exception to my list of exceptions.
219 void GraphKit::add_exception_state(SafePointNode* ex_map) {
220 if (ex_map == NULL || ex_map->control() == top()) {
221 return;
222 }
223 #ifdef ASSERT
224 verify_exception_state(ex_map);
225 if (has_exceptions()) {
226 assert(ex_map->jvms()->same_calls_as(_exceptions->jvms()), "all collected exceptions must come from the same place");
227 }
228 #endif
230 // If there is already an exception of exactly this type, merge with it.
231 // In particular, null-checks and other low-level exceptions common up here.
232 Node* ex_oop = saved_ex_oop(ex_map);
233 const Type* ex_type = _gvn.type(ex_oop);
234 if (ex_oop == top()) {
235 // No action needed.
236 return;
237 }
238 assert(ex_type->isa_instptr(), "exception must be an instance");
239 for (SafePointNode* e2 = _exceptions; e2 != NULL; e2 = e2->next_exception()) {
240 const Type* ex_type2 = _gvn.type(saved_ex_oop(e2));
241 // We check sp also because call bytecodes can generate exceptions
242 // both before and after arguments are popped!
243 if (ex_type2 == ex_type
244 && e2->_jvms->sp() == ex_map->_jvms->sp()) {
245 combine_exception_states(ex_map, e2);
246 return;
247 }
248 }
250 // No pre-existing exception of the same type. Chain it on the list.
251 push_exception_state(ex_map);
252 }
254 //-----------------------add_exception_states_from-----------------------------
255 void GraphKit::add_exception_states_from(JVMState* jvms) {
256 SafePointNode* ex_map = jvms->map()->next_exception();
257 if (ex_map != NULL) {
258 jvms->map()->set_next_exception(NULL);
259 for (SafePointNode* next_map; ex_map != NULL; ex_map = next_map) {
260 next_map = ex_map->next_exception();
261 ex_map->set_next_exception(NULL);
262 add_exception_state(ex_map);
263 }
264 }
265 }
267 //-----------------------transfer_exceptions_into_jvms-------------------------
268 JVMState* GraphKit::transfer_exceptions_into_jvms() {
269 if (map() == NULL) {
270 // We need a JVMS to carry the exceptions, but the map has gone away.
271 // Create a scratch JVMS, cloned from any of the exception states...
272 if (has_exceptions()) {
273 _map = _exceptions;
274 _map = clone_map();
275 _map->set_next_exception(NULL);
276 clear_saved_ex_oop(_map);
277 debug_only(verify_map());
278 } else {
279 // ...or created from scratch
280 JVMState* jvms = new (C) JVMState(_method, NULL);
281 jvms->set_bci(_bci);
282 jvms->set_sp(_sp);
283 jvms->set_map(new (C, TypeFunc::Parms) SafePointNode(TypeFunc::Parms, jvms));
284 set_jvms(jvms);
285 for (uint i = 0; i < map()->req(); i++) map()->init_req(i, top());
286 set_all_memory(top());
287 while (map()->req() < jvms->endoff()) map()->add_req(top());
288 }
289 // (This is a kludge, in case you didn't notice.)
290 set_control(top());
291 }
292 JVMState* jvms = sync_jvms();
293 assert(!jvms->map()->has_exceptions(), "no exceptions on this map yet");
294 jvms->map()->set_next_exception(_exceptions);
295 _exceptions = NULL; // done with this set of exceptions
296 return jvms;
297 }
299 static inline void add_n_reqs(Node* dstphi, Node* srcphi) {
300 assert(is_hidden_merge(dstphi), "must be a special merge node");
301 assert(is_hidden_merge(srcphi), "must be a special merge node");
302 uint limit = srcphi->req();
303 for (uint i = PhiNode::Input; i < limit; i++) {
304 dstphi->add_req(srcphi->in(i));
305 }
306 }
307 static inline void add_one_req(Node* dstphi, Node* src) {
308 assert(is_hidden_merge(dstphi), "must be a special merge node");
309 assert(!is_hidden_merge(src), "must not be a special merge node");
310 dstphi->add_req(src);
311 }
313 //-----------------------combine_exception_states------------------------------
314 // This helper function combines exception states by building phis on a
315 // specially marked state-merging region. These regions and phis are
316 // untransformed, and can build up gradually. The region is marked by
317 // having a control input of its exception map, rather than NULL. Such
318 // regions do not appear except in this function, and in use_exception_state.
319 void GraphKit::combine_exception_states(SafePointNode* ex_map, SafePointNode* phi_map) {
320 if (failing()) return; // dying anyway...
321 JVMState* ex_jvms = ex_map->_jvms;
322 assert(ex_jvms->same_calls_as(phi_map->_jvms), "consistent call chains");
323 assert(ex_jvms->stkoff() == phi_map->_jvms->stkoff(), "matching locals");
324 assert(ex_jvms->sp() == phi_map->_jvms->sp(), "matching stack sizes");
325 assert(ex_jvms->monoff() == phi_map->_jvms->monoff(), "matching JVMS");
326 assert(ex_map->req() == phi_map->req(), "matching maps");
327 uint tos = ex_jvms->stkoff() + ex_jvms->sp();
328 Node* hidden_merge_mark = root();
329 Node* region = phi_map->control();
330 MergeMemNode* phi_mem = phi_map->merged_memory();
331 MergeMemNode* ex_mem = ex_map->merged_memory();
332 if (region->in(0) != hidden_merge_mark) {
333 // The control input is not (yet) a specially-marked region in phi_map.
334 // Make it so, and build some phis.
335 region = new (C, 2) RegionNode(2);
336 _gvn.set_type(region, Type::CONTROL);
337 region->set_req(0, hidden_merge_mark); // marks an internal ex-state
338 region->init_req(1, phi_map->control());
339 phi_map->set_control(region);
340 Node* io_phi = PhiNode::make(region, phi_map->i_o(), Type::ABIO);
341 record_for_igvn(io_phi);
342 _gvn.set_type(io_phi, Type::ABIO);
343 phi_map->set_i_o(io_phi);
344 for (MergeMemStream mms(phi_mem); mms.next_non_empty(); ) {
345 Node* m = mms.memory();
346 Node* m_phi = PhiNode::make(region, m, Type::MEMORY, mms.adr_type(C));
347 record_for_igvn(m_phi);
348 _gvn.set_type(m_phi, Type::MEMORY);
349 mms.set_memory(m_phi);
350 }
351 }
353 // Either or both of phi_map and ex_map might already be converted into phis.
354 Node* ex_control = ex_map->control();
355 // if there is special marking on ex_map also, we add multiple edges from src
356 bool add_multiple = (ex_control->in(0) == hidden_merge_mark);
357 // how wide was the destination phi_map, originally?
358 uint orig_width = region->req();
360 if (add_multiple) {
361 add_n_reqs(region, ex_control);
362 add_n_reqs(phi_map->i_o(), ex_map->i_o());
363 } else {
364 // ex_map has no merges, so we just add single edges everywhere
365 add_one_req(region, ex_control);
366 add_one_req(phi_map->i_o(), ex_map->i_o());
367 }
368 for (MergeMemStream mms(phi_mem, ex_mem); mms.next_non_empty2(); ) {
369 if (mms.is_empty()) {
370 // get a copy of the base memory, and patch some inputs into it
371 const TypePtr* adr_type = mms.adr_type(C);
372 Node* phi = mms.force_memory()->as_Phi()->slice_memory(adr_type);
373 assert(phi->as_Phi()->region() == mms.base_memory()->in(0), "");
374 mms.set_memory(phi);
375 // Prepare to append interesting stuff onto the newly sliced phi:
376 while (phi->req() > orig_width) phi->del_req(phi->req()-1);
377 }
378 // Append stuff from ex_map:
379 if (add_multiple) {
380 add_n_reqs(mms.memory(), mms.memory2());
381 } else {
382 add_one_req(mms.memory(), mms.memory2());
383 }
384 }
385 uint limit = ex_map->req();
386 for (uint i = TypeFunc::Parms; i < limit; i++) {
387 // Skip everything in the JVMS after tos. (The ex_oop follows.)
388 if (i == tos) i = ex_jvms->monoff();
389 Node* src = ex_map->in(i);
390 Node* dst = phi_map->in(i);
391 if (src != dst) {
392 PhiNode* phi;
393 if (dst->in(0) != region) {
394 dst = phi = PhiNode::make(region, dst, _gvn.type(dst));
395 record_for_igvn(phi);
396 _gvn.set_type(phi, phi->type());
397 phi_map->set_req(i, dst);
398 // Prepare to append interesting stuff onto the new phi:
399 while (dst->req() > orig_width) dst->del_req(dst->req()-1);
400 } else {
401 assert(dst->is_Phi(), "nobody else uses a hidden region");
402 phi = (PhiNode*)dst;
403 }
404 if (add_multiple && src->in(0) == ex_control) {
405 // Both are phis.
406 add_n_reqs(dst, src);
407 } else {
408 while (dst->req() < region->req()) add_one_req(dst, src);
409 }
410 const Type* srctype = _gvn.type(src);
411 if (phi->type() != srctype) {
412 const Type* dsttype = phi->type()->meet(srctype);
413 if (phi->type() != dsttype) {
414 phi->set_type(dsttype);
415 _gvn.set_type(phi, dsttype);
416 }
417 }
418 }
419 }
420 }
422 //--------------------------use_exception_state--------------------------------
423 Node* GraphKit::use_exception_state(SafePointNode* phi_map) {
424 if (failing()) { stop(); return top(); }
425 Node* region = phi_map->control();
426 Node* hidden_merge_mark = root();
427 assert(phi_map->jvms()->map() == phi_map, "sanity: 1-1 relation");
428 Node* ex_oop = clear_saved_ex_oop(phi_map);
429 if (region->in(0) == hidden_merge_mark) {
430 // Special marking for internal ex-states. Process the phis now.
431 region->set_req(0, region); // now it's an ordinary region
432 set_jvms(phi_map->jvms()); // ...so now we can use it as a map
433 // Note: Setting the jvms also sets the bci and sp.
434 set_control(_gvn.transform(region));
435 uint tos = jvms()->stkoff() + sp();
436 for (uint i = 1; i < tos; i++) {
437 Node* x = phi_map->in(i);
438 if (x->in(0) == region) {
439 assert(x->is_Phi(), "expected a special phi");
440 phi_map->set_req(i, _gvn.transform(x));
441 }
442 }
443 for (MergeMemStream mms(merged_memory()); mms.next_non_empty(); ) {
444 Node* x = mms.memory();
445 if (x->in(0) == region) {
446 assert(x->is_Phi(), "nobody else uses a hidden region");
447 mms.set_memory(_gvn.transform(x));
448 }
449 }
450 if (ex_oop->in(0) == region) {
451 assert(ex_oop->is_Phi(), "expected a special phi");
452 ex_oop = _gvn.transform(ex_oop);
453 }
454 } else {
455 set_jvms(phi_map->jvms());
456 }
458 assert(!is_hidden_merge(phi_map->control()), "hidden ex. states cleared");
459 assert(!is_hidden_merge(phi_map->i_o()), "hidden ex. states cleared");
460 return ex_oop;
461 }
463 //---------------------------------java_bc-------------------------------------
464 Bytecodes::Code GraphKit::java_bc() const {
465 ciMethod* method = this->method();
466 int bci = this->bci();
467 if (method != NULL && bci != InvocationEntryBci)
468 return method->java_code_at_bci(bci);
469 else
470 return Bytecodes::_illegal;
471 }
473 void GraphKit::uncommon_trap_if_should_post_on_exceptions(Deoptimization::DeoptReason reason,
474 bool must_throw) {
475 // if the exception capability is set, then we will generate code
476 // to check the JavaThread.should_post_on_exceptions flag to see
477 // if we actually need to report exception events (for this
478 // thread). If we don't need to report exception events, we will
479 // take the normal fast path provided by add_exception_events. If
480 // exception event reporting is enabled for this thread, we will
481 // take the uncommon_trap in the BuildCutout below.
483 // first must access the should_post_on_exceptions_flag in this thread's JavaThread
484 Node* jthread = _gvn.transform(new (C, 1) ThreadLocalNode());
485 Node* adr = basic_plus_adr(top(), jthread, in_bytes(JavaThread::should_post_on_exceptions_flag_offset()));
486 Node* should_post_flag = make_load(control(), adr, TypeInt::INT, T_INT, Compile::AliasIdxRaw, false);
488 // Test the should_post_on_exceptions_flag vs. 0
489 Node* chk = _gvn.transform( new (C, 3) CmpINode(should_post_flag, intcon(0)) );
490 Node* tst = _gvn.transform( new (C, 2) BoolNode(chk, BoolTest::eq) );
492 // Branch to slow_path if should_post_on_exceptions_flag was true
493 { BuildCutout unless(this, tst, PROB_MAX);
494 // Do not try anything fancy if we're notifying the VM on every throw.
495 // Cf. case Bytecodes::_athrow in parse2.cpp.
496 uncommon_trap(reason, Deoptimization::Action_none,
497 (ciKlass*)NULL, (char*)NULL, must_throw);
498 }
500 }
502 //------------------------------builtin_throw----------------------------------
503 void GraphKit::builtin_throw(Deoptimization::DeoptReason reason, Node* arg) {
504 bool must_throw = true;
506 if (env()->jvmti_can_post_on_exceptions()) {
507 // check if we must post exception events, take uncommon trap if so
508 uncommon_trap_if_should_post_on_exceptions(reason, must_throw);
509 // here if should_post_on_exceptions is false
510 // continue on with the normal codegen
511 }
513 // If this particular condition has not yet happened at this
514 // bytecode, then use the uncommon trap mechanism, and allow for
515 // a future recompilation if several traps occur here.
516 // If the throw is hot, try to use a more complicated inline mechanism
517 // which keeps execution inside the compiled code.
518 bool treat_throw_as_hot = false;
519 ciMethodData* md = method()->method_data();
521 if (ProfileTraps) {
522 if (too_many_traps(reason)) {
523 treat_throw_as_hot = true;
524 }
525 // (If there is no MDO at all, assume it is early in
526 // execution, and that any deopts are part of the
527 // startup transient, and don't need to be remembered.)
529 // Also, if there is a local exception handler, treat all throws
530 // as hot if there has been at least one in this method.
531 if (C->trap_count(reason) != 0
532 && method()->method_data()->trap_count(reason) != 0
533 && has_ex_handler()) {
534 treat_throw_as_hot = true;
535 }
536 }
538 // If this throw happens frequently, an uncommon trap might cause
539 // a performance pothole. If there is a local exception handler,
540 // and if this particular bytecode appears to be deoptimizing often,
541 // let us handle the throw inline, with a preconstructed instance.
542 // Note: If the deopt count has blown up, the uncommon trap
543 // runtime is going to flush this nmethod, not matter what.
544 if (treat_throw_as_hot
545 && (!StackTraceInThrowable || OmitStackTraceInFastThrow)) {
546 // If the throw is local, we use a pre-existing instance and
547 // punt on the backtrace. This would lead to a missing backtrace
548 // (a repeat of 4292742) if the backtrace object is ever asked
549 // for its backtrace.
550 // Fixing this remaining case of 4292742 requires some flavor of
551 // escape analysis. Leave that for the future.
552 ciInstance* ex_obj = NULL;
553 switch (reason) {
554 case Deoptimization::Reason_null_check:
555 ex_obj = env()->NullPointerException_instance();
556 break;
557 case Deoptimization::Reason_div0_check:
558 ex_obj = env()->ArithmeticException_instance();
559 break;
560 case Deoptimization::Reason_range_check:
561 ex_obj = env()->ArrayIndexOutOfBoundsException_instance();
562 break;
563 case Deoptimization::Reason_class_check:
564 if (java_bc() == Bytecodes::_aastore) {
565 ex_obj = env()->ArrayStoreException_instance();
566 } else {
567 ex_obj = env()->ClassCastException_instance();
568 }
569 break;
570 }
571 if (failing()) { stop(); return; } // exception allocation might fail
572 if (ex_obj != NULL) {
573 // Cheat with a preallocated exception object.
574 if (C->log() != NULL)
575 C->log()->elem("hot_throw preallocated='1' reason='%s'",
576 Deoptimization::trap_reason_name(reason));
577 const TypeInstPtr* ex_con = TypeInstPtr::make(ex_obj);
578 Node* ex_node = _gvn.transform( ConNode::make(C, ex_con) );
580 // Clear the detail message of the preallocated exception object.
581 // Weblogic sometimes mutates the detail message of exceptions
582 // using reflection.
583 int offset = java_lang_Throwable::get_detailMessage_offset();
584 const TypePtr* adr_typ = ex_con->add_offset(offset);
586 Node *adr = basic_plus_adr(ex_node, ex_node, offset);
587 const TypeOopPtr* val_type = TypeOopPtr::make_from_klass(env()->String_klass());
588 Node *store = store_oop_to_object(control(), ex_node, adr, adr_typ, null(), val_type, T_OBJECT);
590 add_exception_state(make_exception_state(ex_node));
591 return;
592 }
593 }
595 // %%% Maybe add entry to OptoRuntime which directly throws the exc.?
596 // It won't be much cheaper than bailing to the interp., since we'll
597 // have to pass up all the debug-info, and the runtime will have to
598 // create the stack trace.
600 // Usual case: Bail to interpreter.
601 // Reserve the right to recompile if we haven't seen anything yet.
603 Deoptimization::DeoptAction action = Deoptimization::Action_maybe_recompile;
604 if (treat_throw_as_hot
605 && (method()->method_data()->trap_recompiled_at(bci())
606 || C->too_many_traps(reason))) {
607 // We cannot afford to take more traps here. Suffer in the interpreter.
608 if (C->log() != NULL)
609 C->log()->elem("hot_throw preallocated='0' reason='%s' mcount='%d'",
610 Deoptimization::trap_reason_name(reason),
611 C->trap_count(reason));
612 action = Deoptimization::Action_none;
613 }
615 // "must_throw" prunes the JVM state to include only the stack, if there
616 // are no local exception handlers. This should cut down on register
617 // allocation time and code size, by drastically reducing the number
618 // of in-edges on the call to the uncommon trap.
620 uncommon_trap(reason, action, (ciKlass*)NULL, (char*)NULL, must_throw);
621 }
624 //----------------------------PreserveJVMState---------------------------------
625 PreserveJVMState::PreserveJVMState(GraphKit* kit, bool clone_map) {
626 debug_only(kit->verify_map());
627 _kit = kit;
628 _map = kit->map(); // preserve the map
629 _sp = kit->sp();
630 kit->set_map(clone_map ? kit->clone_map() : NULL);
631 #ifdef ASSERT
632 _bci = kit->bci();
633 Parse* parser = kit->is_Parse();
634 int block = (parser == NULL || parser->block() == NULL) ? -1 : parser->block()->rpo();
635 _block = block;
636 #endif
637 }
638 PreserveJVMState::~PreserveJVMState() {
639 GraphKit* kit = _kit;
640 #ifdef ASSERT
641 assert(kit->bci() == _bci, "bci must not shift");
642 Parse* parser = kit->is_Parse();
643 int block = (parser == NULL || parser->block() == NULL) ? -1 : parser->block()->rpo();
644 assert(block == _block, "block must not shift");
645 #endif
646 kit->set_map(_map);
647 kit->set_sp(_sp);
648 }
651 //-----------------------------BuildCutout-------------------------------------
652 BuildCutout::BuildCutout(GraphKit* kit, Node* p, float prob, float cnt)
653 : PreserveJVMState(kit)
654 {
655 assert(p->is_Con() || p->is_Bool(), "test must be a bool");
656 SafePointNode* outer_map = _map; // preserved map is caller's
657 SafePointNode* inner_map = kit->map();
658 IfNode* iff = kit->create_and_map_if(outer_map->control(), p, prob, cnt);
659 outer_map->set_control(kit->gvn().transform( new (kit->C, 1) IfTrueNode(iff) ));
660 inner_map->set_control(kit->gvn().transform( new (kit->C, 1) IfFalseNode(iff) ));
661 }
662 BuildCutout::~BuildCutout() {
663 GraphKit* kit = _kit;
664 assert(kit->stopped(), "cutout code must stop, throw, return, etc.");
665 }
667 //---------------------------PreserveReexecuteState----------------------------
668 PreserveReexecuteState::PreserveReexecuteState(GraphKit* kit) {
669 assert(!kit->stopped(), "must call stopped() before");
670 _kit = kit;
671 _sp = kit->sp();
672 _reexecute = kit->jvms()->_reexecute;
673 }
674 PreserveReexecuteState::~PreserveReexecuteState() {
675 if (_kit->stopped()) return;
676 _kit->jvms()->_reexecute = _reexecute;
677 _kit->set_sp(_sp);
678 }
680 //------------------------------clone_map--------------------------------------
681 // Implementation of PreserveJVMState
682 //
683 // Only clone_map(...) here. If this function is only used in the
684 // PreserveJVMState class we may want to get rid of this extra
685 // function eventually and do it all there.
687 SafePointNode* GraphKit::clone_map() {
688 if (map() == NULL) return NULL;
690 // Clone the memory edge first
691 Node* mem = MergeMemNode::make(C, map()->memory());
692 gvn().set_type_bottom(mem);
694 SafePointNode *clonemap = (SafePointNode*)map()->clone();
695 JVMState* jvms = this->jvms();
696 JVMState* clonejvms = jvms->clone_shallow(C);
697 clonemap->set_memory(mem);
698 clonemap->set_jvms(clonejvms);
699 clonejvms->set_map(clonemap);
700 record_for_igvn(clonemap);
701 gvn().set_type_bottom(clonemap);
702 return clonemap;
703 }
706 //-----------------------------set_map_clone-----------------------------------
707 void GraphKit::set_map_clone(SafePointNode* m) {
708 _map = m;
709 _map = clone_map();
710 _map->set_next_exception(NULL);
711 debug_only(verify_map());
712 }
715 //----------------------------kill_dead_locals---------------------------------
716 // Detect any locals which are known to be dead, and force them to top.
717 void GraphKit::kill_dead_locals() {
718 // Consult the liveness information for the locals. If any
719 // of them are unused, then they can be replaced by top(). This
720 // should help register allocation time and cut down on the size
721 // of the deoptimization information.
723 // This call is made from many of the bytecode handling
724 // subroutines called from the Big Switch in do_one_bytecode.
725 // Every bytecode which might include a slow path is responsible
726 // for killing its dead locals. The more consistent we
727 // are about killing deads, the fewer useless phis will be
728 // constructed for them at various merge points.
730 // bci can be -1 (InvocationEntryBci). We return the entry
731 // liveness for the method.
733 if (method() == NULL || method()->code_size() == 0) {
734 // We are building a graph for a call to a native method.
735 // All locals are live.
736 return;
737 }
739 ResourceMark rm;
741 // Consult the liveness information for the locals. If any
742 // of them are unused, then they can be replaced by top(). This
743 // should help register allocation time and cut down on the size
744 // of the deoptimization information.
745 MethodLivenessResult live_locals = method()->liveness_at_bci(bci());
747 int len = (int)live_locals.size();
748 assert(len <= jvms()->loc_size(), "too many live locals");
749 for (int local = 0; local < len; local++) {
750 if (!live_locals.at(local)) {
751 set_local(local, top());
752 }
753 }
754 }
756 #ifdef ASSERT
757 //-------------------------dead_locals_are_killed------------------------------
758 // Return true if all dead locals are set to top in the map.
759 // Used to assert "clean" debug info at various points.
760 bool GraphKit::dead_locals_are_killed() {
761 if (method() == NULL || method()->code_size() == 0) {
762 // No locals need to be dead, so all is as it should be.
763 return true;
764 }
766 // Make sure somebody called kill_dead_locals upstream.
767 ResourceMark rm;
768 for (JVMState* jvms = this->jvms(); jvms != NULL; jvms = jvms->caller()) {
769 if (jvms->loc_size() == 0) continue; // no locals to consult
770 SafePointNode* map = jvms->map();
771 ciMethod* method = jvms->method();
772 int bci = jvms->bci();
773 if (jvms == this->jvms()) {
774 bci = this->bci(); // it might not yet be synched
775 }
776 MethodLivenessResult live_locals = method->liveness_at_bci(bci);
777 int len = (int)live_locals.size();
778 if (!live_locals.is_valid() || len == 0)
779 // This method is trivial, or is poisoned by a breakpoint.
780 return true;
781 assert(len == jvms->loc_size(), "live map consistent with locals map");
782 for (int local = 0; local < len; local++) {
783 if (!live_locals.at(local) && map->local(jvms, local) != top()) {
784 if (PrintMiscellaneous && (Verbose || WizardMode)) {
785 tty->print_cr("Zombie local %d: ", local);
786 jvms->dump();
787 }
788 return false;
789 }
790 }
791 }
792 return true;
793 }
795 #endif //ASSERT
797 // Helper function for enforcing certain bytecodes to reexecute if
798 // deoptimization happens
799 static bool should_reexecute_implied_by_bytecode(JVMState *jvms, bool is_anewarray) {
800 ciMethod* cur_method = jvms->method();
801 int cur_bci = jvms->bci();
802 if (cur_method != NULL && cur_bci != InvocationEntryBci) {
803 Bytecodes::Code code = cur_method->java_code_at_bci(cur_bci);
804 return Interpreter::bytecode_should_reexecute(code) ||
805 is_anewarray && code == Bytecodes::_multianewarray;
806 // Reexecute _multianewarray bytecode which was replaced with
807 // sequence of [a]newarray. See Parse::do_multianewarray().
808 //
809 // Note: interpreter should not have it set since this optimization
810 // is limited by dimensions and guarded by flag so in some cases
811 // multianewarray() runtime calls will be generated and
812 // the bytecode should not be reexecutes (stack will not be reset).
813 } else
814 return false;
815 }
817 // Helper function for adding JVMState and debug information to node
818 void GraphKit::add_safepoint_edges(SafePointNode* call, bool must_throw) {
819 // Add the safepoint edges to the call (or other safepoint).
821 // Make sure dead locals are set to top. This
822 // should help register allocation time and cut down on the size
823 // of the deoptimization information.
824 assert(dead_locals_are_killed(), "garbage in debug info before safepoint");
826 // Walk the inline list to fill in the correct set of JVMState's
827 // Also fill in the associated edges for each JVMState.
829 JVMState* youngest_jvms = sync_jvms();
831 // If we are guaranteed to throw, we can prune everything but the
832 // input to the current bytecode.
833 bool can_prune_locals = false;
834 uint stack_slots_not_pruned = 0;
835 int inputs = 0, depth = 0;
836 if (must_throw) {
837 assert(method() == youngest_jvms->method(), "sanity");
838 if (compute_stack_effects(inputs, depth)) {
839 can_prune_locals = true;
840 stack_slots_not_pruned = inputs;
841 }
842 }
844 if (env()->jvmti_can_access_local_variables()) {
845 // At any safepoint, this method can get breakpointed, which would
846 // then require an immediate deoptimization.
847 can_prune_locals = false; // do not prune locals
848 stack_slots_not_pruned = 0;
849 }
851 // do not scribble on the input jvms
852 JVMState* out_jvms = youngest_jvms->clone_deep(C);
853 call->set_jvms(out_jvms); // Start jvms list for call node
855 // For a known set of bytecodes, the interpreter should reexecute them if
856 // deoptimization happens. We set the reexecute state for them here
857 if (out_jvms->is_reexecute_undefined() && //don't change if already specified
858 should_reexecute_implied_by_bytecode(out_jvms, call->is_AllocateArray())) {
859 out_jvms->set_should_reexecute(true); //NOTE: youngest_jvms not changed
860 }
862 // Presize the call:
863 debug_only(uint non_debug_edges = call->req());
864 call->add_req_batch(top(), youngest_jvms->debug_depth());
865 assert(call->req() == non_debug_edges + youngest_jvms->debug_depth(), "");
867 // Set up edges so that the call looks like this:
868 // Call [state:] ctl io mem fptr retadr
869 // [parms:] parm0 ... parmN
870 // [root:] loc0 ... locN stk0 ... stkSP mon0 obj0 ... monN objN
871 // [...mid:] loc0 ... locN stk0 ... stkSP mon0 obj0 ... monN objN [...]
872 // [young:] loc0 ... locN stk0 ... stkSP mon0 obj0 ... monN objN
873 // Note that caller debug info precedes callee debug info.
875 // Fill pointer walks backwards from "young:" to "root:" in the diagram above:
876 uint debug_ptr = call->req();
878 // Loop over the map input edges associated with jvms, add them
879 // to the call node, & reset all offsets to match call node array.
880 for (JVMState* in_jvms = youngest_jvms; in_jvms != NULL; ) {
881 uint debug_end = debug_ptr;
882 uint debug_start = debug_ptr - in_jvms->debug_size();
883 debug_ptr = debug_start; // back up the ptr
885 uint p = debug_start; // walks forward in [debug_start, debug_end)
886 uint j, k, l;
887 SafePointNode* in_map = in_jvms->map();
888 out_jvms->set_map(call);
890 if (can_prune_locals) {
891 assert(in_jvms->method() == out_jvms->method(), "sanity");
892 // If the current throw can reach an exception handler in this JVMS,
893 // then we must keep everything live that can reach that handler.
894 // As a quick and dirty approximation, we look for any handlers at all.
895 if (in_jvms->method()->has_exception_handlers()) {
896 can_prune_locals = false;
897 }
898 }
900 // Add the Locals
901 k = in_jvms->locoff();
902 l = in_jvms->loc_size();
903 out_jvms->set_locoff(p);
904 if (!can_prune_locals) {
905 for (j = 0; j < l; j++)
906 call->set_req(p++, in_map->in(k+j));
907 } else {
908 p += l; // already set to top above by add_req_batch
909 }
911 // Add the Expression Stack
912 k = in_jvms->stkoff();
913 l = in_jvms->sp();
914 out_jvms->set_stkoff(p);
915 if (!can_prune_locals) {
916 for (j = 0; j < l; j++)
917 call->set_req(p++, in_map->in(k+j));
918 } else if (can_prune_locals && stack_slots_not_pruned != 0) {
919 // Divide stack into {S0,...,S1}, where S0 is set to top.
920 uint s1 = stack_slots_not_pruned;
921 stack_slots_not_pruned = 0; // for next iteration
922 if (s1 > l) s1 = l;
923 uint s0 = l - s1;
924 p += s0; // skip the tops preinstalled by add_req_batch
925 for (j = s0; j < l; j++)
926 call->set_req(p++, in_map->in(k+j));
927 } else {
928 p += l; // already set to top above by add_req_batch
929 }
931 // Add the Monitors
932 k = in_jvms->monoff();
933 l = in_jvms->mon_size();
934 out_jvms->set_monoff(p);
935 for (j = 0; j < l; j++)
936 call->set_req(p++, in_map->in(k+j));
938 // Copy any scalar object fields.
939 k = in_jvms->scloff();
940 l = in_jvms->scl_size();
941 out_jvms->set_scloff(p);
942 for (j = 0; j < l; j++)
943 call->set_req(p++, in_map->in(k+j));
945 // Finish the new jvms.
946 out_jvms->set_endoff(p);
948 assert(out_jvms->endoff() == debug_end, "fill ptr must match");
949 assert(out_jvms->depth() == in_jvms->depth(), "depth must match");
950 assert(out_jvms->loc_size() == in_jvms->loc_size(), "size must match");
951 assert(out_jvms->mon_size() == in_jvms->mon_size(), "size must match");
952 assert(out_jvms->scl_size() == in_jvms->scl_size(), "size must match");
953 assert(out_jvms->debug_size() == in_jvms->debug_size(), "size must match");
955 // Update the two tail pointers in parallel.
956 out_jvms = out_jvms->caller();
957 in_jvms = in_jvms->caller();
958 }
960 assert(debug_ptr == non_debug_edges, "debug info must fit exactly");
962 // Test the correctness of JVMState::debug_xxx accessors:
963 assert(call->jvms()->debug_start() == non_debug_edges, "");
964 assert(call->jvms()->debug_end() == call->req(), "");
965 assert(call->jvms()->debug_depth() == call->req() - non_debug_edges, "");
966 }
968 bool GraphKit::compute_stack_effects(int& inputs, int& depth) {
969 Bytecodes::Code code = java_bc();
970 if (code == Bytecodes::_wide) {
971 code = method()->java_code_at_bci(bci() + 1);
972 }
974 BasicType rtype = T_ILLEGAL;
975 int rsize = 0;
977 if (code != Bytecodes::_illegal) {
978 depth = Bytecodes::depth(code); // checkcast=0, athrow=-1
979 rtype = Bytecodes::result_type(code); // checkcast=P, athrow=V
980 if (rtype < T_CONFLICT)
981 rsize = type2size[rtype];
982 }
984 switch (code) {
985 case Bytecodes::_illegal:
986 return false;
988 case Bytecodes::_ldc:
989 case Bytecodes::_ldc_w:
990 case Bytecodes::_ldc2_w:
991 inputs = 0;
992 break;
994 case Bytecodes::_dup: inputs = 1; break;
995 case Bytecodes::_dup_x1: inputs = 2; break;
996 case Bytecodes::_dup_x2: inputs = 3; break;
997 case Bytecodes::_dup2: inputs = 2; break;
998 case Bytecodes::_dup2_x1: inputs = 3; break;
999 case Bytecodes::_dup2_x2: inputs = 4; break;
1000 case Bytecodes::_swap: inputs = 2; break;
1001 case Bytecodes::_arraylength: inputs = 1; break;
1003 case Bytecodes::_getstatic:
1004 case Bytecodes::_putstatic:
1005 case Bytecodes::_getfield:
1006 case Bytecodes::_putfield:
1007 {
1008 bool is_get = (depth >= 0), is_static = (depth & 1);
1009 bool ignore;
1010 ciBytecodeStream iter(method());
1011 iter.reset_to_bci(bci());
1012 iter.next();
1013 ciField* field = iter.get_field(ignore);
1014 int size = field->type()->size();
1015 inputs = (is_static ? 0 : 1);
1016 if (is_get) {
1017 depth = size - inputs;
1018 } else {
1019 inputs += size; // putxxx pops the value from the stack
1020 depth = - inputs;
1021 }
1022 }
1023 break;
1025 case Bytecodes::_invokevirtual:
1026 case Bytecodes::_invokespecial:
1027 case Bytecodes::_invokestatic:
1028 case Bytecodes::_invokedynamic:
1029 case Bytecodes::_invokeinterface:
1030 {
1031 bool ignore;
1032 ciBytecodeStream iter(method());
1033 iter.reset_to_bci(bci());
1034 iter.next();
1035 ciMethod* method = iter.get_method(ignore);
1036 inputs = method->arg_size_no_receiver();
1037 // Add a receiver argument, maybe:
1038 if (code != Bytecodes::_invokestatic &&
1039 code != Bytecodes::_invokedynamic)
1040 inputs += 1;
1041 // (Do not use ciMethod::arg_size(), because
1042 // it might be an unloaded method, which doesn't
1043 // know whether it is static or not.)
1044 int size = method->return_type()->size();
1045 depth = size - inputs;
1046 }
1047 break;
1049 case Bytecodes::_multianewarray:
1050 {
1051 ciBytecodeStream iter(method());
1052 iter.reset_to_bci(bci());
1053 iter.next();
1054 inputs = iter.get_dimensions();
1055 assert(rsize == 1, "");
1056 depth = rsize - inputs;
1057 }
1058 break;
1060 case Bytecodes::_ireturn:
1061 case Bytecodes::_lreturn:
1062 case Bytecodes::_freturn:
1063 case Bytecodes::_dreturn:
1064 case Bytecodes::_areturn:
1065 assert(rsize = -depth, "");
1066 inputs = rsize;
1067 break;
1069 case Bytecodes::_jsr:
1070 case Bytecodes::_jsr_w:
1071 inputs = 0;
1072 depth = 1; // S.B. depth=1, not zero
1073 break;
1075 default:
1076 // bytecode produces a typed result
1077 inputs = rsize - depth;
1078 assert(inputs >= 0, "");
1079 break;
1080 }
1082 #ifdef ASSERT
1083 // spot check
1084 int outputs = depth + inputs;
1085 assert(outputs >= 0, "sanity");
1086 switch (code) {
1087 case Bytecodes::_checkcast: assert(inputs == 1 && outputs == 1, ""); break;
1088 case Bytecodes::_athrow: assert(inputs == 1 && outputs == 0, ""); break;
1089 case Bytecodes::_aload_0: assert(inputs == 0 && outputs == 1, ""); break;
1090 case Bytecodes::_return: assert(inputs == 0 && outputs == 0, ""); break;
1091 case Bytecodes::_drem: assert(inputs == 4 && outputs == 2, ""); break;
1092 }
1093 #endif //ASSERT
1095 return true;
1096 }
1100 //------------------------------basic_plus_adr---------------------------------
1101 Node* GraphKit::basic_plus_adr(Node* base, Node* ptr, Node* offset) {
1102 // short-circuit a common case
1103 if (offset == intcon(0)) return ptr;
1104 return _gvn.transform( new (C, 4) AddPNode(base, ptr, offset) );
1105 }
1107 Node* GraphKit::ConvI2L(Node* offset) {
1108 // short-circuit a common case
1109 jint offset_con = find_int_con(offset, Type::OffsetBot);
1110 if (offset_con != Type::OffsetBot) {
1111 return longcon((long) offset_con);
1112 }
1113 return _gvn.transform( new (C, 2) ConvI2LNode(offset));
1114 }
1115 Node* GraphKit::ConvL2I(Node* offset) {
1116 // short-circuit a common case
1117 jlong offset_con = find_long_con(offset, (jlong)Type::OffsetBot);
1118 if (offset_con != (jlong)Type::OffsetBot) {
1119 return intcon((int) offset_con);
1120 }
1121 return _gvn.transform( new (C, 2) ConvL2INode(offset));
1122 }
1124 //-------------------------load_object_klass-----------------------------------
1125 Node* GraphKit::load_object_klass(Node* obj) {
1126 // Special-case a fresh allocation to avoid building nodes:
1127 Node* akls = AllocateNode::Ideal_klass(obj, &_gvn);
1128 if (akls != NULL) return akls;
1129 Node* k_adr = basic_plus_adr(obj, oopDesc::klass_offset_in_bytes());
1130 return _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), k_adr, TypeInstPtr::KLASS) );
1131 }
1133 //-------------------------load_array_length-----------------------------------
1134 Node* GraphKit::load_array_length(Node* array) {
1135 // Special-case a fresh allocation to avoid building nodes:
1136 AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(array, &_gvn);
1137 Node *alen;
1138 if (alloc == NULL) {
1139 Node *r_adr = basic_plus_adr(array, arrayOopDesc::length_offset_in_bytes());
1140 alen = _gvn.transform( new (C, 3) LoadRangeNode(0, immutable_memory(), r_adr, TypeInt::POS));
1141 } else {
1142 alen = alloc->Ideal_length();
1143 Node* ccast = alloc->make_ideal_length(_gvn.type(array)->is_oopptr(), &_gvn);
1144 if (ccast != alen) {
1145 alen = _gvn.transform(ccast);
1146 }
1147 }
1148 return alen;
1149 }
1151 //------------------------------do_null_check----------------------------------
1152 // Helper function to do a NULL pointer check. Returned value is
1153 // the incoming address with NULL casted away. You are allowed to use the
1154 // not-null value only if you are control dependent on the test.
1155 extern int explicit_null_checks_inserted,
1156 explicit_null_checks_elided;
1157 Node* GraphKit::null_check_common(Node* value, BasicType type,
1158 // optional arguments for variations:
1159 bool assert_null,
1160 Node* *null_control) {
1161 assert(!assert_null || null_control == NULL, "not both at once");
1162 if (stopped()) return top();
1163 if (!GenerateCompilerNullChecks && !assert_null && null_control == NULL) {
1164 // For some performance testing, we may wish to suppress null checking.
1165 value = cast_not_null(value); // Make it appear to be non-null (4962416).
1166 return value;
1167 }
1168 explicit_null_checks_inserted++;
1170 // Construct NULL check
1171 Node *chk = NULL;
1172 switch(type) {
1173 case T_LONG : chk = new (C, 3) CmpLNode(value, _gvn.zerocon(T_LONG)); break;
1174 case T_INT : chk = new (C, 3) CmpINode( value, _gvn.intcon(0)); break;
1175 case T_ARRAY : // fall through
1176 type = T_OBJECT; // simplify further tests
1177 case T_OBJECT : {
1178 const Type *t = _gvn.type( value );
1180 const TypeOopPtr* tp = t->isa_oopptr();
1181 if (tp != NULL && tp->klass() != NULL && !tp->klass()->is_loaded()
1182 // Only for do_null_check, not any of its siblings:
1183 && !assert_null && null_control == NULL) {
1184 // Usually, any field access or invocation on an unloaded oop type
1185 // will simply fail to link, since the statically linked class is
1186 // likely also to be unloaded. However, in -Xcomp mode, sometimes
1187 // the static class is loaded but the sharper oop type is not.
1188 // Rather than checking for this obscure case in lots of places,
1189 // we simply observe that a null check on an unloaded class
1190 // will always be followed by a nonsense operation, so we
1191 // can just issue the uncommon trap here.
1192 // Our access to the unloaded class will only be correct
1193 // after it has been loaded and initialized, which requires
1194 // a trip through the interpreter.
1195 #ifndef PRODUCT
1196 if (WizardMode) { tty->print("Null check of unloaded "); tp->klass()->print(); tty->cr(); }
1197 #endif
1198 uncommon_trap(Deoptimization::Reason_unloaded,
1199 Deoptimization::Action_reinterpret,
1200 tp->klass(), "!loaded");
1201 return top();
1202 }
1204 if (assert_null) {
1205 // See if the type is contained in NULL_PTR.
1206 // If so, then the value is already null.
1207 if (t->higher_equal(TypePtr::NULL_PTR)) {
1208 explicit_null_checks_elided++;
1209 return value; // Elided null assert quickly!
1210 }
1211 } else {
1212 // See if mixing in the NULL pointer changes type.
1213 // If so, then the NULL pointer was not allowed in the original
1214 // type. In other words, "value" was not-null.
1215 if (t->meet(TypePtr::NULL_PTR) != t) {
1216 // same as: if (!TypePtr::NULL_PTR->higher_equal(t)) ...
1217 explicit_null_checks_elided++;
1218 return value; // Elided null check quickly!
1219 }
1220 }
1221 chk = new (C, 3) CmpPNode( value, null() );
1222 break;
1223 }
1225 default : ShouldNotReachHere();
1226 }
1227 assert(chk != NULL, "sanity check");
1228 chk = _gvn.transform(chk);
1230 BoolTest::mask btest = assert_null ? BoolTest::eq : BoolTest::ne;
1231 BoolNode *btst = new (C, 2) BoolNode( chk, btest);
1232 Node *tst = _gvn.transform( btst );
1234 //-----------
1235 // if peephole optimizations occurred, a prior test existed.
1236 // If a prior test existed, maybe it dominates as we can avoid this test.
1237 if (tst != btst && type == T_OBJECT) {
1238 // At this point we want to scan up the CFG to see if we can
1239 // find an identical test (and so avoid this test altogether).
1240 Node *cfg = control();
1241 int depth = 0;
1242 while( depth < 16 ) { // Limit search depth for speed
1243 if( cfg->Opcode() == Op_IfTrue &&
1244 cfg->in(0)->in(1) == tst ) {
1245 // Found prior test. Use "cast_not_null" to construct an identical
1246 // CastPP (and hence hash to) as already exists for the prior test.
1247 // Return that casted value.
1248 if (assert_null) {
1249 replace_in_map(value, null());
1250 return null(); // do not issue the redundant test
1251 }
1252 Node *oldcontrol = control();
1253 set_control(cfg);
1254 Node *res = cast_not_null(value);
1255 set_control(oldcontrol);
1256 explicit_null_checks_elided++;
1257 return res;
1258 }
1259 cfg = IfNode::up_one_dom(cfg, /*linear_only=*/ true);
1260 if (cfg == NULL) break; // Quit at region nodes
1261 depth++;
1262 }
1263 }
1265 //-----------
1266 // Branch to failure if null
1267 float ok_prob = PROB_MAX; // a priori estimate: nulls never happen
1268 Deoptimization::DeoptReason reason;
1269 if (assert_null)
1270 reason = Deoptimization::Reason_null_assert;
1271 else if (type == T_OBJECT)
1272 reason = Deoptimization::Reason_null_check;
1273 else
1274 reason = Deoptimization::Reason_div0_check;
1276 // %%% Since Reason_unhandled is not recorded on a per-bytecode basis,
1277 // ciMethodData::has_trap_at will return a conservative -1 if any
1278 // must-be-null assertion has failed. This could cause performance
1279 // problems for a method after its first do_null_assert failure.
1280 // Consider using 'Reason_class_check' instead?
1282 // To cause an implicit null check, we set the not-null probability
1283 // to the maximum (PROB_MAX). For an explicit check the probability
1284 // is set to a smaller value.
1285 if (null_control != NULL || too_many_traps(reason)) {
1286 // probability is less likely
1287 ok_prob = PROB_LIKELY_MAG(3);
1288 } else if (!assert_null &&
1289 (ImplicitNullCheckThreshold > 0) &&
1290 method() != NULL &&
1291 (method()->method_data()->trap_count(reason)
1292 >= (uint)ImplicitNullCheckThreshold)) {
1293 ok_prob = PROB_LIKELY_MAG(3);
1294 }
1296 if (null_control != NULL) {
1297 IfNode* iff = create_and_map_if(control(), tst, ok_prob, COUNT_UNKNOWN);
1298 Node* null_true = _gvn.transform( new (C, 1) IfFalseNode(iff));
1299 set_control( _gvn.transform( new (C, 1) IfTrueNode(iff)));
1300 if (null_true == top())
1301 explicit_null_checks_elided++;
1302 (*null_control) = null_true;
1303 } else {
1304 BuildCutout unless(this, tst, ok_prob);
1305 // Check for optimizer eliding test at parse time
1306 if (stopped()) {
1307 // Failure not possible; do not bother making uncommon trap.
1308 explicit_null_checks_elided++;
1309 } else if (assert_null) {
1310 uncommon_trap(reason,
1311 Deoptimization::Action_make_not_entrant,
1312 NULL, "assert_null");
1313 } else {
1314 replace_in_map(value, zerocon(type));
1315 builtin_throw(reason);
1316 }
1317 }
1319 // Must throw exception, fall-thru not possible?
1320 if (stopped()) {
1321 return top(); // No result
1322 }
1324 if (assert_null) {
1325 // Cast obj to null on this path.
1326 replace_in_map(value, zerocon(type));
1327 return zerocon(type);
1328 }
1330 // Cast obj to not-null on this path, if there is no null_control.
1331 // (If there is a null_control, a non-null value may come back to haunt us.)
1332 if (type == T_OBJECT) {
1333 Node* cast = cast_not_null(value, false);
1334 if (null_control == NULL || (*null_control) == top())
1335 replace_in_map(value, cast);
1336 value = cast;
1337 }
1339 return value;
1340 }
1343 //------------------------------cast_not_null----------------------------------
1344 // Cast obj to not-null on this path
1345 Node* GraphKit::cast_not_null(Node* obj, bool do_replace_in_map) {
1346 const Type *t = _gvn.type(obj);
1347 const Type *t_not_null = t->join(TypePtr::NOTNULL);
1348 // Object is already not-null?
1349 if( t == t_not_null ) return obj;
1351 Node *cast = new (C, 2) CastPPNode(obj,t_not_null);
1352 cast->init_req(0, control());
1353 cast = _gvn.transform( cast );
1355 // Scan for instances of 'obj' in the current JVM mapping.
1356 // These instances are known to be not-null after the test.
1357 if (do_replace_in_map)
1358 replace_in_map(obj, cast);
1360 return cast; // Return casted value
1361 }
1364 //--------------------------replace_in_map-------------------------------------
1365 void GraphKit::replace_in_map(Node* old, Node* neww) {
1366 this->map()->replace_edge(old, neww);
1368 // Note: This operation potentially replaces any edge
1369 // on the map. This includes locals, stack, and monitors
1370 // of the current (innermost) JVM state.
1372 // We can consider replacing in caller maps.
1373 // The idea would be that an inlined function's null checks
1374 // can be shared with the entire inlining tree.
1375 // The expense of doing this is that the PreserveJVMState class
1376 // would have to preserve caller states too, with a deep copy.
1377 }
1381 //=============================================================================
1382 //--------------------------------memory---------------------------------------
1383 Node* GraphKit::memory(uint alias_idx) {
1384 MergeMemNode* mem = merged_memory();
1385 Node* p = mem->memory_at(alias_idx);
1386 _gvn.set_type(p, Type::MEMORY); // must be mapped
1387 return p;
1388 }
1390 //-----------------------------reset_memory------------------------------------
1391 Node* GraphKit::reset_memory() {
1392 Node* mem = map()->memory();
1393 // do not use this node for any more parsing!
1394 debug_only( map()->set_memory((Node*)NULL) );
1395 return _gvn.transform( mem );
1396 }
1398 //------------------------------set_all_memory---------------------------------
1399 void GraphKit::set_all_memory(Node* newmem) {
1400 Node* mergemem = MergeMemNode::make(C, newmem);
1401 gvn().set_type_bottom(mergemem);
1402 map()->set_memory(mergemem);
1403 }
1405 //------------------------------set_all_memory_call----------------------------
1406 void GraphKit::set_all_memory_call(Node* call, bool separate_io_proj) {
1407 Node* newmem = _gvn.transform( new (C, 1) ProjNode(call, TypeFunc::Memory, separate_io_proj) );
1408 set_all_memory(newmem);
1409 }
1411 //=============================================================================
1412 //
1413 // parser factory methods for MemNodes
1414 //
1415 // These are layered on top of the factory methods in LoadNode and StoreNode,
1416 // and integrate with the parser's memory state and _gvn engine.
1417 //
1419 // factory methods in "int adr_idx"
1420 Node* GraphKit::make_load(Node* ctl, Node* adr, const Type* t, BasicType bt,
1421 int adr_idx,
1422 bool require_atomic_access) {
1423 assert(adr_idx != Compile::AliasIdxTop, "use other make_load factory" );
1424 const TypePtr* adr_type = NULL; // debug-mode-only argument
1425 debug_only(adr_type = C->get_adr_type(adr_idx));
1426 Node* mem = memory(adr_idx);
1427 Node* ld;
1428 if (require_atomic_access && bt == T_LONG) {
1429 ld = LoadLNode::make_atomic(C, ctl, mem, adr, adr_type, t);
1430 } else {
1431 ld = LoadNode::make(_gvn, ctl, mem, adr, adr_type, t, bt);
1432 }
1433 return _gvn.transform(ld);
1434 }
1436 Node* GraphKit::store_to_memory(Node* ctl, Node* adr, Node *val, BasicType bt,
1437 int adr_idx,
1438 bool require_atomic_access) {
1439 assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
1440 const TypePtr* adr_type = NULL;
1441 debug_only(adr_type = C->get_adr_type(adr_idx));
1442 Node *mem = memory(adr_idx);
1443 Node* st;
1444 if (require_atomic_access && bt == T_LONG) {
1445 st = StoreLNode::make_atomic(C, ctl, mem, adr, adr_type, val);
1446 } else {
1447 st = StoreNode::make(_gvn, ctl, mem, adr, adr_type, val, bt);
1448 }
1449 st = _gvn.transform(st);
1450 set_memory(st, adr_idx);
1451 // Back-to-back stores can only remove intermediate store with DU info
1452 // so push on worklist for optimizer.
1453 if (mem->req() > MemNode::Address && adr == mem->in(MemNode::Address))
1454 record_for_igvn(st);
1456 return st;
1457 }
1460 void GraphKit::pre_barrier(bool do_load,
1461 Node* ctl,
1462 Node* obj,
1463 Node* adr,
1464 uint adr_idx,
1465 Node* val,
1466 const TypeOopPtr* val_type,
1467 Node* pre_val,
1468 BasicType bt) {
1470 BarrierSet* bs = Universe::heap()->barrier_set();
1471 set_control(ctl);
1472 switch (bs->kind()) {
1473 case BarrierSet::G1SATBCT:
1474 case BarrierSet::G1SATBCTLogging:
1475 g1_write_barrier_pre(do_load, obj, adr, adr_idx, val, val_type, pre_val, bt);
1476 break;
1478 case BarrierSet::CardTableModRef:
1479 case BarrierSet::CardTableExtension:
1480 case BarrierSet::ModRef:
1481 break;
1483 case BarrierSet::Other:
1484 default :
1485 ShouldNotReachHere();
1487 }
1488 }
1490 void GraphKit::post_barrier(Node* ctl,
1491 Node* store,
1492 Node* obj,
1493 Node* adr,
1494 uint adr_idx,
1495 Node* val,
1496 BasicType bt,
1497 bool use_precise) {
1498 BarrierSet* bs = Universe::heap()->barrier_set();
1499 set_control(ctl);
1500 switch (bs->kind()) {
1501 case BarrierSet::G1SATBCT:
1502 case BarrierSet::G1SATBCTLogging:
1503 g1_write_barrier_post(store, obj, adr, adr_idx, val, bt, use_precise);
1504 break;
1506 case BarrierSet::CardTableModRef:
1507 case BarrierSet::CardTableExtension:
1508 write_barrier_post(store, obj, adr, adr_idx, val, use_precise);
1509 break;
1511 case BarrierSet::ModRef:
1512 break;
1514 case BarrierSet::Other:
1515 default :
1516 ShouldNotReachHere();
1518 }
1519 }
1521 Node* GraphKit::store_oop(Node* ctl,
1522 Node* obj,
1523 Node* adr,
1524 const TypePtr* adr_type,
1525 Node* val,
1526 const TypeOopPtr* val_type,
1527 BasicType bt,
1528 bool use_precise) {
1530 set_control(ctl);
1531 if (stopped()) return top(); // Dead path ?
1533 assert(bt == T_OBJECT, "sanity");
1534 assert(val != NULL, "not dead path");
1535 uint adr_idx = C->get_alias_index(adr_type);
1536 assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
1538 pre_barrier(true /* do_load */,
1539 control(), obj, adr, adr_idx, val, val_type,
1540 NULL /* pre_val */,
1541 bt);
1543 Node* store = store_to_memory(control(), adr, val, bt, adr_idx);
1544 post_barrier(control(), store, obj, adr, adr_idx, val, bt, use_precise);
1545 return store;
1546 }
1548 // Could be an array or object we don't know at compile time (unsafe ref.)
1549 Node* GraphKit::store_oop_to_unknown(Node* ctl,
1550 Node* obj, // containing obj
1551 Node* adr, // actual adress to store val at
1552 const TypePtr* adr_type,
1553 Node* val,
1554 BasicType bt) {
1555 Compile::AliasType* at = C->alias_type(adr_type);
1556 const TypeOopPtr* val_type = NULL;
1557 if (adr_type->isa_instptr()) {
1558 if (at->field() != NULL) {
1559 // known field. This code is a copy of the do_put_xxx logic.
1560 ciField* field = at->field();
1561 if (!field->type()->is_loaded()) {
1562 val_type = TypeInstPtr::BOTTOM;
1563 } else {
1564 val_type = TypeOopPtr::make_from_klass(field->type()->as_klass());
1565 }
1566 }
1567 } else if (adr_type->isa_aryptr()) {
1568 val_type = adr_type->is_aryptr()->elem()->make_oopptr();
1569 }
1570 if (val_type == NULL) {
1571 val_type = TypeInstPtr::BOTTOM;
1572 }
1573 return store_oop(ctl, obj, adr, adr_type, val, val_type, bt, true);
1574 }
1577 //-------------------------array_element_address-------------------------
1578 Node* GraphKit::array_element_address(Node* ary, Node* idx, BasicType elembt,
1579 const TypeInt* sizetype) {
1580 uint shift = exact_log2(type2aelembytes(elembt));
1581 uint header = arrayOopDesc::base_offset_in_bytes(elembt);
1583 // short-circuit a common case (saves lots of confusing waste motion)
1584 jint idx_con = find_int_con(idx, -1);
1585 if (idx_con >= 0) {
1586 intptr_t offset = header + ((intptr_t)idx_con << shift);
1587 return basic_plus_adr(ary, offset);
1588 }
1590 // must be correct type for alignment purposes
1591 Node* base = basic_plus_adr(ary, header);
1592 #ifdef _LP64
1593 // The scaled index operand to AddP must be a clean 64-bit value.
1594 // Java allows a 32-bit int to be incremented to a negative
1595 // value, which appears in a 64-bit register as a large
1596 // positive number. Using that large positive number as an
1597 // operand in pointer arithmetic has bad consequences.
1598 // On the other hand, 32-bit overflow is rare, and the possibility
1599 // can often be excluded, if we annotate the ConvI2L node with
1600 // a type assertion that its value is known to be a small positive
1601 // number. (The prior range check has ensured this.)
1602 // This assertion is used by ConvI2LNode::Ideal.
1603 int index_max = max_jint - 1; // array size is max_jint, index is one less
1604 if (sizetype != NULL) index_max = sizetype->_hi - 1;
1605 const TypeLong* lidxtype = TypeLong::make(CONST64(0), index_max, Type::WidenMax);
1606 idx = _gvn.transform( new (C, 2) ConvI2LNode(idx, lidxtype) );
1607 #endif
1608 Node* scale = _gvn.transform( new (C, 3) LShiftXNode(idx, intcon(shift)) );
1609 return basic_plus_adr(ary, base, scale);
1610 }
1612 //-------------------------load_array_element-------------------------
1613 Node* GraphKit::load_array_element(Node* ctl, Node* ary, Node* idx, const TypeAryPtr* arytype) {
1614 const Type* elemtype = arytype->elem();
1615 BasicType elembt = elemtype->array_element_basic_type();
1616 Node* adr = array_element_address(ary, idx, elembt, arytype->size());
1617 Node* ld = make_load(ctl, adr, elemtype, elembt, arytype);
1618 return ld;
1619 }
1621 //-------------------------set_arguments_for_java_call-------------------------
1622 // Arguments (pre-popped from the stack) are taken from the JVMS.
1623 void GraphKit::set_arguments_for_java_call(CallJavaNode* call) {
1624 // Add the call arguments:
1625 uint nargs = call->method()->arg_size();
1626 for (uint i = 0; i < nargs; i++) {
1627 Node* arg = argument(i);
1628 call->init_req(i + TypeFunc::Parms, arg);
1629 }
1630 }
1632 //---------------------------set_edges_for_java_call---------------------------
1633 // Connect a newly created call into the current JVMS.
1634 // A return value node (if any) is returned from set_edges_for_java_call.
1635 void GraphKit::set_edges_for_java_call(CallJavaNode* call, bool must_throw, bool separate_io_proj) {
1637 // Add the predefined inputs:
1638 call->init_req( TypeFunc::Control, control() );
1639 call->init_req( TypeFunc::I_O , i_o() );
1640 call->init_req( TypeFunc::Memory , reset_memory() );
1641 call->init_req( TypeFunc::FramePtr, frameptr() );
1642 call->init_req( TypeFunc::ReturnAdr, top() );
1644 add_safepoint_edges(call, must_throw);
1646 Node* xcall = _gvn.transform(call);
1648 if (xcall == top()) {
1649 set_control(top());
1650 return;
1651 }
1652 assert(xcall == call, "call identity is stable");
1654 // Re-use the current map to produce the result.
1656 set_control(_gvn.transform(new (C, 1) ProjNode(call, TypeFunc::Control)));
1657 set_i_o( _gvn.transform(new (C, 1) ProjNode(call, TypeFunc::I_O , separate_io_proj)));
1658 set_all_memory_call(xcall, separate_io_proj);
1660 //return xcall; // no need, caller already has it
1661 }
1663 Node* GraphKit::set_results_for_java_call(CallJavaNode* call, bool separate_io_proj) {
1664 if (stopped()) return top(); // maybe the call folded up?
1666 // Capture the return value, if any.
1667 Node* ret;
1668 if (call->method() == NULL ||
1669 call->method()->return_type()->basic_type() == T_VOID)
1670 ret = top();
1671 else ret = _gvn.transform(new (C, 1) ProjNode(call, TypeFunc::Parms));
1673 // Note: Since any out-of-line call can produce an exception,
1674 // we always insert an I_O projection from the call into the result.
1676 make_slow_call_ex(call, env()->Throwable_klass(), separate_io_proj);
1678 if (separate_io_proj) {
1679 // The caller requested separate projections be used by the fall
1680 // through and exceptional paths, so replace the projections for
1681 // the fall through path.
1682 set_i_o(_gvn.transform( new (C, 1) ProjNode(call, TypeFunc::I_O) ));
1683 set_all_memory(_gvn.transform( new (C, 1) ProjNode(call, TypeFunc::Memory) ));
1684 }
1685 return ret;
1686 }
1688 //--------------------set_predefined_input_for_runtime_call--------------------
1689 // Reading and setting the memory state is way conservative here.
1690 // The real problem is that I am not doing real Type analysis on memory,
1691 // so I cannot distinguish card mark stores from other stores. Across a GC
1692 // point the Store Barrier and the card mark memory has to agree. I cannot
1693 // have a card mark store and its barrier split across the GC point from
1694 // either above or below. Here I get that to happen by reading ALL of memory.
1695 // A better answer would be to separate out card marks from other memory.
1696 // For now, return the input memory state, so that it can be reused
1697 // after the call, if this call has restricted memory effects.
1698 Node* GraphKit::set_predefined_input_for_runtime_call(SafePointNode* call) {
1699 // Set fixed predefined input arguments
1700 Node* memory = reset_memory();
1701 call->init_req( TypeFunc::Control, control() );
1702 call->init_req( TypeFunc::I_O, top() ); // does no i/o
1703 call->init_req( TypeFunc::Memory, memory ); // may gc ptrs
1704 call->init_req( TypeFunc::FramePtr, frameptr() );
1705 call->init_req( TypeFunc::ReturnAdr, top() );
1706 return memory;
1707 }
1709 //-------------------set_predefined_output_for_runtime_call--------------------
1710 // Set control and memory (not i_o) from the call.
1711 // If keep_mem is not NULL, use it for the output state,
1712 // except for the RawPtr output of the call, if hook_mem is TypeRawPtr::BOTTOM.
1713 // If hook_mem is NULL, this call produces no memory effects at all.
1714 // If hook_mem is a Java-visible memory slice (such as arraycopy operands),
1715 // then only that memory slice is taken from the call.
1716 // In the last case, we must put an appropriate memory barrier before
1717 // the call, so as to create the correct anti-dependencies on loads
1718 // preceding the call.
1719 void GraphKit::set_predefined_output_for_runtime_call(Node* call,
1720 Node* keep_mem,
1721 const TypePtr* hook_mem) {
1722 // no i/o
1723 set_control(_gvn.transform( new (C, 1) ProjNode(call,TypeFunc::Control) ));
1724 if (keep_mem) {
1725 // First clone the existing memory state
1726 set_all_memory(keep_mem);
1727 if (hook_mem != NULL) {
1728 // Make memory for the call
1729 Node* mem = _gvn.transform( new (C, 1) ProjNode(call, TypeFunc::Memory) );
1730 // Set the RawPtr memory state only. This covers all the heap top/GC stuff
1731 // We also use hook_mem to extract specific effects from arraycopy stubs.
1732 set_memory(mem, hook_mem);
1733 }
1734 // ...else the call has NO memory effects.
1736 // Make sure the call advertises its memory effects precisely.
1737 // This lets us build accurate anti-dependences in gcm.cpp.
1738 assert(C->alias_type(call->adr_type()) == C->alias_type(hook_mem),
1739 "call node must be constructed correctly");
1740 } else {
1741 assert(hook_mem == NULL, "");
1742 // This is not a "slow path" call; all memory comes from the call.
1743 set_all_memory_call(call);
1744 }
1745 }
1748 // Replace the call with the current state of the kit.
1749 void GraphKit::replace_call(CallNode* call, Node* result) {
1750 JVMState* ejvms = NULL;
1751 if (has_exceptions()) {
1752 ejvms = transfer_exceptions_into_jvms();
1753 }
1755 SafePointNode* final_state = stop();
1757 // Find all the needed outputs of this call
1758 CallProjections callprojs;
1759 call->extract_projections(&callprojs, true);
1761 // Replace all the old call edges with the edges from the inlining result
1762 C->gvn_replace_by(callprojs.fallthrough_catchproj, final_state->in(TypeFunc::Control));
1763 C->gvn_replace_by(callprojs.fallthrough_memproj, final_state->in(TypeFunc::Memory));
1764 C->gvn_replace_by(callprojs.fallthrough_ioproj, final_state->in(TypeFunc::I_O));
1765 Node* final_mem = final_state->in(TypeFunc::Memory);
1767 // Replace the result with the new result if it exists and is used
1768 if (callprojs.resproj != NULL && result != NULL) {
1769 C->gvn_replace_by(callprojs.resproj, result);
1770 }
1772 if (ejvms == NULL) {
1773 // No exception edges to simply kill off those paths
1774 C->gvn_replace_by(callprojs.catchall_catchproj, C->top());
1775 C->gvn_replace_by(callprojs.catchall_memproj, C->top());
1776 C->gvn_replace_by(callprojs.catchall_ioproj, C->top());
1778 // Replace the old exception object with top
1779 if (callprojs.exobj != NULL) {
1780 C->gvn_replace_by(callprojs.exobj, C->top());
1781 }
1782 } else {
1783 GraphKit ekit(ejvms);
1785 // Load my combined exception state into the kit, with all phis transformed:
1786 SafePointNode* ex_map = ekit.combine_and_pop_all_exception_states();
1788 Node* ex_oop = ekit.use_exception_state(ex_map);
1790 C->gvn_replace_by(callprojs.catchall_catchproj, ekit.control());
1791 C->gvn_replace_by(callprojs.catchall_memproj, ekit.reset_memory());
1792 C->gvn_replace_by(callprojs.catchall_ioproj, ekit.i_o());
1794 // Replace the old exception object with the newly created one
1795 if (callprojs.exobj != NULL) {
1796 C->gvn_replace_by(callprojs.exobj, ex_oop);
1797 }
1798 }
1800 // Disconnect the call from the graph
1801 call->disconnect_inputs(NULL);
1802 C->gvn_replace_by(call, C->top());
1804 // Clean up any MergeMems that feed other MergeMems since the
1805 // optimizer doesn't like that.
1806 if (final_mem->is_MergeMem()) {
1807 Node_List wl;
1808 for (SimpleDUIterator i(final_mem); i.has_next(); i.next()) {
1809 Node* m = i.get();
1810 if (m->is_MergeMem() && !wl.contains(m)) {
1811 wl.push(m);
1812 }
1813 }
1814 while (wl.size() > 0) {
1815 _gvn.transform(wl.pop());
1816 }
1817 }
1818 }
1821 //------------------------------increment_counter------------------------------
1822 // for statistics: increment a VM counter by 1
1824 void GraphKit::increment_counter(address counter_addr) {
1825 Node* adr1 = makecon(TypeRawPtr::make(counter_addr));
1826 increment_counter(adr1);
1827 }
1829 void GraphKit::increment_counter(Node* counter_addr) {
1830 int adr_type = Compile::AliasIdxRaw;
1831 Node* ctrl = control();
1832 Node* cnt = make_load(ctrl, counter_addr, TypeInt::INT, T_INT, adr_type);
1833 Node* incr = _gvn.transform(new (C, 3) AddINode(cnt, _gvn.intcon(1)));
1834 store_to_memory( ctrl, counter_addr, incr, T_INT, adr_type );
1835 }
1838 //------------------------------uncommon_trap----------------------------------
1839 // Bail out to the interpreter in mid-method. Implemented by calling the
1840 // uncommon_trap blob. This helper function inserts a runtime call with the
1841 // right debug info.
1842 void GraphKit::uncommon_trap(int trap_request,
1843 ciKlass* klass, const char* comment,
1844 bool must_throw,
1845 bool keep_exact_action) {
1846 if (failing()) stop();
1847 if (stopped()) return; // trap reachable?
1849 // Note: If ProfileTraps is true, and if a deopt. actually
1850 // occurs here, the runtime will make sure an MDO exists. There is
1851 // no need to call method()->ensure_method_data() at this point.
1853 #ifdef ASSERT
1854 if (!must_throw) {
1855 // Make sure the stack has at least enough depth to execute
1856 // the current bytecode.
1857 int inputs, ignore;
1858 if (compute_stack_effects(inputs, ignore)) {
1859 assert(sp() >= inputs, "must have enough JVMS stack to execute");
1860 // It is a frequent error in library_call.cpp to issue an
1861 // uncommon trap with the _sp value already popped.
1862 }
1863 }
1864 #endif
1866 Deoptimization::DeoptReason reason = Deoptimization::trap_request_reason(trap_request);
1867 Deoptimization::DeoptAction action = Deoptimization::trap_request_action(trap_request);
1869 switch (action) {
1870 case Deoptimization::Action_maybe_recompile:
1871 case Deoptimization::Action_reinterpret:
1872 // Temporary fix for 6529811 to allow virtual calls to be sure they
1873 // get the chance to go from mono->bi->mega
1874 if (!keep_exact_action &&
1875 Deoptimization::trap_request_index(trap_request) < 0 &&
1876 too_many_recompiles(reason)) {
1877 // This BCI is causing too many recompilations.
1878 action = Deoptimization::Action_none;
1879 trap_request = Deoptimization::make_trap_request(reason, action);
1880 } else {
1881 C->set_trap_can_recompile(true);
1882 }
1883 break;
1884 case Deoptimization::Action_make_not_entrant:
1885 C->set_trap_can_recompile(true);
1886 break;
1887 #ifdef ASSERT
1888 case Deoptimization::Action_none:
1889 case Deoptimization::Action_make_not_compilable:
1890 break;
1891 default:
1892 assert(false, "bad action");
1893 #endif
1894 }
1896 if (TraceOptoParse) {
1897 char buf[100];
1898 tty->print_cr("Uncommon trap %s at bci:%d",
1899 Deoptimization::format_trap_request(buf, sizeof(buf),
1900 trap_request), bci());
1901 }
1903 CompileLog* log = C->log();
1904 if (log != NULL) {
1905 int kid = (klass == NULL)? -1: log->identify(klass);
1906 log->begin_elem("uncommon_trap bci='%d'", bci());
1907 char buf[100];
1908 log->print(" %s", Deoptimization::format_trap_request(buf, sizeof(buf),
1909 trap_request));
1910 if (kid >= 0) log->print(" klass='%d'", kid);
1911 if (comment != NULL) log->print(" comment='%s'", comment);
1912 log->end_elem();
1913 }
1915 // Make sure any guarding test views this path as very unlikely
1916 Node *i0 = control()->in(0);
1917 if (i0 != NULL && i0->is_If()) { // Found a guarding if test?
1918 IfNode *iff = i0->as_If();
1919 float f = iff->_prob; // Get prob
1920 if (control()->Opcode() == Op_IfTrue) {
1921 if (f > PROB_UNLIKELY_MAG(4))
1922 iff->_prob = PROB_MIN;
1923 } else {
1924 if (f < PROB_LIKELY_MAG(4))
1925 iff->_prob = PROB_MAX;
1926 }
1927 }
1929 // Clear out dead values from the debug info.
1930 kill_dead_locals();
1932 // Now insert the uncommon trap subroutine call
1933 address call_addr = SharedRuntime::uncommon_trap_blob()->entry_point();
1934 const TypePtr* no_memory_effects = NULL;
1935 // Pass the index of the class to be loaded
1936 Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON |
1937 (must_throw ? RC_MUST_THROW : 0),
1938 OptoRuntime::uncommon_trap_Type(),
1939 call_addr, "uncommon_trap", no_memory_effects,
1940 intcon(trap_request));
1941 assert(call->as_CallStaticJava()->uncommon_trap_request() == trap_request,
1942 "must extract request correctly from the graph");
1943 assert(trap_request != 0, "zero value reserved by uncommon_trap_request");
1945 call->set_req(TypeFunc::ReturnAdr, returnadr());
1946 // The debug info is the only real input to this call.
1948 // Halt-and-catch fire here. The above call should never return!
1949 HaltNode* halt = new(C, TypeFunc::Parms) HaltNode(control(), frameptr());
1950 _gvn.set_type_bottom(halt);
1951 root()->add_req(halt);
1953 stop_and_kill_map();
1954 }
1957 //--------------------------just_allocated_object------------------------------
1958 // Report the object that was just allocated.
1959 // It must be the case that there are no intervening safepoints.
1960 // We use this to determine if an object is so "fresh" that
1961 // it does not require card marks.
1962 Node* GraphKit::just_allocated_object(Node* current_control) {
1963 if (C->recent_alloc_ctl() == current_control)
1964 return C->recent_alloc_obj();
1965 return NULL;
1966 }
1969 void GraphKit::round_double_arguments(ciMethod* dest_method) {
1970 // (Note: TypeFunc::make has a cache that makes this fast.)
1971 const TypeFunc* tf = TypeFunc::make(dest_method);
1972 int nargs = tf->_domain->_cnt - TypeFunc::Parms;
1973 for (int j = 0; j < nargs; j++) {
1974 const Type *targ = tf->_domain->field_at(j + TypeFunc::Parms);
1975 if( targ->basic_type() == T_DOUBLE ) {
1976 // If any parameters are doubles, they must be rounded before
1977 // the call, dstore_rounding does gvn.transform
1978 Node *arg = argument(j);
1979 arg = dstore_rounding(arg);
1980 set_argument(j, arg);
1981 }
1982 }
1983 }
1985 void GraphKit::round_double_result(ciMethod* dest_method) {
1986 // A non-strict method may return a double value which has an extended
1987 // exponent, but this must not be visible in a caller which is 'strict'
1988 // If a strict caller invokes a non-strict callee, round a double result
1990 BasicType result_type = dest_method->return_type()->basic_type();
1991 assert( method() != NULL, "must have caller context");
1992 if( result_type == T_DOUBLE && method()->is_strict() && !dest_method->is_strict() ) {
1993 // Destination method's return value is on top of stack
1994 // dstore_rounding() does gvn.transform
1995 Node *result = pop_pair();
1996 result = dstore_rounding(result);
1997 push_pair(result);
1998 }
1999 }
2001 // rounding for strict float precision conformance
2002 Node* GraphKit::precision_rounding(Node* n) {
2003 return UseStrictFP && _method->flags().is_strict()
2004 && UseSSE == 0 && Matcher::strict_fp_requires_explicit_rounding
2005 ? _gvn.transform( new (C, 2) RoundFloatNode(0, n) )
2006 : n;
2007 }
2009 // rounding for strict double precision conformance
2010 Node* GraphKit::dprecision_rounding(Node *n) {
2011 return UseStrictFP && _method->flags().is_strict()
2012 && UseSSE <= 1 && Matcher::strict_fp_requires_explicit_rounding
2013 ? _gvn.transform( new (C, 2) RoundDoubleNode(0, n) )
2014 : n;
2015 }
2017 // rounding for non-strict double stores
2018 Node* GraphKit::dstore_rounding(Node* n) {
2019 return Matcher::strict_fp_requires_explicit_rounding
2020 && UseSSE <= 1
2021 ? _gvn.transform( new (C, 2) RoundDoubleNode(0, n) )
2022 : n;
2023 }
2025 //=============================================================================
2026 // Generate a fast path/slow path idiom. Graph looks like:
2027 // [foo] indicates that 'foo' is a parameter
2028 //
2029 // [in] NULL
2030 // \ /
2031 // CmpP
2032 // Bool ne
2033 // If
2034 // / \
2035 // True False-<2>
2036 // / |
2037 // / cast_not_null
2038 // Load | | ^
2039 // [fast_test] | |
2040 // gvn to opt_test | |
2041 // / \ | <1>
2042 // True False |
2043 // | \\ |
2044 // [slow_call] \[fast_result]
2045 // Ctl Val \ \
2046 // | \ \
2047 // Catch <1> \ \
2048 // / \ ^ \ \
2049 // Ex No_Ex | \ \
2050 // | \ \ | \ <2> \
2051 // ... \ [slow_res] | | \ [null_result]
2052 // \ \--+--+--- | |
2053 // \ | / \ | /
2054 // --------Region Phi
2055 //
2056 //=============================================================================
2057 // Code is structured as a series of driver functions all called 'do_XXX' that
2058 // call a set of helper functions. Helper functions first, then drivers.
2060 //------------------------------null_check_oop---------------------------------
2061 // Null check oop. Set null-path control into Region in slot 3.
2062 // Make a cast-not-nullness use the other not-null control. Return cast.
2063 Node* GraphKit::null_check_oop(Node* value, Node* *null_control,
2064 bool never_see_null) {
2065 // Initial NULL check taken path
2066 (*null_control) = top();
2067 Node* cast = null_check_common(value, T_OBJECT, false, null_control);
2069 // Generate uncommon_trap:
2070 if (never_see_null && (*null_control) != top()) {
2071 // If we see an unexpected null at a check-cast we record it and force a
2072 // recompile; the offending check-cast will be compiled to handle NULLs.
2073 // If we see more than one offending BCI, then all checkcasts in the
2074 // method will be compiled to handle NULLs.
2075 PreserveJVMState pjvms(this);
2076 set_control(*null_control);
2077 replace_in_map(value, null());
2078 uncommon_trap(Deoptimization::Reason_null_check,
2079 Deoptimization::Action_make_not_entrant);
2080 (*null_control) = top(); // NULL path is dead
2081 }
2083 // Cast away null-ness on the result
2084 return cast;
2085 }
2087 //------------------------------opt_iff----------------------------------------
2088 // Optimize the fast-check IfNode. Set the fast-path region slot 2.
2089 // Return slow-path control.
2090 Node* GraphKit::opt_iff(Node* region, Node* iff) {
2091 IfNode *opt_iff = _gvn.transform(iff)->as_If();
2093 // Fast path taken; set region slot 2
2094 Node *fast_taken = _gvn.transform( new (C, 1) IfFalseNode(opt_iff) );
2095 region->init_req(2,fast_taken); // Capture fast-control
2097 // Fast path not-taken, i.e. slow path
2098 Node *slow_taken = _gvn.transform( new (C, 1) IfTrueNode(opt_iff) );
2099 return slow_taken;
2100 }
2102 //-----------------------------make_runtime_call-------------------------------
2103 Node* GraphKit::make_runtime_call(int flags,
2104 const TypeFunc* call_type, address call_addr,
2105 const char* call_name,
2106 const TypePtr* adr_type,
2107 // The following parms are all optional.
2108 // The first NULL ends the list.
2109 Node* parm0, Node* parm1,
2110 Node* parm2, Node* parm3,
2111 Node* parm4, Node* parm5,
2112 Node* parm6, Node* parm7) {
2113 // Slow-path call
2114 int size = call_type->domain()->cnt();
2115 bool is_leaf = !(flags & RC_NO_LEAF);
2116 bool has_io = (!is_leaf && !(flags & RC_NO_IO));
2117 if (call_name == NULL) {
2118 assert(!is_leaf, "must supply name for leaf");
2119 call_name = OptoRuntime::stub_name(call_addr);
2120 }
2121 CallNode* call;
2122 if (!is_leaf) {
2123 call = new(C, size) CallStaticJavaNode(call_type, call_addr, call_name,
2124 bci(), adr_type);
2125 } else if (flags & RC_NO_FP) {
2126 call = new(C, size) CallLeafNoFPNode(call_type, call_addr, call_name, adr_type);
2127 } else {
2128 call = new(C, size) CallLeafNode(call_type, call_addr, call_name, adr_type);
2129 }
2131 // The following is similar to set_edges_for_java_call,
2132 // except that the memory effects of the call are restricted to AliasIdxRaw.
2134 // Slow path call has no side-effects, uses few values
2135 bool wide_in = !(flags & RC_NARROW_MEM);
2136 bool wide_out = (C->get_alias_index(adr_type) == Compile::AliasIdxBot);
2138 Node* prev_mem = NULL;
2139 if (wide_in) {
2140 prev_mem = set_predefined_input_for_runtime_call(call);
2141 } else {
2142 assert(!wide_out, "narrow in => narrow out");
2143 Node* narrow_mem = memory(adr_type);
2144 prev_mem = reset_memory();
2145 map()->set_memory(narrow_mem);
2146 set_predefined_input_for_runtime_call(call);
2147 }
2149 // Hook each parm in order. Stop looking at the first NULL.
2150 if (parm0 != NULL) { call->init_req(TypeFunc::Parms+0, parm0);
2151 if (parm1 != NULL) { call->init_req(TypeFunc::Parms+1, parm1);
2152 if (parm2 != NULL) { call->init_req(TypeFunc::Parms+2, parm2);
2153 if (parm3 != NULL) { call->init_req(TypeFunc::Parms+3, parm3);
2154 if (parm4 != NULL) { call->init_req(TypeFunc::Parms+4, parm4);
2155 if (parm5 != NULL) { call->init_req(TypeFunc::Parms+5, parm5);
2156 if (parm6 != NULL) { call->init_req(TypeFunc::Parms+6, parm6);
2157 if (parm7 != NULL) { call->init_req(TypeFunc::Parms+7, parm7);
2158 /* close each nested if ===> */ } } } } } } } }
2159 assert(call->in(call->req()-1) != NULL, "must initialize all parms");
2161 if (!is_leaf) {
2162 // Non-leaves can block and take safepoints:
2163 add_safepoint_edges(call, ((flags & RC_MUST_THROW) != 0));
2164 }
2165 // Non-leaves can throw exceptions:
2166 if (has_io) {
2167 call->set_req(TypeFunc::I_O, i_o());
2168 }
2170 if (flags & RC_UNCOMMON) {
2171 // Set the count to a tiny probability. Cf. Estimate_Block_Frequency.
2172 // (An "if" probability corresponds roughly to an unconditional count.
2173 // Sort of.)
2174 call->set_cnt(PROB_UNLIKELY_MAG(4));
2175 }
2177 Node* c = _gvn.transform(call);
2178 assert(c == call, "cannot disappear");
2180 if (wide_out) {
2181 // Slow path call has full side-effects.
2182 set_predefined_output_for_runtime_call(call);
2183 } else {
2184 // Slow path call has few side-effects, and/or sets few values.
2185 set_predefined_output_for_runtime_call(call, prev_mem, adr_type);
2186 }
2188 if (has_io) {
2189 set_i_o(_gvn.transform(new (C, 1) ProjNode(call, TypeFunc::I_O)));
2190 }
2191 return call;
2193 }
2195 //------------------------------merge_memory-----------------------------------
2196 // Merge memory from one path into the current memory state.
2197 void GraphKit::merge_memory(Node* new_mem, Node* region, int new_path) {
2198 for (MergeMemStream mms(merged_memory(), new_mem->as_MergeMem()); mms.next_non_empty2(); ) {
2199 Node* old_slice = mms.force_memory();
2200 Node* new_slice = mms.memory2();
2201 if (old_slice != new_slice) {
2202 PhiNode* phi;
2203 if (new_slice->is_Phi() && new_slice->as_Phi()->region() == region) {
2204 phi = new_slice->as_Phi();
2205 #ifdef ASSERT
2206 if (old_slice->is_Phi() && old_slice->as_Phi()->region() == region)
2207 old_slice = old_slice->in(new_path);
2208 // Caller is responsible for ensuring that any pre-existing
2209 // phis are already aware of old memory.
2210 int old_path = (new_path > 1) ? 1 : 2; // choose old_path != new_path
2211 assert(phi->in(old_path) == old_slice, "pre-existing phis OK");
2212 #endif
2213 mms.set_memory(phi);
2214 } else {
2215 phi = PhiNode::make(region, old_slice, Type::MEMORY, mms.adr_type(C));
2216 _gvn.set_type(phi, Type::MEMORY);
2217 phi->set_req(new_path, new_slice);
2218 mms.set_memory(_gvn.transform(phi)); // assume it is complete
2219 }
2220 }
2221 }
2222 }
2224 //------------------------------make_slow_call_ex------------------------------
2225 // Make the exception handler hookups for the slow call
2226 void GraphKit::make_slow_call_ex(Node* call, ciInstanceKlass* ex_klass, bool separate_io_proj) {
2227 if (stopped()) return;
2229 // Make a catch node with just two handlers: fall-through and catch-all
2230 Node* i_o = _gvn.transform( new (C, 1) ProjNode(call, TypeFunc::I_O, separate_io_proj) );
2231 Node* catc = _gvn.transform( new (C, 2) CatchNode(control(), i_o, 2) );
2232 Node* norm = _gvn.transform( new (C, 1) CatchProjNode(catc, CatchProjNode::fall_through_index, CatchProjNode::no_handler_bci) );
2233 Node* excp = _gvn.transform( new (C, 1) CatchProjNode(catc, CatchProjNode::catch_all_index, CatchProjNode::no_handler_bci) );
2235 { PreserveJVMState pjvms(this);
2236 set_control(excp);
2237 set_i_o(i_o);
2239 if (excp != top()) {
2240 // Create an exception state also.
2241 // Use an exact type if the caller has specified a specific exception.
2242 const Type* ex_type = TypeOopPtr::make_from_klass_unique(ex_klass)->cast_to_ptr_type(TypePtr::NotNull);
2243 Node* ex_oop = new (C, 2) CreateExNode(ex_type, control(), i_o);
2244 add_exception_state(make_exception_state(_gvn.transform(ex_oop)));
2245 }
2246 }
2248 // Get the no-exception control from the CatchNode.
2249 set_control(norm);
2250 }
2253 //-------------------------------gen_subtype_check-----------------------------
2254 // Generate a subtyping check. Takes as input the subtype and supertype.
2255 // Returns 2 values: sets the default control() to the true path and returns
2256 // the false path. Only reads invariant memory; sets no (visible) memory.
2257 // The PartialSubtypeCheckNode sets the hidden 1-word cache in the encoding
2258 // but that's not exposed to the optimizer. This call also doesn't take in an
2259 // Object; if you wish to check an Object you need to load the Object's class
2260 // prior to coming here.
2261 Node* GraphKit::gen_subtype_check(Node* subklass, Node* superklass) {
2262 // Fast check for identical types, perhaps identical constants.
2263 // The types can even be identical non-constants, in cases
2264 // involving Array.newInstance, Object.clone, etc.
2265 if (subklass == superklass)
2266 return top(); // false path is dead; no test needed.
2268 if (_gvn.type(superklass)->singleton()) {
2269 ciKlass* superk = _gvn.type(superklass)->is_klassptr()->klass();
2270 ciKlass* subk = _gvn.type(subklass)->is_klassptr()->klass();
2272 // In the common case of an exact superklass, try to fold up the
2273 // test before generating code. You may ask, why not just generate
2274 // the code and then let it fold up? The answer is that the generated
2275 // code will necessarily include null checks, which do not always
2276 // completely fold away. If they are also needless, then they turn
2277 // into a performance loss. Example:
2278 // Foo[] fa = blah(); Foo x = fa[0]; fa[1] = x;
2279 // Here, the type of 'fa' is often exact, so the store check
2280 // of fa[1]=x will fold up, without testing the nullness of x.
2281 switch (static_subtype_check(superk, subk)) {
2282 case SSC_always_false:
2283 {
2284 Node* always_fail = control();
2285 set_control(top());
2286 return always_fail;
2287 }
2288 case SSC_always_true:
2289 return top();
2290 case SSC_easy_test:
2291 {
2292 // Just do a direct pointer compare and be done.
2293 Node* cmp = _gvn.transform( new(C, 3) CmpPNode(subklass, superklass) );
2294 Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::eq) );
2295 IfNode* iff = create_and_xform_if(control(), bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
2296 set_control( _gvn.transform( new(C, 1) IfTrueNode (iff) ) );
2297 return _gvn.transform( new(C, 1) IfFalseNode(iff) );
2298 }
2299 case SSC_full_test:
2300 break;
2301 default:
2302 ShouldNotReachHere();
2303 }
2304 }
2306 // %%% Possible further optimization: Even if the superklass is not exact,
2307 // if the subklass is the unique subtype of the superklass, the check
2308 // will always succeed. We could leave a dependency behind to ensure this.
2310 // First load the super-klass's check-offset
2311 Node *p1 = basic_plus_adr( superklass, superklass, sizeof(oopDesc) + Klass::super_check_offset_offset_in_bytes() );
2312 Node *chk_off = _gvn.transform( new (C, 3) LoadINode( NULL, memory(p1), p1, _gvn.type(p1)->is_ptr() ) );
2313 int cacheoff_con = sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes();
2314 bool might_be_cache = (find_int_con(chk_off, cacheoff_con) == cacheoff_con);
2316 // Load from the sub-klass's super-class display list, or a 1-word cache of
2317 // the secondary superclass list, or a failing value with a sentinel offset
2318 // if the super-klass is an interface or exceptionally deep in the Java
2319 // hierarchy and we have to scan the secondary superclass list the hard way.
2320 // Worst-case type is a little odd: NULL is allowed as a result (usually
2321 // klass loads can never produce a NULL).
2322 Node *chk_off_X = ConvI2X(chk_off);
2323 Node *p2 = _gvn.transform( new (C, 4) AddPNode(subklass,subklass,chk_off_X) );
2324 // For some types like interfaces the following loadKlass is from a 1-word
2325 // cache which is mutable so can't use immutable memory. Other
2326 // types load from the super-class display table which is immutable.
2327 Node *kmem = might_be_cache ? memory(p2) : immutable_memory();
2328 Node *nkls = _gvn.transform( LoadKlassNode::make( _gvn, kmem, p2, _gvn.type(p2)->is_ptr(), TypeKlassPtr::OBJECT_OR_NULL ) );
2330 // Compile speed common case: ARE a subtype and we canNOT fail
2331 if( superklass == nkls )
2332 return top(); // false path is dead; no test needed.
2334 // See if we get an immediate positive hit. Happens roughly 83% of the
2335 // time. Test to see if the value loaded just previously from the subklass
2336 // is exactly the superklass.
2337 Node *cmp1 = _gvn.transform( new (C, 3) CmpPNode( superklass, nkls ) );
2338 Node *bol1 = _gvn.transform( new (C, 2) BoolNode( cmp1, BoolTest::eq ) );
2339 IfNode *iff1 = create_and_xform_if( control(), bol1, PROB_LIKELY(0.83f), COUNT_UNKNOWN );
2340 Node *iftrue1 = _gvn.transform( new (C, 1) IfTrueNode ( iff1 ) );
2341 set_control( _gvn.transform( new (C, 1) IfFalseNode( iff1 ) ) );
2343 // Compile speed common case: Check for being deterministic right now. If
2344 // chk_off is a constant and not equal to cacheoff then we are NOT a
2345 // subklass. In this case we need exactly the 1 test above and we can
2346 // return those results immediately.
2347 if (!might_be_cache) {
2348 Node* not_subtype_ctrl = control();
2349 set_control(iftrue1); // We need exactly the 1 test above
2350 return not_subtype_ctrl;
2351 }
2353 // Gather the various success & failures here
2354 RegionNode *r_ok_subtype = new (C, 4) RegionNode(4);
2355 record_for_igvn(r_ok_subtype);
2356 RegionNode *r_not_subtype = new (C, 3) RegionNode(3);
2357 record_for_igvn(r_not_subtype);
2359 r_ok_subtype->init_req(1, iftrue1);
2361 // Check for immediate negative hit. Happens roughly 11% of the time (which
2362 // is roughly 63% of the remaining cases). Test to see if the loaded
2363 // check-offset points into the subklass display list or the 1-element
2364 // cache. If it points to the display (and NOT the cache) and the display
2365 // missed then it's not a subtype.
2366 Node *cacheoff = _gvn.intcon(cacheoff_con);
2367 Node *cmp2 = _gvn.transform( new (C, 3) CmpINode( chk_off, cacheoff ) );
2368 Node *bol2 = _gvn.transform( new (C, 2) BoolNode( cmp2, BoolTest::ne ) );
2369 IfNode *iff2 = create_and_xform_if( control(), bol2, PROB_LIKELY(0.63f), COUNT_UNKNOWN );
2370 r_not_subtype->init_req(1, _gvn.transform( new (C, 1) IfTrueNode (iff2) ) );
2371 set_control( _gvn.transform( new (C, 1) IfFalseNode(iff2) ) );
2373 // Check for self. Very rare to get here, but it is taken 1/3 the time.
2374 // No performance impact (too rare) but allows sharing of secondary arrays
2375 // which has some footprint reduction.
2376 Node *cmp3 = _gvn.transform( new (C, 3) CmpPNode( subklass, superklass ) );
2377 Node *bol3 = _gvn.transform( new (C, 2) BoolNode( cmp3, BoolTest::eq ) );
2378 IfNode *iff3 = create_and_xform_if( control(), bol3, PROB_LIKELY(0.36f), COUNT_UNKNOWN );
2379 r_ok_subtype->init_req(2, _gvn.transform( new (C, 1) IfTrueNode ( iff3 ) ) );
2380 set_control( _gvn.transform( new (C, 1) IfFalseNode( iff3 ) ) );
2382 // -- Roads not taken here: --
2383 // We could also have chosen to perform the self-check at the beginning
2384 // of this code sequence, as the assembler does. This would not pay off
2385 // the same way, since the optimizer, unlike the assembler, can perform
2386 // static type analysis to fold away many successful self-checks.
2387 // Non-foldable self checks work better here in second position, because
2388 // the initial primary superclass check subsumes a self-check for most
2389 // types. An exception would be a secondary type like array-of-interface,
2390 // which does not appear in its own primary supertype display.
2391 // Finally, we could have chosen to move the self-check into the
2392 // PartialSubtypeCheckNode, and from there out-of-line in a platform
2393 // dependent manner. But it is worthwhile to have the check here,
2394 // where it can be perhaps be optimized. The cost in code space is
2395 // small (register compare, branch).
2397 // Now do a linear scan of the secondary super-klass array. Again, no real
2398 // performance impact (too rare) but it's gotta be done.
2399 // Since the code is rarely used, there is no penalty for moving it
2400 // out of line, and it can only improve I-cache density.
2401 // The decision to inline or out-of-line this final check is platform
2402 // dependent, and is found in the AD file definition of PartialSubtypeCheck.
2403 Node* psc = _gvn.transform(
2404 new (C, 3) PartialSubtypeCheckNode(control(), subklass, superklass) );
2406 Node *cmp4 = _gvn.transform( new (C, 3) CmpPNode( psc, null() ) );
2407 Node *bol4 = _gvn.transform( new (C, 2) BoolNode( cmp4, BoolTest::ne ) );
2408 IfNode *iff4 = create_and_xform_if( control(), bol4, PROB_FAIR, COUNT_UNKNOWN );
2409 r_not_subtype->init_req(2, _gvn.transform( new (C, 1) IfTrueNode (iff4) ) );
2410 r_ok_subtype ->init_req(3, _gvn.transform( new (C, 1) IfFalseNode(iff4) ) );
2412 // Return false path; set default control to true path.
2413 set_control( _gvn.transform(r_ok_subtype) );
2414 return _gvn.transform(r_not_subtype);
2415 }
2417 //----------------------------static_subtype_check-----------------------------
2418 // Shortcut important common cases when superklass is exact:
2419 // (0) superklass is java.lang.Object (can occur in reflective code)
2420 // (1) subklass is already limited to a subtype of superklass => always ok
2421 // (2) subklass does not overlap with superklass => always fail
2422 // (3) superklass has NO subtypes and we can check with a simple compare.
2423 int GraphKit::static_subtype_check(ciKlass* superk, ciKlass* subk) {
2424 if (StressReflectiveCode) {
2425 return SSC_full_test; // Let caller generate the general case.
2426 }
2428 if (superk == env()->Object_klass()) {
2429 return SSC_always_true; // (0) this test cannot fail
2430 }
2432 ciType* superelem = superk;
2433 if (superelem->is_array_klass())
2434 superelem = superelem->as_array_klass()->base_element_type();
2436 if (!subk->is_interface()) { // cannot trust static interface types yet
2437 if (subk->is_subtype_of(superk)) {
2438 return SSC_always_true; // (1) false path dead; no dynamic test needed
2439 }
2440 if (!(superelem->is_klass() && superelem->as_klass()->is_interface()) &&
2441 !superk->is_subtype_of(subk)) {
2442 return SSC_always_false;
2443 }
2444 }
2446 // If casting to an instance klass, it must have no subtypes
2447 if (superk->is_interface()) {
2448 // Cannot trust interfaces yet.
2449 // %%% S.B. superk->nof_implementors() == 1
2450 } else if (superelem->is_instance_klass()) {
2451 ciInstanceKlass* ik = superelem->as_instance_klass();
2452 if (!ik->has_subklass() && !ik->is_interface()) {
2453 if (!ik->is_final()) {
2454 // Add a dependency if there is a chance of a later subclass.
2455 C->dependencies()->assert_leaf_type(ik);
2456 }
2457 return SSC_easy_test; // (3) caller can do a simple ptr comparison
2458 }
2459 } else {
2460 // A primitive array type has no subtypes.
2461 return SSC_easy_test; // (3) caller can do a simple ptr comparison
2462 }
2464 return SSC_full_test;
2465 }
2467 // Profile-driven exact type check:
2468 Node* GraphKit::type_check_receiver(Node* receiver, ciKlass* klass,
2469 float prob,
2470 Node* *casted_receiver) {
2471 const TypeKlassPtr* tklass = TypeKlassPtr::make(klass);
2472 Node* recv_klass = load_object_klass(receiver);
2473 Node* want_klass = makecon(tklass);
2474 Node* cmp = _gvn.transform( new(C, 3) CmpPNode(recv_klass, want_klass) );
2475 Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::eq) );
2476 IfNode* iff = create_and_xform_if(control(), bol, prob, COUNT_UNKNOWN);
2477 set_control( _gvn.transform( new(C, 1) IfTrueNode (iff) ));
2478 Node* fail = _gvn.transform( new(C, 1) IfFalseNode(iff) );
2480 const TypeOopPtr* recv_xtype = tklass->as_instance_type();
2481 assert(recv_xtype->klass_is_exact(), "");
2483 // Subsume downstream occurrences of receiver with a cast to
2484 // recv_xtype, since now we know what the type will be.
2485 Node* cast = new(C, 2) CheckCastPPNode(control(), receiver, recv_xtype);
2486 (*casted_receiver) = _gvn.transform(cast);
2487 // (User must make the replace_in_map call.)
2489 return fail;
2490 }
2493 //------------------------------seems_never_null-------------------------------
2494 // Use null_seen information if it is available from the profile.
2495 // If we see an unexpected null at a type check we record it and force a
2496 // recompile; the offending check will be recompiled to handle NULLs.
2497 // If we see several offending BCIs, then all checks in the
2498 // method will be recompiled.
2499 bool GraphKit::seems_never_null(Node* obj, ciProfileData* data) {
2500 if (UncommonNullCast // Cutout for this technique
2501 && obj != null() // And not the -Xcomp stupid case?
2502 && !too_many_traps(Deoptimization::Reason_null_check)
2503 ) {
2504 if (data == NULL)
2505 // Edge case: no mature data. Be optimistic here.
2506 return true;
2507 // If the profile has not seen a null, assume it won't happen.
2508 assert(java_bc() == Bytecodes::_checkcast ||
2509 java_bc() == Bytecodes::_instanceof ||
2510 java_bc() == Bytecodes::_aastore, "MDO must collect null_seen bit here");
2511 return !data->as_BitData()->null_seen();
2512 }
2513 return false;
2514 }
2516 //------------------------maybe_cast_profiled_receiver-------------------------
2517 // If the profile has seen exactly one type, narrow to exactly that type.
2518 // Subsequent type checks will always fold up.
2519 Node* GraphKit::maybe_cast_profiled_receiver(Node* not_null_obj,
2520 ciProfileData* data,
2521 ciKlass* require_klass) {
2522 if (!UseTypeProfile || !TypeProfileCasts) return NULL;
2523 if (data == NULL) return NULL;
2525 // Make sure we haven't already deoptimized from this tactic.
2526 if (too_many_traps(Deoptimization::Reason_class_check))
2527 return NULL;
2529 // (No, this isn't a call, but it's enough like a virtual call
2530 // to use the same ciMethod accessor to get the profile info...)
2531 ciCallProfile profile = method()->call_profile_at_bci(bci());
2532 if (profile.count() >= 0 && // no cast failures here
2533 profile.has_receiver(0) &&
2534 profile.morphism() == 1) {
2535 ciKlass* exact_kls = profile.receiver(0);
2536 if (require_klass == NULL ||
2537 static_subtype_check(require_klass, exact_kls) == SSC_always_true) {
2538 // If we narrow the type to match what the type profile sees,
2539 // we can then remove the rest of the cast.
2540 // This is a win, even if the exact_kls is very specific,
2541 // because downstream operations, such as method calls,
2542 // will often benefit from the sharper type.
2543 Node* exact_obj = not_null_obj; // will get updated in place...
2544 Node* slow_ctl = type_check_receiver(exact_obj, exact_kls, 1.0,
2545 &exact_obj);
2546 { PreserveJVMState pjvms(this);
2547 set_control(slow_ctl);
2548 uncommon_trap(Deoptimization::Reason_class_check,
2549 Deoptimization::Action_maybe_recompile);
2550 }
2551 replace_in_map(not_null_obj, exact_obj);
2552 return exact_obj;
2553 }
2554 // assert(ssc == SSC_always_true)... except maybe the profile lied to us.
2555 }
2557 return NULL;
2558 }
2561 //-------------------------------gen_instanceof--------------------------------
2562 // Generate an instance-of idiom. Used by both the instance-of bytecode
2563 // and the reflective instance-of call.
2564 Node* GraphKit::gen_instanceof(Node* obj, Node* superklass) {
2565 kill_dead_locals(); // Benefit all the uncommon traps
2566 assert( !stopped(), "dead parse path should be checked in callers" );
2567 assert(!TypePtr::NULL_PTR->higher_equal(_gvn.type(superklass)->is_klassptr()),
2568 "must check for not-null not-dead klass in callers");
2570 // Make the merge point
2571 enum { _obj_path = 1, _fail_path, _null_path, PATH_LIMIT };
2572 RegionNode* region = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT);
2573 Node* phi = new(C, PATH_LIMIT) PhiNode(region, TypeInt::BOOL);
2574 C->set_has_split_ifs(true); // Has chance for split-if optimization
2576 ciProfileData* data = NULL;
2577 if (java_bc() == Bytecodes::_instanceof) { // Only for the bytecode
2578 data = method()->method_data()->bci_to_data(bci());
2579 }
2580 bool never_see_null = (ProfileDynamicTypes // aggressive use of profile
2581 && seems_never_null(obj, data));
2583 // Null check; get casted pointer; set region slot 3
2584 Node* null_ctl = top();
2585 Node* not_null_obj = null_check_oop(obj, &null_ctl, never_see_null);
2587 // If not_null_obj is dead, only null-path is taken
2588 if (stopped()) { // Doing instance-of on a NULL?
2589 set_control(null_ctl);
2590 return intcon(0);
2591 }
2592 region->init_req(_null_path, null_ctl);
2593 phi ->init_req(_null_path, intcon(0)); // Set null path value
2594 if (null_ctl == top()) {
2595 // Do this eagerly, so that pattern matches like is_diamond_phi
2596 // will work even during parsing.
2597 assert(_null_path == PATH_LIMIT-1, "delete last");
2598 region->del_req(_null_path);
2599 phi ->del_req(_null_path);
2600 }
2602 if (ProfileDynamicTypes && data != NULL) {
2603 Node* cast_obj = maybe_cast_profiled_receiver(not_null_obj, data, NULL);
2604 if (stopped()) { // Profile disagrees with this path.
2605 set_control(null_ctl); // Null is the only remaining possibility.
2606 return intcon(0);
2607 }
2608 if (cast_obj != NULL)
2609 not_null_obj = cast_obj;
2610 }
2612 // Load the object's klass
2613 Node* obj_klass = load_object_klass(not_null_obj);
2615 // Generate the subtype check
2616 Node* not_subtype_ctrl = gen_subtype_check(obj_klass, superklass);
2618 // Plug in the success path to the general merge in slot 1.
2619 region->init_req(_obj_path, control());
2620 phi ->init_req(_obj_path, intcon(1));
2622 // Plug in the failing path to the general merge in slot 2.
2623 region->init_req(_fail_path, not_subtype_ctrl);
2624 phi ->init_req(_fail_path, intcon(0));
2626 // Return final merged results
2627 set_control( _gvn.transform(region) );
2628 record_for_igvn(region);
2629 return _gvn.transform(phi);
2630 }
2632 //-------------------------------gen_checkcast---------------------------------
2633 // Generate a checkcast idiom. Used by both the checkcast bytecode and the
2634 // array store bytecode. Stack must be as-if BEFORE doing the bytecode so the
2635 // uncommon-trap paths work. Adjust stack after this call.
2636 // If failure_control is supplied and not null, it is filled in with
2637 // the control edge for the cast failure. Otherwise, an appropriate
2638 // uncommon trap or exception is thrown.
2639 Node* GraphKit::gen_checkcast(Node *obj, Node* superklass,
2640 Node* *failure_control) {
2641 kill_dead_locals(); // Benefit all the uncommon traps
2642 const TypeKlassPtr *tk = _gvn.type(superklass)->is_klassptr();
2643 const Type *toop = TypeOopPtr::make_from_klass(tk->klass());
2645 // Fast cutout: Check the case that the cast is vacuously true.
2646 // This detects the common cases where the test will short-circuit
2647 // away completely. We do this before we perform the null check,
2648 // because if the test is going to turn into zero code, we don't
2649 // want a residual null check left around. (Causes a slowdown,
2650 // for example, in some objArray manipulations, such as a[i]=a[j].)
2651 if (tk->singleton()) {
2652 const TypeOopPtr* objtp = _gvn.type(obj)->isa_oopptr();
2653 if (objtp != NULL && objtp->klass() != NULL) {
2654 switch (static_subtype_check(tk->klass(), objtp->klass())) {
2655 case SSC_always_true:
2656 return obj;
2657 case SSC_always_false:
2658 // It needs a null check because a null will *pass* the cast check.
2659 // A non-null value will always produce an exception.
2660 return do_null_assert(obj, T_OBJECT);
2661 }
2662 }
2663 }
2665 ciProfileData* data = NULL;
2666 if (failure_control == NULL) { // use MDO in regular case only
2667 assert(java_bc() == Bytecodes::_aastore ||
2668 java_bc() == Bytecodes::_checkcast,
2669 "interpreter profiles type checks only for these BCs");
2670 data = method()->method_data()->bci_to_data(bci());
2671 }
2673 // Make the merge point
2674 enum { _obj_path = 1, _null_path, PATH_LIMIT };
2675 RegionNode* region = new (C, PATH_LIMIT) RegionNode(PATH_LIMIT);
2676 Node* phi = new (C, PATH_LIMIT) PhiNode(region, toop);
2677 C->set_has_split_ifs(true); // Has chance for split-if optimization
2679 // Use null-cast information if it is available
2680 bool never_see_null = ((failure_control == NULL) // regular case only
2681 && seems_never_null(obj, data));
2683 // Null check; get casted pointer; set region slot 3
2684 Node* null_ctl = top();
2685 Node* not_null_obj = null_check_oop(obj, &null_ctl, never_see_null);
2687 // If not_null_obj is dead, only null-path is taken
2688 if (stopped()) { // Doing instance-of on a NULL?
2689 set_control(null_ctl);
2690 return null();
2691 }
2692 region->init_req(_null_path, null_ctl);
2693 phi ->init_req(_null_path, null()); // Set null path value
2694 if (null_ctl == top()) {
2695 // Do this eagerly, so that pattern matches like is_diamond_phi
2696 // will work even during parsing.
2697 assert(_null_path == PATH_LIMIT-1, "delete last");
2698 region->del_req(_null_path);
2699 phi ->del_req(_null_path);
2700 }
2702 Node* cast_obj = NULL;
2703 if (data != NULL &&
2704 // Counter has never been decremented (due to cast failure).
2705 // ...This is a reasonable thing to expect. It is true of
2706 // all casts inserted by javac to implement generic types.
2707 data->as_CounterData()->count() >= 0) {
2708 cast_obj = maybe_cast_profiled_receiver(not_null_obj, data, tk->klass());
2709 if (cast_obj != NULL) {
2710 if (failure_control != NULL) // failure is now impossible
2711 (*failure_control) = top();
2712 // adjust the type of the phi to the exact klass:
2713 phi->raise_bottom_type(_gvn.type(cast_obj)->meet(TypePtr::NULL_PTR));
2714 }
2715 }
2717 if (cast_obj == NULL) {
2718 // Load the object's klass
2719 Node* obj_klass = load_object_klass(not_null_obj);
2721 // Generate the subtype check
2722 Node* not_subtype_ctrl = gen_subtype_check( obj_klass, superklass );
2724 // Plug in success path into the merge
2725 cast_obj = _gvn.transform(new (C, 2) CheckCastPPNode(control(),
2726 not_null_obj, toop));
2727 // Failure path ends in uncommon trap (or may be dead - failure impossible)
2728 if (failure_control == NULL) {
2729 if (not_subtype_ctrl != top()) { // If failure is possible
2730 PreserveJVMState pjvms(this);
2731 set_control(not_subtype_ctrl);
2732 builtin_throw(Deoptimization::Reason_class_check, obj_klass);
2733 }
2734 } else {
2735 (*failure_control) = not_subtype_ctrl;
2736 }
2737 }
2739 region->init_req(_obj_path, control());
2740 phi ->init_req(_obj_path, cast_obj);
2742 // A merge of NULL or Casted-NotNull obj
2743 Node* res = _gvn.transform(phi);
2745 // Note I do NOT always 'replace_in_map(obj,result)' here.
2746 // if( tk->klass()->can_be_primary_super() )
2747 // This means that if I successfully store an Object into an array-of-String
2748 // I 'forget' that the Object is really now known to be a String. I have to
2749 // do this because we don't have true union types for interfaces - if I store
2750 // a Baz into an array-of-Interface and then tell the optimizer it's an
2751 // Interface, I forget that it's also a Baz and cannot do Baz-like field
2752 // references to it. FIX THIS WHEN UNION TYPES APPEAR!
2753 // replace_in_map( obj, res );
2755 // Return final merged results
2756 set_control( _gvn.transform(region) );
2757 record_for_igvn(region);
2758 return res;
2759 }
2761 //------------------------------next_monitor-----------------------------------
2762 // What number should be given to the next monitor?
2763 int GraphKit::next_monitor() {
2764 int current = jvms()->monitor_depth()* C->sync_stack_slots();
2765 int next = current + C->sync_stack_slots();
2766 // Keep the toplevel high water mark current:
2767 if (C->fixed_slots() < next) C->set_fixed_slots(next);
2768 return current;
2769 }
2771 //------------------------------insert_mem_bar---------------------------------
2772 // Memory barrier to avoid floating things around
2773 // The membar serves as a pinch point between both control and all memory slices.
2774 Node* GraphKit::insert_mem_bar(int opcode, Node* precedent) {
2775 MemBarNode* mb = MemBarNode::make(C, opcode, Compile::AliasIdxBot, precedent);
2776 mb->init_req(TypeFunc::Control, control());
2777 mb->init_req(TypeFunc::Memory, reset_memory());
2778 Node* membar = _gvn.transform(mb);
2779 set_control(_gvn.transform(new (C, 1) ProjNode(membar,TypeFunc::Control) ));
2780 set_all_memory_call(membar);
2781 return membar;
2782 }
2784 //-------------------------insert_mem_bar_volatile----------------------------
2785 // Memory barrier to avoid floating things around
2786 // The membar serves as a pinch point between both control and memory(alias_idx).
2787 // If you want to make a pinch point on all memory slices, do not use this
2788 // function (even with AliasIdxBot); use insert_mem_bar() instead.
2789 Node* GraphKit::insert_mem_bar_volatile(int opcode, int alias_idx, Node* precedent) {
2790 // When Parse::do_put_xxx updates a volatile field, it appends a series
2791 // of MemBarVolatile nodes, one for *each* volatile field alias category.
2792 // The first membar is on the same memory slice as the field store opcode.
2793 // This forces the membar to follow the store. (Bug 6500685 broke this.)
2794 // All the other membars (for other volatile slices, including AliasIdxBot,
2795 // which stands for all unknown volatile slices) are control-dependent
2796 // on the first membar. This prevents later volatile loads or stores
2797 // from sliding up past the just-emitted store.
2799 MemBarNode* mb = MemBarNode::make(C, opcode, alias_idx, precedent);
2800 mb->set_req(TypeFunc::Control,control());
2801 if (alias_idx == Compile::AliasIdxBot) {
2802 mb->set_req(TypeFunc::Memory, merged_memory()->base_memory());
2803 } else {
2804 assert(!(opcode == Op_Initialize && alias_idx != Compile::AliasIdxRaw), "fix caller");
2805 mb->set_req(TypeFunc::Memory, memory(alias_idx));
2806 }
2807 Node* membar = _gvn.transform(mb);
2808 set_control(_gvn.transform(new (C, 1) ProjNode(membar, TypeFunc::Control)));
2809 if (alias_idx == Compile::AliasIdxBot) {
2810 merged_memory()->set_base_memory(_gvn.transform(new (C, 1) ProjNode(membar, TypeFunc::Memory)));
2811 } else {
2812 set_memory(_gvn.transform(new (C, 1) ProjNode(membar, TypeFunc::Memory)),alias_idx);
2813 }
2814 return membar;
2815 }
2817 //------------------------------shared_lock------------------------------------
2818 // Emit locking code.
2819 FastLockNode* GraphKit::shared_lock(Node* obj) {
2820 // bci is either a monitorenter bc or InvocationEntryBci
2821 // %%% SynchronizationEntryBCI is redundant; use InvocationEntryBci in interfaces
2822 assert(SynchronizationEntryBCI == InvocationEntryBci, "");
2824 if( !GenerateSynchronizationCode )
2825 return NULL; // Not locking things?
2826 if (stopped()) // Dead monitor?
2827 return NULL;
2829 assert(dead_locals_are_killed(), "should kill locals before sync. point");
2831 // Box the stack location
2832 Node* box = _gvn.transform(new (C, 1) BoxLockNode(next_monitor()));
2833 Node* mem = reset_memory();
2835 FastLockNode * flock = _gvn.transform(new (C, 3) FastLockNode(0, obj, box) )->as_FastLock();
2836 if (PrintPreciseBiasedLockingStatistics) {
2837 // Create the counters for this fast lock.
2838 flock->create_lock_counter(sync_jvms()); // sync_jvms used to get current bci
2839 }
2840 // Add monitor to debug info for the slow path. If we block inside the
2841 // slow path and de-opt, we need the monitor hanging around
2842 map()->push_monitor( flock );
2844 const TypeFunc *tf = LockNode::lock_type();
2845 LockNode *lock = new (C, tf->domain()->cnt()) LockNode(C, tf);
2847 lock->init_req( TypeFunc::Control, control() );
2848 lock->init_req( TypeFunc::Memory , mem );
2849 lock->init_req( TypeFunc::I_O , top() ) ; // does no i/o
2850 lock->init_req( TypeFunc::FramePtr, frameptr() );
2851 lock->init_req( TypeFunc::ReturnAdr, top() );
2853 lock->init_req(TypeFunc::Parms + 0, obj);
2854 lock->init_req(TypeFunc::Parms + 1, box);
2855 lock->init_req(TypeFunc::Parms + 2, flock);
2856 add_safepoint_edges(lock);
2858 lock = _gvn.transform( lock )->as_Lock();
2860 // lock has no side-effects, sets few values
2861 set_predefined_output_for_runtime_call(lock, mem, TypeRawPtr::BOTTOM);
2863 insert_mem_bar(Op_MemBarAcquire);
2865 // Add this to the worklist so that the lock can be eliminated
2866 record_for_igvn(lock);
2868 #ifndef PRODUCT
2869 if (PrintLockStatistics) {
2870 // Update the counter for this lock. Don't bother using an atomic
2871 // operation since we don't require absolute accuracy.
2872 lock->create_lock_counter(map()->jvms());
2873 increment_counter(lock->counter()->addr());
2874 }
2875 #endif
2877 return flock;
2878 }
2881 //------------------------------shared_unlock----------------------------------
2882 // Emit unlocking code.
2883 void GraphKit::shared_unlock(Node* box, Node* obj) {
2884 // bci is either a monitorenter bc or InvocationEntryBci
2885 // %%% SynchronizationEntryBCI is redundant; use InvocationEntryBci in interfaces
2886 assert(SynchronizationEntryBCI == InvocationEntryBci, "");
2888 if( !GenerateSynchronizationCode )
2889 return;
2890 if (stopped()) { // Dead monitor?
2891 map()->pop_monitor(); // Kill monitor from debug info
2892 return;
2893 }
2895 // Memory barrier to avoid floating things down past the locked region
2896 insert_mem_bar(Op_MemBarRelease);
2898 const TypeFunc *tf = OptoRuntime::complete_monitor_exit_Type();
2899 UnlockNode *unlock = new (C, tf->domain()->cnt()) UnlockNode(C, tf);
2900 uint raw_idx = Compile::AliasIdxRaw;
2901 unlock->init_req( TypeFunc::Control, control() );
2902 unlock->init_req( TypeFunc::Memory , memory(raw_idx) );
2903 unlock->init_req( TypeFunc::I_O , top() ) ; // does no i/o
2904 unlock->init_req( TypeFunc::FramePtr, frameptr() );
2905 unlock->init_req( TypeFunc::ReturnAdr, top() );
2907 unlock->init_req(TypeFunc::Parms + 0, obj);
2908 unlock->init_req(TypeFunc::Parms + 1, box);
2909 unlock = _gvn.transform(unlock)->as_Unlock();
2911 Node* mem = reset_memory();
2913 // unlock has no side-effects, sets few values
2914 set_predefined_output_for_runtime_call(unlock, mem, TypeRawPtr::BOTTOM);
2916 // Kill monitor from debug info
2917 map()->pop_monitor( );
2918 }
2920 //-------------------------------get_layout_helper-----------------------------
2921 // If the given klass is a constant or known to be an array,
2922 // fetch the constant layout helper value into constant_value
2923 // and return (Node*)NULL. Otherwise, load the non-constant
2924 // layout helper value, and return the node which represents it.
2925 // This two-faced routine is useful because allocation sites
2926 // almost always feature constant types.
2927 Node* GraphKit::get_layout_helper(Node* klass_node, jint& constant_value) {
2928 const TypeKlassPtr* inst_klass = _gvn.type(klass_node)->isa_klassptr();
2929 if (!StressReflectiveCode && inst_klass != NULL) {
2930 ciKlass* klass = inst_klass->klass();
2931 bool xklass = inst_klass->klass_is_exact();
2932 if (xklass || klass->is_array_klass()) {
2933 jint lhelper = klass->layout_helper();
2934 if (lhelper != Klass::_lh_neutral_value) {
2935 constant_value = lhelper;
2936 return (Node*) NULL;
2937 }
2938 }
2939 }
2940 constant_value = Klass::_lh_neutral_value; // put in a known value
2941 Node* lhp = basic_plus_adr(klass_node, klass_node, Klass::layout_helper_offset_in_bytes() + sizeof(oopDesc));
2942 return make_load(NULL, lhp, TypeInt::INT, T_INT);
2943 }
2945 // We just put in an allocate/initialize with a big raw-memory effect.
2946 // Hook selected additional alias categories on the initialization.
2947 static void hook_memory_on_init(GraphKit& kit, int alias_idx,
2948 MergeMemNode* init_in_merge,
2949 Node* init_out_raw) {
2950 DEBUG_ONLY(Node* init_in_raw = init_in_merge->base_memory());
2951 assert(init_in_merge->memory_at(alias_idx) == init_in_raw, "");
2953 Node* prevmem = kit.memory(alias_idx);
2954 init_in_merge->set_memory_at(alias_idx, prevmem);
2955 kit.set_memory(init_out_raw, alias_idx);
2956 }
2958 //---------------------------set_output_for_allocation-------------------------
2959 Node* GraphKit::set_output_for_allocation(AllocateNode* alloc,
2960 const TypeOopPtr* oop_type,
2961 bool raw_mem_only) {
2962 int rawidx = Compile::AliasIdxRaw;
2963 alloc->set_req( TypeFunc::FramePtr, frameptr() );
2964 add_safepoint_edges(alloc);
2965 Node* allocx = _gvn.transform(alloc);
2966 set_control( _gvn.transform(new (C, 1) ProjNode(allocx, TypeFunc::Control) ) );
2967 // create memory projection for i_o
2968 set_memory ( _gvn.transform( new (C, 1) ProjNode(allocx, TypeFunc::Memory, true) ), rawidx );
2969 make_slow_call_ex(allocx, env()->OutOfMemoryError_klass(), true);
2971 // create a memory projection as for the normal control path
2972 Node* malloc = _gvn.transform(new (C, 1) ProjNode(allocx, TypeFunc::Memory));
2973 set_memory(malloc, rawidx);
2975 // a normal slow-call doesn't change i_o, but an allocation does
2976 // we create a separate i_o projection for the normal control path
2977 set_i_o(_gvn.transform( new (C, 1) ProjNode(allocx, TypeFunc::I_O, false) ) );
2978 Node* rawoop = _gvn.transform( new (C, 1) ProjNode(allocx, TypeFunc::Parms) );
2980 // put in an initialization barrier
2981 InitializeNode* init = insert_mem_bar_volatile(Op_Initialize, rawidx,
2982 rawoop)->as_Initialize();
2983 assert(alloc->initialization() == init, "2-way macro link must work");
2984 assert(init ->allocation() == alloc, "2-way macro link must work");
2985 if (ReduceFieldZeroing && !raw_mem_only) {
2986 // Extract memory strands which may participate in the new object's
2987 // initialization, and source them from the new InitializeNode.
2988 // This will allow us to observe initializations when they occur,
2989 // and link them properly (as a group) to the InitializeNode.
2990 assert(init->in(InitializeNode::Memory) == malloc, "");
2991 MergeMemNode* minit_in = MergeMemNode::make(C, malloc);
2992 init->set_req(InitializeNode::Memory, minit_in);
2993 record_for_igvn(minit_in); // fold it up later, if possible
2994 Node* minit_out = memory(rawidx);
2995 assert(minit_out->is_Proj() && minit_out->in(0) == init, "");
2996 if (oop_type->isa_aryptr()) {
2997 const TypePtr* telemref = oop_type->add_offset(Type::OffsetBot);
2998 int elemidx = C->get_alias_index(telemref);
2999 hook_memory_on_init(*this, elemidx, minit_in, minit_out);
3000 } else if (oop_type->isa_instptr()) {
3001 ciInstanceKlass* ik = oop_type->klass()->as_instance_klass();
3002 for (int i = 0, len = ik->nof_nonstatic_fields(); i < len; i++) {
3003 ciField* field = ik->nonstatic_field_at(i);
3004 if (field->offset() >= TrackedInitializationLimit * HeapWordSize)
3005 continue; // do not bother to track really large numbers of fields
3006 // Find (or create) the alias category for this field:
3007 int fieldidx = C->alias_type(field)->index();
3008 hook_memory_on_init(*this, fieldidx, minit_in, minit_out);
3009 }
3010 }
3011 }
3013 // Cast raw oop to the real thing...
3014 Node* javaoop = new (C, 2) CheckCastPPNode(control(), rawoop, oop_type);
3015 javaoop = _gvn.transform(javaoop);
3016 C->set_recent_alloc(control(), javaoop);
3017 assert(just_allocated_object(control()) == javaoop, "just allocated");
3019 #ifdef ASSERT
3020 { // Verify that the AllocateNode::Ideal_allocation recognizers work:
3021 assert(AllocateNode::Ideal_allocation(rawoop, &_gvn) == alloc,
3022 "Ideal_allocation works");
3023 assert(AllocateNode::Ideal_allocation(javaoop, &_gvn) == alloc,
3024 "Ideal_allocation works");
3025 if (alloc->is_AllocateArray()) {
3026 assert(AllocateArrayNode::Ideal_array_allocation(rawoop, &_gvn) == alloc->as_AllocateArray(),
3027 "Ideal_allocation works");
3028 assert(AllocateArrayNode::Ideal_array_allocation(javaoop, &_gvn) == alloc->as_AllocateArray(),
3029 "Ideal_allocation works");
3030 } else {
3031 assert(alloc->in(AllocateNode::ALength)->is_top(), "no length, please");
3032 }
3033 }
3034 #endif //ASSERT
3036 return javaoop;
3037 }
3039 //---------------------------new_instance--------------------------------------
3040 // This routine takes a klass_node which may be constant (for a static type)
3041 // or may be non-constant (for reflective code). It will work equally well
3042 // for either, and the graph will fold nicely if the optimizer later reduces
3043 // the type to a constant.
3044 // The optional arguments are for specialized use by intrinsics:
3045 // - If 'extra_slow_test' if not null is an extra condition for the slow-path.
3046 // - If 'raw_mem_only', do not cast the result to an oop.
3047 // - If 'return_size_val', report the the total object size to the caller.
3048 Node* GraphKit::new_instance(Node* klass_node,
3049 Node* extra_slow_test,
3050 bool raw_mem_only, // affect only raw memory
3051 Node* *return_size_val) {
3052 // Compute size in doublewords
3053 // The size is always an integral number of doublewords, represented
3054 // as a positive bytewise size stored in the klass's layout_helper.
3055 // The layout_helper also encodes (in a low bit) the need for a slow path.
3056 jint layout_con = Klass::_lh_neutral_value;
3057 Node* layout_val = get_layout_helper(klass_node, layout_con);
3058 int layout_is_con = (layout_val == NULL);
3060 if (extra_slow_test == NULL) extra_slow_test = intcon(0);
3061 // Generate the initial go-slow test. It's either ALWAYS (return a
3062 // Node for 1) or NEVER (return a NULL) or perhaps (in the reflective
3063 // case) a computed value derived from the layout_helper.
3064 Node* initial_slow_test = NULL;
3065 if (layout_is_con) {
3066 assert(!StressReflectiveCode, "stress mode does not use these paths");
3067 bool must_go_slow = Klass::layout_helper_needs_slow_path(layout_con);
3068 initial_slow_test = must_go_slow? intcon(1): extra_slow_test;
3070 } else { // reflective case
3071 // This reflective path is used by Unsafe.allocateInstance.
3072 // (It may be stress-tested by specifying StressReflectiveCode.)
3073 // Basically, we want to get into the VM is there's an illegal argument.
3074 Node* bit = intcon(Klass::_lh_instance_slow_path_bit);
3075 initial_slow_test = _gvn.transform( new (C, 3) AndINode(layout_val, bit) );
3076 if (extra_slow_test != intcon(0)) {
3077 initial_slow_test = _gvn.transform( new (C, 3) OrINode(initial_slow_test, extra_slow_test) );
3078 }
3079 // (Macro-expander will further convert this to a Bool, if necessary.)
3080 }
3082 // Find the size in bytes. This is easy; it's the layout_helper.
3083 // The size value must be valid even if the slow path is taken.
3084 Node* size = NULL;
3085 if (layout_is_con) {
3086 size = MakeConX(Klass::layout_helper_size_in_bytes(layout_con));
3087 } else { // reflective case
3088 // This reflective path is used by clone and Unsafe.allocateInstance.
3089 size = ConvI2X(layout_val);
3091 // Clear the low bits to extract layout_helper_size_in_bytes:
3092 assert((int)Klass::_lh_instance_slow_path_bit < BytesPerLong, "clear bit");
3093 Node* mask = MakeConX(~ (intptr_t)right_n_bits(LogBytesPerLong));
3094 size = _gvn.transform( new (C, 3) AndXNode(size, mask) );
3095 }
3096 if (return_size_val != NULL) {
3097 (*return_size_val) = size;
3098 }
3100 // This is a precise notnull oop of the klass.
3101 // (Actually, it need not be precise if this is a reflective allocation.)
3102 // It's what we cast the result to.
3103 const TypeKlassPtr* tklass = _gvn.type(klass_node)->isa_klassptr();
3104 if (!tklass) tklass = TypeKlassPtr::OBJECT;
3105 const TypeOopPtr* oop_type = tklass->as_instance_type();
3107 // Now generate allocation code
3109 // The entire memory state is needed for slow path of the allocation
3110 // since GC and deoptimization can happened.
3111 Node *mem = reset_memory();
3112 set_all_memory(mem); // Create new memory state
3114 AllocateNode* alloc
3115 = new (C, AllocateNode::ParmLimit)
3116 AllocateNode(C, AllocateNode::alloc_type(),
3117 control(), mem, i_o(),
3118 size, klass_node,
3119 initial_slow_test);
3121 return set_output_for_allocation(alloc, oop_type, raw_mem_only);
3122 }
3124 //-------------------------------new_array-------------------------------------
3125 // helper for both newarray and anewarray
3126 // The 'length' parameter is (obviously) the length of the array.
3127 // See comments on new_instance for the meaning of the other arguments.
3128 Node* GraphKit::new_array(Node* klass_node, // array klass (maybe variable)
3129 Node* length, // number of array elements
3130 int nargs, // number of arguments to push back for uncommon trap
3131 bool raw_mem_only, // affect only raw memory
3132 Node* *return_size_val) {
3133 jint layout_con = Klass::_lh_neutral_value;
3134 Node* layout_val = get_layout_helper(klass_node, layout_con);
3135 int layout_is_con = (layout_val == NULL);
3137 if (!layout_is_con && !StressReflectiveCode &&
3138 !too_many_traps(Deoptimization::Reason_class_check)) {
3139 // This is a reflective array creation site.
3140 // Optimistically assume that it is a subtype of Object[],
3141 // so that we can fold up all the address arithmetic.
3142 layout_con = Klass::array_layout_helper(T_OBJECT);
3143 Node* cmp_lh = _gvn.transform( new(C, 3) CmpINode(layout_val, intcon(layout_con)) );
3144 Node* bol_lh = _gvn.transform( new(C, 2) BoolNode(cmp_lh, BoolTest::eq) );
3145 { BuildCutout unless(this, bol_lh, PROB_MAX);
3146 _sp += nargs;
3147 uncommon_trap(Deoptimization::Reason_class_check,
3148 Deoptimization::Action_maybe_recompile);
3149 }
3150 layout_val = NULL;
3151 layout_is_con = true;
3152 }
3154 // Generate the initial go-slow test. Make sure we do not overflow
3155 // if length is huge (near 2Gig) or negative! We do not need
3156 // exact double-words here, just a close approximation of needed
3157 // double-words. We can't add any offset or rounding bits, lest we
3158 // take a size -1 of bytes and make it positive. Use an unsigned
3159 // compare, so negative sizes look hugely positive.
3160 int fast_size_limit = FastAllocateSizeLimit;
3161 if (layout_is_con) {
3162 assert(!StressReflectiveCode, "stress mode does not use these paths");
3163 // Increase the size limit if we have exact knowledge of array type.
3164 int log2_esize = Klass::layout_helper_log2_element_size(layout_con);
3165 fast_size_limit <<= (LogBytesPerLong - log2_esize);
3166 }
3168 Node* initial_slow_cmp = _gvn.transform( new (C, 3) CmpUNode( length, intcon( fast_size_limit ) ) );
3169 Node* initial_slow_test = _gvn.transform( new (C, 2) BoolNode( initial_slow_cmp, BoolTest::gt ) );
3170 if (initial_slow_test->is_Bool()) {
3171 // Hide it behind a CMoveI, or else PhaseIdealLoop::split_up will get sick.
3172 initial_slow_test = initial_slow_test->as_Bool()->as_int_value(&_gvn);
3173 }
3175 // --- Size Computation ---
3176 // array_size = round_to_heap(array_header + (length << elem_shift));
3177 // where round_to_heap(x) == round_to(x, MinObjAlignmentInBytes)
3178 // and round_to(x, y) == ((x + y-1) & ~(y-1))
3179 // The rounding mask is strength-reduced, if possible.
3180 int round_mask = MinObjAlignmentInBytes - 1;
3181 Node* header_size = NULL;
3182 int header_size_min = arrayOopDesc::base_offset_in_bytes(T_BYTE);
3183 // (T_BYTE has the weakest alignment and size restrictions...)
3184 if (layout_is_con) {
3185 int hsize = Klass::layout_helper_header_size(layout_con);
3186 int eshift = Klass::layout_helper_log2_element_size(layout_con);
3187 BasicType etype = Klass::layout_helper_element_type(layout_con);
3188 if ((round_mask & ~right_n_bits(eshift)) == 0)
3189 round_mask = 0; // strength-reduce it if it goes away completely
3190 assert((hsize & right_n_bits(eshift)) == 0, "hsize is pre-rounded");
3191 assert(header_size_min <= hsize, "generic minimum is smallest");
3192 header_size_min = hsize;
3193 header_size = intcon(hsize + round_mask);
3194 } else {
3195 Node* hss = intcon(Klass::_lh_header_size_shift);
3196 Node* hsm = intcon(Klass::_lh_header_size_mask);
3197 Node* hsize = _gvn.transform( new(C, 3) URShiftINode(layout_val, hss) );
3198 hsize = _gvn.transform( new(C, 3) AndINode(hsize, hsm) );
3199 Node* mask = intcon(round_mask);
3200 header_size = _gvn.transform( new(C, 3) AddINode(hsize, mask) );
3201 }
3203 Node* elem_shift = NULL;
3204 if (layout_is_con) {
3205 int eshift = Klass::layout_helper_log2_element_size(layout_con);
3206 if (eshift != 0)
3207 elem_shift = intcon(eshift);
3208 } else {
3209 // There is no need to mask or shift this value.
3210 // The semantics of LShiftINode include an implicit mask to 0x1F.
3211 assert(Klass::_lh_log2_element_size_shift == 0, "use shift in place");
3212 elem_shift = layout_val;
3213 }
3215 // Transition to native address size for all offset calculations:
3216 Node* lengthx = ConvI2X(length);
3217 Node* headerx = ConvI2X(header_size);
3218 #ifdef _LP64
3219 { const TypeLong* tllen = _gvn.find_long_type(lengthx);
3220 if (tllen != NULL && tllen->_lo < 0) {
3221 // Add a manual constraint to a positive range. Cf. array_element_address.
3222 jlong size_max = arrayOopDesc::max_array_length(T_BYTE);
3223 if (size_max > tllen->_hi) size_max = tllen->_hi;
3224 const TypeLong* tlcon = TypeLong::make(CONST64(0), size_max, Type::WidenMin);
3225 lengthx = _gvn.transform( new (C, 2) ConvI2LNode(length, tlcon));
3226 }
3227 }
3228 #endif
3230 // Combine header size (plus rounding) and body size. Then round down.
3231 // This computation cannot overflow, because it is used only in two
3232 // places, one where the length is sharply limited, and the other
3233 // after a successful allocation.
3234 Node* abody = lengthx;
3235 if (elem_shift != NULL)
3236 abody = _gvn.transform( new(C, 3) LShiftXNode(lengthx, elem_shift) );
3237 Node* size = _gvn.transform( new(C, 3) AddXNode(headerx, abody) );
3238 if (round_mask != 0) {
3239 Node* mask = MakeConX(~round_mask);
3240 size = _gvn.transform( new(C, 3) AndXNode(size, mask) );
3241 }
3242 // else if round_mask == 0, the size computation is self-rounding
3244 if (return_size_val != NULL) {
3245 // This is the size
3246 (*return_size_val) = size;
3247 }
3249 // Now generate allocation code
3251 // The entire memory state is needed for slow path of the allocation
3252 // since GC and deoptimization can happened.
3253 Node *mem = reset_memory();
3254 set_all_memory(mem); // Create new memory state
3256 // Create the AllocateArrayNode and its result projections
3257 AllocateArrayNode* alloc
3258 = new (C, AllocateArrayNode::ParmLimit)
3259 AllocateArrayNode(C, AllocateArrayNode::alloc_type(),
3260 control(), mem, i_o(),
3261 size, klass_node,
3262 initial_slow_test,
3263 length);
3265 // Cast to correct type. Note that the klass_node may be constant or not,
3266 // and in the latter case the actual array type will be inexact also.
3267 // (This happens via a non-constant argument to inline_native_newArray.)
3268 // In any case, the value of klass_node provides the desired array type.
3269 const TypeInt* length_type = _gvn.find_int_type(length);
3270 const TypeOopPtr* ary_type = _gvn.type(klass_node)->is_klassptr()->as_instance_type();
3271 if (ary_type->isa_aryptr() && length_type != NULL) {
3272 // Try to get a better type than POS for the size
3273 ary_type = ary_type->is_aryptr()->cast_to_size(length_type);
3274 }
3276 Node* javaoop = set_output_for_allocation(alloc, ary_type, raw_mem_only);
3278 // Cast length on remaining path to be as narrow as possible
3279 if (map()->find_edge(length) >= 0) {
3280 Node* ccast = alloc->make_ideal_length(ary_type, &_gvn);
3281 if (ccast != length) {
3282 _gvn.set_type_bottom(ccast);
3283 record_for_igvn(ccast);
3284 replace_in_map(length, ccast);
3285 }
3286 }
3288 return javaoop;
3289 }
3291 // The following "Ideal_foo" functions are placed here because they recognize
3292 // the graph shapes created by the functions immediately above.
3294 //---------------------------Ideal_allocation----------------------------------
3295 // Given an oop pointer or raw pointer, see if it feeds from an AllocateNode.
3296 AllocateNode* AllocateNode::Ideal_allocation(Node* ptr, PhaseTransform* phase) {
3297 if (ptr == NULL) { // reduce dumb test in callers
3298 return NULL;
3299 }
3300 if (ptr->is_CheckCastPP()) { // strip a raw-to-oop cast
3301 ptr = ptr->in(1);
3302 if (ptr == NULL) return NULL;
3303 }
3304 if (ptr->is_Proj()) {
3305 Node* allo = ptr->in(0);
3306 if (allo != NULL && allo->is_Allocate()) {
3307 return allo->as_Allocate();
3308 }
3309 }
3310 // Report failure to match.
3311 return NULL;
3312 }
3314 // Fancy version which also strips off an offset (and reports it to caller).
3315 AllocateNode* AllocateNode::Ideal_allocation(Node* ptr, PhaseTransform* phase,
3316 intptr_t& offset) {
3317 Node* base = AddPNode::Ideal_base_and_offset(ptr, phase, offset);
3318 if (base == NULL) return NULL;
3319 return Ideal_allocation(base, phase);
3320 }
3322 // Trace Initialize <- Proj[Parm] <- Allocate
3323 AllocateNode* InitializeNode::allocation() {
3324 Node* rawoop = in(InitializeNode::RawAddress);
3325 if (rawoop->is_Proj()) {
3326 Node* alloc = rawoop->in(0);
3327 if (alloc->is_Allocate()) {
3328 return alloc->as_Allocate();
3329 }
3330 }
3331 return NULL;
3332 }
3334 // Trace Allocate -> Proj[Parm] -> Initialize
3335 InitializeNode* AllocateNode::initialization() {
3336 ProjNode* rawoop = proj_out(AllocateNode::RawAddress);
3337 if (rawoop == NULL) return NULL;
3338 for (DUIterator_Fast imax, i = rawoop->fast_outs(imax); i < imax; i++) {
3339 Node* init = rawoop->fast_out(i);
3340 if (init->is_Initialize()) {
3341 assert(init->as_Initialize()->allocation() == this, "2-way link");
3342 return init->as_Initialize();
3343 }
3344 }
3345 return NULL;
3346 }
3348 //----------------------------- loop predicates ---------------------------
3350 //------------------------------add_predicate_impl----------------------------
3351 void GraphKit::add_predicate_impl(Deoptimization::DeoptReason reason, int nargs) {
3352 // Too many traps seen?
3353 if (too_many_traps(reason)) {
3354 #ifdef ASSERT
3355 if (TraceLoopPredicate) {
3356 int tc = C->trap_count(reason);
3357 tty->print("too many traps=%s tcount=%d in ",
3358 Deoptimization::trap_reason_name(reason), tc);
3359 method()->print(); // which method has too many predicate traps
3360 tty->cr();
3361 }
3362 #endif
3363 // We cannot afford to take more traps here,
3364 // do not generate predicate.
3365 return;
3366 }
3368 Node *cont = _gvn.intcon(1);
3369 Node* opq = _gvn.transform(new (C, 2) Opaque1Node(C, cont));
3370 Node *bol = _gvn.transform(new (C, 2) Conv2BNode(opq));
3371 IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
3372 Node* iffalse = _gvn.transform(new (C, 1) IfFalseNode(iff));
3373 C->add_predicate_opaq(opq);
3374 {
3375 PreserveJVMState pjvms(this);
3376 set_control(iffalse);
3377 _sp += nargs;
3378 uncommon_trap(reason, Deoptimization::Action_maybe_recompile);
3379 }
3380 Node* iftrue = _gvn.transform(new (C, 1) IfTrueNode(iff));
3381 set_control(iftrue);
3382 }
3384 //------------------------------add_predicate---------------------------------
3385 void GraphKit::add_predicate(int nargs) {
3386 if (UseLoopPredicate) {
3387 add_predicate_impl(Deoptimization::Reason_predicate, nargs);
3388 }
3389 }
3391 //----------------------------- store barriers ----------------------------
3392 #define __ ideal.
3394 void GraphKit::sync_kit(IdealKit& ideal) {
3395 // Final sync IdealKit and graphKit.
3396 __ drain_delay_transform();
3397 set_all_memory(__ merged_memory());
3398 set_control(__ ctrl());
3399 }
3401 // vanilla/CMS post barrier
3402 // Insert a write-barrier store. This is to let generational GC work; we have
3403 // to flag all oop-stores before the next GC point.
3404 void GraphKit::write_barrier_post(Node* oop_store,
3405 Node* obj,
3406 Node* adr,
3407 uint adr_idx,
3408 Node* val,
3409 bool use_precise) {
3410 // No store check needed if we're storing a NULL or an old object
3411 // (latter case is probably a string constant). The concurrent
3412 // mark sweep garbage collector, however, needs to have all nonNull
3413 // oop updates flagged via card-marks.
3414 if (val != NULL && val->is_Con()) {
3415 // must be either an oop or NULL
3416 const Type* t = val->bottom_type();
3417 if (t == TypePtr::NULL_PTR || t == Type::TOP)
3418 // stores of null never (?) need barriers
3419 return;
3420 ciObject* con = t->is_oopptr()->const_oop();
3421 if (con != NULL
3422 && con->is_perm()
3423 && Universe::heap()->can_elide_permanent_oop_store_barriers())
3424 // no store barrier needed, because no old-to-new ref created
3425 return;
3426 }
3428 if (use_ReduceInitialCardMarks()
3429 && obj == just_allocated_object(control())) {
3430 // We can skip marks on a freshly-allocated object in Eden.
3431 // Keep this code in sync with new_store_pre_barrier() in runtime.cpp.
3432 // That routine informs GC to take appropriate compensating steps,
3433 // upon a slow-path allocation, so as to make this card-mark
3434 // elision safe.
3435 return;
3436 }
3438 if (!use_precise) {
3439 // All card marks for a (non-array) instance are in one place:
3440 adr = obj;
3441 }
3442 // (Else it's an array (or unknown), and we want more precise card marks.)
3443 assert(adr != NULL, "");
3445 IdealKit ideal(gvn(), control(), merged_memory(), true);
3447 // Convert the pointer to an int prior to doing math on it
3448 Node* cast = __ CastPX(__ ctrl(), adr);
3450 // Divide by card size
3451 assert(Universe::heap()->barrier_set()->kind() == BarrierSet::CardTableModRef,
3452 "Only one we handle so far.");
3453 Node* card_offset = __ URShiftX( cast, __ ConI(CardTableModRefBS::card_shift) );
3455 // Combine card table base and card offset
3456 Node* card_adr = __ AddP(__ top(), byte_map_base_node(), card_offset );
3458 // Get the alias_index for raw card-mark memory
3459 int adr_type = Compile::AliasIdxRaw;
3460 // Smash zero into card
3461 Node* zero = __ ConI(0);
3462 BasicType bt = T_BYTE;
3463 if( !UseConcMarkSweepGC ) {
3464 __ store(__ ctrl(), card_adr, zero, bt, adr_type);
3465 } else {
3466 // Specialized path for CM store barrier
3467 __ storeCM(__ ctrl(), card_adr, zero, oop_store, adr_idx, bt, adr_type);
3468 }
3470 // Final sync IdealKit and GraphKit.
3471 sync_kit(ideal);
3472 }
3474 // G1 pre/post barriers
3475 void GraphKit::g1_write_barrier_pre(bool do_load,
3476 Node* obj,
3477 Node* adr,
3478 uint alias_idx,
3479 Node* val,
3480 const TypeOopPtr* val_type,
3481 Node* pre_val,
3482 BasicType bt) {
3484 // Some sanity checks
3485 // Note: val is unused in this routine.
3487 if (do_load) {
3488 // We need to generate the load of the previous value
3489 assert(obj != NULL, "must have a base");
3490 assert(adr != NULL, "where are loading from?");
3491 assert(pre_val == NULL, "loaded already?");
3492 assert(val_type != NULL, "need a type");
3493 } else {
3494 // In this case both val_type and alias_idx are unused.
3495 assert(pre_val != NULL, "must be loaded already");
3496 assert(pre_val->bottom_type()->basic_type() == T_OBJECT, "or we shouldn't be here");
3497 }
3498 assert(bt == T_OBJECT, "or we shouldn't be here");
3500 IdealKit ideal(gvn(), control(), merged_memory(), true);
3502 Node* tls = __ thread(); // ThreadLocalStorage
3504 Node* no_ctrl = NULL;
3505 Node* no_base = __ top();
3506 Node* zero = __ ConI(0);
3508 float likely = PROB_LIKELY(0.999);
3509 float unlikely = PROB_UNLIKELY(0.999);
3511 BasicType active_type = in_bytes(PtrQueue::byte_width_of_active()) == 4 ? T_INT : T_BYTE;
3512 assert(in_bytes(PtrQueue::byte_width_of_active()) == 4 || in_bytes(PtrQueue::byte_width_of_active()) == 1, "flag width");
3514 // Offsets into the thread
3515 const int marking_offset = in_bytes(JavaThread::satb_mark_queue_offset() + // 648
3516 PtrQueue::byte_offset_of_active());
3517 const int index_offset = in_bytes(JavaThread::satb_mark_queue_offset() + // 656
3518 PtrQueue::byte_offset_of_index());
3519 const int buffer_offset = in_bytes(JavaThread::satb_mark_queue_offset() + // 652
3520 PtrQueue::byte_offset_of_buf());
3522 // Now the actual pointers into the thread
3523 Node* marking_adr = __ AddP(no_base, tls, __ ConX(marking_offset));
3524 Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset));
3525 Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset));
3527 // Now some of the values
3528 Node* marking = __ load(__ ctrl(), marking_adr, TypeInt::INT, active_type, Compile::AliasIdxRaw);
3530 // if (!marking)
3531 __ if_then(marking, BoolTest::ne, zero); {
3532 Node* index = __ load(__ ctrl(), index_adr, TypeInt::INT, T_INT, Compile::AliasIdxRaw);
3534 if (do_load) {
3535 // load original value
3536 // alias_idx correct??
3537 pre_val = __ load(no_ctrl, adr, val_type, bt, alias_idx);
3538 }
3540 // if (pre_val != NULL)
3541 __ if_then(pre_val, BoolTest::ne, null()); {
3542 Node* buffer = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
3544 // is the queue for this thread full?
3545 __ if_then(index, BoolTest::ne, zero, likely); {
3547 // decrement the index
3548 Node* next_index = __ SubI(index, __ ConI(sizeof(intptr_t)));
3549 Node* next_indexX = next_index;
3550 #ifdef _LP64
3551 // We could refine the type for what it's worth
3552 // const TypeLong* lidxtype = TypeLong::make(CONST64(0), get_size_from_queue);
3553 next_indexX = _gvn.transform( new (C, 2) ConvI2LNode(next_index, TypeLong::make(0, max_jlong, Type::WidenMax)) );
3554 #endif
3556 // Now get the buffer location we will log the previous value into and store it
3557 Node *log_addr = __ AddP(no_base, buffer, next_indexX);
3558 __ store(__ ctrl(), log_addr, pre_val, T_OBJECT, Compile::AliasIdxRaw);
3559 // update the index
3560 __ store(__ ctrl(), index_adr, next_index, T_INT, Compile::AliasIdxRaw);
3562 } __ else_(); {
3564 // logging buffer is full, call the runtime
3565 const TypeFunc *tf = OptoRuntime::g1_wb_pre_Type();
3566 __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), "g1_wb_pre", pre_val, tls);
3567 } __ end_if(); // (!index)
3568 } __ end_if(); // (pre_val != NULL)
3569 } __ end_if(); // (!marking)
3571 // Final sync IdealKit and GraphKit.
3572 sync_kit(ideal);
3573 }
3575 //
3576 // Update the card table and add card address to the queue
3577 //
3578 void GraphKit::g1_mark_card(IdealKit& ideal,
3579 Node* card_adr,
3580 Node* oop_store,
3581 uint oop_alias_idx,
3582 Node* index,
3583 Node* index_adr,
3584 Node* buffer,
3585 const TypeFunc* tf) {
3587 Node* zero = __ ConI(0);
3588 Node* no_base = __ top();
3589 BasicType card_bt = T_BYTE;
3590 // Smash zero into card. MUST BE ORDERED WRT TO STORE
3591 __ storeCM(__ ctrl(), card_adr, zero, oop_store, oop_alias_idx, card_bt, Compile::AliasIdxRaw);
3593 // Now do the queue work
3594 __ if_then(index, BoolTest::ne, zero); {
3596 Node* next_index = __ SubI(index, __ ConI(sizeof(intptr_t)));
3597 Node* next_indexX = next_index;
3598 #ifdef _LP64
3599 // We could refine the type for what it's worth
3600 // const TypeLong* lidxtype = TypeLong::make(CONST64(0), get_size_from_queue);
3601 next_indexX = _gvn.transform( new (C, 2) ConvI2LNode(next_index, TypeLong::make(0, max_jlong, Type::WidenMax)) );
3602 #endif // _LP64
3603 Node* log_addr = __ AddP(no_base, buffer, next_indexX);
3605 __ store(__ ctrl(), log_addr, card_adr, T_ADDRESS, Compile::AliasIdxRaw);
3606 __ store(__ ctrl(), index_adr, next_index, T_INT, Compile::AliasIdxRaw);
3608 } __ else_(); {
3609 __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), "g1_wb_post", card_adr, __ thread());
3610 } __ end_if();
3612 }
3614 void GraphKit::g1_write_barrier_post(Node* oop_store,
3615 Node* obj,
3616 Node* adr,
3617 uint alias_idx,
3618 Node* val,
3619 BasicType bt,
3620 bool use_precise) {
3621 // If we are writing a NULL then we need no post barrier
3623 if (val != NULL && val->is_Con() && val->bottom_type() == TypePtr::NULL_PTR) {
3624 // Must be NULL
3625 const Type* t = val->bottom_type();
3626 assert(t == Type::TOP || t == TypePtr::NULL_PTR, "must be NULL");
3627 // No post barrier if writing NULLx
3628 return;
3629 }
3631 if (!use_precise) {
3632 // All card marks for a (non-array) instance are in one place:
3633 adr = obj;
3634 }
3635 // (Else it's an array (or unknown), and we want more precise card marks.)
3636 assert(adr != NULL, "");
3638 IdealKit ideal(gvn(), control(), merged_memory(), true);
3640 Node* tls = __ thread(); // ThreadLocalStorage
3642 Node* no_base = __ top();
3643 float likely = PROB_LIKELY(0.999);
3644 float unlikely = PROB_UNLIKELY(0.999);
3645 Node* zero = __ ConI(0);
3646 Node* zeroX = __ ConX(0);
3648 // Get the alias_index for raw card-mark memory
3649 const TypePtr* card_type = TypeRawPtr::BOTTOM;
3651 const TypeFunc *tf = OptoRuntime::g1_wb_post_Type();
3653 // Offsets into the thread
3654 const int index_offset = in_bytes(JavaThread::dirty_card_queue_offset() +
3655 PtrQueue::byte_offset_of_index());
3656 const int buffer_offset = in_bytes(JavaThread::dirty_card_queue_offset() +
3657 PtrQueue::byte_offset_of_buf());
3659 // Pointers into the thread
3661 Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset));
3662 Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset));
3664 // Now some values
3665 // Use ctrl to avoid hoisting these values past a safepoint, which could
3666 // potentially reset these fields in the JavaThread.
3667 Node* index = __ load(__ ctrl(), index_adr, TypeInt::INT, T_INT, Compile::AliasIdxRaw);
3668 Node* buffer = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
3670 // Convert the store obj pointer to an int prior to doing math on it
3671 // Must use ctrl to prevent "integerized oop" existing across safepoint
3672 Node* cast = __ CastPX(__ ctrl(), adr);
3674 // Divide pointer by card size
3675 Node* card_offset = __ URShiftX( cast, __ ConI(CardTableModRefBS::card_shift) );
3677 // Combine card table base and card offset
3678 Node* card_adr = __ AddP(no_base, byte_map_base_node(), card_offset );
3680 // If we know the value being stored does it cross regions?
3682 if (val != NULL) {
3683 // Does the store cause us to cross regions?
3685 // Should be able to do an unsigned compare of region_size instead of
3686 // and extra shift. Do we have an unsigned compare??
3687 // Node* region_size = __ ConI(1 << HeapRegion::LogOfHRGrainBytes);
3688 Node* xor_res = __ URShiftX ( __ XorX( cast, __ CastPX(__ ctrl(), val)), __ ConI(HeapRegion::LogOfHRGrainBytes));
3690 // if (xor_res == 0) same region so skip
3691 __ if_then(xor_res, BoolTest::ne, zeroX); {
3693 // No barrier if we are storing a NULL
3694 __ if_then(val, BoolTest::ne, null(), unlikely); {
3696 // Ok must mark the card if not already dirty
3698 // load the original value of the card
3699 Node* card_val = __ load(__ ctrl(), card_adr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw);
3701 __ if_then(card_val, BoolTest::ne, zero); {
3702 g1_mark_card(ideal, card_adr, oop_store, alias_idx, index, index_adr, buffer, tf);
3703 } __ end_if();
3704 } __ end_if();
3705 } __ end_if();
3706 } else {
3707 // Object.clone() instrinsic uses this path.
3708 g1_mark_card(ideal, card_adr, oop_store, alias_idx, index, index_adr, buffer, tf);
3709 }
3711 // Final sync IdealKit and GraphKit.
3712 sync_kit(ideal);
3713 }
3714 #undef __