104 // Find trailing Region |
104 // Find trailing Region |
105 Block *pb = get_block_for_node(in0); // Block-projection already has basic block |
105 Block *pb = get_block_for_node(in0); // Block-projection already has basic block |
106 uint j = 0; |
106 uint j = 0; |
107 if (pb->_num_succs != 1) { // More then 1 successor? |
107 if (pb->_num_succs != 1) { // More then 1 successor? |
108 // Search for successor |
108 // Search for successor |
109 uint max = pb->_nodes.size(); |
109 uint max = pb->number_of_nodes(); |
110 assert( max > 1, "" ); |
110 assert( max > 1, "" ); |
111 uint start = max - pb->_num_succs; |
111 uint start = max - pb->_num_succs; |
112 // Find which output path belongs to projection |
112 // Find which output path belongs to projection |
113 for (j = start; j < max; j++) { |
113 for (j = start; j < max; j++) { |
114 if( pb->_nodes[j] == in0 ) |
114 if( pb->get_node(j) == in0 ) |
115 break; |
115 break; |
116 } |
116 } |
117 assert( j < max, "must find" ); |
117 assert( j < max, "must find" ); |
118 // Change control to match head of successor basic block |
118 // Change control to match head of successor basic block |
119 j -= start; |
119 j -= start; |
1029 Block* PhaseCFG::hoist_to_cheaper_block(Block* LCA, Block* early, Node* self) { |
1029 Block* PhaseCFG::hoist_to_cheaper_block(Block* LCA, Block* early, Node* self) { |
1030 const double delta = 1+PROB_UNLIKELY_MAG(4); |
1030 const double delta = 1+PROB_UNLIKELY_MAG(4); |
1031 Block* least = LCA; |
1031 Block* least = LCA; |
1032 double least_freq = least->_freq; |
1032 double least_freq = least->_freq; |
1033 uint target = get_latency_for_node(self); |
1033 uint target = get_latency_for_node(self); |
1034 uint start_latency = get_latency_for_node(LCA->_nodes[0]); |
1034 uint start_latency = get_latency_for_node(LCA->head()); |
1035 uint end_latency = get_latency_for_node(LCA->_nodes[LCA->end_idx()]); |
1035 uint end_latency = get_latency_for_node(LCA->get_node(LCA->end_idx())); |
1036 bool in_latency = (target <= start_latency); |
1036 bool in_latency = (target <= start_latency); |
1037 const Block* root_block = get_block_for_node(_root); |
1037 const Block* root_block = get_block_for_node(_root); |
1038 |
1038 |
1039 // Turn off latency scheduling if scheduling is just plain off |
1039 // Turn off latency scheduling if scheduling is just plain off |
1040 if (!C->do_scheduling()) |
1040 if (!C->do_scheduling()) |
1051 if (trace_opto_pipelining()) { |
1051 if (trace_opto_pipelining()) { |
1052 tty->print("# Find cheaper block for latency %d: ", get_latency_for_node(self)); |
1052 tty->print("# Find cheaper block for latency %d: ", get_latency_for_node(self)); |
1053 self->dump(); |
1053 self->dump(); |
1054 tty->print_cr("# B%d: start latency for [%4d]=%d, end latency for [%4d]=%d, freq=%g", |
1054 tty->print_cr("# B%d: start latency for [%4d]=%d, end latency for [%4d]=%d, freq=%g", |
1055 LCA->_pre_order, |
1055 LCA->_pre_order, |
1056 LCA->_nodes[0]->_idx, |
1056 LCA->head()->_idx, |
1057 start_latency, |
1057 start_latency, |
1058 LCA->_nodes[LCA->end_idx()]->_idx, |
1058 LCA->get_node(LCA->end_idx())->_idx, |
1059 end_latency, |
1059 end_latency, |
1060 least_freq); |
1060 least_freq); |
1061 } |
1061 } |
1062 #endif |
1062 #endif |
1063 |
1063 |
1076 |
1076 |
1077 // Don't hoist machine instructions to the root basic block |
1077 // Don't hoist machine instructions to the root basic block |
1078 if (mach && LCA == root_block) |
1078 if (mach && LCA == root_block) |
1079 break; |
1079 break; |
1080 |
1080 |
1081 uint start_lat = get_latency_for_node(LCA->_nodes[0]); |
1081 uint start_lat = get_latency_for_node(LCA->head()); |
1082 uint end_idx = LCA->end_idx(); |
1082 uint end_idx = LCA->end_idx(); |
1083 uint end_lat = get_latency_for_node(LCA->_nodes[end_idx]); |
1083 uint end_lat = get_latency_for_node(LCA->get_node(end_idx)); |
1084 double LCA_freq = LCA->_freq; |
1084 double LCA_freq = LCA->_freq; |
1085 #ifndef PRODUCT |
1085 #ifndef PRODUCT |
1086 if (trace_opto_pipelining()) { |
1086 if (trace_opto_pipelining()) { |
1087 tty->print_cr("# B%d: start latency for [%4d]=%d, end latency for [%4d]=%d, freq=%g", |
1087 tty->print_cr("# B%d: start latency for [%4d]=%d, end latency for [%4d]=%d, freq=%g", |
1088 LCA->_pre_order, LCA->_nodes[0]->_idx, start_lat, end_idx, end_lat, LCA_freq); |
1088 LCA->_pre_order, LCA->head()->_idx, start_lat, end_idx, end_lat, LCA_freq); |
1089 } |
1089 } |
1090 #endif |
1090 #endif |
1091 cand_cnt++; |
1091 cand_cnt++; |
1092 if (LCA_freq < least_freq || // Better Frequency |
1092 if (LCA_freq < least_freq || // Better Frequency |
1093 (StressGCM && Compile::randomized_select(cand_cnt)) || // Should be randomly accepted in stress mode |
1093 (StressGCM && Compile::randomized_select(cand_cnt)) || // Should be randomly accepted in stress mode |
1344 // for( int i=0; i < (int)matcher._null_check_tests.size(); i+=2 ) { |
1344 // for( int i=0; i < (int)matcher._null_check_tests.size(); i+=2 ) { |
1345 for (int i = _matcher._null_check_tests.size() - 2; i >= 0; i -= 2) { |
1345 for (int i = _matcher._null_check_tests.size() - 2; i >= 0; i -= 2) { |
1346 Node* proj = _matcher._null_check_tests[i]; |
1346 Node* proj = _matcher._null_check_tests[i]; |
1347 Node* val = _matcher._null_check_tests[i + 1]; |
1347 Node* val = _matcher._null_check_tests[i + 1]; |
1348 Block* block = get_block_for_node(proj); |
1348 Block* block = get_block_for_node(proj); |
1349 block->implicit_null_check(this, proj, val, allowed_reasons); |
1349 implicit_null_check(block, proj, val, allowed_reasons); |
1350 // The implicit_null_check will only perform the transformation |
1350 // The implicit_null_check will only perform the transformation |
1351 // if the null branch is truly uncommon, *and* it leads to an |
1351 // if the null branch is truly uncommon, *and* it leads to an |
1352 // uncommon trap. Combined with the too_many_traps guards |
1352 // uncommon trap. Combined with the too_many_traps guards |
1353 // above, this prevents SEGV storms reported in 6366351, |
1353 // above, this prevents SEGV storms reported in 6366351, |
1354 // by recompiling offending methods without this optimization. |
1354 // by recompiling offending methods without this optimization. |
1365 // Later, do a real latency aware scheduler. |
1365 // Later, do a real latency aware scheduler. |
1366 GrowableArray<int> ready_cnt(C->unique(), C->unique(), -1); |
1366 GrowableArray<int> ready_cnt(C->unique(), C->unique(), -1); |
1367 visited.Clear(); |
1367 visited.Clear(); |
1368 for (uint i = 0; i < number_of_blocks(); i++) { |
1368 for (uint i = 0; i < number_of_blocks(); i++) { |
1369 Block* block = get_block(i); |
1369 Block* block = get_block(i); |
1370 if (!block->schedule_local(this, _matcher, ready_cnt, visited)) { |
1370 if (!schedule_local(block, ready_cnt, visited)) { |
1371 if (!C->failure_reason_is(C2Compiler::retry_no_subsuming_loads())) { |
1371 if (!C->failure_reason_is(C2Compiler::retry_no_subsuming_loads())) { |
1372 C->record_method_not_compilable("local schedule failed"); |
1372 C->record_method_not_compilable("local schedule failed"); |
1373 } |
1373 } |
1374 return; |
1374 return; |
1375 } |
1375 } |
1377 |
1377 |
1378 // If we inserted any instructions between a Call and his CatchNode, |
1378 // If we inserted any instructions between a Call and his CatchNode, |
1379 // clone the instructions on all paths below the Catch. |
1379 // clone the instructions on all paths below the Catch. |
1380 for (uint i = 0; i < number_of_blocks(); i++) { |
1380 for (uint i = 0; i < number_of_blocks(); i++) { |
1381 Block* block = get_block(i); |
1381 Block* block = get_block(i); |
1382 block->call_catch_cleanup(this, C); |
1382 call_catch_cleanup(block); |
1383 } |
1383 } |
1384 |
1384 |
1385 #ifndef PRODUCT |
1385 #ifndef PRODUCT |
1386 if (trace_opto_pipelining()) { |
1386 if (trace_opto_pipelining()) { |
1387 tty->print("\n---- After GlobalCodeMotion ----\n"); |
1387 tty->print("\n---- After GlobalCodeMotion ----\n"); |
1728 |
1728 |
1729 //------------------------------succ_prob------------------------------------- |
1729 //------------------------------succ_prob------------------------------------- |
1730 // Determine the probability of reaching successor 'i' from the receiver block. |
1730 // Determine the probability of reaching successor 'i' from the receiver block. |
1731 float Block::succ_prob(uint i) { |
1731 float Block::succ_prob(uint i) { |
1732 int eidx = end_idx(); |
1732 int eidx = end_idx(); |
1733 Node *n = _nodes[eidx]; // Get ending Node |
1733 Node *n = get_node(eidx); // Get ending Node |
1734 |
1734 |
1735 int op = n->Opcode(); |
1735 int op = n->Opcode(); |
1736 if (n->is_Mach()) { |
1736 if (n->is_Mach()) { |
1737 if (n->is_MachNullCheck()) { |
1737 if (n->is_MachNullCheck()) { |
1738 // Can only reach here if called after lcm. The original Op_If is gone, |
1738 // Can only reach here if called after lcm. The original Op_If is gone, |
1763 assert (i < 2, "just checking"); |
1763 assert (i < 2, "just checking"); |
1764 // Conditionals pass on only part of their frequency |
1764 // Conditionals pass on only part of their frequency |
1765 float prob = n->as_MachIf()->_prob; |
1765 float prob = n->as_MachIf()->_prob; |
1766 assert(prob >= 0.0 && prob <= 1.0, "out of range probability"); |
1766 assert(prob >= 0.0 && prob <= 1.0, "out of range probability"); |
1767 // If succ[i] is the FALSE branch, invert path info |
1767 // If succ[i] is the FALSE branch, invert path info |
1768 if( _nodes[i + eidx + 1]->Opcode() == Op_IfFalse ) { |
1768 if( get_node(i + eidx + 1)->Opcode() == Op_IfFalse ) { |
1769 return 1.0f - prob; // not taken |
1769 return 1.0f - prob; // not taken |
1770 } else { |
1770 } else { |
1771 return prob; // taken |
1771 return prob; // taken |
1772 } |
1772 } |
1773 } |
1773 } |
1775 case Op_Jump: |
1775 case Op_Jump: |
1776 // Divide the frequency between all successors evenly |
1776 // Divide the frequency between all successors evenly |
1777 return 1.0f/_num_succs; |
1777 return 1.0f/_num_succs; |
1778 |
1778 |
1779 case Op_Catch: { |
1779 case Op_Catch: { |
1780 const CatchProjNode *ci = _nodes[i + eidx + 1]->as_CatchProj(); |
1780 const CatchProjNode *ci = get_node(i + eidx + 1)->as_CatchProj(); |
1781 if (ci->_con == CatchProjNode::fall_through_index) { |
1781 if (ci->_con == CatchProjNode::fall_through_index) { |
1782 // Fall-thru path gets the lion's share. |
1782 // Fall-thru path gets the lion's share. |
1783 return 1.0f - PROB_UNLIKELY_MAG(5)*_num_succs; |
1783 return 1.0f - PROB_UNLIKELY_MAG(5)*_num_succs; |
1784 } else { |
1784 } else { |
1785 // Presume exceptional paths are equally unlikely |
1785 // Presume exceptional paths are equally unlikely |
1812 |
1812 |
1813 //------------------------------num_fall_throughs----------------------------- |
1813 //------------------------------num_fall_throughs----------------------------- |
1814 // Return the number of fall-through candidates for a block |
1814 // Return the number of fall-through candidates for a block |
1815 int Block::num_fall_throughs() { |
1815 int Block::num_fall_throughs() { |
1816 int eidx = end_idx(); |
1816 int eidx = end_idx(); |
1817 Node *n = _nodes[eidx]; // Get ending Node |
1817 Node *n = get_node(eidx); // Get ending Node |
1818 |
1818 |
1819 int op = n->Opcode(); |
1819 int op = n->Opcode(); |
1820 if (n->is_Mach()) { |
1820 if (n->is_Mach()) { |
1821 if (n->is_MachNullCheck()) { |
1821 if (n->is_MachNullCheck()) { |
1822 // In theory, either side can fall-thru, for simplicity sake, |
1822 // In theory, either side can fall-thru, for simplicity sake, |
1864 |
1864 |
1865 //------------------------------succ_fall_through----------------------------- |
1865 //------------------------------succ_fall_through----------------------------- |
1866 // Return true if a specific successor could be fall-through target. |
1866 // Return true if a specific successor could be fall-through target. |
1867 bool Block::succ_fall_through(uint i) { |
1867 bool Block::succ_fall_through(uint i) { |
1868 int eidx = end_idx(); |
1868 int eidx = end_idx(); |
1869 Node *n = _nodes[eidx]; // Get ending Node |
1869 Node *n = get_node(eidx); // Get ending Node |
1870 |
1870 |
1871 int op = n->Opcode(); |
1871 int op = n->Opcode(); |
1872 if (n->is_Mach()) { |
1872 if (n->is_Mach()) { |
1873 if (n->is_MachNullCheck()) { |
1873 if (n->is_MachNullCheck()) { |
1874 // In theory, either side can fall-thru, for simplicity sake, |
1874 // In theory, either side can fall-thru, for simplicity sake, |
1875 // let's say only the false branch can now. |
1875 // let's say only the false branch can now. |
1876 return _nodes[i + eidx + 1]->Opcode() == Op_IfFalse; |
1876 return get_node(i + eidx + 1)->Opcode() == Op_IfFalse; |
1877 } |
1877 } |
1878 op = n->as_Mach()->ideal_Opcode(); |
1878 op = n->as_Mach()->ideal_Opcode(); |
1879 } |
1879 } |
1880 |
1880 |
1881 // Switch on branch type |
1881 // Switch on branch type |
1909 |
1909 |
1910 //------------------------------update_uncommon_branch------------------------ |
1910 //------------------------------update_uncommon_branch------------------------ |
1911 // Update the probability of a two-branch to be uncommon |
1911 // Update the probability of a two-branch to be uncommon |
1912 void Block::update_uncommon_branch(Block* ub) { |
1912 void Block::update_uncommon_branch(Block* ub) { |
1913 int eidx = end_idx(); |
1913 int eidx = end_idx(); |
1914 Node *n = _nodes[eidx]; // Get ending Node |
1914 Node *n = get_node(eidx); // Get ending Node |
1915 |
1915 |
1916 int op = n->as_Mach()->ideal_Opcode(); |
1916 int op = n->as_Mach()->ideal_Opcode(); |
1917 |
1917 |
1918 assert(op == Op_CountedLoopEnd || op == Op_If, "must be a If"); |
1918 assert(op == Op_CountedLoopEnd || op == Op_If, "must be a If"); |
1919 assert(num_fall_throughs() == 2, "must be a two way branch block"); |
1919 assert(num_fall_throughs() == 2, "must be a two way branch block"); |
1925 } |
1925 } |
1926 assert(s < 2, "uncommon successor must be found"); |
1926 assert(s < 2, "uncommon successor must be found"); |
1927 |
1927 |
1928 // If ub is the true path, make the proability small, else |
1928 // If ub is the true path, make the proability small, else |
1929 // ub is the false path, and make the probability large |
1929 // ub is the false path, and make the probability large |
1930 bool invert = (_nodes[s + eidx + 1]->Opcode() == Op_IfFalse); |
1930 bool invert = (get_node(s + eidx + 1)->Opcode() == Op_IfFalse); |
1931 |
1931 |
1932 // Get existing probability |
1932 // Get existing probability |
1933 float p = n->as_MachIf()->_prob; |
1933 float p = n->as_MachIf()->_prob; |
1934 |
1934 |
1935 if (invert) p = 1.0 - p; |
1935 if (invert) p = 1.0 - p; |