src/share/vm/opto/parse3.cpp

Wed, 03 Jun 2015 14:22:57 +0200

author
roland
date
Wed, 03 Jun 2015 14:22:57 +0200
changeset 7859
c1c199dde5c9
parent 6507
752ba2e5f6d0
child 7994
04ff2f6cd0eb
permissions
-rw-r--r--

8077504: Unsafe load can loose control dependency and cause crash
Summary: Node::depends_only_on_test() should return false for Unsafe loads
Reviewed-by: kvn, adinn

     1 /*
     2  * Copyright (c) 1998, 2013, Oracle and/or its affiliates. All rights reserved.
     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
     4  *
     5  * This code is free software; you can redistribute it and/or modify it
     6  * under the terms of the GNU General Public License version 2 only, as
     7  * published by the Free Software Foundation.
     8  *
     9  * This code is distributed in the hope that it will be useful, but WITHOUT
    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
    12  * version 2 for more details (a copy is included in the LICENSE file that
    13  * accompanied this code).
    14  *
    15  * You should have received a copy of the GNU General Public License version
    16  * 2 along with this work; if not, write to the Free Software Foundation,
    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
    18  *
    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
    20  * or visit www.oracle.com if you need additional information or have any
    21  * questions.
    22  *
    23  */
    25 #include "precompiled.hpp"
    26 #include "compiler/compileLog.hpp"
    27 #include "interpreter/linkResolver.hpp"
    28 #include "memory/universe.inline.hpp"
    29 #include "oops/objArrayKlass.hpp"
    30 #include "opto/addnode.hpp"
    31 #include "opto/memnode.hpp"
    32 #include "opto/parse.hpp"
    33 #include "opto/rootnode.hpp"
    34 #include "opto/runtime.hpp"
    35 #include "opto/subnode.hpp"
    36 #include "runtime/deoptimization.hpp"
    37 #include "runtime/handles.inline.hpp"
    39 //=============================================================================
    40 // Helper methods for _get* and _put* bytecodes
    41 //=============================================================================
    42 bool Parse::static_field_ok_in_clinit(ciField *field, ciMethod *method) {
    43   // Could be the field_holder's <clinit> method, or <clinit> for a subklass.
    44   // Better to check now than to Deoptimize as soon as we execute
    45   assert( field->is_static(), "Only check if field is static");
    46   // is_being_initialized() is too generous.  It allows access to statics
    47   // by threads that are not running the <clinit> before the <clinit> finishes.
    48   // return field->holder()->is_being_initialized();
    50   // The following restriction is correct but conservative.
    51   // It is also desirable to allow compilation of methods called from <clinit>
    52   // but this generated code will need to be made safe for execution by
    53   // other threads, or the transition from interpreted to compiled code would
    54   // need to be guarded.
    55   ciInstanceKlass *field_holder = field->holder();
    57   bool access_OK = false;
    58   if (method->holder()->is_subclass_of(field_holder)) {
    59     if (method->is_static()) {
    60       if (method->name() == ciSymbol::class_initializer_name()) {
    61         // OK to access static fields inside initializer
    62         access_OK = true;
    63       }
    64     } else {
    65       if (method->name() == ciSymbol::object_initializer_name()) {
    66         // It's also OK to access static fields inside a constructor,
    67         // because any thread calling the constructor must first have
    68         // synchronized on the class by executing a '_new' bytecode.
    69         access_OK = true;
    70       }
    71     }
    72   }
    74   return access_OK;
    76 }
    79 void Parse::do_field_access(bool is_get, bool is_field) {
    80   bool will_link;
    81   ciField* field = iter().get_field(will_link);
    82   assert(will_link, "getfield: typeflow responsibility");
    84   ciInstanceKlass* field_holder = field->holder();
    86   if (is_field == field->is_static()) {
    87     // Interpreter will throw java_lang_IncompatibleClassChangeError
    88     // Check this before allowing <clinit> methods to access static fields
    89     uncommon_trap(Deoptimization::Reason_unhandled,
    90                   Deoptimization::Action_none);
    91     return;
    92   }
    94   if (!is_field && !field_holder->is_initialized()) {
    95     if (!static_field_ok_in_clinit(field, method())) {
    96       uncommon_trap(Deoptimization::Reason_uninitialized,
    97                     Deoptimization::Action_reinterpret,
    98                     NULL, "!static_field_ok_in_clinit");
    99       return;
   100     }
   101   }
   103   // Deoptimize on putfield writes to call site target field.
   104   if (!is_get && field->is_call_site_target()) {
   105     uncommon_trap(Deoptimization::Reason_unhandled,
   106                   Deoptimization::Action_reinterpret,
   107                   NULL, "put to call site target field");
   108     return;
   109   }
   111   assert(field->will_link(method()->holder(), bc()), "getfield: typeflow responsibility");
   113   // Note:  We do not check for an unloaded field type here any more.
   115   // Generate code for the object pointer.
   116   Node* obj;
   117   if (is_field) {
   118     int obj_depth = is_get ? 0 : field->type()->size();
   119     obj = null_check(peek(obj_depth));
   120     // Compile-time detect of null-exception?
   121     if (stopped())  return;
   123 #ifdef ASSERT
   124     const TypeInstPtr *tjp = TypeInstPtr::make(TypePtr::NotNull, iter().get_declared_field_holder());
   125     assert(_gvn.type(obj)->higher_equal(tjp), "cast_up is no longer needed");
   126 #endif
   128     if (is_get) {
   129       (void) pop();  // pop receiver before getting
   130       do_get_xxx(obj, field, is_field);
   131     } else {
   132       do_put_xxx(obj, field, is_field);
   133       (void) pop();  // pop receiver after putting
   134     }
   135   } else {
   136     const TypeInstPtr* tip = TypeInstPtr::make(field_holder->java_mirror());
   137     obj = _gvn.makecon(tip);
   138     if (is_get) {
   139       do_get_xxx(obj, field, is_field);
   140     } else {
   141       do_put_xxx(obj, field, is_field);
   142     }
   143   }
   144 }
   147 void Parse::do_get_xxx(Node* obj, ciField* field, bool is_field) {
   148   // Does this field have a constant value?  If so, just push the value.
   149   if (field->is_constant()) {
   150     // final or stable field
   151     const Type* stable_type = NULL;
   152     if (FoldStableValues && field->is_stable()) {
   153       stable_type = Type::get_const_type(field->type());
   154       if (field->type()->is_array_klass()) {
   155         int stable_dimension = field->type()->as_array_klass()->dimension();
   156         stable_type = stable_type->is_aryptr()->cast_to_stable(true, stable_dimension);
   157       }
   158     }
   159     if (field->is_static()) {
   160       // final static field
   161       if (C->eliminate_boxing()) {
   162         // The pointers in the autobox arrays are always non-null.
   163         ciSymbol* klass_name = field->holder()->name();
   164         if (field->name() == ciSymbol::cache_field_name() &&
   165             field->holder()->uses_default_loader() &&
   166             (klass_name == ciSymbol::java_lang_Character_CharacterCache() ||
   167              klass_name == ciSymbol::java_lang_Byte_ByteCache() ||
   168              klass_name == ciSymbol::java_lang_Short_ShortCache() ||
   169              klass_name == ciSymbol::java_lang_Integer_IntegerCache() ||
   170              klass_name == ciSymbol::java_lang_Long_LongCache())) {
   171           bool require_const = true;
   172           bool autobox_cache = true;
   173           if (push_constant(field->constant_value(), require_const, autobox_cache)) {
   174             return;
   175           }
   176         }
   177       }
   178       if (push_constant(field->constant_value(), false, false, stable_type))
   179         return;
   180     } else {
   181       // final or stable non-static field
   182       // Treat final non-static fields of trusted classes (classes in
   183       // java.lang.invoke and sun.invoke packages and subpackages) as
   184       // compile time constants.
   185       if (obj->is_Con()) {
   186         const TypeOopPtr* oop_ptr = obj->bottom_type()->isa_oopptr();
   187         ciObject* constant_oop = oop_ptr->const_oop();
   188         ciConstant constant = field->constant_value_of(constant_oop);
   189         if (FoldStableValues && field->is_stable() && constant.is_null_or_zero()) {
   190           // fall through to field load; the field is not yet initialized
   191         } else {
   192           if (push_constant(constant, true, false, stable_type))
   193             return;
   194         }
   195       }
   196     }
   197   }
   199   ciType* field_klass = field->type();
   200   bool is_vol = field->is_volatile();
   202   // Compute address and memory type.
   203   int offset = field->offset_in_bytes();
   204   const TypePtr* adr_type = C->alias_type(field)->adr_type();
   205   Node *adr = basic_plus_adr(obj, obj, offset);
   206   BasicType bt = field->layout_type();
   208   // Build the resultant type of the load
   209   const Type *type;
   211   bool must_assert_null = false;
   213   if( bt == T_OBJECT ) {
   214     if (!field->type()->is_loaded()) {
   215       type = TypeInstPtr::BOTTOM;
   216       must_assert_null = true;
   217     } else if (field->is_constant() && field->is_static()) {
   218       // This can happen if the constant oop is non-perm.
   219       ciObject* con = field->constant_value().as_object();
   220       // Do not "join" in the previous type; it doesn't add value,
   221       // and may yield a vacuous result if the field is of interface type.
   222       type = TypeOopPtr::make_from_constant(con)->isa_oopptr();
   223       assert(type != NULL, "field singleton type must be consistent");
   224     } else {
   225       type = TypeOopPtr::make_from_klass(field_klass->as_klass());
   226     }
   227   } else {
   228     type = Type::get_const_basic_type(bt);
   229   }
   230   if (support_IRIW_for_not_multiple_copy_atomic_cpu && field->is_volatile()) {
   231     insert_mem_bar(Op_MemBarVolatile);   // StoreLoad barrier
   232   }
   233   // Build the load.
   234   //
   235   MemNode::MemOrd mo = is_vol ? MemNode::acquire : MemNode::unordered;
   236   Node* ld = make_load(NULL, adr, type, bt, adr_type, mo, LoadNode::DependsOnlyOnTest, is_vol);
   238   // Adjust Java stack
   239   if (type2size[bt] == 1)
   240     push(ld);
   241   else
   242     push_pair(ld);
   244   if (must_assert_null) {
   245     // Do not take a trap here.  It's possible that the program
   246     // will never load the field's class, and will happily see
   247     // null values in this field forever.  Don't stumble into a
   248     // trap for such a program, or we might get a long series
   249     // of useless recompilations.  (Or, we might load a class
   250     // which should not be loaded.)  If we ever see a non-null
   251     // value, we will then trap and recompile.  (The trap will
   252     // not need to mention the class index, since the class will
   253     // already have been loaded if we ever see a non-null value.)
   254     // uncommon_trap(iter().get_field_signature_index());
   255 #ifndef PRODUCT
   256     if (PrintOpto && (Verbose || WizardMode)) {
   257       method()->print_name(); tty->print_cr(" asserting nullness of field at bci: %d", bci());
   258     }
   259 #endif
   260     if (C->log() != NULL) {
   261       C->log()->elem("assert_null reason='field' klass='%d'",
   262                      C->log()->identify(field->type()));
   263     }
   264     // If there is going to be a trap, put it at the next bytecode:
   265     set_bci(iter().next_bci());
   266     null_assert(peek());
   267     set_bci(iter().cur_bci()); // put it back
   268   }
   270   // If reference is volatile, prevent following memory ops from
   271   // floating up past the volatile read.  Also prevents commoning
   272   // another volatile read.
   273   if (field->is_volatile()) {
   274     // Memory barrier includes bogus read of value to force load BEFORE membar
   275     insert_mem_bar(Op_MemBarAcquire, ld);
   276   }
   277 }
   279 void Parse::do_put_xxx(Node* obj, ciField* field, bool is_field) {
   280   bool is_vol = field->is_volatile();
   281   // If reference is volatile, prevent following memory ops from
   282   // floating down past the volatile write.  Also prevents commoning
   283   // another volatile read.
   284   if (is_vol)  insert_mem_bar(Op_MemBarRelease);
   286   // Compute address and memory type.
   287   int offset = field->offset_in_bytes();
   288   const TypePtr* adr_type = C->alias_type(field)->adr_type();
   289   Node* adr = basic_plus_adr(obj, obj, offset);
   290   BasicType bt = field->layout_type();
   291   // Value to be stored
   292   Node* val = type2size[bt] == 1 ? pop() : pop_pair();
   293   // Round doubles before storing
   294   if (bt == T_DOUBLE)  val = dstore_rounding(val);
   296   // Conservatively release stores of object references.
   297   const MemNode::MemOrd mo =
   298     is_vol ?
   299     // Volatile fields need releasing stores.
   300     MemNode::release :
   301     // Non-volatile fields also need releasing stores if they hold an
   302     // object reference, because the object reference might point to
   303     // a freshly created object.
   304     StoreNode::release_if_reference(bt);
   306   // Store the value.
   307   Node* store;
   308   if (bt == T_OBJECT) {
   309     const TypeOopPtr* field_type;
   310     if (!field->type()->is_loaded()) {
   311       field_type = TypeInstPtr::BOTTOM;
   312     } else {
   313       field_type = TypeOopPtr::make_from_klass(field->type()->as_klass());
   314     }
   315     store = store_oop_to_object(control(), obj, adr, adr_type, val, field_type, bt, mo);
   316   } else {
   317     store = store_to_memory(control(), adr, val, bt, adr_type, mo, is_vol);
   318   }
   320   // If reference is volatile, prevent following volatiles ops from
   321   // floating up before the volatile write.
   322   if (is_vol) {
   323     // If not multiple copy atomic, we do the MemBarVolatile before the load.
   324     if (!support_IRIW_for_not_multiple_copy_atomic_cpu) {
   325       insert_mem_bar(Op_MemBarVolatile); // Use fat membar
   326     }
   327     // Remember we wrote a volatile field.
   328     // For not multiple copy atomic cpu (ppc64) a barrier should be issued
   329     // in constructors which have such stores. See do_exits() in parse1.cpp.
   330     if (is_field) {
   331       set_wrote_volatile(true);
   332     }
   333   }
   335   // If the field is final, the rules of Java say we are in <init> or <clinit>.
   336   // Note the presence of writes to final non-static fields, so that we
   337   // can insert a memory barrier later on to keep the writes from floating
   338   // out of the constructor.
   339   // Any method can write a @Stable field; insert memory barriers after those also.
   340   if (is_field && (field->is_final() || field->is_stable())) {
   341     set_wrote_final(true);
   342     // Preserve allocation ptr to create precedent edge to it in membar
   343     // generated on exit from constructor.
   344     if (C->eliminate_boxing() &&
   345         adr_type->isa_oopptr() && adr_type->is_oopptr()->is_ptr_to_boxed_value() &&
   346         AllocateNode::Ideal_allocation(obj, &_gvn) != NULL) {
   347       set_alloc_with_final(obj);
   348     }
   349   }
   350 }
   354 bool Parse::push_constant(ciConstant constant, bool require_constant, bool is_autobox_cache, const Type* stable_type) {
   355   const Type* con_type = Type::make_from_constant(constant, require_constant, is_autobox_cache);
   356   switch (constant.basic_type()) {
   357   case T_ARRAY:
   358   case T_OBJECT:
   359     // cases:
   360     //   can_be_constant    = (oop not scavengable || ScavengeRootsInCode != 0)
   361     //   should_be_constant = (oop not scavengable || ScavengeRootsInCode >= 2)
   362     // An oop is not scavengable if it is in the perm gen.
   363     if (stable_type != NULL && con_type != NULL && con_type->isa_oopptr())
   364       con_type = con_type->join_speculative(stable_type);
   365     break;
   367   case T_ILLEGAL:
   368     // Invalid ciConstant returned due to OutOfMemoryError in the CI
   369     assert(C->env()->failing(), "otherwise should not see this");
   370     // These always occur because of object types; we are going to
   371     // bail out anyway, so make the stack depths match up
   372     push( zerocon(T_OBJECT) );
   373     return false;
   374   }
   376   if (con_type == NULL)
   377     // we cannot inline the oop, but we can use it later to narrow a type
   378     return false;
   380   push_node(constant.basic_type(), makecon(con_type));
   381   return true;
   382 }
   385 //=============================================================================
   386 void Parse::do_anewarray() {
   387   bool will_link;
   388   ciKlass* klass = iter().get_klass(will_link);
   390   // Uncommon Trap when class that array contains is not loaded
   391   // we need the loaded class for the rest of graph; do not
   392   // initialize the container class (see Java spec)!!!
   393   assert(will_link, "anewarray: typeflow responsibility");
   395   ciObjArrayKlass* array_klass = ciObjArrayKlass::make(klass);
   396   // Check that array_klass object is loaded
   397   if (!array_klass->is_loaded()) {
   398     // Generate uncommon_trap for unloaded array_class
   399     uncommon_trap(Deoptimization::Reason_unloaded,
   400                   Deoptimization::Action_reinterpret,
   401                   array_klass);
   402     return;
   403   }
   405   kill_dead_locals();
   407   const TypeKlassPtr* array_klass_type = TypeKlassPtr::make(array_klass);
   408   Node* count_val = pop();
   409   Node* obj = new_array(makecon(array_klass_type), count_val, 1);
   410   push(obj);
   411 }
   414 void Parse::do_newarray(BasicType elem_type) {
   415   kill_dead_locals();
   417   Node*   count_val = pop();
   418   const TypeKlassPtr* array_klass = TypeKlassPtr::make(ciTypeArrayKlass::make(elem_type));
   419   Node*   obj = new_array(makecon(array_klass), count_val, 1);
   420   // Push resultant oop onto stack
   421   push(obj);
   422 }
   424 // Expand simple expressions like new int[3][5] and new Object[2][nonConLen].
   425 // Also handle the degenerate 1-dimensional case of anewarray.
   426 Node* Parse::expand_multianewarray(ciArrayKlass* array_klass, Node* *lengths, int ndimensions, int nargs) {
   427   Node* length = lengths[0];
   428   assert(length != NULL, "");
   429   Node* array = new_array(makecon(TypeKlassPtr::make(array_klass)), length, nargs);
   430   if (ndimensions > 1) {
   431     jint length_con = find_int_con(length, -1);
   432     guarantee(length_con >= 0, "non-constant multianewarray");
   433     ciArrayKlass* array_klass_1 = array_klass->as_obj_array_klass()->element_klass()->as_array_klass();
   434     const TypePtr* adr_type = TypeAryPtr::OOPS;
   435     const TypeOopPtr*    elemtype = _gvn.type(array)->is_aryptr()->elem()->make_oopptr();
   436     const intptr_t header   = arrayOopDesc::base_offset_in_bytes(T_OBJECT);
   437     for (jint i = 0; i < length_con; i++) {
   438       Node*    elem   = expand_multianewarray(array_klass_1, &lengths[1], ndimensions-1, nargs);
   439       intptr_t offset = header + ((intptr_t)i << LogBytesPerHeapOop);
   440       Node*    eaddr  = basic_plus_adr(array, offset);
   441       store_oop_to_array(control(), array, eaddr, adr_type, elem, elemtype, T_OBJECT, MemNode::unordered);
   442     }
   443   }
   444   return array;
   445 }
   447 void Parse::do_multianewarray() {
   448   int ndimensions = iter().get_dimensions();
   450   // the m-dimensional array
   451   bool will_link;
   452   ciArrayKlass* array_klass = iter().get_klass(will_link)->as_array_klass();
   453   assert(will_link, "multianewarray: typeflow responsibility");
   455   // Note:  Array classes are always initialized; no is_initialized check.
   457   kill_dead_locals();
   459   // get the lengths from the stack (first dimension is on top)
   460   Node** length = NEW_RESOURCE_ARRAY(Node*, ndimensions + 1);
   461   length[ndimensions] = NULL;  // terminating null for make_runtime_call
   462   int j;
   463   for (j = ndimensions-1; j >= 0 ; j--) length[j] = pop();
   465   // The original expression was of this form: new T[length0][length1]...
   466   // It is often the case that the lengths are small (except the last).
   467   // If that happens, use the fast 1-d creator a constant number of times.
   468   const jint expand_limit = MIN2((juint)MultiArrayExpandLimit, (juint)100);
   469   jint expand_count = 1;        // count of allocations in the expansion
   470   jint expand_fanout = 1;       // running total fanout
   471   for (j = 0; j < ndimensions-1; j++) {
   472     jint dim_con = find_int_con(length[j], -1);
   473     expand_fanout *= dim_con;
   474     expand_count  += expand_fanout; // count the level-J sub-arrays
   475     if (dim_con <= 0
   476         || dim_con > expand_limit
   477         || expand_count > expand_limit) {
   478       expand_count = 0;
   479       break;
   480     }
   481   }
   483   // Can use multianewarray instead of [a]newarray if only one dimension,
   484   // or if all non-final dimensions are small constants.
   485   if (ndimensions == 1 || (1 <= expand_count && expand_count <= expand_limit)) {
   486     Node* obj = NULL;
   487     // Set the original stack and the reexecute bit for the interpreter
   488     // to reexecute the multianewarray bytecode if deoptimization happens.
   489     // Do it unconditionally even for one dimension multianewarray.
   490     // Note: the reexecute bit will be set in GraphKit::add_safepoint_edges()
   491     // when AllocateArray node for newarray is created.
   492     { PreserveReexecuteState preexecs(this);
   493       inc_sp(ndimensions);
   494       // Pass 0 as nargs since uncommon trap code does not need to restore stack.
   495       obj = expand_multianewarray(array_klass, &length[0], ndimensions, 0);
   496     } //original reexecute and sp are set back here
   497     push(obj);
   498     return;
   499   }
   501   address fun = NULL;
   502   switch (ndimensions) {
   503   case 1: ShouldNotReachHere(); break;
   504   case 2: fun = OptoRuntime::multianewarray2_Java(); break;
   505   case 3: fun = OptoRuntime::multianewarray3_Java(); break;
   506   case 4: fun = OptoRuntime::multianewarray4_Java(); break;
   507   case 5: fun = OptoRuntime::multianewarray5_Java(); break;
   508   };
   509   Node* c = NULL;
   511   if (fun != NULL) {
   512     c = make_runtime_call(RC_NO_LEAF | RC_NO_IO,
   513                           OptoRuntime::multianewarray_Type(ndimensions),
   514                           fun, NULL, TypeRawPtr::BOTTOM,
   515                           makecon(TypeKlassPtr::make(array_klass)),
   516                           length[0], length[1], length[2],
   517                           (ndimensions > 2) ? length[3] : NULL,
   518                           (ndimensions > 3) ? length[4] : NULL);
   519   } else {
   520     // Create a java array for dimension sizes
   521     Node* dims = NULL;
   522     { PreserveReexecuteState preexecs(this);
   523       inc_sp(ndimensions);
   524       Node* dims_array_klass = makecon(TypeKlassPtr::make(ciArrayKlass::make(ciType::make(T_INT))));
   525       dims = new_array(dims_array_klass, intcon(ndimensions), 0);
   527       // Fill-in it with values
   528       for (j = 0; j < ndimensions; j++) {
   529         Node *dims_elem = array_element_address(dims, intcon(j), T_INT);
   530         store_to_memory(control(), dims_elem, length[j], T_INT, TypeAryPtr::INTS, MemNode::unordered);
   531       }
   532     }
   534     c = make_runtime_call(RC_NO_LEAF | RC_NO_IO,
   535                           OptoRuntime::multianewarrayN_Type(),
   536                           OptoRuntime::multianewarrayN_Java(), NULL, TypeRawPtr::BOTTOM,
   537                           makecon(TypeKlassPtr::make(array_klass)),
   538                           dims);
   539   }
   540   make_slow_call_ex(c, env()->Throwable_klass(), false);
   542   Node* res = _gvn.transform(new (C) ProjNode(c, TypeFunc::Parms));
   544   const Type* type = TypeOopPtr::make_from_klass_raw(array_klass);
   546   // Improve the type:  We know it's not null, exact, and of a given length.
   547   type = type->is_ptr()->cast_to_ptr_type(TypePtr::NotNull);
   548   type = type->is_aryptr()->cast_to_exactness(true);
   550   const TypeInt* ltype = _gvn.find_int_type(length[0]);
   551   if (ltype != NULL)
   552     type = type->is_aryptr()->cast_to_size(ltype);
   554     // We cannot sharpen the nested sub-arrays, since the top level is mutable.
   556   Node* cast = _gvn.transform( new (C) CheckCastPPNode(control(), res, type) );
   557   push(cast);
   559   // Possible improvements:
   560   // - Make a fast path for small multi-arrays.  (W/ implicit init. loops.)
   561   // - Issue CastII against length[*] values, to TypeInt::POS.
   562 }

mercurial