163 |
163 |
164 bool inline_native_newArray(); |
164 bool inline_native_newArray(); |
165 bool inline_native_getLength(); |
165 bool inline_native_getLength(); |
166 bool inline_array_copyOf(bool is_copyOfRange); |
166 bool inline_array_copyOf(bool is_copyOfRange); |
167 bool inline_array_equals(); |
167 bool inline_array_equals(); |
|
168 void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark); |
168 bool inline_native_clone(bool is_virtual); |
169 bool inline_native_clone(bool is_virtual); |
169 bool inline_native_Reflection_getCallerClass(); |
170 bool inline_native_Reflection_getCallerClass(); |
170 bool inline_native_AtomicLong_get(); |
171 bool inline_native_AtomicLong_get(); |
171 bool inline_native_AtomicLong_attemptUpdate(); |
172 bool inline_native_AtomicLong_attemptUpdate(); |
172 bool is_method_invoke_or_aux_frame(JVMState* jvms); |
173 bool is_method_invoke_or_aux_frame(JVMState* jvms); |
179 void generate_arraycopy(const TypePtr* adr_type, |
180 void generate_arraycopy(const TypePtr* adr_type, |
180 BasicType basic_elem_type, |
181 BasicType basic_elem_type, |
181 Node* src, Node* src_offset, |
182 Node* src, Node* src_offset, |
182 Node* dest, Node* dest_offset, |
183 Node* dest, Node* dest_offset, |
183 Node* copy_length, |
184 Node* copy_length, |
184 int nargs, // arguments on stack for debug info |
|
185 bool disjoint_bases = false, |
185 bool disjoint_bases = false, |
186 bool length_never_negative = false, |
186 bool length_never_negative = false, |
187 RegionNode* slow_region = NULL); |
187 RegionNode* slow_region = NULL); |
188 AllocateArrayNode* tightly_coupled_allocation(Node* ptr, |
188 AllocateArrayNode* tightly_coupled_allocation(Node* ptr, |
189 RegionNode* slow_region); |
189 RegionNode* slow_region); |
200 Node* dest, Node* dest_offset, |
200 Node* dest, Node* dest_offset, |
201 Node* dest_size); |
201 Node* dest_size); |
202 void generate_slow_arraycopy(const TypePtr* adr_type, |
202 void generate_slow_arraycopy(const TypePtr* adr_type, |
203 Node* src, Node* src_offset, |
203 Node* src, Node* src_offset, |
204 Node* dest, Node* dest_offset, |
204 Node* dest, Node* dest_offset, |
205 Node* copy_length, |
205 Node* copy_length); |
206 int nargs); |
|
207 Node* generate_checkcast_arraycopy(const TypePtr* adr_type, |
206 Node* generate_checkcast_arraycopy(const TypePtr* adr_type, |
208 Node* dest_elem_klass, |
207 Node* dest_elem_klass, |
209 Node* src, Node* src_offset, |
208 Node* src, Node* src_offset, |
210 Node* dest, Node* dest_offset, |
209 Node* dest, Node* dest_offset, |
211 Node* copy_length, int nargs); |
210 Node* copy_length); |
212 Node* generate_generic_arraycopy(const TypePtr* adr_type, |
211 Node* generate_generic_arraycopy(const TypePtr* adr_type, |
213 Node* src, Node* src_offset, |
212 Node* src, Node* src_offset, |
214 Node* dest, Node* dest_offset, |
213 Node* dest, Node* dest_offset, |
215 Node* copy_length, int nargs); |
214 Node* copy_length); |
216 void generate_unchecked_arraycopy(const TypePtr* adr_type, |
215 void generate_unchecked_arraycopy(const TypePtr* adr_type, |
217 BasicType basic_elem_type, |
216 BasicType basic_elem_type, |
218 bool disjoint_bases, |
217 bool disjoint_bases, |
219 Node* src, Node* src_offset, |
218 Node* src, Node* src_offset, |
220 Node* dest, Node* dest_offset, |
219 Node* dest, Node* dest_offset, |
3227 // How many elements will we copy from the original? |
3226 // How many elements will we copy from the original? |
3228 // The answer is MinI(orig_length - start, length). |
3227 // The answer is MinI(orig_length - start, length). |
3229 Node* orig_tail = _gvn.transform( new(C, 3) SubINode(orig_length, start) ); |
3228 Node* orig_tail = _gvn.transform( new(C, 3) SubINode(orig_length, start) ); |
3230 Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length); |
3229 Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length); |
3231 |
3230 |
3232 Node* newcopy = new_array(klass_node, length, nargs); |
3231 const bool raw_mem_only = true; |
|
3232 Node* newcopy = new_array(klass_node, length, nargs, raw_mem_only); |
3233 |
3233 |
3234 // Generate a direct call to the right arraycopy function(s). |
3234 // Generate a direct call to the right arraycopy function(s). |
3235 // We know the copy is disjoint but we might not know if the |
3235 // We know the copy is disjoint but we might not know if the |
3236 // oop stores need checking. |
3236 // oop stores need checking. |
3237 // Extreme case: Arrays.copyOf((Integer[])x, 10, String[].class). |
3237 // Extreme case: Arrays.copyOf((Integer[])x, 10, String[].class). |
3238 // This will fail a store-check if x contains any non-nulls. |
3238 // This will fail a store-check if x contains any non-nulls. |
3239 bool disjoint_bases = true; |
3239 bool disjoint_bases = true; |
3240 bool length_never_negative = true; |
3240 bool length_never_negative = true; |
3241 generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT, |
3241 generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT, |
3242 original, start, newcopy, intcon(0), moved, |
3242 original, start, newcopy, intcon(0), moved, |
3243 nargs, disjoint_bases, length_never_negative); |
3243 disjoint_bases, length_never_negative); |
3244 |
3244 |
3245 push(newcopy); |
3245 push(newcopy); |
3246 } |
3246 } |
3247 |
3247 |
3248 C->set_has_split_ifs(true); // Has chance for split-if optimization |
3248 C->set_has_split_ifs(true); // Has chance for split-if optimization |
3880 insert_mem_bar(Op_MemBarCPUOrder); |
3880 insert_mem_bar(Op_MemBarCPUOrder); |
3881 |
3881 |
3882 return true; |
3882 return true; |
3883 } |
3883 } |
3884 |
3884 |
|
3885 //------------------------clone_coping----------------------------------- |
|
3886 // Helper function for inline_native_clone. |
|
3887 void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) { |
|
3888 assert(obj_size != NULL, ""); |
|
3889 Node* raw_obj = alloc_obj->in(1); |
|
3890 assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); |
|
3891 |
|
3892 if (ReduceBulkZeroing) { |
|
3893 // We will be completely responsible for initializing this object - |
|
3894 // mark Initialize node as complete. |
|
3895 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); |
|
3896 // The object was just allocated - there should be no any stores! |
|
3897 guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), ""); |
|
3898 } |
|
3899 |
|
3900 // Cast to Object for arraycopy. |
|
3901 // We can't use the original CheckCastPP since it should be moved |
|
3902 // after the arraycopy to prevent stores flowing above it. |
|
3903 Node* new_obj = new(C, 2) CheckCastPPNode(alloc_obj->in(0), raw_obj, |
|
3904 TypeInstPtr::NOTNULL); |
|
3905 new_obj = _gvn.transform(new_obj); |
|
3906 // Substitute in the locally valid dest_oop. |
|
3907 replace_in_map(alloc_obj, new_obj); |
|
3908 |
|
3909 // Copy the fastest available way. |
|
3910 // TODO: generate fields copies for small objects instead. |
|
3911 Node* src = obj; |
|
3912 Node* dest = new_obj; |
|
3913 Node* size = _gvn.transform(obj_size); |
|
3914 |
|
3915 // Exclude the header but include array length to copy by 8 bytes words. |
|
3916 // Can't use base_offset_in_bytes(bt) since basic type is unknown. |
|
3917 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() : |
|
3918 instanceOopDesc::base_offset_in_bytes(); |
|
3919 // base_off: |
|
3920 // 8 - 32-bit VM |
|
3921 // 12 - 64-bit VM, compressed oops |
|
3922 // 16 - 64-bit VM, normal oops |
|
3923 if (base_off % BytesPerLong != 0) { |
|
3924 assert(UseCompressedOops, ""); |
|
3925 if (is_array) { |
|
3926 // Exclude length to copy by 8 bytes words. |
|
3927 base_off += sizeof(int); |
|
3928 } else { |
|
3929 // Include klass to copy by 8 bytes words. |
|
3930 base_off = instanceOopDesc::klass_offset_in_bytes(); |
|
3931 } |
|
3932 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment"); |
|
3933 } |
|
3934 src = basic_plus_adr(src, base_off); |
|
3935 dest = basic_plus_adr(dest, base_off); |
|
3936 |
|
3937 // Compute the length also, if needed: |
|
3938 Node* countx = size; |
|
3939 countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(base_off)) ); |
|
3940 countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) )); |
|
3941 |
|
3942 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; |
|
3943 bool disjoint_bases = true; |
|
3944 generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases, |
|
3945 src, NULL, dest, NULL, countx); |
|
3946 |
|
3947 // If necessary, emit some card marks afterwards. (Non-arrays only.) |
|
3948 if (card_mark) { |
|
3949 assert(!is_array, ""); |
|
3950 // Put in store barrier for any and all oops we are sticking |
|
3951 // into this object. (We could avoid this if we could prove |
|
3952 // that the object type contains no oop fields at all.) |
|
3953 Node* no_particular_value = NULL; |
|
3954 Node* no_particular_field = NULL; |
|
3955 int raw_adr_idx = Compile::AliasIdxRaw; |
|
3956 post_barrier(control(), |
|
3957 memory(raw_adr_type), |
|
3958 new_obj, |
|
3959 no_particular_field, |
|
3960 raw_adr_idx, |
|
3961 no_particular_value, |
|
3962 T_OBJECT, |
|
3963 false); |
|
3964 } |
|
3965 |
|
3966 // Move the original CheckCastPP after arraycopy. |
|
3967 _gvn.hash_delete(alloc_obj); |
|
3968 alloc_obj->set_req(0, control()); |
|
3969 // Replace raw memory edge with new CheckCastPP to have a live oop |
|
3970 // at safepoints instead of raw value. |
|
3971 assert(new_obj->is_CheckCastPP() && new_obj->in(1) == alloc_obj->in(1), "sanity"); |
|
3972 alloc_obj->set_req(1, new_obj); // cast to the original type |
|
3973 _gvn.hash_find_insert(alloc_obj); // put back into GVN table |
|
3974 // Restore in the locally valid dest_oop. |
|
3975 replace_in_map(new_obj, alloc_obj); |
|
3976 } |
3885 |
3977 |
3886 //------------------------inline_native_clone---------------------------- |
3978 //------------------------inline_native_clone---------------------------- |
3887 // Here are the simple edge cases: |
3979 // Here are the simple edge cases: |
3888 // null receiver => normal trap |
3980 // null receiver => normal trap |
3889 // virtual and clone was overridden => slow path to out-of-line clone |
3981 // virtual and clone was overridden => slow path to out-of-line clone |
3914 insert_mem_bar(Op_MemBarCPUOrder); |
4006 insert_mem_bar(Op_MemBarCPUOrder); |
3915 |
4007 |
3916 // paths into result_reg: |
4008 // paths into result_reg: |
3917 enum { |
4009 enum { |
3918 _slow_path = 1, // out-of-line call to clone method (virtual or not) |
4010 _slow_path = 1, // out-of-line call to clone method (virtual or not) |
3919 _objArray_path, // plain allocation, plus arrayof_oop_arraycopy |
4011 _objArray_path, // plain array allocation, plus arrayof_oop_arraycopy |
3920 _fast_path, // plain allocation, plus a CopyArray operation |
4012 _array_path, // plain array allocation, plus arrayof_long_arraycopy |
|
4013 _instance_path, // plain instance allocation, plus arrayof_long_arraycopy |
3921 PATH_LIMIT |
4014 PATH_LIMIT |
3922 }; |
4015 }; |
3923 RegionNode* result_reg = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT); |
4016 RegionNode* result_reg = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT); |
3924 PhiNode* result_val = new(C, PATH_LIMIT) PhiNode(result_reg, |
4017 PhiNode* result_val = new(C, PATH_LIMIT) PhiNode(result_reg, |
3925 TypeInstPtr::NOTNULL); |
4018 TypeInstPtr::NOTNULL); |
3930 |
4023 |
3931 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; |
4024 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; |
3932 int raw_adr_idx = Compile::AliasIdxRaw; |
4025 int raw_adr_idx = Compile::AliasIdxRaw; |
3933 const bool raw_mem_only = true; |
4026 const bool raw_mem_only = true; |
3934 |
4027 |
3935 // paths into alloc_reg (on the fast path, just before the CopyArray): |
|
3936 enum { _typeArray_alloc = 1, _instance_alloc, ALLOC_LIMIT }; |
|
3937 RegionNode* alloc_reg = new(C, ALLOC_LIMIT) RegionNode(ALLOC_LIMIT); |
|
3938 PhiNode* alloc_val = new(C, ALLOC_LIMIT) PhiNode(alloc_reg, raw_adr_type); |
|
3939 PhiNode* alloc_siz = new(C, ALLOC_LIMIT) PhiNode(alloc_reg, TypeX_X); |
|
3940 PhiNode* alloc_i_o = new(C, ALLOC_LIMIT) PhiNode(alloc_reg, Type::ABIO); |
|
3941 PhiNode* alloc_mem = new(C, ALLOC_LIMIT) PhiNode(alloc_reg, Type::MEMORY, |
|
3942 raw_adr_type); |
|
3943 record_for_igvn(alloc_reg); |
|
3944 |
|
3945 bool card_mark = false; // (see below) |
|
3946 |
|
3947 Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL); |
4028 Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL); |
3948 if (array_ctl != NULL) { |
4029 if (array_ctl != NULL) { |
3949 // It's an array. |
4030 // It's an array. |
3950 PreserveJVMState pjvms(this); |
4031 PreserveJVMState pjvms(this); |
3951 set_control(array_ctl); |
4032 set_control(array_ctl); |
3952 Node* obj_length = load_array_length(obj); |
4033 Node* obj_length = load_array_length(obj); |
3953 Node* obj_size = NULL; |
4034 Node* obj_size = NULL; |
3954 Node* alloc_obj = new_array(obj_klass, obj_length, nargs, |
4035 Node* alloc_obj = new_array(obj_klass, obj_length, nargs, |
3955 raw_mem_only, &obj_size); |
4036 raw_mem_only, &obj_size); |
3956 assert(obj_size != NULL, ""); |
|
3957 Node* raw_obj = alloc_obj->in(1); |
|
3958 assert(raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); |
|
3959 if (ReduceBulkZeroing) { |
|
3960 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); |
|
3961 if (alloc != NULL) { |
|
3962 // We will be completely responsible for initializing this object. |
|
3963 alloc->maybe_set_complete(&_gvn); |
|
3964 } |
|
3965 } |
|
3966 |
4037 |
3967 if (!use_ReduceInitialCardMarks()) { |
4038 if (!use_ReduceInitialCardMarks()) { |
3968 // If it is an oop array, it requires very special treatment, |
4039 // If it is an oop array, it requires very special treatment, |
3969 // because card marking is required on each card of the array. |
4040 // because card marking is required on each card of the array. |
3970 Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL); |
4041 Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL); |
3974 // Generate a direct call to the right arraycopy function(s). |
4045 // Generate a direct call to the right arraycopy function(s). |
3975 bool disjoint_bases = true; |
4046 bool disjoint_bases = true; |
3976 bool length_never_negative = true; |
4047 bool length_never_negative = true; |
3977 generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT, |
4048 generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT, |
3978 obj, intcon(0), alloc_obj, intcon(0), |
4049 obj, intcon(0), alloc_obj, intcon(0), |
3979 obj_length, nargs, |
4050 obj_length, |
3980 disjoint_bases, length_never_negative); |
4051 disjoint_bases, length_never_negative); |
3981 result_reg->init_req(_objArray_path, control()); |
4052 result_reg->init_req(_objArray_path, control()); |
3982 result_val->init_req(_objArray_path, alloc_obj); |
4053 result_val->init_req(_objArray_path, alloc_obj); |
3983 result_i_o ->set_req(_objArray_path, i_o()); |
4054 result_i_o ->set_req(_objArray_path, i_o()); |
3984 result_mem ->set_req(_objArray_path, reset_memory()); |
4055 result_mem ->set_req(_objArray_path, reset_memory()); |
3989 // causes the non-eden paths to simulate a fresh allocation, |
4060 // causes the non-eden paths to simulate a fresh allocation, |
3990 // insofar that no further card marks are required to initialize |
4061 // insofar that no further card marks are required to initialize |
3991 // the object. |
4062 // the object. |
3992 |
4063 |
3993 // Otherwise, there are no card marks to worry about. |
4064 // Otherwise, there are no card marks to worry about. |
3994 alloc_val->init_req(_typeArray_alloc, raw_obj); |
4065 |
3995 alloc_siz->init_req(_typeArray_alloc, obj_size); |
4066 if (!stopped()) { |
3996 alloc_reg->init_req(_typeArray_alloc, control()); |
4067 copy_to_clone(obj, alloc_obj, obj_size, true, false); |
3997 alloc_i_o->init_req(_typeArray_alloc, i_o()); |
4068 |
3998 alloc_mem->init_req(_typeArray_alloc, memory(raw_adr_type)); |
4069 // Present the results of the copy. |
3999 } |
4070 result_reg->init_req(_array_path, control()); |
4000 |
4071 result_val->init_req(_array_path, alloc_obj); |
4001 // We only go to the fast case code if we pass a number of guards. |
4072 result_i_o ->set_req(_array_path, i_o()); |
|
4073 result_mem ->set_req(_array_path, reset_memory()); |
|
4074 } |
|
4075 } |
|
4076 |
|
4077 // We only go to the instance fast case code if we pass a number of guards. |
4002 // The paths which do not pass are accumulated in the slow_region. |
4078 // The paths which do not pass are accumulated in the slow_region. |
4003 RegionNode* slow_region = new (C, 1) RegionNode(1); |
4079 RegionNode* slow_region = new (C, 1) RegionNode(1); |
4004 record_for_igvn(slow_region); |
4080 record_for_igvn(slow_region); |
4005 if (!stopped()) { |
4081 if (!stopped()) { |
4006 // It's an instance. Make the slow-path tests. |
4082 // It's an instance (we did array above). Make the slow-path tests. |
4007 // If this is a virtual call, we generate a funny guard. We grab |
4083 // If this is a virtual call, we generate a funny guard. We grab |
4008 // the vtable entry corresponding to clone() from the target object. |
4084 // the vtable entry corresponding to clone() from the target object. |
4009 // If the target method which we are calling happens to be the |
4085 // If the target method which we are calling happens to be the |
4010 // Object clone() method, we pass the guard. We do not need this |
4086 // Object clone() method, we pass the guard. We do not need this |
4011 // guard for non-virtual calls; the caller is known to be the native |
4087 // guard for non-virtual calls; the caller is known to be the native |
4028 if (!stopped()) { |
4104 if (!stopped()) { |
4029 // It's an instance, and it passed the slow-path tests. |
4105 // It's an instance, and it passed the slow-path tests. |
4030 PreserveJVMState pjvms(this); |
4106 PreserveJVMState pjvms(this); |
4031 Node* obj_size = NULL; |
4107 Node* obj_size = NULL; |
4032 Node* alloc_obj = new_instance(obj_klass, NULL, raw_mem_only, &obj_size); |
4108 Node* alloc_obj = new_instance(obj_klass, NULL, raw_mem_only, &obj_size); |
4033 assert(obj_size != NULL, ""); |
4109 |
4034 Node* raw_obj = alloc_obj->in(1); |
4110 copy_to_clone(obj, alloc_obj, obj_size, false, !use_ReduceInitialCardMarks()); |
4035 assert(raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); |
4111 |
4036 if (ReduceBulkZeroing) { |
4112 // Present the results of the slow call. |
4037 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); |
4113 result_reg->init_req(_instance_path, control()); |
4038 if (alloc != NULL && !alloc->maybe_set_complete(&_gvn)) |
4114 result_val->init_req(_instance_path, alloc_obj); |
4039 alloc = NULL; |
4115 result_i_o ->set_req(_instance_path, i_o()); |
4040 } |
4116 result_mem ->set_req(_instance_path, reset_memory()); |
4041 if (!use_ReduceInitialCardMarks()) { |
|
4042 // Put in store barrier for any and all oops we are sticking |
|
4043 // into this object. (We could avoid this if we could prove |
|
4044 // that the object type contains no oop fields at all.) |
|
4045 card_mark = true; |
|
4046 } |
|
4047 alloc_val->init_req(_instance_alloc, raw_obj); |
|
4048 alloc_siz->init_req(_instance_alloc, obj_size); |
|
4049 alloc_reg->init_req(_instance_alloc, control()); |
|
4050 alloc_i_o->init_req(_instance_alloc, i_o()); |
|
4051 alloc_mem->init_req(_instance_alloc, memory(raw_adr_type)); |
|
4052 } |
4117 } |
4053 |
4118 |
4054 // Generate code for the slow case. We make a call to clone(). |
4119 // Generate code for the slow case. We make a call to clone(). |
4055 set_control(_gvn.transform(slow_region)); |
4120 set_control(_gvn.transform(slow_region)); |
4056 if (!stopped()) { |
4121 if (!stopped()) { |
4062 result_val->init_req(_slow_path, slow_result); |
4127 result_val->init_req(_slow_path, slow_result); |
4063 result_i_o ->set_req(_slow_path, i_o()); |
4128 result_i_o ->set_req(_slow_path, i_o()); |
4064 result_mem ->set_req(_slow_path, reset_memory()); |
4129 result_mem ->set_req(_slow_path, reset_memory()); |
4065 } |
4130 } |
4066 |
4131 |
4067 // The object is allocated, as an array and/or an instance. Now copy it. |
|
4068 set_control( _gvn.transform(alloc_reg) ); |
|
4069 set_i_o( _gvn.transform(alloc_i_o) ); |
|
4070 set_memory( _gvn.transform(alloc_mem), raw_adr_type ); |
|
4071 Node* raw_obj = _gvn.transform(alloc_val); |
|
4072 |
|
4073 if (!stopped()) { |
|
4074 // Copy the fastest available way. |
|
4075 // (No need for PreserveJVMState, since we're using it all up now.) |
|
4076 // TODO: generate fields/elements copies for small objects instead. |
|
4077 Node* src = obj; |
|
4078 Node* dest = raw_obj; |
|
4079 Node* size = _gvn.transform(alloc_siz); |
|
4080 |
|
4081 // Exclude the header. |
|
4082 int base_off = instanceOopDesc::base_offset_in_bytes(); |
|
4083 if (UseCompressedOops) { |
|
4084 assert(base_off % BytesPerLong != 0, "base with compressed oops"); |
|
4085 // With compressed oops base_offset_in_bytes is 12 which creates |
|
4086 // the gap since countx is rounded by 8 bytes below. |
|
4087 // Copy klass and the gap. |
|
4088 base_off = instanceOopDesc::klass_offset_in_bytes(); |
|
4089 } |
|
4090 src = basic_plus_adr(src, base_off); |
|
4091 dest = basic_plus_adr(dest, base_off); |
|
4092 |
|
4093 // Compute the length also, if needed: |
|
4094 Node* countx = size; |
|
4095 countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(base_off)) ); |
|
4096 countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) )); |
|
4097 |
|
4098 // Select an appropriate instruction to initialize the range. |
|
4099 // The CopyArray instruction (if supported) can be optimized |
|
4100 // into a discrete set of scalar loads and stores. |
|
4101 bool disjoint_bases = true; |
|
4102 generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases, |
|
4103 src, NULL, dest, NULL, countx); |
|
4104 |
|
4105 // Now that the object is properly initialized, type it as an oop. |
|
4106 // Use a secondary InitializeNode memory barrier. |
|
4107 InitializeNode* init = insert_mem_bar_volatile(Op_Initialize, raw_adr_idx, |
|
4108 raw_obj)->as_Initialize(); |
|
4109 init->set_complete(&_gvn); // (there is no corresponding AllocateNode) |
|
4110 Node* new_obj = new(C, 2) CheckCastPPNode(control(), raw_obj, |
|
4111 TypeInstPtr::NOTNULL); |
|
4112 new_obj = _gvn.transform(new_obj); |
|
4113 |
|
4114 // If necessary, emit some card marks afterwards. (Non-arrays only.) |
|
4115 if (card_mark) { |
|
4116 Node* no_particular_value = NULL; |
|
4117 Node* no_particular_field = NULL; |
|
4118 post_barrier(control(), |
|
4119 memory(raw_adr_type), |
|
4120 new_obj, |
|
4121 no_particular_field, |
|
4122 raw_adr_idx, |
|
4123 no_particular_value, |
|
4124 T_OBJECT, |
|
4125 false); |
|
4126 } |
|
4127 // Present the results of the slow call. |
|
4128 result_reg->init_req(_fast_path, control()); |
|
4129 result_val->init_req(_fast_path, new_obj); |
|
4130 result_i_o ->set_req(_fast_path, i_o()); |
|
4131 result_mem ->set_req(_fast_path, reset_memory()); |
|
4132 } |
|
4133 |
|
4134 // Return the combined state. |
4132 // Return the combined state. |
4135 set_control( _gvn.transform(result_reg) ); |
4133 set_control( _gvn.transform(result_reg) ); |
4136 set_i_o( _gvn.transform(result_i_o) ); |
4134 set_i_o( _gvn.transform(result_i_o) ); |
4137 set_all_memory( _gvn.transform(result_mem) ); |
4135 set_all_memory( _gvn.transform(result_mem) ); |
4138 |
4136 |
4139 // Cast the result to a sharper type, since we know what clone does. |
4137 push(_gvn.transform(result_val)); |
4140 Node* new_obj = _gvn.transform(result_val); |
|
4141 Node* cast = new (C, 2) CheckCastPPNode(control(), new_obj, toop); |
|
4142 push(_gvn.transform(cast)); |
|
4143 |
4138 |
4144 return true; |
4139 return true; |
4145 } |
4140 } |
4146 |
4141 |
4147 |
4142 |
4276 // Do not let writes into the source float below the arraycopy. |
4271 // Do not let writes into the source float below the arraycopy. |
4277 insert_mem_bar(Op_MemBarCPUOrder); |
4272 insert_mem_bar(Op_MemBarCPUOrder); |
4278 |
4273 |
4279 // Call StubRoutines::generic_arraycopy stub. |
4274 // Call StubRoutines::generic_arraycopy stub. |
4280 generate_arraycopy(TypeRawPtr::BOTTOM, T_CONFLICT, |
4275 generate_arraycopy(TypeRawPtr::BOTTOM, T_CONFLICT, |
4281 src, src_offset, dest, dest_offset, length, |
4276 src, src_offset, dest, dest_offset, length); |
4282 nargs); |
|
4283 |
4277 |
4284 // Do not let reads from the destination float above the arraycopy. |
4278 // Do not let reads from the destination float above the arraycopy. |
4285 // Since we cannot type the arrays, we don't know which slices |
4279 // Since we cannot type the arrays, we don't know which slices |
4286 // might be affected. We could restrict this barrier only to those |
4280 // might be affected. We could restrict this barrier only to those |
4287 // memory slices which pertain to array elements--but don't bother. |
4281 // memory slices which pertain to array elements--but don't bother. |
4300 |
4294 |
4301 if (src_elem != dest_elem || dest_elem == T_VOID) { |
4295 if (src_elem != dest_elem || dest_elem == T_VOID) { |
4302 // The component types are not the same or are not recognized. Punt. |
4296 // The component types are not the same or are not recognized. Punt. |
4303 // (But, avoid the native method wrapper to JVM_ArrayCopy.) |
4297 // (But, avoid the native method wrapper to JVM_ArrayCopy.) |
4304 generate_slow_arraycopy(TypePtr::BOTTOM, |
4298 generate_slow_arraycopy(TypePtr::BOTTOM, |
4305 src, src_offset, dest, dest_offset, length, |
4299 src, src_offset, dest, dest_offset, length); |
4306 nargs); |
|
4307 return true; |
4300 return true; |
4308 } |
4301 } |
4309 |
4302 |
4310 //--------------------------------------------------------------------------- |
4303 //--------------------------------------------------------------------------- |
4311 // We will make a fast path for this call to arraycopy. |
4304 // We will make a fast path for this call to arraycopy. |
4358 |
4351 |
4359 // This is where the memory effects are placed: |
4352 // This is where the memory effects are placed: |
4360 const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem); |
4353 const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem); |
4361 generate_arraycopy(adr_type, dest_elem, |
4354 generate_arraycopy(adr_type, dest_elem, |
4362 src, src_offset, dest, dest_offset, length, |
4355 src, src_offset, dest, dest_offset, length, |
4363 nargs, false, false, slow_region); |
4356 false, false, slow_region); |
4364 |
4357 |
4365 return true; |
4358 return true; |
4366 } |
4359 } |
4367 |
4360 |
4368 //-----------------------------generate_arraycopy---------------------- |
4361 //-----------------------------generate_arraycopy---------------------- |
4403 LibraryCallKit::generate_arraycopy(const TypePtr* adr_type, |
4396 LibraryCallKit::generate_arraycopy(const TypePtr* adr_type, |
4404 BasicType basic_elem_type, |
4397 BasicType basic_elem_type, |
4405 Node* src, Node* src_offset, |
4398 Node* src, Node* src_offset, |
4406 Node* dest, Node* dest_offset, |
4399 Node* dest, Node* dest_offset, |
4407 Node* copy_length, |
4400 Node* copy_length, |
4408 int nargs, |
|
4409 bool disjoint_bases, |
4401 bool disjoint_bases, |
4410 bool length_never_negative, |
4402 bool length_never_negative, |
4411 RegionNode* slow_region) { |
4403 RegionNode* slow_region) { |
4412 |
4404 |
4413 if (slow_region == NULL) { |
4405 if (slow_region == NULL) { |
4434 && _gvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0 |
4425 && _gvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0 |
4435 && alloc->maybe_set_complete(&_gvn)) { |
4426 && alloc->maybe_set_complete(&_gvn)) { |
4436 // "You break it, you buy it." |
4427 // "You break it, you buy it." |
4437 InitializeNode* init = alloc->initialization(); |
4428 InitializeNode* init = alloc->initialization(); |
4438 assert(init->is_complete(), "we just did this"); |
4429 assert(init->is_complete(), "we just did this"); |
4439 assert(dest->Opcode() == Op_CheckCastPP, "sanity"); |
4430 assert(dest->is_CheckCastPP(), "sanity"); |
4440 assert(dest->in(0)->in(0) == init, "dest pinned"); |
4431 assert(dest->in(0)->in(0) == init, "dest pinned"); |
4441 raw_dest = dest->in(1); // grab the raw pointer! |
4432 |
4442 original_dest = dest; |
4433 // Cast to Object for arraycopy. |
4443 dest = raw_dest; |
4434 // We can't use the original CheckCastPP since it should be moved |
|
4435 // after the arraycopy to prevent stores flowing above it. |
|
4436 Node* new_obj = new(C, 2) CheckCastPPNode(dest->in(0), dest->in(1), |
|
4437 TypeInstPtr::NOTNULL); |
|
4438 dest = _gvn.transform(new_obj); |
|
4439 // Substitute in the locally valid dest_oop. |
|
4440 replace_in_map(original_dest, dest); |
4444 adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory |
4441 adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory |
4445 // Decouple the original InitializeNode, turning it into a simple membar. |
|
4446 // We will build a new one at the end of this routine. |
|
4447 init->set_req(InitializeNode::RawAddress, top()); |
|
4448 // From this point on, every exit path is responsible for |
4442 // From this point on, every exit path is responsible for |
4449 // initializing any non-copied parts of the object to zero. |
4443 // initializing any non-copied parts of the object to zero. |
4450 must_clear_dest = true; |
4444 must_clear_dest = true; |
4451 } else { |
4445 } else { |
4452 // No zeroing elimination here. |
4446 // No zeroing elimination here. |
4485 |
4479 |
4486 if (basic_elem_type == T_CONFLICT) { |
4480 if (basic_elem_type == T_CONFLICT) { |
4487 assert(!must_clear_dest, ""); |
4481 assert(!must_clear_dest, ""); |
4488 Node* cv = generate_generic_arraycopy(adr_type, |
4482 Node* cv = generate_generic_arraycopy(adr_type, |
4489 src, src_offset, dest, dest_offset, |
4483 src, src_offset, dest, dest_offset, |
4490 copy_length, nargs); |
4484 copy_length); |
4491 if (cv == NULL) cv = intcon(-1); // failure (no stub available) |
4485 if (cv == NULL) cv = intcon(-1); // failure (no stub available) |
4492 checked_control = control(); |
4486 checked_control = control(); |
4493 checked_i_o = i_o(); |
4487 checked_i_o = i_o(); |
4494 checked_mem = memory(adr_type); |
4488 checked_mem = memory(adr_type); |
4495 checked_value = cv; |
4489 checked_value = cv; |
4516 generate_clear_array(adr_type, dest, basic_elem_type, |
4510 generate_clear_array(adr_type, dest, basic_elem_type, |
4517 intcon(0), NULL, |
4511 intcon(0), NULL, |
4518 alloc->in(AllocateNode::AllocSize)); |
4512 alloc->in(AllocateNode::AllocSize)); |
4519 } |
4513 } |
4520 } |
4514 } |
|
4515 |
|
4516 // Use a secondary InitializeNode as raw memory barrier. |
|
4517 // Currently it is needed only on this path since other |
|
4518 // paths have stub or runtime calls as raw memory barriers. |
|
4519 InitializeNode* init = insert_mem_bar_volatile(Op_Initialize, |
|
4520 Compile::AliasIdxRaw, |
|
4521 top())->as_Initialize(); |
|
4522 init->set_complete(&_gvn); // (there is no corresponding AllocateNode) |
4521 |
4523 |
4522 // Present the results of the fast call. |
4524 // Present the results of the fast call. |
4523 result_region->init_req(zero_path, control()); |
4525 result_region->init_req(zero_path, control()); |
4524 result_i_o ->init_req(zero_path, i_o()); |
4526 result_i_o ->init_req(zero_path, i_o()); |
4525 result_memory->init_req(zero_path, memory(adr_type)); |
4527 result_memory->init_req(zero_path, memory(adr_type)); |
4635 Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM); |
4637 Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM); |
4636 Node* dest_elem_klass = _gvn.transform(n1); |
4638 Node* dest_elem_klass = _gvn.transform(n1); |
4637 Node* cv = generate_checkcast_arraycopy(adr_type, |
4639 Node* cv = generate_checkcast_arraycopy(adr_type, |
4638 dest_elem_klass, |
4640 dest_elem_klass, |
4639 src, src_offset, dest, dest_offset, |
4641 src, src_offset, dest, dest_offset, |
4640 copy_length, |
4642 copy_length); |
4641 nargs); |
|
4642 if (cv == NULL) cv = intcon(-1); // failure (no stub available) |
4643 if (cv == NULL) cv = intcon(-1); // failure (no stub available) |
4643 checked_control = control(); |
4644 checked_control = control(); |
4644 checked_i_o = i_o(); |
4645 checked_i_o = i_o(); |
4645 checked_mem = memory(adr_type); |
4646 checked_mem = memory(adr_type); |
4646 checked_value = cv; |
4647 checked_value = cv; |
4698 record_for_igvn(slow_reg2); |
4699 record_for_igvn(slow_reg2); |
4699 slow_reg2 ->init_req(1, slow_control); |
4700 slow_reg2 ->init_req(1, slow_control); |
4700 slow_i_o2 ->init_req(1, slow_i_o); |
4701 slow_i_o2 ->init_req(1, slow_i_o); |
4701 slow_mem2 ->init_req(1, slow_mem); |
4702 slow_mem2 ->init_req(1, slow_mem); |
4702 slow_reg2 ->init_req(2, control()); |
4703 slow_reg2 ->init_req(2, control()); |
4703 slow_i_o2 ->init_req(2, i_o()); |
4704 slow_i_o2 ->init_req(2, checked_i_o); |
4704 slow_mem2 ->init_req(2, memory(adr_type)); |
4705 slow_mem2 ->init_req(2, checked_mem); |
4705 |
4706 |
4706 slow_control = _gvn.transform(slow_reg2); |
4707 slow_control = _gvn.transform(slow_reg2); |
4707 slow_i_o = _gvn.transform(slow_i_o2); |
4708 slow_i_o = _gvn.transform(slow_i_o2); |
4708 slow_mem = _gvn.transform(slow_mem2); |
4709 slow_mem = _gvn.transform(slow_mem2); |
4709 |
4710 |
4744 generate_clear_array(adr_type, dest, basic_elem_type, |
4745 generate_clear_array(adr_type, dest, basic_elem_type, |
4745 intcon(0), NULL, |
4746 intcon(0), NULL, |
4746 alloc->in(AllocateNode::AllocSize)); |
4747 alloc->in(AllocateNode::AllocSize)); |
4747 } |
4748 } |
4748 |
4749 |
4749 if (dest != original_dest) { |
|
4750 // Promote from rawptr to oop, so it looks right in the call's GC map. |
|
4751 dest = _gvn.transform( new(C,2) CheckCastPPNode(control(), dest, |
|
4752 TypeInstPtr::NOTNULL) ); |
|
4753 |
|
4754 // Edit the call's debug-info to avoid referring to original_dest. |
|
4755 // (The problem with original_dest is that it isn't ready until |
|
4756 // after the InitializeNode completes, but this stuff is before.) |
|
4757 // Substitute in the locally valid dest_oop. |
|
4758 replace_in_map(original_dest, dest); |
|
4759 } |
|
4760 |
|
4761 generate_slow_arraycopy(adr_type, |
4750 generate_slow_arraycopy(adr_type, |
4762 src, src_offset, dest, dest_offset, |
4751 src, src_offset, dest, dest_offset, |
4763 copy_length, nargs); |
4752 copy_length); |
4764 |
4753 |
4765 result_region->init_req(slow_call_path, control()); |
4754 result_region->init_req(slow_call_path, control()); |
4766 result_i_o ->init_req(slow_call_path, i_o()); |
4755 result_i_o ->init_req(slow_call_path, i_o()); |
4767 result_memory->init_req(slow_call_path, memory(adr_type)); |
4756 result_memory->init_req(slow_call_path, memory(adr_type)); |
4768 } |
4757 } |
4778 set_i_o( _gvn.transform(result_i_o) ); |
4767 set_i_o( _gvn.transform(result_i_o) ); |
4779 set_memory( _gvn.transform(result_memory), adr_type ); |
4768 set_memory( _gvn.transform(result_memory), adr_type ); |
4780 |
4769 |
4781 if (dest != original_dest) { |
4770 if (dest != original_dest) { |
4782 // Pin the "finished" array node after the arraycopy/zeroing operations. |
4771 // Pin the "finished" array node after the arraycopy/zeroing operations. |
4783 // Use a secondary InitializeNode memory barrier. |
|
4784 InitializeNode* init = insert_mem_bar_volatile(Op_Initialize, |
|
4785 Compile::AliasIdxRaw, |
|
4786 raw_dest)->as_Initialize(); |
|
4787 init->set_complete(&_gvn); // (there is no corresponding AllocateNode) |
|
4788 _gvn.hash_delete(original_dest); |
4772 _gvn.hash_delete(original_dest); |
4789 original_dest->set_req(0, control()); |
4773 original_dest->set_req(0, control()); |
|
4774 // Replace raw memory edge with new CheckCastPP to have a live oop |
|
4775 // at safepoints instead of raw value. |
|
4776 assert(dest->is_CheckCastPP() && dest->in(1) == original_dest->in(1), "sanity"); |
|
4777 original_dest->set_req(1, dest); // cast to the original type |
4790 _gvn.hash_find_insert(original_dest); // put back into GVN table |
4778 _gvn.hash_find_insert(original_dest); // put back into GVN table |
4791 } |
4779 // Restore in the locally valid dest_oop. |
4792 |
4780 replace_in_map(dest, original_dest); |
|
4781 } |
4793 // The memory edges above are precise in order to model effects around |
4782 // The memory edges above are precise in order to model effects around |
4794 // array copies accurately to allow value numbering of field loads around |
4783 // array copies accurately to allow value numbering of field loads around |
4795 // arraycopy. Such field loads, both before and after, are common in Java |
4784 // arraycopy. Such field loads, both before and after, are common in Java |
4796 // collections and similar classes involving header/array data structures. |
4785 // collections and similar classes involving header/array data structures. |
4797 // |
4786 // |
5071 // but without the native wrapper overhead. |
5060 // but without the native wrapper overhead. |
5072 void |
5061 void |
5073 LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type, |
5062 LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type, |
5074 Node* src, Node* src_offset, |
5063 Node* src, Node* src_offset, |
5075 Node* dest, Node* dest_offset, |
5064 Node* dest, Node* dest_offset, |
5076 Node* copy_length, |
5065 Node* copy_length) { |
5077 int nargs) { |
|
5078 _sp += nargs; // any deopt will start just before call to enclosing method |
|
5079 Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON, |
5066 Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON, |
5080 OptoRuntime::slow_arraycopy_Type(), |
5067 OptoRuntime::slow_arraycopy_Type(), |
5081 OptoRuntime::slow_arraycopy_Java(), |
5068 OptoRuntime::slow_arraycopy_Java(), |
5082 "slow_arraycopy", adr_type, |
5069 "slow_arraycopy", adr_type, |
5083 src, src_offset, dest, dest_offset, |
5070 src, src_offset, dest, dest_offset, |
5084 copy_length); |
5071 copy_length); |
5085 _sp -= nargs; |
|
5086 |
5072 |
5087 // Handle exceptions thrown by this fellow: |
5073 // Handle exceptions thrown by this fellow: |
5088 make_slow_call_ex(call, env()->Throwable_klass(), false); |
5074 make_slow_call_ex(call, env()->Throwable_klass(), false); |
5089 } |
5075 } |
5090 |
5076 |
5092 Node* |
5078 Node* |
5093 LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type, |
5079 LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type, |
5094 Node* dest_elem_klass, |
5080 Node* dest_elem_klass, |
5095 Node* src, Node* src_offset, |
5081 Node* src, Node* src_offset, |
5096 Node* dest, Node* dest_offset, |
5082 Node* dest, Node* dest_offset, |
5097 Node* copy_length, |
5083 Node* copy_length) { |
5098 int nargs) { |
|
5099 if (stopped()) return NULL; |
5084 if (stopped()) return NULL; |
5100 |
5085 |
5101 address copyfunc_addr = StubRoutines::checkcast_arraycopy(); |
5086 address copyfunc_addr = StubRoutines::checkcast_arraycopy(); |
5102 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. |
5087 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. |
5103 return NULL; |
5088 return NULL; |
5134 // Helper function; generates code for cases requiring runtime checks. |
5119 // Helper function; generates code for cases requiring runtime checks. |
5135 Node* |
5120 Node* |
5136 LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type, |
5121 LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type, |
5137 Node* src, Node* src_offset, |
5122 Node* src, Node* src_offset, |
5138 Node* dest, Node* dest_offset, |
5123 Node* dest, Node* dest_offset, |
5139 Node* copy_length, |
5124 Node* copy_length) { |
5140 int nargs) { |
|
5141 if (stopped()) return NULL; |
5125 if (stopped()) return NULL; |
5142 |
5126 |
5143 address copyfunc_addr = StubRoutines::generic_arraycopy(); |
5127 address copyfunc_addr = StubRoutines::generic_arraycopy(); |
5144 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. |
5128 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. |
5145 return NULL; |
5129 return NULL; |