Mon, 03 Dec 2018 07:29:54 -0500
8141491: Unaligned memory access in Bits.c
Summary: Introduce alignment-safe Copy::conjoint_swap and JVM_CopySwapMemory
Reviewed-by: mikael, dholmes
1.1 --- a/make/aix/makefiles/mapfile-vers-debug Wed Nov 28 13:49:11 2018 +0000 1.2 +++ b/make/aix/makefiles/mapfile-vers-debug Mon Dec 03 07:29:54 2018 -0500 1.3 @@ -1,5 +1,5 @@ 1.4 # 1.5 -# Copyright (c) 2002, 2013, Oracle and/or its affiliates. All rights reserved. 1.6 +# Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved. 1.7 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 1.8 # 1.9 # This code is free software; you can redistribute it and/or modify it 1.10 @@ -63,6 +63,7 @@ 1.11 JVM_ConstantPoolGetSize; 1.12 JVM_ConstantPoolGetStringAt; 1.13 JVM_ConstantPoolGetUTF8At; 1.14 + JVM_CopySwapMemory; 1.15 JVM_CountStackFrames; 1.16 JVM_CurrentClassLoader; 1.17 JVM_CurrentLoadedClass;
2.1 --- a/make/aix/makefiles/mapfile-vers-product Wed Nov 28 13:49:11 2018 +0000 2.2 +++ b/make/aix/makefiles/mapfile-vers-product Mon Dec 03 07:29:54 2018 -0500 2.3 @@ -1,5 +1,5 @@ 2.4 # 2.5 -# Copyright (c) 2002, 2013, Oracle and/or its affiliates. All rights reserved. 2.6 +# Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved. 2.7 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 2.8 # 2.9 # This code is free software; you can redistribute it and/or modify it 2.10 @@ -63,6 +63,7 @@ 2.11 JVM_ConstantPoolGetSize; 2.12 JVM_ConstantPoolGetStringAt; 2.13 JVM_ConstantPoolGetUTF8At; 2.14 + JVM_CopySwapMemory; 2.15 JVM_CountStackFrames; 2.16 JVM_CurrentClassLoader; 2.17 JVM_CurrentLoadedClass;
3.1 --- a/make/bsd/makefiles/mapfile-vers-debug Wed Nov 28 13:49:11 2018 +0000 3.2 +++ b/make/bsd/makefiles/mapfile-vers-debug Mon Dec 03 07:29:54 2018 -0500 3.3 @@ -1,5 +1,5 @@ 3.4 # 3.5 -# Copyright (c) 2002, 2014, Oracle and/or its affiliates. All rights reserved. 3.6 +# Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved. 3.7 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 3.8 # 3.9 # This code is free software; you can redistribute it and/or modify it 3.10 @@ -61,6 +61,7 @@ 3.11 _JVM_ConstantPoolGetSize 3.12 _JVM_ConstantPoolGetStringAt 3.13 _JVM_ConstantPoolGetUTF8At 3.14 + _JVM_CopySwapMemory 3.15 _JVM_CountStackFrames 3.16 _JVM_CurrentClassLoader 3.17 _JVM_CurrentLoadedClass
4.1 --- a/make/bsd/makefiles/mapfile-vers-product Wed Nov 28 13:49:11 2018 +0000 4.2 +++ b/make/bsd/makefiles/mapfile-vers-product Mon Dec 03 07:29:54 2018 -0500 4.3 @@ -1,5 +1,5 @@ 4.4 # 4.5 -# Copyright (c) 2002, 2014, Oracle and/or its affiliates. All rights reserved. 4.6 +# Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved. 4.7 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4.8 # 4.9 # This code is free software; you can redistribute it and/or modify it 4.10 @@ -61,6 +61,7 @@ 4.11 _JVM_ConstantPoolGetSize 4.12 _JVM_ConstantPoolGetStringAt 4.13 _JVM_ConstantPoolGetUTF8At 4.14 + _JVM_CopySwapMemory 4.15 _JVM_CountStackFrames 4.16 _JVM_CurrentClassLoader 4.17 _JVM_CurrentLoadedClass
5.1 --- a/make/linux/makefiles/mapfile-vers-debug Wed Nov 28 13:49:11 2018 +0000 5.2 +++ b/make/linux/makefiles/mapfile-vers-debug Mon Dec 03 07:29:54 2018 -0500 5.3 @@ -1,5 +1,5 @@ 5.4 # 5.5 -# Copyright (c) 2002, 2014, Oracle and/or its affiliates. All rights reserved. 5.6 +# Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved. 5.7 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5.8 # 5.9 # This code is free software; you can redistribute it and/or modify it 5.10 @@ -63,6 +63,7 @@ 5.11 JVM_ConstantPoolGetSize; 5.12 JVM_ConstantPoolGetStringAt; 5.13 JVM_ConstantPoolGetUTF8At; 5.14 + JVM_CopySwapMemory; 5.15 JVM_CountStackFrames; 5.16 JVM_CurrentClassLoader; 5.17 JVM_CurrentLoadedClass;
6.1 --- a/make/linux/makefiles/mapfile-vers-product Wed Nov 28 13:49:11 2018 +0000 6.2 +++ b/make/linux/makefiles/mapfile-vers-product Mon Dec 03 07:29:54 2018 -0500 6.3 @@ -1,5 +1,5 @@ 6.4 # 6.5 -# Copyright (c) 2002, 2014, Oracle and/or its affiliates. All rights reserved. 6.6 +# Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved. 6.7 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6.8 # 6.9 # This code is free software; you can redistribute it and/or modify it 6.10 @@ -63,6 +63,7 @@ 6.11 JVM_ConstantPoolGetSize; 6.12 JVM_ConstantPoolGetStringAt; 6.13 JVM_ConstantPoolGetUTF8At; 6.14 + JVM_CopySwapMemory; 6.15 JVM_CountStackFrames; 6.16 JVM_CurrentClassLoader; 6.17 JVM_CurrentLoadedClass;
7.1 --- a/make/solaris/makefiles/mapfile-vers Wed Nov 28 13:49:11 2018 +0000 7.2 +++ b/make/solaris/makefiles/mapfile-vers Mon Dec 03 07:29:54 2018 -0500 7.3 @@ -1,5 +1,5 @@ 7.4 # 7.5 -# Copyright (c) 2000, 2014, Oracle and/or its affiliates. All rights reserved. 7.6 +# Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved. 7.7 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 7.8 # 7.9 # This code is free software; you can redistribute it and/or modify it 7.10 @@ -64,6 +64,7 @@ 7.11 JVM_ConstantPoolGetStringAt; 7.12 JVM_ConstantPoolGetUTF8At; 7.13 JVM_CountStackFrames; 7.14 + JVM_CopySwapMemory; 7.15 JVM_CurrentClassLoader; 7.16 JVM_CurrentLoadedClass; 7.17 JVM_CurrentThread;
8.1 --- a/src/share/vm/prims/jvm.cpp Wed Nov 28 13:49:11 2018 +0000 8.2 +++ b/src/share/vm/prims/jvm.cpp Mon Dec 03 07:29:54 2018 -0500 8.3 @@ -759,6 +759,79 @@ 8.4 JVM_END 8.5 8.6 8.7 +// java.nio.Bits /////////////////////////////////////////////////////////////// 8.8 + 8.9 +#define MAX_OBJECT_SIZE \ 8.10 + ( arrayOopDesc::header_size(T_DOUBLE) * HeapWordSize \ 8.11 + + ((julong)max_jint * sizeof(double)) ) 8.12 + 8.13 +static inline jlong field_offset_to_byte_offset(jlong field_offset) { 8.14 + return field_offset; 8.15 +} 8.16 + 8.17 +static inline void assert_field_offset_sane(oop p, jlong field_offset) { 8.18 +#ifdef ASSERT 8.19 + jlong byte_offset = field_offset_to_byte_offset(field_offset); 8.20 + 8.21 + if (p != NULL) { 8.22 + assert(byte_offset >= 0 && byte_offset <= (jlong)MAX_OBJECT_SIZE, "sane offset"); 8.23 + if (byte_offset == (jint)byte_offset) { 8.24 + void* ptr_plus_disp = (address)p + byte_offset; 8.25 + assert((void*)p->obj_field_addr<oop>((jint)byte_offset) == ptr_plus_disp, 8.26 + "raw [ptr+disp] must be consistent with oop::field_base"); 8.27 + } 8.28 + jlong p_size = HeapWordSize * (jlong)(p->size()); 8.29 + assert(byte_offset < p_size, err_msg("Unsafe access: offset " INT64_FORMAT 8.30 + " > object's size " INT64_FORMAT, 8.31 + (int64_t)byte_offset, (int64_t)p_size)); 8.32 + } 8.33 +#endif 8.34 +} 8.35 + 8.36 +static inline void* index_oop_from_field_offset_long(oop p, jlong field_offset) { 8.37 + assert_field_offset_sane(p, field_offset); 8.38 + jlong byte_offset = field_offset_to_byte_offset(field_offset); 8.39 + 8.40 + if (sizeof(char*) == sizeof(jint)) { // (this constant folds!) 8.41 + return (address)p + (jint) byte_offset; 8.42 + } else { 8.43 + return (address)p + byte_offset; 8.44 + } 8.45 +} 8.46 + 8.47 +// This function is a leaf since if the source and destination are both in native memory 8.48 +// the copy may potentially be very large, and we don't want to disable GC if we can avoid it. 8.49 +// If either source or destination (or both) are on the heap, the function will enter VM using 8.50 +// JVM_ENTRY_FROM_LEAF 8.51 +JVM_LEAF(void, JVM_CopySwapMemory(JNIEnv *env, jobject srcObj, jlong srcOffset, 8.52 + jobject dstObj, jlong dstOffset, jlong size, 8.53 + jlong elemSize)) { 8.54 + 8.55 + size_t sz = (size_t)size; 8.56 + size_t esz = (size_t)elemSize; 8.57 + 8.58 + if (srcObj == NULL && dstObj == NULL) { 8.59 + // Both src & dst are in native memory 8.60 + address src = (address)srcOffset; 8.61 + address dst = (address)dstOffset; 8.62 + 8.63 + Copy::conjoint_swap(src, dst, sz, esz); 8.64 + } else { 8.65 + // At least one of src/dst are on heap, transition to VM to access raw pointers 8.66 + 8.67 + JVM_ENTRY_FROM_LEAF(env, void, JVM_CopySwapMemory) { 8.68 + oop srcp = JNIHandles::resolve(srcObj); 8.69 + oop dstp = JNIHandles::resolve(dstObj); 8.70 + 8.71 + address src = (address)index_oop_from_field_offset_long(srcp, srcOffset); 8.72 + address dst = (address)index_oop_from_field_offset_long(dstp, dstOffset); 8.73 + 8.74 + Copy::conjoint_swap(src, dst, sz, esz); 8.75 + } JVM_END 8.76 + } 8.77 +} JVM_END 8.78 + 8.79 + 8.80 // Misc. class handling /////////////////////////////////////////////////////////// 8.81 8.82
9.1 --- a/src/share/vm/prims/jvm.h Wed Nov 28 13:49:11 2018 +0000 9.2 +++ b/src/share/vm/prims/jvm.h Mon Dec 03 07:29:54 2018 -0500 9.3 @@ -1,5 +1,5 @@ 9.4 /* 9.5 - * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved. 9.6 + * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved. 9.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 9.8 * 9.9 * This code is free software; you can redistribute it and/or modify it 9.10 @@ -145,6 +145,14 @@ 9.11 JVM_OnExit(void (*func)(void)); 9.12 9.13 /* 9.14 + * java.nio.Bits 9.15 + */ 9.16 +JNIEXPORT void JNICALL 9.17 +JVM_CopySwapMemory(JNIEnv *env, jobject srcObj, jlong srcOffset, 9.18 + jobject dstObj, jlong dstOffset, jlong size, 9.19 + jlong elemSize); 9.20 + 9.21 +/* 9.22 * java.lang.Runtime 9.23 */ 9.24 JNIEXPORT void JNICALL
10.1 --- a/src/share/vm/runtime/interfaceSupport.hpp Wed Nov 28 13:49:11 2018 +0000 10.2 +++ b/src/share/vm/runtime/interfaceSupport.hpp Mon Dec 03 07:29:54 2018 -0500 10.3 @@ -1,5 +1,5 @@ 10.4 /* 10.5 - * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved. 10.6 + * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved. 10.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 10.8 * 10.9 * This code is free software; you can redistribute it and/or modify it 10.10 @@ -431,6 +431,14 @@ 10.11 os::verify_stack_alignment(); \ 10.12 /* begin of body */ 10.13 10.14 +#define VM_ENTRY_BASE_FROM_LEAF(result_type, header, thread) \ 10.15 + TRACE_CALL(result_type, header) \ 10.16 + debug_only(ResetNoHandleMark __rnhm;) \ 10.17 + HandleMarkCleaner __hm(thread); \ 10.18 + Thread* THREAD = thread; \ 10.19 + os::verify_stack_alignment(); \ 10.20 + /* begin of body */ 10.21 + 10.22 10.23 // ENTRY routines may lock, GC and throw exceptions 10.24 10.25 @@ -592,6 +600,14 @@ 10.26 VM_LEAF_BASE(result_type, header) 10.27 10.28 10.29 +#define JVM_ENTRY_FROM_LEAF(env, result_type, header) \ 10.30 + { { \ 10.31 + JavaThread* thread=JavaThread::thread_from_jni_environment(env); \ 10.32 + ThreadInVMfromNative __tiv(thread); \ 10.33 + debug_only(VMNativeEntryWrapper __vew;) \ 10.34 + VM_ENTRY_BASE_FROM_LEAF(result_type, header, thread) 10.35 + 10.36 + 10.37 #define JVM_END } } 10.38 10.39 #endif // SHARE_VM_RUNTIME_INTERFACESUPPORT_HPP
11.1 --- a/src/share/vm/utilities/copy.cpp Wed Nov 28 13:49:11 2018 +0000 11.2 +++ b/src/share/vm/utilities/copy.cpp Mon Dec 03 07:29:54 2018 -0500 11.3 @@ -1,5 +1,5 @@ 11.4 /* 11.5 - * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved. 11.6 + * Copyright (c) 2006, 2018, Oracle and/or its affiliates. All rights reserved. 11.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 11.8 * 11.9 * This code is free software; you can redistribute it and/or modify it 11.10 @@ -53,6 +53,175 @@ 11.11 } 11.12 } 11.13 11.14 +class CopySwap : AllStatic { 11.15 +public: 11.16 + /** 11.17 + * Copy and byte swap elements 11.18 + * 11.19 + * @param src address of source 11.20 + * @param dst address of destination 11.21 + * @param byte_count number of bytes to copy 11.22 + * @param elem_size size of the elements to copy-swap 11.23 + */ 11.24 + static void conjoint_swap(address src, address dst, size_t byte_count, size_t elem_size) { 11.25 + assert(src != NULL, "address must not be NULL"); 11.26 + assert(dst != NULL, "address must not be NULL"); 11.27 + assert(elem_size == 2 || elem_size == 4 || elem_size == 8, 11.28 + err_msg("incorrect element size: " SIZE_FORMAT, elem_size)); 11.29 + assert(is_size_aligned(byte_count, elem_size), 11.30 + err_msg("byte_count " SIZE_FORMAT " must be multiple of element size " SIZE_FORMAT, byte_count, elem_size)); 11.31 + 11.32 + address src_end = src + byte_count; 11.33 + 11.34 + if (dst <= src || dst >= src_end) { 11.35 + do_conjoint_swap<RIGHT>(src, dst, byte_count, elem_size); 11.36 + } else { 11.37 + do_conjoint_swap<LEFT>(src, dst, byte_count, elem_size); 11.38 + } 11.39 + } 11.40 + 11.41 +private: 11.42 + /** 11.43 + * Byte swap a 16-bit value 11.44 + */ 11.45 + static uint16_t byte_swap(uint16_t x) { 11.46 + return (x << 8) | (x >> 8); 11.47 + } 11.48 + 11.49 + /** 11.50 + * Byte swap a 32-bit value 11.51 + */ 11.52 + static uint32_t byte_swap(uint32_t x) { 11.53 + uint16_t lo = (uint16_t)x; 11.54 + uint16_t hi = (uint16_t)(x >> 16); 11.55 + 11.56 + return ((uint32_t)byte_swap(lo) << 16) | (uint32_t)byte_swap(hi); 11.57 + } 11.58 + 11.59 + /** 11.60 + * Byte swap a 64-bit value 11.61 + */ 11.62 + static uint64_t byte_swap(uint64_t x) { 11.63 + uint32_t lo = (uint32_t)x; 11.64 + uint32_t hi = (uint32_t)(x >> 32); 11.65 + 11.66 + return ((uint64_t)byte_swap(lo) << 32) | (uint64_t)byte_swap(hi); 11.67 + } 11.68 + 11.69 + enum CopyDirection { 11.70 + RIGHT, // lower -> higher address 11.71 + LEFT // higher -> lower address 11.72 + }; 11.73 + 11.74 + /** 11.75 + * Copy and byte swap elements 11.76 + * 11.77 + * <T> - type of element to copy 11.78 + * <D> - copy direction 11.79 + * <is_src_aligned> - true if src argument is aligned to element size 11.80 + * <is_dst_aligned> - true if dst argument is aligned to element size 11.81 + * 11.82 + * @param src address of source 11.83 + * @param dst address of destination 11.84 + * @param byte_count number of bytes to copy 11.85 + */ 11.86 + template <typename T, CopyDirection D, bool is_src_aligned, bool is_dst_aligned> 11.87 + static void do_conjoint_swap(address src, address dst, size_t byte_count) { 11.88 + address cur_src, cur_dst; 11.89 + 11.90 + switch (D) { 11.91 + case RIGHT: 11.92 + cur_src = src; 11.93 + cur_dst = dst; 11.94 + break; 11.95 + case LEFT: 11.96 + cur_src = src + byte_count - sizeof(T); 11.97 + cur_dst = dst + byte_count - sizeof(T); 11.98 + break; 11.99 + } 11.100 + 11.101 + for (size_t i = 0; i < byte_count / sizeof(T); i++) { 11.102 + T tmp; 11.103 + 11.104 + if (is_src_aligned) { 11.105 + tmp = *(T*)cur_src; 11.106 + } else { 11.107 + memcpy(&tmp, cur_src, sizeof(T)); 11.108 + } 11.109 + 11.110 + tmp = byte_swap(tmp); 11.111 + 11.112 + if (is_dst_aligned) { 11.113 + *(T*)cur_dst = tmp; 11.114 + } else { 11.115 + memcpy(cur_dst, &tmp, sizeof(T)); 11.116 + } 11.117 + 11.118 + switch (D) { 11.119 + case RIGHT: 11.120 + cur_src += sizeof(T); 11.121 + cur_dst += sizeof(T); 11.122 + break; 11.123 + case LEFT: 11.124 + cur_src -= sizeof(T); 11.125 + cur_dst -= sizeof(T); 11.126 + break; 11.127 + } 11.128 + } 11.129 + } 11.130 + 11.131 + /** 11.132 + * Copy and byte swap elements 11.133 + * 11.134 + * <T> - type of element to copy 11.135 + * <D> - copy direction 11.136 + * 11.137 + * @param src address of source 11.138 + * @param dst address of destination 11.139 + * @param byte_count number of bytes to copy 11.140 + */ 11.141 + template <typename T, CopyDirection direction> 11.142 + static void do_conjoint_swap(address src, address dst, size_t byte_count) { 11.143 + if (is_ptr_aligned(src, sizeof(T))) { 11.144 + if (is_ptr_aligned(dst, sizeof(T))) { 11.145 + do_conjoint_swap<T,direction,true,true>(src, dst, byte_count); 11.146 + } else { 11.147 + do_conjoint_swap<T,direction,true,false>(src, dst, byte_count); 11.148 + } 11.149 + } else { 11.150 + if (is_ptr_aligned(dst, sizeof(T))) { 11.151 + do_conjoint_swap<T,direction,false,true>(src, dst, byte_count); 11.152 + } else { 11.153 + do_conjoint_swap<T,direction,false,false>(src, dst, byte_count); 11.154 + } 11.155 + } 11.156 + } 11.157 + 11.158 + 11.159 + /** 11.160 + * Copy and byte swap elements 11.161 + * 11.162 + * <D> - copy direction 11.163 + * 11.164 + * @param src address of source 11.165 + * @param dst address of destination 11.166 + * @param byte_count number of bytes to copy 11.167 + * @param elem_size size of the elements to copy-swap 11.168 + */ 11.169 + template <CopyDirection D> 11.170 + static void do_conjoint_swap(address src, address dst, size_t byte_count, size_t elem_size) { 11.171 + switch (elem_size) { 11.172 + case 2: do_conjoint_swap<uint16_t,D>(src, dst, byte_count); break; 11.173 + case 4: do_conjoint_swap<uint32_t,D>(src, dst, byte_count); break; 11.174 + case 8: do_conjoint_swap<uint64_t,D>(src, dst, byte_count); break; 11.175 + default: guarantee(false, err_msg("do_conjoint_swap: Invalid elem_size %zd\n", elem_size)); 11.176 + } 11.177 + } 11.178 +}; 11.179 + 11.180 +void Copy::conjoint_swap(address src, address dst, size_t byte_count, size_t elem_size) { 11.181 + CopySwap::conjoint_swap(src, dst, byte_count, elem_size); 11.182 +} 11.183 11.184 // Fill bytes; larger units are filled atomically if everything is aligned. 11.185 void Copy::fill_to_memory_atomic(void* to, size_t size, jubyte value) {
12.1 --- a/src/share/vm/utilities/copy.hpp Wed Nov 28 13:49:11 2018 +0000 12.2 +++ b/src/share/vm/utilities/copy.hpp Mon Dec 03 07:29:54 2018 -0500 12.3 @@ -1,5 +1,5 @@ 12.4 /* 12.5 - * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved. 12.6 + * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved. 12.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 12.8 * 12.9 * This code is free software; you can redistribute it and/or modify it 12.10 @@ -227,6 +227,16 @@ 12.11 } 12.12 } 12.13 12.14 + /** 12.15 + * Copy and *unconditionally* byte swap elements 12.16 + * 12.17 + * @param src address of source 12.18 + * @param dst address of destination 12.19 + * @param byte_count number of bytes to copy 12.20 + * @param elem_size size of the elements to copy-swap 12.21 + */ 12.22 + static void conjoint_swap(address src, address dst, size_t byte_count, size_t elem_size); 12.23 + 12.24 // Fill methods 12.25 12.26 // Fill word-aligned words, not atomic on each word