• file
• revisions
• annotate
• diff
• comparison
• raw

src/os_cpu/linux_x86/vm/copy_linux_x86.inline.hpp@f03d0a26bf83 (annotated)

src/os_cpu/linux_x86/vm/copy_linux_x86.inline.hpp

Thu, 22 Apr 2010 13:23:15 -0700

author

jcoomes

date

Thu, 22 Apr 2010 13:23:15 -0700

changeset 1845

f03d0a26bf83

parent 435

a61af66fc99e

child 1907

c18cbe5936b8

permissions

-rw-r--r--

6888954: argument formatting for assert() and friends
Reviewed-by: kvn, twisti, apetrusenko, never, dcubed

duke@435	1	/*
duke@435	2	* Copyright 2003-2004 Sun Microsystems, Inc. All Rights Reserved.
duke@435	3	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
duke@435	4	*
duke@435	5	* This code is free software; you can redistribute it and/or modify it
duke@435	6	* under the terms of the GNU General Public License version 2 only, as
duke@435	7	* published by the Free Software Foundation.
duke@435	8	*
duke@435	9	* This code is distributed in the hope that it will be useful, but WITHOUT
duke@435	10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
duke@435	11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
duke@435	12	* version 2 for more details (a copy is included in the LICENSE file that
duke@435	13	* accompanied this code).
duke@435	14	*
duke@435	15	* You should have received a copy of the GNU General Public License version
duke@435	16	* 2 along with this work; if not, write to the Free Software Foundation,
duke@435	17	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
duke@435	18	*
duke@435	19	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
duke@435	20	* CA 95054 USA or visit www.sun.com if you need additional information or
duke@435	21	* have any questions.
duke@435	22	*
duke@435	23	*/
duke@435	24
duke@435	25	static void pd_conjoint_words(HeapWord* from, HeapWord* to, size_t count) {
duke@435	26	#ifdef AMD64
duke@435	27	(void)memmove(to, from, count * HeapWordSize);
duke@435	28	#else
duke@435	29	// Same as pd_aligned_conjoint_words, except includes a zero-count check.
duke@435	30	intx temp;
duke@435	31	__asm__ volatile(" testl %6,%6 ;"
duke@435	32	" jz 7f ;"
duke@435	33	" cmpl %4,%5 ;"
duke@435	34	" leal -4(%4,%6,4),%3;"
duke@435	35	" jbe 1f ;"
duke@435	36	" cmpl %7,%5 ;"
duke@435	37	" jbe 4f ;"
duke@435	38	"1: cmpl $32,%6 ;"
duke@435	39	" ja 3f ;"
duke@435	40	" subl %4,%1 ;"
duke@435	41	"2: movl (%4),%3 ;"
duke@435	42	" movl %7,(%5,%4,1) ;"
duke@435	43	" addl $4,%0 ;"
duke@435	44	" subl $1,%2 ;"
duke@435	45	" jnz 2b ;"
duke@435	46	" jmp 7f ;"
duke@435	47	"3: rep; smovl ;"
duke@435	48	" jmp 7f ;"
duke@435	49	"4: cmpl $32,%2 ;"
duke@435	50	" movl %7,%0 ;"
duke@435	51	" leal -4(%5,%6,4),%1;"
duke@435	52	" ja 6f ;"
duke@435	53	" subl %4,%1 ;"
duke@435	54	"5: movl (%4),%3 ;"
duke@435	55	" movl %7,(%5,%4,1) ;"
duke@435	56	" subl $4,%0 ;"
duke@435	57	" subl $1,%2 ;"
duke@435	58	" jnz 5b ;"
duke@435	59	" jmp 7f ;"
duke@435	60	"6: std ;"
duke@435	61	" rep; smovl ;"
duke@435	62	" cld ;"
duke@435	63	"7: nop "
duke@435	64	: "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
duke@435	65	: "0" (from), "1" (to), "2" (count), "3" (temp)
duke@435	66	: "memory", "flags");
duke@435	67	#endif // AMD64
duke@435	68	}
duke@435	69
duke@435	70	static void pd_disjoint_words(HeapWord* from, HeapWord* to, size_t count) {
duke@435	71	#ifdef AMD64
duke@435	72	switch (count) {
duke@435	73	case 8: to[7] = from[7];
duke@435	74	case 7: to[6] = from[6];
duke@435	75	case 6: to[5] = from[5];
duke@435	76	case 5: to[4] = from[4];
duke@435	77	case 4: to[3] = from[3];
duke@435	78	case 3: to[2] = from[2];
duke@435	79	case 2: to[1] = from[1];
duke@435	80	case 1: to[0] = from[0];
duke@435	81	case 0: break;
duke@435	82	default:
duke@435	83	(void)memcpy(to, from, count * HeapWordSize);
duke@435	84	break;
duke@435	85	}
duke@435	86	#else
duke@435	87	// Same as pd_aligned_disjoint_words, except includes a zero-count check.
duke@435	88	intx temp;
duke@435	89	__asm__ volatile(" testl %6,%6 ;"
duke@435	90	" jz 3f ;"
duke@435	91	" cmpl $32,%6 ;"
duke@435	92	" ja 2f ;"
duke@435	93	" subl %4,%1 ;"
duke@435	94	"1: movl (%4),%3 ;"
duke@435	95	" movl %7,(%5,%4,1);"
duke@435	96	" addl $4,%0 ;"
duke@435	97	" subl $1,%2 ;"
duke@435	98	" jnz 1b ;"
duke@435	99	" jmp 3f ;"
duke@435	100	"2: rep; smovl ;"
duke@435	101	"3: nop "
duke@435	102	: "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
duke@435	103	: "0" (from), "1" (to), "2" (count), "3" (temp)
duke@435	104	: "memory", "cc");
duke@435	105	#endif // AMD64
duke@435	106	}
duke@435	107
duke@435	108	static void pd_disjoint_words_atomic(HeapWord* from, HeapWord* to, size_t count) {
duke@435	109	#ifdef AMD64
duke@435	110	switch (count) {
duke@435	111	case 8: to[7] = from[7];
duke@435	112	case 7: to[6] = from[6];
duke@435	113	case 6: to[5] = from[5];
duke@435	114	case 5: to[4] = from[4];
duke@435	115	case 4: to[3] = from[3];
duke@435	116	case 3: to[2] = from[2];
duke@435	117	case 2: to[1] = from[1];
duke@435	118	case 1: to[0] = from[0];
duke@435	119	case 0: break;
duke@435	120	default:
duke@435	121	while (count-- > 0) {
duke@435	122	*to++ = *from++;
duke@435	123	}
duke@435	124	break;
duke@435	125	}
duke@435	126	#else
duke@435	127	// pd_disjoint_words is word-atomic in this implementation.
duke@435	128	pd_disjoint_words(from, to, count);
duke@435	129	#endif // AMD64
duke@435	130	}
duke@435	131
duke@435	132	static void pd_aligned_conjoint_words(HeapWord* from, HeapWord* to, size_t count) {
duke@435	133	#ifdef AMD64
duke@435	134	(void)memmove(to, from, count * HeapWordSize);
duke@435	135	#else
duke@435	136	// Same as pd_conjoint_words, except no zero-count check.
duke@435	137	intx temp;
duke@435	138	__asm__ volatile(" cmpl %4,%5 ;"
duke@435	139	" leal -4(%4,%6,4),%3;"
duke@435	140	" jbe 1f ;"
duke@435	141	" cmpl %7,%5 ;"
duke@435	142	" jbe 4f ;"
duke@435	143	"1: cmpl $32,%6 ;"
duke@435	144	" ja 3f ;"
duke@435	145	" subl %4,%1 ;"
duke@435	146	"2: movl (%4),%3 ;"
duke@435	147	" movl %7,(%5,%4,1) ;"
duke@435	148	" addl $4,%0 ;"
duke@435	149	" subl $1,%2 ;"
duke@435	150	" jnz 2b ;"
duke@435	151	" jmp 7f ;"
duke@435	152	"3: rep; smovl ;"
duke@435	153	" jmp 7f ;"
duke@435	154	"4: cmpl $32,%2 ;"
duke@435	155	" movl %7,%0 ;"
duke@435	156	" leal -4(%5,%6,4),%1;"
duke@435	157	" ja 6f ;"
duke@435	158	" subl %4,%1 ;"
duke@435	159	"5: movl (%4),%3 ;"
duke@435	160	" movl %7,(%5,%4,1) ;"
duke@435	161	" subl $4,%0 ;"
duke@435	162	" subl $1,%2 ;"
duke@435	163	" jnz 5b ;"
duke@435	164	" jmp 7f ;"
duke@435	165	"6: std ;"
duke@435	166	" rep; smovl ;"
duke@435	167	" cld ;"
duke@435	168	"7: nop "
duke@435	169	: "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
duke@435	170	: "0" (from), "1" (to), "2" (count), "3" (temp)
duke@435	171	: "memory", "flags");
duke@435	172	#endif // AMD64
duke@435	173	}
duke@435	174
duke@435	175	static void pd_aligned_disjoint_words(HeapWord* from, HeapWord* to, size_t count) {
duke@435	176	#ifdef AMD64
duke@435	177	pd_disjoint_words(from, to, count);
duke@435	178	#else
duke@435	179	// Same as pd_disjoint_words, except no zero-count check.
duke@435	180	intx temp;
duke@435	181	__asm__ volatile(" cmpl $32,%6 ;"
duke@435	182	" ja 2f ;"
duke@435	183	" subl %4,%1 ;"
duke@435	184	"1: movl (%4),%3 ;"
duke@435	185	" movl %7,(%5,%4,1);"
duke@435	186	" addl $4,%0 ;"
duke@435	187	" subl $1,%2 ;"
duke@435	188	" jnz 1b ;"
duke@435	189	" jmp 3f ;"
duke@435	190	"2: rep; smovl ;"
duke@435	191	"3: nop "
duke@435	192	: "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
duke@435	193	: "0" (from), "1" (to), "2" (count), "3" (temp)
duke@435	194	: "memory", "cc");
duke@435	195	#endif // AMD64
duke@435	196	}
duke@435	197
duke@435	198	static void pd_conjoint_bytes(void* from, void* to, size_t count) {
duke@435	199	#ifdef AMD64
duke@435	200	(void)memmove(to, from, count);
duke@435	201	#else
duke@435	202	intx temp;
duke@435	203	__asm__ volatile(" testl %6,%6 ;"
duke@435	204	" jz 13f ;"
duke@435	205	" cmpl %4,%5 ;"
duke@435	206	" leal -1(%4,%6),%3 ;"
duke@435	207	" jbe 1f ;"
duke@435	208	" cmpl %7,%5 ;"
duke@435	209	" jbe 8f ;"
duke@435	210	"1: cmpl $3,%6 ;"
duke@435	211	" jbe 6f ;"
duke@435	212	" movl %6,%3 ;"
duke@435	213	" movl $4,%2 ;"
duke@435	214	" subl %4,%2 ;"
duke@435	215	" andl $3,%2 ;"
duke@435	216	" jz 2f ;"
duke@435	217	" subl %6,%3 ;"
duke@435	218	" rep; smovb ;"
duke@435	219	"2: movl %7,%2 ;"
duke@435	220	" shrl $2,%2 ;"
duke@435	221	" jz 5f ;"
duke@435	222	" cmpl $32,%2 ;"
duke@435	223	" ja 4f ;"
duke@435	224	" subl %4,%1 ;"
duke@435	225	"3: movl (%4),%%edx ;"
duke@435	226	" movl %%edx,(%5,%4,1);"
duke@435	227	" addl $4,%0 ;"
duke@435	228	" subl $1,%2 ;"
duke@435	229	" jnz 3b ;"
duke@435	230	" addl %4,%1 ;"
duke@435	231	" jmp 5f ;"
duke@435	232	"4: rep; smovl ;"
duke@435	233	"5: movl %7,%2 ;"
duke@435	234	" andl $3,%2 ;"
duke@435	235	" jz 13f ;"
duke@435	236	"6: xorl %7,%3 ;"
duke@435	237	"7: movb (%4,%7,1),%%dl ;"
duke@435	238	" movb %%dl,(%5,%7,1) ;"
duke@435	239	" addl $1,%3 ;"
duke@435	240	" subl $1,%2 ;"
duke@435	241	" jnz 7b ;"
duke@435	242	" jmp 13f ;"
duke@435	243	"8: std ;"
duke@435	244	" cmpl $12,%2 ;"
duke@435	245	" ja 9f ;"
duke@435	246	" movl %7,%0 ;"
duke@435	247	" leal -1(%6,%5),%1 ;"
duke@435	248	" jmp 11f ;"
duke@435	249	"9: xchgl %3,%2 ;"
duke@435	250	" movl %6,%0 ;"
duke@435	251	" addl $1,%2 ;"
duke@435	252	" leal -1(%7,%5),%1 ;"
duke@435	253	" andl $3,%2 ;"
duke@435	254	" jz 10f ;"
duke@435	255	" subl %6,%3 ;"
duke@435	256	" rep; smovb ;"
duke@435	257	"10: movl %7,%2 ;"
duke@435	258	" subl $3,%0 ;"
duke@435	259	" shrl $2,%2 ;"
duke@435	260	" subl $3,%1 ;"
duke@435	261	" rep; smovl ;"
duke@435	262	" andl $3,%3 ;"
duke@435	263	" jz 12f ;"
duke@435	264	" movl %7,%2 ;"
duke@435	265	" addl $3,%0 ;"
duke@435	266	" addl $3,%1 ;"
duke@435	267	"11: rep; smovb ;"
duke@435	268	"12: cld ;"
duke@435	269	"13: nop ;"
duke@435	270	: "=S" (from), "=D" (to), "=c" (count), "=r" (temp)
duke@435	271	: "0" (from), "1" (to), "2" (count), "3" (temp)
duke@435	272	: "memory", "flags", "%edx");
duke@435	273	#endif // AMD64
duke@435	274	}
duke@435	275
duke@435	276	static void pd_conjoint_bytes_atomic(void* from, void* to, size_t count) {
duke@435	277	pd_conjoint_bytes(from, to, count);
duke@435	278	}
duke@435	279
duke@435	280	static void pd_conjoint_jshorts_atomic(jshort* from, jshort* to, size_t count) {
duke@435	281	_Copy_conjoint_jshorts_atomic(from, to, count);
duke@435	282	}
duke@435	283
duke@435	284	static void pd_conjoint_jints_atomic(jint* from, jint* to, size_t count) {
duke@435	285	#ifdef AMD64
duke@435	286	_Copy_conjoint_jints_atomic(from, to, count);
duke@435	287	#else
duke@435	288	assert(HeapWordSize == BytesPerInt, "heapwords and jints must be the same size");
duke@435	289	// pd_conjoint_words is word-atomic in this implementation.
duke@435	290	pd_conjoint_words((HeapWord*)from, (HeapWord*)to, count);
duke@435	291	#endif // AMD64
duke@435	292	}
duke@435	293
duke@435	294	static void pd_conjoint_jlongs_atomic(jlong* from, jlong* to, size_t count) {
duke@435	295	#ifdef AMD64
duke@435	296	_Copy_conjoint_jlongs_atomic(from, to, count);
duke@435	297	#else
duke@435	298	// Guarantee use of fild/fistp or xmm regs via some asm code, because compilers won't.
duke@435	299	if (from > to) {
duke@435	300	while (count-- > 0) {
duke@435	301	__asm__ volatile("fildll (%0); fistpll (%1)"
duke@435	302	:
duke@435	303	: "r" (from), "r" (to)
duke@435	304	: "memory" );
duke@435	305	++from;
duke@435	306	++to;
duke@435	307	}
duke@435	308	} else {
duke@435	309	while (count-- > 0) {
duke@435	310	__asm__ volatile("fildll (%0,%2,8); fistpll (%1,%2,8)"
duke@435	311	:
duke@435	312	: "r" (from), "r" (to), "r" (count)
duke@435	313	: "memory" );
duke@435	314	}
duke@435	315	}
duke@435	316	#endif // AMD64
duke@435	317	}
duke@435	318
duke@435	319	static void pd_conjoint_oops_atomic(oop* from, oop* to, size_t count) {
duke@435	320	#ifdef AMD64
duke@435	321	assert(BytesPerLong == BytesPerOop, "jlongs and oops must be the same size");
duke@435	322	_Copy_conjoint_jlongs_atomic((jlong*)from, (jlong*)to, count);
duke@435	323	#else
duke@435	324	assert(HeapWordSize == BytesPerOop, "heapwords and oops must be the same size");
duke@435	325	// pd_conjoint_words is word-atomic in this implementation.
duke@435	326	pd_conjoint_words((HeapWord*)from, (HeapWord*)to, count);
duke@435	327	#endif // AMD64
duke@435	328	}
duke@435	329
duke@435	330	static void pd_arrayof_conjoint_bytes(HeapWord* from, HeapWord* to, size_t count) {
duke@435	331	_Copy_arrayof_conjoint_bytes(from, to, count);
duke@435	332	}
duke@435	333
duke@435	334	static void pd_arrayof_conjoint_jshorts(HeapWord* from, HeapWord* to, size_t count) {
duke@435	335	_Copy_arrayof_conjoint_jshorts(from, to, count);
duke@435	336	}
duke@435	337
duke@435	338	static void pd_arrayof_conjoint_jints(HeapWord* from, HeapWord* to, size_t count) {
duke@435	339	#ifdef AMD64
duke@435	340	_Copy_arrayof_conjoint_jints(from, to, count);
duke@435	341	#else
duke@435	342	pd_conjoint_jints_atomic((jint*)from, (jint*)to, count);
duke@435	343	#endif // AMD64
duke@435	344	}
duke@435	345
duke@435	346	static void pd_arrayof_conjoint_jlongs(HeapWord* from, HeapWord* to, size_t count) {
duke@435	347	#ifdef AMD64
duke@435	348	_Copy_arrayof_conjoint_jlongs(from, to, count);
duke@435	349	#else
duke@435	350	pd_conjoint_jlongs_atomic((jlong*)from, (jlong*)to, count);
duke@435	351	#endif // AMD64
duke@435	352	}
duke@435	353
duke@435	354	static void pd_arrayof_conjoint_oops(HeapWord* from, HeapWord* to, size_t count) {
duke@435	355	#ifdef AMD64
duke@435	356	assert(BytesPerLong == BytesPerOop, "jlongs and oops must be the same size");
duke@435	357	_Copy_arrayof_conjoint_jlongs(from, to, count);
duke@435	358	#else
duke@435	359	pd_conjoint_oops_atomic((oop*)from, (oop*)to, count);
duke@435	360	#endif // AMD64
duke@435	361	}