xref: /onnv-gate/usr/src/lib/libc/amd64/gen/memcpy.s (revision 7298:b69e27387f74)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright (c) 2008, Intel Corporation
 * All rights reserved.
 */

/*
 * memcpy.s - copies two blocks of memory
 *	Implements memcpy() and memmove() libc primitives.
 */

	.file	"memcpy.s"

#include <sys/asm_linkage.h>

	ANSI_PRAGMA_WEAK(memmove,function)
	ANSI_PRAGMA_WEAK(memcpy,function)

#include "cache.h"
#include "proc64_id.h"

#define L(s) .memcpy/**/s

/*
 * memcpy algorithm overview:
 *
 * Thresholds used below were determined experimentally.
 *
 * Pseudo code:
 *
 * If (size <= 128 bytes) {
 *	do unrolled code (primarily 8-byte loads/stores) regardless of
 *	alignment.
 * } else {
 *	Align destination to 16-byte boundary
 *
 *      if (NO_SSE) {
 *		If (size > half of the largest level cache) {
 *			Use 8-byte non-temporal stores (64-bytes/loop)
 *		} else {
 *			if (size > 4K && size <= half l1 cache size) {
 *				Use rep movsq
 *			} else {
 *				Use 8-byte loads/stores (64 bytes per loop)
 *			}
 *		}
 *
 *	} else { **USE SSE**
 *		If (size > half of the largest level cache) {
 *			Use 16-byte non-temporal stores (128-bytes per loop)
 *		} else {
 *			If (both source and destination are aligned) {
 *			    Use 16-byte aligned loads and stores (128 bytes/loop)
 *			} else {
 *			    use pairs of xmm registers with SSE2 or SSSE3
 *			    instructions to concatenate and shift appropriately
 *			    to account for source unalignment. This enables
 *			    16-byte aligned loads to be done.
 *			}
 *		}
	}
 *
 *	Finish any remaining bytes via unrolled code above.
 * }
 *
 * memmove overview:
 *	memmove is the same as memcpy except one case where copy needs to be
 *	done backwards. The copy backwards code is done in a similar manner.
 */

	ENTRY(memmove)
	cmp	%rsi,%rdi		# if dst <= src
	jbe	L(CopyForward)		# then do copy forward
	mov	%rsi,%r9		# move src to r9
	add	%rdx,%r9		# add len to get addr of end of src
	cmp	%r9,%rdi		# if dst < end of src
	jb	L(CopyBackwards)	# then do copy backwards
	jmp	L(CopyForward)

	ENTRY (memcpy)
L(CopyForward):
	mov    %rdx,%r8
	mov    %rdi,%rcx
	mov    %rsi,%rdx
	mov    %rdi,%rax
	lea    L(fwdPxQx)(%rip),%r11
	cmp    $0x80,%r8		# 128
	jg     L(ck_use_sse2)
	add    %r8,%rcx
	add    %r8,%rdx

	movslq (%r11,%r8,4),%r10
	lea    (%r10,%r11,1),%r11
	jmpq   *%r11

	.balign 16
L(ShrtAlignNew):
	lea    L(AliPxQx)(%rip),%r11
	mov    %rcx,%r9
	and    $0xf,%r9

	movslq (%r11,%r9,4),%r10
	lea    (%r10,%r11,1),%r11
	jmpq   *%r11

	.balign 16
L(fwdPxQx): .int       L(P0Q0)-L(fwdPxQx)
           .int        L(P1Q0)-L(fwdPxQx)
           .int        L(P2Q0)-L(fwdPxQx)
           .int        L(P3Q0)-L(fwdPxQx)
           .int        L(P4Q0)-L(fwdPxQx)
           .int        L(P5Q0)-L(fwdPxQx)
           .int        L(P6Q0)-L(fwdPxQx)
           .int        L(P7Q0)-L(fwdPxQx)

           .int        L(P0Q1)-L(fwdPxQx)
           .int        L(P1Q1)-L(fwdPxQx)
           .int        L(P2Q1)-L(fwdPxQx)
           .int        L(P3Q1)-L(fwdPxQx)
           .int        L(P4Q1)-L(fwdPxQx)
           .int        L(P5Q1)-L(fwdPxQx)
           .int        L(P6Q1)-L(fwdPxQx)
           .int        L(P7Q1)-L(fwdPxQx)

           .int        L(P0Q2)-L(fwdPxQx)
           .int        L(P1Q2)-L(fwdPxQx)
           .int        L(P2Q2)-L(fwdPxQx)
           .int        L(P3Q2)-L(fwdPxQx)
           .int        L(P4Q2)-L(fwdPxQx)
           .int        L(P5Q2)-L(fwdPxQx)
           .int        L(P6Q2)-L(fwdPxQx)
           .int        L(P7Q2)-L(fwdPxQx)

           .int        L(P0Q3)-L(fwdPxQx)
           .int        L(P1Q3)-L(fwdPxQx)
           .int        L(P2Q3)-L(fwdPxQx)
           .int        L(P3Q3)-L(fwdPxQx)
           .int        L(P4Q3)-L(fwdPxQx)
           .int        L(P5Q3)-L(fwdPxQx)
           .int        L(P6Q3)-L(fwdPxQx)
           .int        L(P7Q3)-L(fwdPxQx)

           .int        L(P0Q4)-L(fwdPxQx)
           .int        L(P1Q4)-L(fwdPxQx)
           .int        L(P2Q4)-L(fwdPxQx)
           .int        L(P3Q4)-L(fwdPxQx)
           .int        L(P4Q4)-L(fwdPxQx)
           .int        L(P5Q4)-L(fwdPxQx)
           .int        L(P6Q4)-L(fwdPxQx)
           .int        L(P7Q4)-L(fwdPxQx)

           .int        L(P0Q5)-L(fwdPxQx)
           .int        L(P1Q5)-L(fwdPxQx)
           .int        L(P2Q5)-L(fwdPxQx)
           .int        L(P3Q5)-L(fwdPxQx)
           .int        L(P4Q5)-L(fwdPxQx)
           .int        L(P5Q5)-L(fwdPxQx)
           .int        L(P6Q5)-L(fwdPxQx)
           .int        L(P7Q5)-L(fwdPxQx)

           .int        L(P0Q6)-L(fwdPxQx)
           .int        L(P1Q6)-L(fwdPxQx)
           .int        L(P2Q6)-L(fwdPxQx)
           .int        L(P3Q6)-L(fwdPxQx)
           .int        L(P4Q6)-L(fwdPxQx)
           .int        L(P5Q6)-L(fwdPxQx)
           .int        L(P6Q6)-L(fwdPxQx)
           .int        L(P7Q6)-L(fwdPxQx)

           .int        L(P0Q7)-L(fwdPxQx)
           .int        L(P1Q7)-L(fwdPxQx)
           .int        L(P2Q7)-L(fwdPxQx)
           .int        L(P3Q7)-L(fwdPxQx)
           .int        L(P4Q7)-L(fwdPxQx)
           .int        L(P5Q7)-L(fwdPxQx)
           .int        L(P6Q7)-L(fwdPxQx)
           .int        L(P7Q7)-L(fwdPxQx)

           .int        L(P0Q8)-L(fwdPxQx)
           .int        L(P1Q8)-L(fwdPxQx)
           .int        L(P2Q8)-L(fwdPxQx)
           .int        L(P3Q8)-L(fwdPxQx)
           .int        L(P4Q8)-L(fwdPxQx)
           .int        L(P5Q8)-L(fwdPxQx)
           .int        L(P6Q8)-L(fwdPxQx)
           .int        L(P7Q8)-L(fwdPxQx)

           .int        L(P0Q9)-L(fwdPxQx)
           .int        L(P1Q9)-L(fwdPxQx)
           .int        L(P2Q9)-L(fwdPxQx)
           .int        L(P3Q9)-L(fwdPxQx)
           .int        L(P4Q9)-L(fwdPxQx)
           .int        L(P5Q9)-L(fwdPxQx)
           .int        L(P6Q9)-L(fwdPxQx)
           .int        L(P7Q9)-L(fwdPxQx)

           .int        L(P0QA)-L(fwdPxQx)
           .int        L(P1QA)-L(fwdPxQx)
           .int        L(P2QA)-L(fwdPxQx)
           .int        L(P3QA)-L(fwdPxQx)
           .int        L(P4QA)-L(fwdPxQx)
           .int        L(P5QA)-L(fwdPxQx)
           .int        L(P6QA)-L(fwdPxQx)
           .int        L(P7QA)-L(fwdPxQx)

           .int        L(P0QB)-L(fwdPxQx)
           .int        L(P1QB)-L(fwdPxQx)
           .int        L(P2QB)-L(fwdPxQx)
           .int        L(P3QB)-L(fwdPxQx)
           .int        L(P4QB)-L(fwdPxQx)
           .int        L(P5QB)-L(fwdPxQx)
           .int        L(P6QB)-L(fwdPxQx)
           .int        L(P7QB)-L(fwdPxQx)

           .int        L(P0QC)-L(fwdPxQx)
           .int        L(P1QC)-L(fwdPxQx)
           .int        L(P2QC)-L(fwdPxQx)
           .int        L(P3QC)-L(fwdPxQx)
           .int        L(P4QC)-L(fwdPxQx)
           .int        L(P5QC)-L(fwdPxQx)
           .int        L(P6QC)-L(fwdPxQx)
           .int        L(P7QC)-L(fwdPxQx)

           .int        L(P0QD)-L(fwdPxQx)
           .int        L(P1QD)-L(fwdPxQx)
           .int        L(P2QD)-L(fwdPxQx)
           .int        L(P3QD)-L(fwdPxQx)
           .int        L(P4QD)-L(fwdPxQx)
           .int        L(P5QD)-L(fwdPxQx)
           .int        L(P6QD)-L(fwdPxQx)
           .int        L(P7QD)-L(fwdPxQx)

           .int        L(P0QE)-L(fwdPxQx)
           .int        L(P1QE)-L(fwdPxQx)
           .int        L(P2QE)-L(fwdPxQx)
           .int        L(P3QE)-L(fwdPxQx)
           .int        L(P4QE)-L(fwdPxQx)
           .int        L(P5QE)-L(fwdPxQx)
           .int        L(P6QE)-L(fwdPxQx)
           .int        L(P7QE)-L(fwdPxQx)

           .int        L(P0QF)-L(fwdPxQx)
           .int        L(P1QF)-L(fwdPxQx)
           .int        L(P2QF)-L(fwdPxQx)
           .int        L(P3QF)-L(fwdPxQx)
           .int        L(P4QF)-L(fwdPxQx)
           .int        L(P5QF)-L(fwdPxQx)
           .int        L(P6QF)-L(fwdPxQx)
           .int        L(P7QF)-L(fwdPxQx)

           .int        L(P0QG)-L(fwdPxQx)	# 0x80

	   .balign 16
L(AliPxQx): .int   L(now_qw_aligned)-L(AliPxQx)
           .int        L(A1Q0)-L(AliPxQx)
           .int        L(A2Q0)-L(AliPxQx)
           .int        L(A3Q0)-L(AliPxQx)
           .int        L(A4Q0)-L(AliPxQx)
           .int        L(A5Q0)-L(AliPxQx)
           .int        L(A6Q0)-L(AliPxQx)
           .int        L(A7Q0)-L(AliPxQx)
           .int        L(A0Q1)-L(AliPxQx)
           .int        L(A1Q1)-L(AliPxQx)
           .int        L(A2Q1)-L(AliPxQx)
           .int        L(A3Q1)-L(AliPxQx)
           .int        L(A4Q1)-L(AliPxQx)
           .int        L(A5Q1)-L(AliPxQx)
           .int        L(A6Q1)-L(AliPxQx)
           .int        L(A7Q1)-L(AliPxQx)

	.balign 16
L(A1Q0):			# ; need to move 8+ 7=1+2+4 bytes
	movzbq (%rdx),%r11
	sub    $0xf,%r8
	mov    %r11b,(%rcx)

	movzwq 0x1(%rdx),%r10
	mov    %r10w,0x1(%rcx)

	mov    0x3(%rdx),%r9d
	mov    %r9d,0x3(%rcx)

	mov    0x7(%rdx),%r11
	add    $0xf,%rdx
	mov    %r11,0x7(%rcx)

	add    $0xf,%rcx
	jmp    L(now_qw_aligned)

	.balign 16
L(A2Q0):			# ; need to move 8+ 6=2+4 bytes
	movzwq (%rdx),%r10
	sub    $0xe,%r8
	mov    %r10w,(%rcx)

	mov    0x2(%rdx),%r9d
	mov    %r9d,0x2(%rcx)

	mov    0x6(%rdx),%r11
	add    $0xe,%rdx
	mov    %r11,0x6(%rcx)
	add    $0xe,%rcx
	jmp    L(now_qw_aligned)

	.balign 16
L(A3Q0):			# ; need to move 8+ 5=1+4 bytes
	movzbq (%rdx),%r11
	sub    $0xd,%r8
	mov    %r11b,(%rcx)

	mov    0x1(%rdx),%r9d
	mov    %r9d,0x1(%rcx)

	mov    0x5(%rdx),%r10
	add    $0xd,%rdx
	mov    %r10,0x5(%rcx)

	add    $0xd,%rcx
	jmp    L(now_qw_aligned)

	.balign 16
L(A4Q0):			# ; need to move 8+4 bytes
	mov    (%rdx),%r9d
	sub    $0xc,%r8
	mov    %r9d,(%rcx)

	mov    0x4(%rdx),%r10
	add    $0xc,%rdx
	mov    %r10,0x4(%rcx)

	add    $0xc,%rcx
	jmp    L(now_qw_aligned)

	.balign 16
L(A5Q0):			# ; need to move 8+ 3=1+2 bytes
	movzbq (%rdx),%r11
	sub    $0xb,%r8
	mov    %r11b,(%rcx)

	movzwq 0x1(%rdx),%r10
	mov    %r10w,0x1(%rcx)

	mov    0x3(%rdx),%r9
	add    $0xb,%rdx
	mov    %r9,0x3(%rcx)

	add    $0xb,%rcx
	jmp    L(now_qw_aligned)

	.balign 16
L(A6Q0):			# ; need to move 8+2 bytes
	movzwq (%rdx),%r10
	sub    $0xa,%r8
	mov    %r10w,(%rcx)

	mov    0x2(%rdx),%r9
	add    $0xa,%rdx
	mov    %r9,0x2(%rcx)

	add    $0xa,%rcx
	jmp    L(now_qw_aligned)

	.balign 16
L(A7Q0):			# ; need to move 8+1 byte
	movzbq (%rdx),%r11
	sub    $0x9,%r8
	mov    %r11b,(%rcx)

	mov    0x1(%rdx),%r10
	add    $0x9,%rdx
	mov    %r10,0x1(%rcx)

	add    $0x9,%rcx
	jmp    L(now_qw_aligned)

	.balign 16
L(A0Q1):			# ; need to move 8 bytes

	mov    (%rdx),%r10
	add    $0x8,%rdx
	sub    $0x8,%r8
	mov    %r10,(%rcx)

	add    $0x8,%rcx
	jmp    L(now_qw_aligned)

	.balign 16
L(A1Q1):			# ; need to move 7=1+2+4 bytes
	movzbq (%rdx),%r11
	sub    $0x7,%r8
	mov    %r11b,(%rcx)

	movzwq 0x1(%rdx),%r10
	mov    %r10w,0x1(%rcx)

	mov    0x3(%rdx),%r9d
	add    $0x7,%rdx
	mov    %r9d,0x3(%rcx)
	add    $0x7,%rcx
	jmp    L(now_qw_aligned)

	.balign 16
L(A2Q1):			# ; need to move 6=2+4 bytes
	movzwq (%rdx),%r10
	sub    $0x6,%r8
	mov    %r10w,(%rcx)
	mov    0x2(%rdx),%r9d
	add    $0x6,%rdx
	mov    %r9d,0x2(%rcx)
	add    $0x6,%rcx
	jmp    L(now_qw_aligned)

	.balign 16
L(A3Q1):			# ; need to move 5=1+4 bytes
	movzbq (%rdx),%r11
	sub    $0x5,%r8
	mov    %r11b,(%rcx)
	mov    0x1(%rdx),%r9d
	add    $0x5,%rdx
	mov    %r9d,0x1(%rcx)
	add    $0x5,%rcx
	jmp    L(now_qw_aligned)

	.balign 16
L(A4Q1):			# ; need to move 4 bytes
	mov    (%rdx),%r9d
	sub    $0x4,%r8
	add    $0x4,%rdx
	mov    %r9d,(%rcx)
	add    $0x4,%rcx
	jmp    L(now_qw_aligned)

	.balign 16
L(A5Q1):			# ; need to move 3=1+2 bytes
	movzbq (%rdx),%r11
	sub    $0x3,%r8
	mov    %r11b,(%rcx)

	movzwq 0x1(%rdx),%r10
	add    $0x3,%rdx
	mov    %r10w,0x1(%rcx)

	add    $0x3,%rcx
	jmp    L(now_qw_aligned)

	.balign 16
L(A6Q1):			# ; need to move 2 bytes
	movzwq (%rdx),%r10
	sub    $0x2,%r8
	add    $0x2,%rdx
	mov    %r10w,(%rcx)
	add    $0x2,%rcx
	jmp    L(now_qw_aligned)

	.balign 16
L(A7Q1):			# ; need to move 1 byte
	movzbq (%rdx),%r11
	dec    %r8
	inc    %rdx
	mov    %r11b,(%rcx)
	inc    %rcx
	jmp    L(now_qw_aligned)


	.balign 16
L(P0QG):
	mov    -0x80(%rdx),%r9
	mov    %r9,-0x80(%rcx)
L(P0QF):
	mov    -0x78(%rdx),%r10
	mov    %r10,-0x78(%rcx)
L(P0QE):
	mov    -0x70(%rdx),%r9
	mov    %r9,-0x70(%rcx)
L(P0QD):
	mov    -0x68(%rdx),%r10
	mov    %r10,-0x68(%rcx)
L(P0QC):
	mov    -0x60(%rdx),%r9
	mov    %r9,-0x60(%rcx)
L(P0QB):
	mov    -0x58(%rdx),%r10
	mov    %r10,-0x58(%rcx)
L(P0QA):
	mov    -0x50(%rdx),%r9
	mov    %r9,-0x50(%rcx)
L(P0Q9):
	mov    -0x48(%rdx),%r10
	mov    %r10,-0x48(%rcx)
L(P0Q8):
	mov    -0x40(%rdx),%r9
	mov    %r9,-0x40(%rcx)
L(P0Q7):
	mov    -0x38(%rdx),%r10
	mov    %r10,-0x38(%rcx)
L(P0Q6):
	mov    -0x30(%rdx),%r9
	mov    %r9,-0x30(%rcx)
L(P0Q5):
	mov    -0x28(%rdx),%r10
	mov    %r10,-0x28(%rcx)
L(P0Q4):
	mov    -0x20(%rdx),%r9
	mov    %r9,-0x20(%rcx)
L(P0Q3):
	mov    -0x18(%rdx),%r10
	mov    %r10,-0x18(%rcx)
L(P0Q2):
	mov    -0x10(%rdx),%r9
	mov    %r9,-0x10(%rcx)
L(P0Q1):
	mov    -0x8(%rdx),%r10
	mov    %r10,-0x8(%rcx)
L(P0Q0):
	ret

	.balign 16
L(P1QF):
	mov    -0x79(%rdx),%r9
	mov    %r9,-0x79(%rcx)
L(P1QE):
	mov    -0x71(%rdx),%r11
	mov    %r11,-0x71(%rcx)
L(P1QD):
	mov    -0x69(%rdx),%r10
	mov    %r10,-0x69(%rcx)
L(P1QC):
	mov    -0x61(%rdx),%r9
	mov    %r9,-0x61(%rcx)
L(P1QB):
	mov    -0x59(%rdx),%r11
	mov    %r11,-0x59(%rcx)
L(P1QA):
	mov    -0x51(%rdx),%r10
	mov    %r10,-0x51(%rcx)
L(P1Q9):
	mov    -0x49(%rdx),%r9
	mov    %r9,-0x49(%rcx)
L(P1Q8):
	mov    -0x41(%rdx),%r11
	mov    %r11,-0x41(%rcx)
L(P1Q7):
	mov    -0x39(%rdx),%r10
	mov    %r10,-0x39(%rcx)
L(P1Q6):
	mov    -0x31(%rdx),%r9
	mov    %r9,-0x31(%rcx)
L(P1Q5):
	mov    -0x29(%rdx),%r11
	mov    %r11,-0x29(%rcx)
L(P1Q4):
	mov    -0x21(%rdx),%r10
	mov    %r10,-0x21(%rcx)
L(P1Q3):
	mov    -0x19(%rdx),%r9
	mov    %r9,-0x19(%rcx)
L(P1Q2):
	mov    -0x11(%rdx),%r11
	mov    %r11,-0x11(%rcx)
L(P1Q1):
	mov    -0x9(%rdx),%r10
	mov    %r10,-0x9(%rcx)
L(P1Q0):
	movzbq -0x1(%rdx),%r9
	mov    %r9b,-0x1(%rcx)
	ret

	.balign 16
L(P2QF):
	mov    -0x7a(%rdx),%r9
	mov    %r9,-0x7a(%rcx)
L(P2QE):
	mov    -0x72(%rdx),%r11
	mov    %r11,-0x72(%rcx)
L(P2QD):
	mov    -0x6a(%rdx),%r10
	mov    %r10,-0x6a(%rcx)
L(P2QC):
	mov    -0x62(%rdx),%r9
	mov    %r9,-0x62(%rcx)
L(P2QB):
	mov    -0x5a(%rdx),%r11
	mov    %r11,-0x5a(%rcx)
L(P2QA):
	mov    -0x52(%rdx),%r10
	mov    %r10,-0x52(%rcx)
L(P2Q9):
	mov    -0x4a(%rdx),%r9
	mov    %r9,-0x4a(%rcx)
L(P2Q8):
	mov    -0x42(%rdx),%r11
	mov    %r11,-0x42(%rcx)
L(P2Q7):
	mov    -0x3a(%rdx),%r10
	mov    %r10,-0x3a(%rcx)
L(P2Q6):
	mov    -0x32(%rdx),%r9
	mov    %r9,-0x32(%rcx)
L(P2Q5):
	mov    -0x2a(%rdx),%r11
	mov    %r11,-0x2a(%rcx)
L(P2Q4):
	mov    -0x22(%rdx),%r10
	mov    %r10,-0x22(%rcx)
L(P2Q3):
	mov    -0x1a(%rdx),%r9
	mov    %r9,-0x1a(%rcx)
L(P2Q2):
	mov    -0x12(%rdx),%r11
	mov    %r11,-0x12(%rcx)
L(P2Q1):
	mov    -0xa(%rdx),%r10
	mov    %r10,-0xa(%rcx)
L(P2Q0):
	movzwq -0x2(%rdx),%r9
	mov    %r9w,-0x2(%rcx)
	ret

	.balign 16
L(P3QF):
	mov    -0x7b(%rdx),%r9
	mov    %r9,-0x7b(%rcx)
L(P3QE):
	mov    -0x73(%rdx),%r11
	mov    %r11,-0x73(%rcx)
L(P3QD):
	mov    -0x6b(%rdx),%r10
	mov    %r10,-0x6b(%rcx)
L(P3QC):
	mov    -0x63(%rdx),%r9
	mov    %r9,-0x63(%rcx)
L(P3QB):
	mov    -0x5b(%rdx),%r11
	mov    %r11,-0x5b(%rcx)
L(P3QA):
	mov    -0x53(%rdx),%r10
	mov    %r10,-0x53(%rcx)
L(P3Q9):
	mov    -0x4b(%rdx),%r9
	mov    %r9,-0x4b(%rcx)
L(P3Q8):
	mov    -0x43(%rdx),%r11
	mov    %r11,-0x43(%rcx)
L(P3Q7):
	mov    -0x3b(%rdx),%r10
	mov    %r10,-0x3b(%rcx)
L(P3Q6):
	mov    -0x33(%rdx),%r9
	mov    %r9,-0x33(%rcx)
L(P3Q5):
	mov    -0x2b(%rdx),%r11
	mov    %r11,-0x2b(%rcx)
L(P3Q4):
	mov    -0x23(%rdx),%r10
	mov    %r10,-0x23(%rcx)
L(P3Q3):
	mov    -0x1b(%rdx),%r9
	mov    %r9,-0x1b(%rcx)
L(P3Q2):
	mov    -0x13(%rdx),%r11
	mov    %r11,-0x13(%rcx)
L(P3Q1):
	mov    -0xb(%rdx),%r10
	mov    %r10,-0xb(%rcx)
	/*
	 * These trailing loads/stores have to do all their loads 1st,
	 * then do the stores.
	 */
L(P3Q0):
	movzwq -0x3(%rdx),%r9
	movzbq -0x1(%rdx),%r10
	mov    %r9w,-0x3(%rcx)
	mov    %r10b,-0x1(%rcx)
	ret

	.balign 16
L(P4QF):
	mov    -0x7c(%rdx),%r9
	mov    %r9,-0x7c(%rcx)
L(P4QE):
	mov    -0x74(%rdx),%r11
	mov    %r11,-0x74(%rcx)
L(P4QD):
	mov    -0x6c(%rdx),%r10
	mov    %r10,-0x6c(%rcx)
L(P4QC):
	mov    -0x64(%rdx),%r9
	mov    %r9,-0x64(%rcx)
L(P4QB):
	mov    -0x5c(%rdx),%r11
	mov    %r11,-0x5c(%rcx)
L(P4QA):
	mov    -0x54(%rdx),%r10
	mov    %r10,-0x54(%rcx)
L(P4Q9):
	mov    -0x4c(%rdx),%r9
	mov    %r9,-0x4c(%rcx)
L(P4Q8):
	mov    -0x44(%rdx),%r11
	mov    %r11,-0x44(%rcx)
L(P4Q7):
	mov    -0x3c(%rdx),%r10
	mov    %r10,-0x3c(%rcx)
L(P4Q6):
	mov    -0x34(%rdx),%r9
	mov    %r9,-0x34(%rcx)
L(P4Q5):
	mov    -0x2c(%rdx),%r11
	mov    %r11,-0x2c(%rcx)
L(P4Q4):
	mov    -0x24(%rdx),%r10
	mov    %r10,-0x24(%rcx)
L(P4Q3):
	mov    -0x1c(%rdx),%r9
	mov    %r9,-0x1c(%rcx)
L(P4Q2):
	mov    -0x14(%rdx),%r11
	mov    %r11,-0x14(%rcx)
L(P4Q1):
	mov    -0xc(%rdx),%r10
	mov    %r10,-0xc(%rcx)
L(P4Q0):
	mov    -0x4(%rdx),%r9d
	mov    %r9d,-0x4(%rcx)
	ret

	.balign 16
L(P5QF):
	mov    -0x7d(%rdx),%r9
	mov    %r9,-0x7d(%rcx)
L(P5QE):
	mov    -0x75(%rdx),%r11
	mov    %r11,-0x75(%rcx)
L(P5QD):
	mov    -0x6d(%rdx),%r10
	mov    %r10,-0x6d(%rcx)
L(P5QC):
	mov    -0x65(%rdx),%r9
	mov    %r9,-0x65(%rcx)
L(P5QB):
	mov    -0x5d(%rdx),%r11
	mov    %r11,-0x5d(%rcx)
L(P5QA):
	mov    -0x55(%rdx),%r10
	mov    %r10,-0x55(%rcx)
L(P5Q9):
	mov    -0x4d(%rdx),%r9
	mov    %r9,-0x4d(%rcx)
L(P5Q8):
	mov    -0x45(%rdx),%r11
	mov    %r11,-0x45(%rcx)
L(P5Q7):
	mov    -0x3d(%rdx),%r10
	mov    %r10,-0x3d(%rcx)
L(P5Q6):
	mov    -0x35(%rdx),%r9
	mov    %r9,-0x35(%rcx)
L(P5Q5):
	mov    -0x2d(%rdx),%r11
	mov    %r11,-0x2d(%rcx)
L(P5Q4):
	mov    -0x25(%rdx),%r10
	mov    %r10,-0x25(%rcx)
L(P5Q3):
	mov    -0x1d(%rdx),%r9
	mov    %r9,-0x1d(%rcx)
L(P5Q2):
	mov    -0x15(%rdx),%r11
	mov    %r11,-0x15(%rcx)
L(P5Q1):
	mov    -0xd(%rdx),%r10
	mov    %r10,-0xd(%rcx)
	/*
	 * These trailing loads/stores have to do all their loads 1st,
	 * then do the stores.
	 */
L(P5Q0):
	mov    -0x5(%rdx),%r9d
	movzbq -0x1(%rdx),%r10
	mov    %r9d,-0x5(%rcx)
	mov    %r10b,-0x1(%rcx)
	ret

	.balign 16
L(P6QF):
	mov    -0x7e(%rdx),%r9
	mov    %r9,-0x7e(%rcx)
L(P6QE):
	mov    -0x76(%rdx),%r11
	mov    %r11,-0x76(%rcx)
L(P6QD):
	mov    -0x6e(%rdx),%r10
	mov    %r10,-0x6e(%rcx)
L(P6QC):
	mov    -0x66(%rdx),%r9
	mov    %r9,-0x66(%rcx)
L(P6QB):
	mov    -0x5e(%rdx),%r11
	mov    %r11,-0x5e(%rcx)
L(P6QA):
	mov    -0x56(%rdx),%r10
	mov    %r10,-0x56(%rcx)
L(P6Q9):
	mov    -0x4e(%rdx),%r9
	mov    %r9,-0x4e(%rcx)
L(P6Q8):
	mov    -0x46(%rdx),%r11
	mov    %r11,-0x46(%rcx)
L(P6Q7):
	mov    -0x3e(%rdx),%r10
	mov    %r10,-0x3e(%rcx)
L(P6Q6):
	mov    -0x36(%rdx),%r9
	mov    %r9,-0x36(%rcx)
L(P6Q5):
	mov    -0x2e(%rdx),%r11
	mov    %r11,-0x2e(%rcx)
L(P6Q4):
	mov    -0x26(%rdx),%r10
	mov    %r10,-0x26(%rcx)
L(P6Q3):
	mov    -0x1e(%rdx),%r9
	mov    %r9,-0x1e(%rcx)
L(P6Q2):
	mov    -0x16(%rdx),%r11
	mov    %r11,-0x16(%rcx)
L(P6Q1):
	mov    -0xe(%rdx),%r10
	mov    %r10,-0xe(%rcx)
	/*
	 * These trailing loads/stores have to do all their loads 1st,
	 * then do the stores.
	 */
L(P6Q0):
	mov    -0x6(%rdx),%r9d
	movzwq -0x2(%rdx),%r10
	mov    %r9d,-0x6(%rcx)
	mov    %r10w,-0x2(%rcx)
	ret

	.balign 16
L(P7QF):
	mov    -0x7f(%rdx),%r9
	mov    %r9,-0x7f(%rcx)
L(P7QE):
	mov    -0x77(%rdx),%r11
	mov    %r11,-0x77(%rcx)
L(P7QD):
	mov    -0x6f(%rdx),%r10
	mov    %r10,-0x6f(%rcx)
L(P7QC):
	mov    -0x67(%rdx),%r9
	mov    %r9,-0x67(%rcx)
L(P7QB):
	mov    -0x5f(%rdx),%r11
	mov    %r11,-0x5f(%rcx)
L(P7QA):
	mov    -0x57(%rdx),%r10
	mov    %r10,-0x57(%rcx)
L(P7Q9):
	mov    -0x4f(%rdx),%r9
	mov    %r9,-0x4f(%rcx)
L(P7Q8):
	mov    -0x47(%rdx),%r11
	mov    %r11,-0x47(%rcx)
L(P7Q7):
	mov    -0x3f(%rdx),%r10
	mov    %r10,-0x3f(%rcx)
L(P7Q6):
	mov    -0x37(%rdx),%r9
	mov    %r9,-0x37(%rcx)
L(P7Q5):
	mov    -0x2f(%rdx),%r11
	mov    %r11,-0x2f(%rcx)
L(P7Q4):
	mov    -0x27(%rdx),%r10
	mov    %r10,-0x27(%rcx)
L(P7Q3):
	mov    -0x1f(%rdx),%r9
	mov    %r9,-0x1f(%rcx)
L(P7Q2):
	mov    -0x17(%rdx),%r11
	mov    %r11,-0x17(%rcx)
L(P7Q1):
	mov    -0xf(%rdx),%r10
	mov    %r10,-0xf(%rcx)
	/*
	 * These trailing loads/stores have to do all their loads 1st,
	 * then do the stores.
	 */
L(P7Q0):
	mov    -0x7(%rdx),%r9d
	movzwq -0x3(%rdx),%r10
	movzbq -0x1(%rdx),%r11
	mov    %r9d,-0x7(%rcx)
	mov    %r10w,-0x3(%rcx)
	mov    %r11b,-0x1(%rcx)
	ret

	.balign 16
L(ck_use_sse2):
	/*
	 * Align dest to 16 byte boundary.
	 */
	test   $0xf,%rcx
	jnz    L(ShrtAlignNew)

L(now_qw_aligned):
	cmpl   $NO_SSE,.memops_method(%rip)
	je     L(Loop8byte_pre)

	/*
	 * The fall-through path is to do SSE2 16-byte load/stores
	 */

	/*
	 * If current move size is larger than half of the highest level cache
	 * size, then do non-temporal moves.
	 */
	mov    .largest_level_cache_size(%rip),%r9d
	shr    %r9		# take half of it
	cmp    %r9,%r8
	jg     L(sse2_nt_move)

	/*
	 * If both the source and dest are aligned, then use the both aligned
	 * logic. Well aligned data should reap the rewards.
	 */
	test   $0xf,%rdx
	jz     L(pre_both_aligned)

	lea    L(SSE_src)(%rip),%r10		# SSE2 (default)
	testl  $USE_SSSE3,.memops_method(%rip)
	jz     1f
	lea    L(SSSE3_src)(%rip),%r10		# SSSE3

1:
	/*
	 * if the src is not 16 byte aligned...
	 */
	mov    %rdx,%r11
	and    $0xf,%r11
	movdqu (%rdx),%xmm0
	movdqa %xmm0,(%rcx)
	add    $0x10,%rdx
	sub    %r11,%rdx
	add    $0x10,%rcx
	sub    $0x10,%r8
	movdqa (%rdx),%xmm1

	movslq (%r10,%r11,4),%r9
	lea    (%r9,%r10,1),%r10
	jmpq   *%r10

	    .balign 16
L(SSSE3_src): .int	L(pre_both_aligned)-L(SSSE3_src)
	    .int        L(mov3dqa1) -L(SSSE3_src)
	    .int        L(mov3dqa2) -L(SSSE3_src)
	    .int        L(mov3dqa3) -L(SSSE3_src)
	    .int        L(mov3dqa4) -L(SSSE3_src)
	    .int        L(mov3dqa5) -L(SSSE3_src)
	    .int        L(mov3dqa6) -L(SSSE3_src)
	    .int        L(mov3dqa7) -L(SSSE3_src)
	    .int        L(movdqa8)  -L(SSSE3_src)
	    .int        L(mov3dqa9) -L(SSSE3_src)
	    .int        L(mov3dqa10)-L(SSSE3_src)
	    .int        L(mov3dqa11)-L(SSSE3_src)
	    .int        L(mov3dqa12)-L(SSSE3_src)
	    .int        L(mov3dqa13)-L(SSSE3_src)
	    .int        L(mov3dqa14)-L(SSSE3_src)
	    .int        L(mov3dqa15)-L(SSSE3_src)
L(SSE_src): .int    L(pre_both_aligned)-L(SSE_src)
	    .int        L(movdqa1) -L(SSE_src)
	    .int        L(movdqa2) -L(SSE_src)
	    .int        L(movdqa3) -L(SSE_src)
	    .int        L(movdqa4) -L(SSE_src)
	    .int        L(movdqa5) -L(SSE_src)
	    .int        L(movdqa6) -L(SSE_src)
	    .int        L(movdqa7) -L(SSE_src)
	    .int        L(movdqa8) -L(SSE_src)
	    .int        L(movdqa9) -L(SSE_src)
	    .int        L(movdqa10)-L(SSE_src)
	    .int        L(movdqa11)-L(SSE_src)
	    .int        L(movdqa12)-L(SSE_src)
	    .int        L(movdqa13)-L(SSE_src)
	    .int        L(movdqa14)-L(SSE_src)
	    .int        L(movdqa15)-L(SSE_src)

	.balign 16
L(movdqa1):
	movdqa 0x10(%rdx),%xmm3 # load the upper source buffer
	movdqa 0x20(%rdx),%xmm0 # load the upper source buffer
	lea    0x20(%rdx),%rdx
	lea    -0x20(%r8),%r8

	psrldq $0x1,%xmm1  # shift right prev buffer (saved from last iteration)
	movdqa %xmm3,%xmm2 # store off xmm reg for use next iteration
	pslldq $0xf,%xmm3  # shift the current buffer left (shift in zeros)
	por    %xmm1,%xmm3 # OR them together
	cmp    $0x20,%r8

	psrldq $0x1,%xmm2  # shift right prev buffer (saved from last iteration)
	movdqa %xmm0,%xmm1 # store off xmm reg for use next iteration
	pslldq $0xf,%xmm0  # shift the current buffer left (shift in zeros)
	por    %xmm2,%xmm0 # OR them together
	movdqa %xmm3,(%rcx)     # store it
	movdqa %xmm0,0x10(%rcx) # store it
	lea    0x20(%rcx),%rcx

	jge    L(movdqa1)
	jmp    L(movdqa_epi)

	.balign 16
L(movdqa2):
	sub    $0x20,%r8
	movdqa 0x10(%rdx),%xmm3
	movdqa 0x20(%rdx),%xmm0
	add    $0x20,%rdx

	psrldq $0x2,%xmm1
	movdqa %xmm3,%xmm2
	pslldq $0xe,%xmm3
	por    %xmm1,%xmm3

	psrldq $0x2,%xmm2
	movdqa %xmm0,%xmm1
	pslldq $0xe,%xmm0
	por    %xmm2,%xmm0
	movdqa %xmm3,(%rcx)
	movdqa %xmm0,0x10(%rcx)

	add    $0x20,%rcx
	cmp    $0x20,%r8
	jge    L(movdqa2)
	jmp    L(movdqa_epi)

	.balign 16
L(movdqa3):
	sub    $0x20,%r8
	movdqa 0x10(%rdx),%xmm3
	movdqa 0x20(%rdx),%xmm0
	add    $0x20,%rdx

	psrldq $0x3,%xmm1
	movdqa %xmm3,%xmm2
	pslldq $0xd,%xmm3
	por    %xmm1,%xmm3

	psrldq $0x3,%xmm2
	movdqa %xmm0,%xmm1
	pslldq $0xd,%xmm0
	por    %xmm2,%xmm0
	movdqa %xmm3,(%rcx)
	movdqa %xmm0,0x10(%rcx)

	add    $0x20,%rcx
	cmp    $0x20,%r8
	jge    L(movdqa3)
	jmp    L(movdqa_epi)

	.balign 16
L(movdqa4):
	sub    $0x20,%r8
	movdqa 0x10(%rdx),%xmm3
	movdqa 0x20(%rdx),%xmm0
	add    $0x20,%rdx

	psrldq $0x4,%xmm1
	movdqa %xmm3,%xmm2
	pslldq $0xc,%xmm3
	por    %xmm1,%xmm3

	psrldq $0x4,%xmm2
	movdqa %xmm0,%xmm1
	pslldq $0xc,%xmm0
	por    %xmm2,%xmm0

	movdqa %xmm3,(%rcx)
	movdqa %xmm0,0x10(%rcx)

	add    $0x20,%rcx
	cmp    $0x20,%r8
	jge    L(movdqa4)
	jmp    L(movdqa_epi)

	.balign 16
L(movdqa5):
	sub    $0x20,%r8
	movdqa 0x10(%rdx),%xmm3
	movdqa 0x20(%rdx),%xmm0
	add    $0x20,%rdx

	psrldq $0x5,%xmm1
	movdqa %xmm3,%xmm2
	pslldq $0xb,%xmm3
	por    %xmm1,%xmm3

	psrldq $0x5,%xmm2
	movdqa %xmm0,%xmm1
	pslldq $0xb,%xmm0
	por    %xmm2,%xmm0

	movdqa %xmm3,(%rcx)
	movdqa %xmm0,0x10(%rcx)

	add    $0x20,%rcx
	cmp    $0x20,%r8
	jge    L(movdqa5)
	jmp    L(movdqa_epi)

	.balign 16
L(movdqa6):
	sub    $0x20,%r8
	movdqa 0x10(%rdx),%xmm3
	movdqa 0x20(%rdx),%xmm0
	add    $0x20,%rdx

	psrldq $0x6,%xmm1
	movdqa %xmm3,%xmm2
	pslldq $0xa,%xmm3
	por    %xmm1,%xmm3

	psrldq $0x6,%xmm2
	movdqa %xmm0,%xmm1
	pslldq $0xa,%xmm0
	por    %xmm2,%xmm0
	movdqa %xmm3,(%rcx)
	movdqa %xmm0,0x10(%rcx)

	add    $0x20,%rcx
	cmp    $0x20,%r8
	jge    L(movdqa6)
	jmp    L(movdqa_epi)

	.balign 16
L(movdqa7):
	sub    $0x20,%r8
	movdqa 0x10(%rdx),%xmm3
	movdqa 0x20(%rdx),%xmm0
	add    $0x20,%rdx

	psrldq $0x7,%xmm1
	movdqa %xmm3,%xmm2
	pslldq $0x9,%xmm3
	por    %xmm1,%xmm3

	psrldq $0x7,%xmm2
	movdqa %xmm0,%xmm1
	pslldq $0x9,%xmm0
	por    %xmm2,%xmm0
	movdqa %xmm3,(%rcx)
	movdqa %xmm0,0x10(%rcx)

	add    $0x20,%rcx
	cmp    $0x20,%r8
	jge    L(movdqa7)
	jmp    L(movdqa_epi)

	.balign 16
L(movdqa8):
	movdqa 0x10(%rdx),%xmm3
	sub    $0x30,%r8
	movdqa 0x20(%rdx),%xmm0
	movdqa 0x30(%rdx),%xmm5
	lea    0x30(%rdx),%rdx

	shufpd $0x1,%xmm3,%xmm1
	movdqa %xmm1,(%rcx)

	cmp    $0x30,%r8

	shufpd $0x1,%xmm0,%xmm3
	movdqa %xmm3,0x10(%rcx)

	movdqa %xmm5,%xmm1
	shufpd $0x1,%xmm5,%xmm0
	movdqa %xmm0,0x20(%rcx)

	lea    0x30(%rcx),%rcx

	jge    L(movdqa8)
	jmp    L(movdqa_epi)

	.balign 16
L(movdqa9):
	sub    $0x20,%r8
	movdqa 0x10(%rdx),%xmm3
	movdqa 0x20(%rdx),%xmm0
	add    $0x20,%rdx

	psrldq $0x9,%xmm1
	movdqa %xmm3,%xmm2
	pslldq $0x7,%xmm3
	por    %xmm1,%xmm3

	psrldq $0x9,%xmm2
	movdqa %xmm0,%xmm1
	pslldq $0x7,%xmm0
	por    %xmm2,%xmm0
	movdqa %xmm3,(%rcx)
	movdqa %xmm0,0x10(%rcx)

	add    $0x20,%rcx
	cmp    $0x20,%r8
	jge    L(movdqa9)
	jmp    L(movdqa_epi)

	.balign 16
L(movdqa10):
	sub    $0x20,%r8
	movdqa 0x10(%rdx),%xmm3
	movdqa 0x20(%rdx),%xmm0
	add    $0x20,%rdx

	psrldq $0xa,%xmm1
	movdqa %xmm3,%xmm2
	pslldq $0x6,%xmm3
	por    %xmm1,%xmm3

	psrldq $0xa,%xmm2
	movdqa %xmm0,%xmm1
	pslldq $0x6,%xmm0
	por    %xmm2,%xmm0
	movdqa %xmm3,(%rcx)
	movdqa %xmm0,0x10(%rcx)

	add    $0x20,%rcx
	cmp    $0x20,%r8
	jge    L(movdqa10)
	jmp    L(movdqa_epi)

	.balign 16
L(movdqa11):
	sub    $0x20,%r8
	movdqa 0x10(%rdx),%xmm3
	movdqa 0x20(%rdx),%xmm0
	add    $0x20,%rdx

	psrldq $0xb,%xmm1
	movdqa %xmm3,%xmm2
	pslldq $0x5,%xmm3
	por    %xmm1,%xmm3

	psrldq $0xb,%xmm2
	movdqa %xmm0,%xmm1
	pslldq $0x5,%xmm0
	por    %xmm2,%xmm0
	movdqa %xmm3,(%rcx)
	movdqa %xmm0,0x10(%rcx)

	add    $0x20,%rcx
	cmp    $0x20,%r8
	jge    L(movdqa11)
	jmp    L(movdqa_epi)

	.balign 16
L(movdqa12):
	sub    $0x20,%r8
	movdqa 0x10(%rdx),%xmm3
	movdqa 0x20(%rdx),%xmm0
	add    $0x20,%rdx

	psrldq $0xc,%xmm1
	movdqa %xmm3,%xmm2
	pslldq $0x4,%xmm3
	por    %xmm1,%xmm3

	psrldq $0xc,%xmm2
	movdqa %xmm0,%xmm1
	pslldq $0x4,%xmm0
	por    %xmm2,%xmm0
	movdqa %xmm3,(%rcx)
	movdqa %xmm0,0x10(%rcx)

	add    $0x20,%rcx
	cmp    $0x20,%r8
	jge    L(movdqa12)
	jmp    L(movdqa_epi)

	.balign 16
L(movdqa13):
	sub    $0x20,%r8
	movdqa 0x10(%rdx),%xmm3
	movdqa 0x20(%rdx),%xmm0
	add    $0x20,%rdx

	psrldq $0xd,%xmm1
	movdqa %xmm3,%xmm2
	pslldq $0x3,%xmm3
	por    %xmm1,%xmm3

	psrldq $0xd,%xmm2
	movdqa %xmm0,%xmm1
	pslldq $0x3,%xmm0
	por    %xmm2,%xmm0
	movdqa %xmm3,(%rcx)
	movdqa %xmm0,0x10(%rcx)

	add    $0x20,%rcx
	cmp    $0x20,%r8
	jge    L(movdqa13)
	jmp    L(movdqa_epi)

	.balign 16
L(movdqa14):
	sub    $0x20,%r8
	movdqa 0x10(%rdx),%xmm3
	movdqa 0x20(%rdx),%xmm0
	add    $0x20,%rdx

	psrldq $0xe,%xmm1
	movdqa %xmm3,%xmm2
	pslldq $0x2,%xmm3
	por    %xmm1,%xmm3

	psrldq $0xe,%xmm2
	movdqa %xmm0,%xmm1
	pslldq $0x2,%xmm0
	por    %xmm2,%xmm0
	movdqa %xmm3,(%rcx)
	movdqa %xmm0,0x10(%rcx)

	add    $0x20,%rcx
	cmp    $0x20,%r8
	jge    L(movdqa14)
	jmp    L(movdqa_epi)

	.balign 16
L(movdqa15):
	sub    $0x20,%r8
	movdqa 0x10(%rdx),%xmm3
	movdqa 0x20(%rdx),%xmm0
	add    $0x20,%rdx

	psrldq $0xf,%xmm1
	movdqa %xmm3,%xmm2
	pslldq $0x1,%xmm3
	por    %xmm1,%xmm3

	psrldq $0xf,%xmm2
	movdqa %xmm0,%xmm1
	pslldq $0x1,%xmm0
	por    %xmm2,%xmm0
	movdqa %xmm3,(%rcx)
	movdqa %xmm0,0x10(%rcx)

	add    $0x20,%rcx
	cmp    $0x20,%r8
	jge    L(movdqa15)
	#jmp   L(movdqa_epi)

	.balign 16
L(movdqa_epi):
	lea    L(fwdPxQx)(%rip),%r10
	add    %r11,%rdx # bump rdx to the right addr (it lagged behind in the above loop)
	add    %r8,%rcx
	add    %r8,%rdx

	movslq (%r10,%r8,4),%r9
	lea    (%r9,%r10,1),%r10
	jmpq   *%r10

	.balign 16
L(mov3dqa1):
	movdqa	0x10(%rdx),%xmm3 # load the upper source buffer
	sub	$0x30,%r8
	movdqa	0x20(%rdx),%xmm0 # load the upper source buffer
	movdqa	0x30(%rdx),%xmm5 # load the upper source buffer
	lea	0x30(%rdx),%rdx
	cmp	$0x30,%r8

	movdqa	%xmm3,%xmm2       # store off xmm reg for use next iteration
	#palignr	$0x1,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x01
	movdqa	%xmm3,(%rcx)      # store it

	movdqa	%xmm0,%xmm4       # store off xmm reg for use next iteration
	#palignr	$0x1,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x01
	movdqa	%xmm0,0x10(%rcx)  # store it

	movdqa	%xmm5,%xmm1       # store off xmm reg for use next iteration
	#palignr	$0x1,%xmm4,%xmm5
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xec,0x01
	movdqa	%xmm5,0x20(%rcx)  # store it

	lea	0x30(%rcx),%rcx
	jge	L(mov3dqa1)

	cmp	$0x10,%r8
	jl	L(movdqa_epi)
	movdqa	0x10(%rdx),%xmm3	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	movdqa	%xmm3,%xmm2		# save for use next concat
	#palignr	$0x1,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x01

	cmp	$0x10,%r8
	movdqa	%xmm3,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jl	L(movdqa_epi)

	movdqa	0x10(%rdx),%xmm0	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	#palignr	$0x1,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x01
	movdqa	%xmm0,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jmp	L(movdqa_epi)

	.balign 16
L(mov3dqa2):
	movdqa	0x10(%rdx),%xmm3
	sub	$0x30,%r8
	movdqa	0x20(%rdx),%xmm0
	movdqa	0x30(%rdx),%xmm5
	lea	0x30(%rdx),%rdx
	cmp	$0x30,%r8

	movdqa	%xmm3,%xmm2
	#palignr	$0x2,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x02
	movdqa	%xmm3,(%rcx)

	movdqa	%xmm0,%xmm4
	#palignr	$0x2,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x02
	movdqa	%xmm0,0x10(%rcx)

	movdqa	%xmm5,%xmm1
	#palignr	$0x2,%xmm4,%xmm5
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xec,0x02
	movdqa	%xmm5,0x20(%rcx)

	lea	0x30(%rcx),%rcx
	jge	L(mov3dqa2)

	cmp	$0x10,%r8
	jl	L(movdqa_epi)
	movdqa	0x10(%rdx),%xmm3	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	movdqa	%xmm3,%xmm2		# save for use next concat
	#palignr	$0x2,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x02

	cmp	$0x10,%r8
	movdqa	%xmm3,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jl	L(movdqa_epi)

	movdqa	0x10(%rdx),%xmm0	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	#palignr	$0x2,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x02
	movdqa	%xmm0,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jmp	L(movdqa_epi)

	.balign 16
L(mov3dqa3):
	movdqa	0x10(%rdx),%xmm3
	sub	$0x30,%r8
	movdqa	0x20(%rdx),%xmm0
	movdqa	0x30(%rdx),%xmm5
	lea	0x30(%rdx),%rdx
	cmp	$0x30,%r8

	movdqa	%xmm3,%xmm2
	#palignr	$0x3,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x03
	movdqa	%xmm3,(%rcx)

	movdqa	%xmm0,%xmm4
	#palignr	$0x3,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x03
	movdqa	%xmm0,0x10(%rcx)

	movdqa	%xmm5,%xmm1
	#palignr	$0x3,%xmm4,%xmm5
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xec,0x03
	movdqa	%xmm5,0x20(%rcx)

	lea	0x30(%rcx),%rcx
	jge	L(mov3dqa3)

	cmp	$0x10,%r8
	jl	L(movdqa_epi)
	movdqa	0x10(%rdx),%xmm3	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	movdqa	%xmm3,%xmm2		# save for use next concat
	#palignr	$0x3,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x03

	cmp	$0x10,%r8
	movdqa	%xmm3,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jl	L(movdqa_epi)

	movdqa	0x10(%rdx),%xmm0	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	#palignr	$0x3,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x03
	movdqa	%xmm0,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jmp	L(movdqa_epi)

	.balign 16
L(mov3dqa4):
	movdqa	0x10(%rdx),%xmm3
	sub	$0x30,%r8
	movdqa	0x20(%rdx),%xmm0
	movdqa	0x30(%rdx),%xmm5
	lea	0x30(%rdx),%rdx
	cmp	$0x30,%r8

	movdqa	%xmm3,%xmm2
	#palignr	$0x4,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x04
	movdqa	%xmm3,(%rcx)

	movdqa	%xmm0,%xmm4
	#palignr	$0x4,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x04
	movdqa	%xmm0,0x10(%rcx)

	movdqa	%xmm5,%xmm1
	#palignr	$0x4,%xmm4,%xmm5
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xec,0x04
	movdqa	%xmm5,0x20(%rcx)

	lea	0x30(%rcx),%rcx
	jge	L(mov3dqa4)

	cmp	$0x10,%r8
	jl	L(movdqa_epi)
	movdqa	0x10(%rdx),%xmm3	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	movdqa	%xmm3,%xmm2		# save for use next concat
	#palignr	$0x4,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x04

	cmp	$0x10,%r8
	movdqa	%xmm3,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jl	L(movdqa_epi)

	movdqa	0x10(%rdx),%xmm0	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	#palignr	$0x4,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x04
	movdqa	%xmm0,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jmp	L(movdqa_epi)

	.balign 16
L(mov3dqa5):
	movdqa	0x10(%rdx),%xmm3
	sub	$0x30,%r8
	movdqa	0x20(%rdx),%xmm0
	movdqa	0x30(%rdx),%xmm5
	lea	0x30(%rdx),%rdx
	cmp	$0x30,%r8

	movdqa	%xmm3,%xmm2
	#palignr	$0x5,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x05
	movdqa	%xmm3,(%rcx)

	movdqa	%xmm0,%xmm4
	#palignr	$0x5,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x05
	movdqa	%xmm0,0x10(%rcx)

	movdqa	%xmm5,%xmm1
	#palignr	$0x5,%xmm4,%xmm5
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xec,0x05
	movdqa	%xmm5,0x20(%rcx)

	lea	0x30(%rcx),%rcx
	jge	L(mov3dqa5)

	cmp	$0x10,%r8
	jl	L(movdqa_epi)
	movdqa	0x10(%rdx),%xmm3	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	movdqa	%xmm3,%xmm2		# save for use next concat
	#palignr	$0x5,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x05

	cmp	$0x10,%r8
	movdqa	%xmm3,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jl	L(movdqa_epi)

	movdqa	0x10(%rdx),%xmm0	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	#palignr	$0x5,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x05
	movdqa	%xmm0,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jmp	L(movdqa_epi)

	.balign 16
L(mov3dqa6):
	movdqa	0x10(%rdx),%xmm3
	sub	$0x30,%r8
	movdqa	0x20(%rdx),%xmm0
	movdqa	0x30(%rdx),%xmm5
	lea	0x30(%rdx),%rdx
	cmp	$0x30,%r8

	movdqa	%xmm3,%xmm2
	#palignr	$0x6,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x06
	movdqa	%xmm3,(%rcx)

	movdqa	%xmm0,%xmm4
	#palignr	$0x6,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x06
	movdqa	%xmm0,0x10(%rcx)

	movdqa	%xmm5,%xmm1
	#palignr	$0x6,%xmm4,%xmm5
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xec,0x06
	movdqa	%xmm5,0x20(%rcx)

	lea	0x30(%rcx),%rcx
	jge	L(mov3dqa6)

	cmp	$0x10,%r8
	jl	L(movdqa_epi)
	movdqa	0x10(%rdx),%xmm3	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	movdqa	%xmm3,%xmm2		# save for use next concat
	#palignr	$0x6,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x06

	cmp	$0x10,%r8
	movdqa	%xmm3,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jl	L(movdqa_epi)

	movdqa	0x10(%rdx),%xmm0	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	#palignr	$0x6,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x06
	movdqa	%xmm0,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jmp	L(movdqa_epi)

	.balign 16
L(mov3dqa7):
	movdqa	0x10(%rdx),%xmm3
	sub	$0x30,%r8
	movdqa	0x20(%rdx),%xmm0
	movdqa	0x30(%rdx),%xmm5
	lea	0x30(%rdx),%rdx
	cmp	$0x30,%r8

	movdqa	%xmm3,%xmm2
	#palignr	$0x7,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x07
	movdqa	%xmm3,(%rcx)

	movdqa	%xmm0,%xmm4
	#palignr	$0x7,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x07
	movdqa	%xmm0,0x10(%rcx)

	movdqa	%xmm5,%xmm1
	#palignr	$0x7,%xmm4,%xmm5
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xec,0x07
	movdqa	%xmm5,0x20(%rcx)

	lea	0x30(%rcx),%rcx
	jge	L(mov3dqa7)

	cmp	$0x10,%r8
	jl	L(movdqa_epi)
	movdqa	0x10(%rdx),%xmm3	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	movdqa	%xmm3,%xmm2		# save for use next concat
	#palignr	$0x7,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x07

	cmp	$0x10,%r8
	movdqa	%xmm3,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jl	L(movdqa_epi)

	movdqa	0x10(%rdx),%xmm0	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	#palignr	$0x7,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x07
	movdqa	%xmm0,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jmp	L(movdqa_epi)

	.balign 16
L(mov3dqa9):
	movdqa	0x10(%rdx),%xmm3
	sub	$0x30,%r8
	movdqa	0x20(%rdx),%xmm0
	movdqa	0x30(%rdx),%xmm5
	lea	0x30(%rdx),%rdx
	cmp	$0x30,%r8

	movdqa	%xmm3,%xmm2
	#palignr	$0x9,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x09
	movdqa	%xmm3,(%rcx)

	movdqa	%xmm0,%xmm4
	#palignr	$0x9,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x09
	movdqa	%xmm0,0x10(%rcx)

	movdqa	%xmm5,%xmm1
	#palignr	$0x9,%xmm4,%xmm5
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xec,0x09
	movdqa	%xmm5,0x20(%rcx)

	lea	0x30(%rcx),%rcx
	jge	L(mov3dqa9)

	cmp	$0x10,%r8
	jl	L(movdqa_epi)
	movdqa	0x10(%rdx),%xmm3	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	movdqa	%xmm3,%xmm2		# save for use next concat
	#palignr	$0x9,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x09

	cmp	$0x10,%r8
	movdqa	%xmm3,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jl	L(movdqa_epi)

	movdqa	0x10(%rdx),%xmm0	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	#palignr	$0x9,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x09
	movdqa	%xmm0,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jmp	L(movdqa_epi)

	.balign 16
L(mov3dqa10):
	movdqa	0x10(%rdx),%xmm3
	sub	$0x30,%r8
	movdqa	0x20(%rdx),%xmm0
	movdqa	0x30(%rdx),%xmm5
	lea	0x30(%rdx),%rdx
	cmp	$0x30,%r8

	movdqa	%xmm3,%xmm2
	#palignr	$0xa,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x0a
	movdqa	%xmm3,(%rcx)

	movdqa	%xmm0,%xmm4
	#palignr	$0xa,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x0a
	movdqa	%xmm0,0x10(%rcx)

	movdqa	%xmm5,%xmm1
	#palignr	$0xa,%xmm4,%xmm5
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xec,0x0a
	movdqa	%xmm5,0x20(%rcx)

	lea	0x30(%rcx),%rcx
	jge	L(mov3dqa10)

	cmp	$0x10,%r8
	jl	L(movdqa_epi)
	movdqa	0x10(%rdx),%xmm3	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	movdqa	%xmm3,%xmm2		# save for use next concat
	#palignr	$0xa,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x0a

	cmp	$0x10,%r8
	movdqa	%xmm3,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jl	L(movdqa_epi)

	movdqa	0x10(%rdx),%xmm0	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	#palignr	$0xa,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x0a
	movdqa	%xmm0,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jmp	L(movdqa_epi)

	.balign 16
L(mov3dqa11):
	movdqa	0x10(%rdx),%xmm3
	sub	$0x30,%r8
	movdqa	0x20(%rdx),%xmm0
	movdqa	0x30(%rdx),%xmm5
	lea	0x30(%rdx),%rdx
	cmp	$0x30,%r8

	movdqa	%xmm3,%xmm2
	#palignr	$0xb,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x0b
	movdqa	%xmm3,(%rcx)

	movdqa	%xmm0,%xmm4
	#palignr	$0xb,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x0b
	movdqa	%xmm0,0x10(%rcx)

	movdqa	%xmm5,%xmm1
	#palignr	$0xb,%xmm4,%xmm5
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xec,0x0b
	movdqa	%xmm5,0x20(%rcx)

	lea	0x30(%rcx),%rcx
	jge	L(mov3dqa11)

	cmp	$0x10,%r8
	jl	L(movdqa_epi)
	movdqa	0x10(%rdx),%xmm3	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	movdqa	%xmm3,%xmm2		# save for use next concat
	#palignr	$0xb,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x0b

	cmp	$0x10,%r8
	movdqa	%xmm3,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jl	L(movdqa_epi)

	movdqa	0x10(%rdx),%xmm0	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	#palignr	$0xb,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x0b
	movdqa	%xmm0,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jmp	L(movdqa_epi)

	.balign 16
L(mov3dqa12):
	movdqa	0x10(%rdx),%xmm3
	sub	$0x30,%r8
	movdqa	0x20(%rdx),%xmm0
	movdqa	0x30(%rdx),%xmm5
	lea	0x30(%rdx),%rdx
	cmp	$0x30,%r8

	movdqa	%xmm3,%xmm2
	#palignr	$0xc,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x0c
	movdqa	%xmm3,(%rcx)

	movdqa	%xmm0,%xmm4
	#palignr	$0xc,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x0c
	movdqa	%xmm0,0x10(%rcx)

	movdqa	%xmm5,%xmm1
	#palignr	$0xc,%xmm4,%xmm5
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xec,0x0c
	movdqa	%xmm5,0x20(%rcx)

	lea	0x30(%rcx),%rcx
	jge	L(mov3dqa12)

	cmp	$0x10,%r8
	jl	L(movdqa_epi)
	movdqa	0x10(%rdx),%xmm3	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	movdqa	%xmm3,%xmm2		# save for use next concat
	#palignr	$0xc,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x0c

	cmp	$0x10,%r8
	movdqa	%xmm3,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jl	L(movdqa_epi)

	movdqa	0x10(%rdx),%xmm0	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	#palignr	$0xc,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x0c
	movdqa	%xmm0,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jmp	L(movdqa_epi)

	.balign 16
L(mov3dqa13):
	movdqa	0x10(%rdx),%xmm3
	sub	$0x30,%r8
	movdqa	0x20(%rdx),%xmm0
	movdqa	0x30(%rdx),%xmm5
	lea	0x30(%rdx),%rdx
	cmp	$0x30,%r8

	movdqa	%xmm3,%xmm2
	#palignr	$0xd,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x0d
	movdqa	%xmm3,(%rcx)

	movdqa	%xmm0,%xmm4
	#palignr	$0xd,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x0d
	movdqa	%xmm0,0x10(%rcx)

	movdqa	%xmm5,%xmm1
	#palignr	$0xd,%xmm4,%xmm5
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xec,0x0d
	movdqa	%xmm5,0x20(%rcx)

	lea	0x30(%rcx),%rcx
	jge	L(mov3dqa13)

	cmp	$0x10,%r8
	jl	L(movdqa_epi)
	movdqa	0x10(%rdx),%xmm3	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	movdqa	%xmm3,%xmm2		# save for use next concat
	#palignr	$0xd,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x0d

	cmp	$0x10,%r8
	movdqa	%xmm3,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jl	L(movdqa_epi)

	movdqa	0x10(%rdx),%xmm0	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	#palignr	$0xd,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x0d
	movdqa	%xmm0,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jmp	L(movdqa_epi)

	.balign 16
L(mov3dqa14):
	movdqa	0x10(%rdx),%xmm3
	sub	$0x30,%r8
	movdqa	0x20(%rdx),%xmm0
	movdqa	0x30(%rdx),%xmm5
	lea	0x30(%rdx),%rdx
	cmp	$0x30,%r8

	movdqa	%xmm3,%xmm2
	#palignr	$0xe,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x0e
	movdqa	%xmm3,(%rcx)

	movdqa	%xmm0,%xmm4
	#palignr	$0xe,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x0e
	movdqa	%xmm0,0x10(%rcx)

	movdqa	%xmm5,%xmm1
	#palignr	$0xe,%xmm4,%xmm5
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xec,0x0e
	movdqa	%xmm5,0x20(%rcx)

	lea	0x30(%rcx),%rcx
	jge	L(mov3dqa14)

	cmp	$0x10,%r8
	jl	L(movdqa_epi)
	movdqa	0x10(%rdx),%xmm3	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	movdqa	%xmm3,%xmm2		# save for use next concat
	#palignr	$0xe,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x0e

	cmp	$0x10,%r8
	movdqa	%xmm3,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jl	L(movdqa_epi)

	movdqa	0x10(%rdx),%xmm0	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	#palignr	$0xe,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x0e
	movdqa	%xmm0,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jmp	L(movdqa_epi)

	.balign 16
L(mov3dqa15):
	movdqa	0x10(%rdx),%xmm3
	sub	$0x30,%r8
	movdqa	0x20(%rdx),%xmm0
	movdqa	0x30(%rdx),%xmm5
	lea	0x30(%rdx),%rdx
	cmp	$0x30,%r8

	movdqa	%xmm3,%xmm2
	#palignr	$0xf,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x0f
	movdqa	%xmm3,(%rcx)

	movdqa	%xmm0,%xmm4
	#palignr	$0xf,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x0f
	movdqa	%xmm0,0x10(%rcx)

	movdqa	%xmm5,%xmm1
	#palignr	$0xf,%xmm4,%xmm5
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xec,0x0f
	movdqa	%xmm5,0x20(%rcx)

	lea	0x30(%rcx),%rcx
	jge	L(mov3dqa15)

	cmp	$0x10,%r8
	jl	L(movdqa_epi)
	movdqa	0x10(%rdx),%xmm3	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	movdqa	%xmm3,%xmm2		# save for use next concat
	#palignr	$0xf,%xmm1,%xmm3
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xd9,0x0f

	cmp	$0x10,%r8
	movdqa	%xmm3,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jl	L(movdqa_epi)

	movdqa	0x10(%rdx),%xmm0	# load the upper source buffer
	sub	$0x10,%r8
	lea	0x10(%rdx),%rdx
	#palignr	$0xf,%xmm2,%xmm0
	.byte	0x66,0x0f,0x3a,0x0f
	.byte	0xc2,0x0f
	movdqa	%xmm0,(%rcx)      	# store it
	lea	0x10(%rcx),%rcx
	jmp	L(movdqa_epi)

	.balign 16
L(sse2_nt_move):
	lea	0x40(%rcx),%rcx
	lea	0x40(%rdx),%rdx
	lea	-0x40(%r8),%r8

	/*
	 * doesn't matter if source is aligned for stuff out of cache.
	 * the mis-aligned penalty is masked by the slowness of main memory.
	 */
	prefetchnta 0x180(%rdx)
	movdqu	-0x40(%rdx),%xmm0
	movdqu	-0x30(%rdx),%xmm1

	cmp	$0x40,%r8
	movntdq	%xmm0,-0x40(%rcx)
	movntdq	%xmm1,-0x30(%rcx)

	movdqu	-0x20(%rdx),%xmm2
	movdqu	-0x10(%rdx),%xmm3

	movntdq	%xmm2,-0x20(%rcx)
	movntdq	%xmm3,-0x10(%rcx)

	jge	L(sse2_nt_move)

	lea	L(Fix16EndTable)(%rip),%r10
	mov	%r8,%r9
	and	$0xFFFFFFFFFFFFFFF0,%r9
	add	%r9,%rcx
	add	%r9,%rdx
	sub	%r9,%r8
	shr	$0x4,%r9
	sfence

	movslq	(%r10,%r9,4),%r11
	lea	(%r11,%r10,1),%r10
	jmpq	*%r10

	.balign 16
L(Fix16EndTable):
	.int    L(fix16_0)-L(Fix16EndTable)
	.int    L(fix16_1)-L(Fix16EndTable)
	.int    L(fix16_2)-L(Fix16EndTable)
	.int    L(fix16_3)-L(Fix16EndTable)

	.balign 16
L(fix16_3):
	movdqu -0x30(%rdx),%xmm1
	movdqa %xmm1,-0x30(%rcx)
L(fix16_2):
	movdqu -0x20(%rdx),%xmm2
	movdqa %xmm2,-0x20(%rcx)
L(fix16_1):
	movdqu -0x10(%rdx),%xmm3
	movdqa %xmm3,-0x10(%rcx)
L(fix16_0):
	lea    L(fwdPxQx)(%rip),%r10
	add    %r8,%rdx
	add    %r8,%rcx

	movslq (%r10,%r8,4),%r9
	lea    (%r9,%r10,1),%r10
	jmpq   *%r10

	.balign 16
L(pre_both_aligned):
	cmp    $0x80,%r8
	jl     L(fix_16b)

	.balign 16
L(both_aligned):

	/*
	 * this 'paired' load/load/store/store seems to do best.
	 */
	movdqa (%rdx),%xmm0
	movdqa 0x10(%rdx),%xmm1

	movdqa %xmm0,(%rcx)
	movdqa %xmm1,0x10(%rcx)
	lea    -0x80(%r8),%r8

	movdqa 0x20(%rdx),%xmm2
	movdqa 0x30(%rdx),%xmm3

	movdqa %xmm2,0x20(%rcx)
	movdqa %xmm3,0x30(%rcx)

	movdqa 0x40(%rdx),%xmm0
	movdqa 0x50(%rdx),%xmm1
	cmp    $0x80,%r8

	movdqa %xmm0,0x40(%rcx)
	movdqa %xmm1,0x50(%rcx)

	movdqa 0x60(%rdx),%xmm2
	movdqa 0x70(%rdx),%xmm3
	lea    0x80(%rdx),%rdx
	movdqa %xmm2,0x60(%rcx)
	movdqa %xmm3,0x70(%rcx)
	lea    0x80(%rcx),%rcx
	jge    L(both_aligned)

L(fix_16b):
	add    %r8,%rcx
	lea    L(fwdPxQx)(%rip),%r10
	add    %r8,%rdx

	movslq (%r10,%r8,4),%r9
	lea    (%r9,%r10,1),%r10
	jmpq   *%r10

	.balign 16
L(Loop8byte_pre):
	# Use 8-byte moves
	mov    .largest_level_cache_size(%rip),%r9d
	shr    %r9		# take half of it
	cmp    %r9,%r8
	jg     L(byte8_nt_top)
	# Find out whether to use rep movsq
	cmp    $4096,%r8
	jle    L(byte8_top)
	mov    .amd64cache1half(%rip),%r9d	# half of l1 cache
	cmp    %r9,%r8
	jle    L(use_rep)

	.balign     16
L(byte8_top):
	mov    (%rdx),%r9
	mov    0x8(%rdx),%r10
	lea    -0x40(%r8),%r8
	mov    %r9,(%rcx)
	mov    %r10,0x8(%rcx)
	mov    0x10(%rdx),%r11
	mov    0x18(%rdx),%r9
	mov    %r11,0x10(%rcx)
	mov    %r9,0x18(%rcx)

	cmp    $0x40,%r8
	mov    0x20(%rdx),%r10
	mov    0x28(%rdx),%r11
	mov    %r10,0x20(%rcx)
	mov    %r11,0x28(%rcx)
	mov    0x30(%rdx),%r9
	mov    0x38(%rdx),%r10
	lea    0x40(%rdx),%rdx
	mov    %r9,0x30(%rcx)
	mov    %r10,0x38(%rcx)
	lea    0x40(%rcx),%rcx
	jg     L(byte8_top)

L(byte8_end):
	lea    L(fwdPxQx)(%rip),%r10
	lea    (%rdx,%r8,1),%rdx
	lea    (%rcx,%r8,1),%rcx

	movslq (%r10,%r8,4),%r9
	lea    (%r9,%r10,1),%r10
	jmpq   *%r10

	.balign	16
L(use_rep):
	mov    %rdx,%rsi		# %rsi = source
	mov    %rcx,%rdi		# %rdi = destination
	mov    %r8,%rcx			# %rcx = count
	shrq   $3,%rcx			# 8-byte word count
	rep
	  movsq
	mov    %rsi,%rdx		# source
	mov    %rdi,%rcx		# destination
	andq   $7,%r8			# remainder
	jnz    L(byte8_end)
	ret

	.balign 16
L(byte8_nt_top):
	sub    $0x40,%r8
	prefetchnta 0x180(%rdx)
	mov    (%rdx),%r9
	movnti %r9,(%rcx)
	mov    0x8(%rdx),%r10
	movnti %r10,0x8(%rcx)
	mov    0x10(%rdx),%r11
	movnti %r11,0x10(%rcx)
	mov    0x18(%rdx),%r9
	movnti %r9,0x18(%rcx)
	mov    0x20(%rdx),%r10
	movnti %r10,0x20(%rcx)
	mov    0x28(%rdx),%r11
	movnti %r11,0x28(%rcx)
	mov    0x30(%rdx),%r9
	movnti %r9,0x30(%rcx)
	mov    0x38(%rdx),%r10
	movnti %r10,0x38(%rcx)

	lea    0x40(%rdx),%rdx
	lea    0x40(%rcx),%rcx
	cmp    $0x40,%r8
	jge    L(byte8_nt_top)
	sfence
	jmp    L(byte8_end)

	SET_SIZE(memcpy)

	.balign 16
L(CopyBackwards):
	mov    %rdx,%r8
	mov    %rdi,%rcx
	mov    %rsi,%rdx
	mov    %rdi,%rax		# return value

	# ck alignment of last byte
	lea    (%rcx,%r8,1),%rcx
	test   $0x7,%rcx
	lea    (%rdx,%r8,1),%rdx
	jne    L(bk_align)

L(bk_qw_aligned):
	lea    L(bkPxQx)(%rip),%r10

	cmp    $0x90,%r8		# 144
	jg     L(bk_ck_sse2_alignment)

	sub    %r8,%rcx
	sub    %r8,%rdx

	movslq (%r10,%r8,4),%r9
	lea    (%r9,%r10,1),%r10
	jmpq   *%r10

	.balign 16
L(bk_align):
	# only align if len > 8
	cmp    $8,%r8
	jle    L(bk_qw_aligned)
	test   $0x1,%rcx
	je     L(bk_tst2)
	dec    %rcx
	dec    %rdx
	dec    %r8
	mov    (%rdx),%r9b
	mov    %r9b,(%rcx)

L(bk_tst2):
	test   $0x2,%rcx
	je     L(bk_tst3)

L(bk_got2):
	sub    $0x2,%rcx
	sub    $0x2,%rdx
	sub    $0x2,%r8
	movzwq (%rdx),%r9
	mov    %r9w,(%rcx)

L(bk_tst3):
	test   $0x4,%rcx
	je     L(bk_qw_aligned)

L(bk_got3):
	sub    $0x4,%rcx
	sub    $0x4,%rdx
	sub    $0x4,%r8
	mov    (%rdx),%r9d
	mov    %r9d,(%rcx)
	jmp    L(bk_qw_aligned)

	.balign 16
L(bk_ck_sse2_alignment):
	cmpl   $NO_SSE,.memops_method(%rip)
	je     L(bk_use_rep)
	# check alignment of last byte
	test   $0xf,%rcx
	jz     L(bk_sse2_cpy)

L(bk_sse2_align):
	# only here if already aligned on at least a qword bndry
	sub    $0x8,%rcx
	sub    $0x8,%rdx
	sub    $0x8,%r8
	mov    (%rdx),%r9
	mov    %r9,(%rcx)
	#jmp   L(bk_sse2_cpy)

	.balign 16
L(bk_sse2_cpy):
	sub    $0x80,%rcx		# 128
	sub    $0x80,%rdx
	movdqu 0x70(%rdx),%xmm3
	movdqu 0x60(%rdx),%xmm2
	movdqa %xmm3,0x70(%rcx)
	movdqa %xmm2,0x60(%rcx)
	sub    $0x80,%r8
	movdqu 0x50(%rdx),%xmm1
	movdqu 0x40(%rdx),%xmm0
	movdqa %xmm1,0x50(%rcx)
	movdqa %xmm0,0x40(%rcx)

	cmp    $0x80,%r8
	movdqu 0x30(%rdx),%xmm3
	movdqu 0x20(%rdx),%xmm2
	movdqa %xmm3,0x30(%rcx)
	movdqa %xmm2,0x20(%rcx)
	movdqu 0x10(%rdx),%xmm1
	movdqu (%rdx),%xmm0
	movdqa %xmm1,0x10(%rcx)
	movdqa %xmm0,(%rcx)
	jge    L(bk_sse2_cpy)

L(bk_sse2_cpy_end):
	lea    L(bkPxQx)(%rip),%r10
	sub    %r8,%rdx
	sub    %r8,%rcx
	movslq (%r10,%r8,4),%r9
	lea    (%r9,%r10,1),%r10
	jmpq   *%r10

	.balign 16
L(bk_use_rep):
	xchg   %rcx,%r9
	mov    %rdx,%rsi		# source
	mov    %r9,%rdi			# destination
	mov    %r8,%rcx			# count
	sub    $8,%rsi
	sub    $8,%rdi
	shr    $3,%rcx
	std				# reverse direction
	rep
	  movsq
	cld				# reset direction flag

	xchg   %rcx,%r9
	lea    L(bkPxQx)(%rip),%r10
	sub    %r8,%rdx
	sub    %r8,%rcx
	andq   $7,%r8			# remainder
	jz     2f
	movslq (%r10,%r8,4),%r9
	lea    (%r9,%r10,1),%r10
	jmpq   *%r10
2:
	ret

	.balign 16
L(bkP0QI):
	mov    0x88(%rdx),%r10
	mov    %r10,0x88(%rcx)
L(bkP0QH):
	mov    0x80(%rdx),%r10
	mov    %r10,0x80(%rcx)
L(bkP0QG):
	mov    0x78(%rdx),%r9
	mov    %r9,0x78(%rcx)
L(bkP0QF):
	mov    0x70(%rdx),%r11
	mov    %r11,0x70(%rcx)
L(bkP0QE):
	mov    0x68(%rdx),%r10
	mov    %r10,0x68(%rcx)
L(bkP0QD):
	mov    0x60(%rdx),%r9
	mov    %r9,0x60(%rcx)
L(bkP0QC):
	mov    0x58(%rdx),%r11
	mov    %r11,0x58(%rcx)
L(bkP0QB):
	mov    0x50(%rdx),%r10
	mov    %r10,0x50(%rcx)
L(bkP0QA):
	mov    0x48(%rdx),%r9
	mov    %r9,0x48(%rcx)
L(bkP0Q9):
	mov    0x40(%rdx),%r11
	mov    %r11,0x40(%rcx)
L(bkP0Q8):
	mov    0x38(%rdx),%r10
	mov    %r10,0x38(%rcx)
L(bkP0Q7):
	mov    0x30(%rdx),%r9
	mov    %r9,0x30(%rcx)
L(bkP0Q6):
	mov    0x28(%rdx),%r11
	mov    %r11,0x28(%rcx)
L(bkP0Q5):
	mov    0x20(%rdx),%r10
	mov    %r10,0x20(%rcx)
L(bkP0Q4):
	mov    0x18(%rdx),%r9
	mov    %r9,0x18(%rcx)
L(bkP0Q3):
	mov    0x10(%rdx),%r11
	mov    %r11,0x10(%rcx)
L(bkP0Q2):
	mov    0x8(%rdx),%r10
	mov    %r10,0x8(%rcx)
L(bkP0Q1):
	mov    (%rdx),%r9
	mov    %r9,(%rcx)
L(bkP0Q0):
	ret

	.balign 16
L(bkP1QI):
	mov    0x89(%rdx),%r10
	mov    %r10,0x89(%rcx)
L(bkP1QH):
	mov    0x81(%rdx),%r11
	mov    %r11,0x81(%rcx)
L(bkP1QG):
	mov    0x79(%rdx),%r10
	mov    %r10,0x79(%rcx)
L(bkP1QF):
	mov    0x71(%rdx),%r9
	mov    %r9,0x71(%rcx)
L(bkP1QE):
	mov    0x69(%rdx),%r11
	mov    %r11,0x69(%rcx)
L(bkP1QD):
	mov    0x61(%rdx),%r10
	mov    %r10,0x61(%rcx)
L(bkP1QC):
	mov    0x59(%rdx),%r9
	mov    %r9,0x59(%rcx)
L(bkP1QB):
	mov    0x51(%rdx),%r11
	mov    %r11,0x51(%rcx)
L(bkP1QA):
	mov    0x49(%rdx),%r10
	mov    %r10,0x49(%rcx)
L(bkP1Q9):
	mov    0x41(%rdx),%r9
	mov    %r9,0x41(%rcx)
L(bkP1Q8):
	mov    0x39(%rdx),%r11
	mov    %r11,0x39(%rcx)
L(bkP1Q7):
	mov    0x31(%rdx),%r10
	mov    %r10,0x31(%rcx)
L(bkP1Q6):
	mov    0x29(%rdx),%r9
	mov    %r9,0x29(%rcx)
L(bkP1Q5):
	mov    0x21(%rdx),%r11
	mov    %r11,0x21(%rcx)
L(bkP1Q4):
	mov    0x19(%rdx),%r10
	mov    %r10,0x19(%rcx)
L(bkP1Q3):
	mov    0x11(%rdx),%r9
	mov    %r9,0x11(%rcx)
L(bkP1Q2):
	mov    0x9(%rdx),%r11
	mov    %r11,0x9(%rcx)
L(bkP1Q1):
	mov    0x1(%rdx),%r10
	mov    %r10,0x1(%rcx)
L(bkP1Q0):
	mov    (%rdx),%r9b
	mov    %r9b,(%rcx)
	ret

	.balign 16
L(bkP2QI):
	mov    0x8a(%rdx),%r10
	mov    %r10,0x8a(%rcx)
L(bkP2QH):
	mov    0x82(%rdx),%r11
	mov    %r11,0x82(%rcx)
L(bkP2QG):
	mov    0x7a(%rdx),%r10
	mov    %r10,0x7a(%rcx)
L(bkP2QF):
	mov    0x72(%rdx),%r9
	mov    %r9,0x72(%rcx)
L(bkP2QE):
	mov    0x6a(%rdx),%r11
	mov    %r11,0x6a(%rcx)
L(bkP2QD):
	mov    0x62(%rdx),%r10
	mov    %r10,0x62(%rcx)
L(bkP2QC):
	mov    0x5a(%rdx),%r9
	mov    %r9,0x5a(%rcx)
L(bkP2QB):
	mov    0x52(%rdx),%r11
	mov    %r11,0x52(%rcx)
L(bkP2QA):
	mov    0x4a(%rdx),%r10
	mov    %r10,0x4a(%rcx)
L(bkP2Q9):
	mov    0x42(%rdx),%r9
	mov    %r9,0x42(%rcx)
L(bkP2Q8):
	mov    0x3a(%rdx),%r11
	mov    %r11,0x3a(%rcx)
L(bkP2Q7):
	mov    0x32(%rdx),%r10
	mov    %r10,0x32(%rcx)
L(bkP2Q6):
	mov    0x2a(%rdx),%r9
	mov    %r9,0x2a(%rcx)
L(bkP2Q5):
	mov    0x22(%rdx),%r11
	mov    %r11,0x22(%rcx)
L(bkP2Q4):
	mov    0x1a(%rdx),%r10
	mov    %r10,0x1a(%rcx)
L(bkP2Q3):
	mov    0x12(%rdx),%r9
	mov    %r9,0x12(%rcx)
L(bkP2Q2):
	mov    0xa(%rdx),%r11
	mov    %r11,0xa(%rcx)
L(bkP2Q1):
	mov    0x2(%rdx),%r10
	mov    %r10,0x2(%rcx)
L(bkP2Q0):
	mov    (%rdx),%r9w
	mov    %r9w,(%rcx)
	ret

	.balign 16
L(bkP3QI):
	mov    0x8b(%rdx),%r10
	mov    %r10,0x8b(%rcx)
L(bkP3QH):
	mov    0x83(%rdx),%r11
	mov    %r11,0x83(%rcx)
L(bkP3QG):
	mov    0x7b(%rdx),%r10
	mov    %r10,0x7b(%rcx)
L(bkP3QF):
	mov    0x73(%rdx),%r9
	mov    %r9,0x73(%rcx)
L(bkP3QE):
	mov    0x6b(%rdx),%r11
	mov    %r11,0x6b(%rcx)
L(bkP3QD):
	mov    0x63(%rdx),%r10
	mov    %r10,0x63(%rcx)
L(bkP3QC):
	mov    0x5b(%rdx),%r9
	mov    %r9,0x5b(%rcx)
L(bkP3QB):
	mov    0x53(%rdx),%r11
	mov    %r11,0x53(%rcx)
L(bkP3QA):
	mov    0x4b(%rdx),%r10
	mov    %r10,0x4b(%rcx)
L(bkP3Q9):
	mov    0x43(%rdx),%r9
	mov    %r9,0x43(%rcx)
L(bkP3Q8):
	mov    0x3b(%rdx),%r11
	mov    %r11,0x3b(%rcx)
L(bkP3Q7):
	mov    0x33(%rdx),%r10
	mov    %r10,0x33(%rcx)
L(bkP3Q6):
	mov    0x2b(%rdx),%r9
	mov    %r9,0x2b(%rcx)
L(bkP3Q5):
	mov    0x23(%rdx),%r11
	mov    %r11,0x23(%rcx)
L(bkP3Q4):
	mov    0x1b(%rdx),%r10
	mov    %r10,0x1b(%rcx)
L(bkP3Q3):
	mov    0x13(%rdx),%r9
	mov    %r9,0x13(%rcx)
L(bkP3Q2):
	mov    0xb(%rdx),%r11
	mov    %r11,0xb(%rcx)
L(bkP3Q1):
	mov    0x3(%rdx),%r10
	mov    %r10,0x3(%rcx)
L(bkP3Q0): # trailing loads/stores do all their loads 1st, then do the stores
	mov    0x1(%rdx),%r9w
	mov    %r9w,0x1(%rcx)
	mov    (%rdx),%r10b
	mov    %r10b,(%rcx)
	ret

	.balign 16
L(bkP4QI):
	mov    0x8c(%rdx),%r10
	mov    %r10,0x8c(%rcx)
L(bkP4QH):
	mov    0x84(%rdx),%r11
	mov    %r11,0x84(%rcx)
L(bkP4QG):
	mov    0x7c(%rdx),%r10
	mov    %r10,0x7c(%rcx)
L(bkP4QF):
	mov    0x74(%rdx),%r9
	mov    %r9,0x74(%rcx)
L(bkP4QE):
	mov    0x6c(%rdx),%r11
	mov    %r11,0x6c(%rcx)
L(bkP4QD):
	mov    0x64(%rdx),%r10
	mov    %r10,0x64(%rcx)
L(bkP4QC):
	mov    0x5c(%rdx),%r9
	mov    %r9,0x5c(%rcx)
L(bkP4QB):
	mov    0x54(%rdx),%r11
	mov    %r11,0x54(%rcx)
L(bkP4QA):
	mov    0x4c(%rdx),%r10
	mov    %r10,0x4c(%rcx)
L(bkP4Q9):
	mov    0x44(%rdx),%r9
	mov    %r9,0x44(%rcx)
L(bkP4Q8):
	mov    0x3c(%rdx),%r11
	mov    %r11,0x3c(%rcx)
L(bkP4Q7):
	mov    0x34(%rdx),%r10
	mov    %r10,0x34(%rcx)
L(bkP4Q6):
	mov    0x2c(%rdx),%r9
	mov    %r9,0x2c(%rcx)
L(bkP4Q5):
	mov    0x24(%rdx),%r11
	mov    %r11,0x24(%rcx)
L(bkP4Q4):
	mov    0x1c(%rdx),%r10
	mov    %r10,0x1c(%rcx)
L(bkP4Q3):
	mov    0x14(%rdx),%r9
	mov    %r9,0x14(%rcx)
L(bkP4Q2):
	mov    0xc(%rdx),%r11
	mov    %r11,0xc(%rcx)
L(bkP4Q1):
	mov    0x4(%rdx),%r10
	mov    %r10,0x4(%rcx)
L(bkP4Q0):
	mov    (%rdx),%r9d
	mov    %r9d,(%rcx)
	ret

	.balign 16
L(bkP5QI):
	mov    0x8d(%rdx),%r10
	mov    %r10,0x8d(%rcx)
L(bkP5QH):
	mov    0x85(%rdx),%r9
	mov    %r9,0x85(%rcx)
L(bkP5QG):
	mov    0x7d(%rdx),%r11
	mov    %r11,0x7d(%rcx)
L(bkP5QF):
	mov    0x75(%rdx),%r10
	mov    %r10,0x75(%rcx)
L(bkP5QE):
	mov    0x6d(%rdx),%r9
	mov    %r9,0x6d(%rcx)
L(bkP5QD):
	mov    0x65(%rdx),%r11
	mov    %r11,0x65(%rcx)
L(bkP5QC):
	mov    0x5d(%rdx),%r10
	mov    %r10,0x5d(%rcx)
L(bkP5QB):
	mov    0x55(%rdx),%r9
	mov    %r9,0x55(%rcx)
L(bkP5QA):
	mov    0x4d(%rdx),%r11
	mov    %r11,0x4d(%rcx)
L(bkP5Q9):
	mov    0x45(%rdx),%r10
	mov    %r10,0x45(%rcx)
L(bkP5Q8):
	mov    0x3d(%rdx),%r9
	mov    %r9,0x3d(%rcx)
L(bkP5Q7):
	mov    0x35(%rdx),%r11
	mov    %r11,0x35(%rcx)
L(bkP5Q6):
	mov    0x2d(%rdx),%r10
	mov    %r10,0x2d(%rcx)
L(bkP5Q5):
	mov    0x25(%rdx),%r9
	mov    %r9,0x25(%rcx)
L(bkP5Q4):
	mov    0x1d(%rdx),%r11
	mov    %r11,0x1d(%rcx)
L(bkP5Q3):
	mov    0x15(%rdx),%r10
	mov    %r10,0x15(%rcx)
L(bkP5Q2):
	mov    0xd(%rdx),%r9
	mov    %r9,0xd(%rcx)
L(bkP5Q1):
	mov    0x5(%rdx),%r11
	mov    %r11,0x5(%rcx)
L(bkP5Q0): # trailing loads/stores do all their loads 1st, then do the stores
	mov    0x1(%rdx),%r9d
	mov    %r9d,0x1(%rcx)
	mov    (%rdx),%r10b
	mov    %r10b,(%rcx)
	ret

	.balign 16
L(bkP6QI):
	mov    0x8e(%rdx),%r10
	mov    %r10,0x8e(%rcx)
L(bkP6QH):
	mov    0x86(%rdx),%r11
	mov    %r11,0x86(%rcx)
L(bkP6QG):
	mov    0x7e(%rdx),%r10
	mov    %r10,0x7e(%rcx)
L(bkP6QF):
	mov    0x76(%rdx),%r9
	mov    %r9,0x76(%rcx)
L(bkP6QE):
	mov    0x6e(%rdx),%r11
	mov    %r11,0x6e(%rcx)
L(bkP6QD):
	mov    0x66(%rdx),%r10
	mov    %r10,0x66(%rcx)
L(bkP6QC):
	mov    0x5e(%rdx),%r9
	mov    %r9,0x5e(%rcx)
L(bkP6QB):
	mov    0x56(%rdx),%r11
	mov    %r11,0x56(%rcx)
L(bkP6QA):
	mov    0x4e(%rdx),%r10
	mov    %r10,0x4e(%rcx)
L(bkP6Q9):
	mov    0x46(%rdx),%r9
	mov    %r9,0x46(%rcx)
L(bkP6Q8):
	mov    0x3e(%rdx),%r11
	mov    %r11,0x3e(%rcx)
L(bkP6Q7):
	mov    0x36(%rdx),%r10
	mov    %r10,0x36(%rcx)
L(bkP6Q6):
	mov    0x2e(%rdx),%r9
	mov    %r9,0x2e(%rcx)
L(bkP6Q5):
	mov    0x26(%rdx),%r11
	mov    %r11,0x26(%rcx)
L(bkP6Q4):
	mov    0x1e(%rdx),%r10
	mov    %r10,0x1e(%rcx)
L(bkP6Q3):
	mov    0x16(%rdx),%r9
	mov    %r9,0x16(%rcx)
L(bkP6Q2):
	mov    0xe(%rdx),%r11
	mov    %r11,0xe(%rcx)
L(bkP6Q1):
	mov    0x6(%rdx),%r10
	mov    %r10,0x6(%rcx)
L(bkP6Q0): # trailing loads/stores do all their loads 1st, then do the stores
	mov    0x2(%rdx),%r9d
	mov    %r9d,0x2(%rcx)
	mov    (%rdx),%r10w
	mov    %r10w,(%rcx)
	ret

	.balign 16
L(bkP7QI):
	mov    0x8f(%rdx),%r10
	mov    %r10,0x8f(%rcx)
L(bkP7QH):
	mov    0x87(%rdx),%r11
	mov    %r11,0x87(%rcx)
L(bkP7QG):
	mov    0x7f(%rdx),%r10
	mov    %r10,0x7f(%rcx)
L(bkP7QF):
	mov    0x77(%rdx),%r9
	mov    %r9,0x77(%rcx)
L(bkP7QE):
	mov    0x6f(%rdx),%r11
	mov    %r11,0x6f(%rcx)
L(bkP7QD):
	mov    0x67(%rdx),%r10
	mov    %r10,0x67(%rcx)
L(bkP7QC):
	mov    0x5f(%rdx),%r9
	mov    %r9,0x5f(%rcx)
L(bkP7QB):
	mov    0x57(%rdx),%r11
	mov    %r11,0x57(%rcx)
L(bkP7QA):
	mov    0x4f(%rdx),%r10
	mov    %r10,0x4f(%rcx)
L(bkP7Q9):
	mov    0x47(%rdx),%r9
	mov    %r9,0x47(%rcx)
L(bkP7Q8):
	mov    0x3f(%rdx),%r11
	mov    %r11,0x3f(%rcx)
L(bkP7Q7):
	mov    0x37(%rdx),%r10
	mov    %r10,0x37(%rcx)
L(bkP7Q6):
	mov    0x2f(%rdx),%r9
	mov    %r9,0x2f(%rcx)
L(bkP7Q5):
	mov    0x27(%rdx),%r11
	mov    %r11,0x27(%rcx)
L(bkP7Q4):
	mov    0x1f(%rdx),%r10
	mov    %r10,0x1f(%rcx)
L(bkP7Q3):
	mov    0x17(%rdx),%r9
	mov    %r9,0x17(%rcx)
L(bkP7Q2):
	mov    0xf(%rdx),%r11
	mov    %r11,0xf(%rcx)
L(bkP7Q1):
	mov    0x7(%rdx),%r10
	mov    %r10,0x7(%rcx)
L(bkP7Q0): # trailing loads/stores do all their loads 1st, then do the stores
	mov    0x3(%rdx),%r9d
	mov    %r9d,0x3(%rcx)
	mov    0x1(%rdx),%r10w
	mov    %r10w,0x1(%rcx)
	mov    (%rdx),%r11b
	mov    %r11b,(%rcx)
	ret

		.balign 16
L(bkPxQx):	.int L(bkP0Q0)-L(bkPxQx)
		.int L(bkP1Q0)-L(bkPxQx)
		.int L(bkP2Q0)-L(bkPxQx)
		.int L(bkP3Q0)-L(bkPxQx)
		.int L(bkP4Q0)-L(bkPxQx)
		.int L(bkP5Q0)-L(bkPxQx)
		.int L(bkP6Q0)-L(bkPxQx)
		.int L(bkP7Q0)-L(bkPxQx)

		.int L(bkP0Q1)-L(bkPxQx)
		.int L(bkP1Q1)-L(bkPxQx)
		.int L(bkP2Q1)-L(bkPxQx)
		.int L(bkP3Q1)-L(bkPxQx)
		.int L(bkP4Q1)-L(bkPxQx)
		.int L(bkP5Q1)-L(bkPxQx)
		.int L(bkP6Q1)-L(bkPxQx)
		.int L(bkP7Q1)-L(bkPxQx)

		.int L(bkP0Q2)-L(bkPxQx)
		.int L(bkP1Q2)-L(bkPxQx)
		.int L(bkP2Q2)-L(bkPxQx)
		.int L(bkP3Q2)-L(bkPxQx)
		.int L(bkP4Q2)-L(bkPxQx)
		.int L(bkP5Q2)-L(bkPxQx)
		.int L(bkP6Q2)-L(bkPxQx)
		.int L(bkP7Q2)-L(bkPxQx)

		.int L(bkP0Q3)-L(bkPxQx)
		.int L(bkP1Q3)-L(bkPxQx)
		.int L(bkP2Q3)-L(bkPxQx)
		.int L(bkP3Q3)-L(bkPxQx)
		.int L(bkP4Q3)-L(bkPxQx)
		.int L(bkP5Q3)-L(bkPxQx)
		.int L(bkP6Q3)-L(bkPxQx)
		.int L(bkP7Q3)-L(bkPxQx)

		.int L(bkP0Q4)-L(bkPxQx)
		.int L(bkP1Q4)-L(bkPxQx)
		.int L(bkP2Q4)-L(bkPxQx)
		.int L(bkP3Q4)-L(bkPxQx)
		.int L(bkP4Q4)-L(bkPxQx)
		.int L(bkP5Q4)-L(bkPxQx)
		.int L(bkP6Q4)-L(bkPxQx)
		.int L(bkP7Q4)-L(bkPxQx)

		.int L(bkP0Q5)-L(bkPxQx)
		.int L(bkP1Q5)-L(bkPxQx)
		.int L(bkP2Q5)-L(bkPxQx)
		.int L(bkP3Q5)-L(bkPxQx)
		.int L(bkP4Q5)-L(bkPxQx)
		.int L(bkP5Q5)-L(bkPxQx)
		.int L(bkP6Q5)-L(bkPxQx)
		.int L(bkP7Q5)-L(bkPxQx)

		.int L(bkP0Q6)-L(bkPxQx)
		.int L(bkP1Q6)-L(bkPxQx)
		.int L(bkP2Q6)-L(bkPxQx)
		.int L(bkP3Q6)-L(bkPxQx)
		.int L(bkP4Q6)-L(bkPxQx)
		.int L(bkP5Q6)-L(bkPxQx)
		.int L(bkP6Q6)-L(bkPxQx)
		.int L(bkP7Q6)-L(bkPxQx)

		.int L(bkP0Q7)-L(bkPxQx)
		.int L(bkP1Q7)-L(bkPxQx)
		.int L(bkP2Q7)-L(bkPxQx)
		.int L(bkP3Q7)-L(bkPxQx)
		.int L(bkP4Q7)-L(bkPxQx)
		.int L(bkP5Q7)-L(bkPxQx)
		.int L(bkP6Q7)-L(bkPxQx)
		.int L(bkP7Q7)-L(bkPxQx)

		.int L(bkP0Q8)-L(bkPxQx)
		.int L(bkP1Q8)-L(bkPxQx)
		.int L(bkP2Q8)-L(bkPxQx)
		.int L(bkP3Q8)-L(bkPxQx)
		.int L(bkP4Q8)-L(bkPxQx)
		.int L(bkP5Q8)-L(bkPxQx)
		.int L(bkP6Q8)-L(bkPxQx)
		.int L(bkP7Q8)-L(bkPxQx)

		.int L(bkP0Q9)-L(bkPxQx)
		.int L(bkP1Q9)-L(bkPxQx)
		.int L(bkP2Q9)-L(bkPxQx)
		.int L(bkP3Q9)-L(bkPxQx)
		.int L(bkP4Q9)-L(bkPxQx)
		.int L(bkP5Q9)-L(bkPxQx)
		.int L(bkP6Q9)-L(bkPxQx)
		.int L(bkP7Q9)-L(bkPxQx)

		.int L(bkP0QA)-L(bkPxQx)
		.int L(bkP1QA)-L(bkPxQx)
		.int L(bkP2QA)-L(bkPxQx)
		.int L(bkP3QA)-L(bkPxQx)
		.int L(bkP4QA)-L(bkPxQx)
		.int L(bkP5QA)-L(bkPxQx)
		.int L(bkP6QA)-L(bkPxQx)
		.int L(bkP7QA)-L(bkPxQx)

		.int L(bkP0QB)-L(bkPxQx)
		.int L(bkP1QB)-L(bkPxQx)
		.int L(bkP2QB)-L(bkPxQx)
		.int L(bkP3QB)-L(bkPxQx)
		.int L(bkP4QB)-L(bkPxQx)
		.int L(bkP5QB)-L(bkPxQx)
		.int L(bkP6QB)-L(bkPxQx)
		.int L(bkP7QB)-L(bkPxQx)

		.int L(bkP0QC)-L(bkPxQx)
		.int L(bkP1QC)-L(bkPxQx)
		.int L(bkP2QC)-L(bkPxQx)
		.int L(bkP3QC)-L(bkPxQx)
		.int L(bkP4QC)-L(bkPxQx)
		.int L(bkP5QC)-L(bkPxQx)
		.int L(bkP6QC)-L(bkPxQx)
		.int L(bkP7QC)-L(bkPxQx)

		.int L(bkP0QD)-L(bkPxQx)
		.int L(bkP1QD)-L(bkPxQx)
		.int L(bkP2QD)-L(bkPxQx)
		.int L(bkP3QD)-L(bkPxQx)
		.int L(bkP4QD)-L(bkPxQx)
		.int L(bkP5QD)-L(bkPxQx)
		.int L(bkP6QD)-L(bkPxQx)
		.int L(bkP7QD)-L(bkPxQx)

		.int L(bkP0QE)-L(bkPxQx)
		.int L(bkP1QE)-L(bkPxQx)
		.int L(bkP2QE)-L(bkPxQx)
		.int L(bkP3QE)-L(bkPxQx)
		.int L(bkP4QE)-L(bkPxQx)
		.int L(bkP5QE)-L(bkPxQx)
		.int L(bkP6QE)-L(bkPxQx)
		.int L(bkP7QE)-L(bkPxQx)

		.int L(bkP0QF)-L(bkPxQx)
		.int L(bkP1QF)-L(bkPxQx)
		.int L(bkP2QF)-L(bkPxQx)
		.int L(bkP3QF)-L(bkPxQx)
		.int L(bkP4QF)-L(bkPxQx)
		.int L(bkP5QF)-L(bkPxQx)
		.int L(bkP6QF)-L(bkPxQx)
		.int L(bkP7QF)-L(bkPxQx)

		.int L(bkP0QG)-L(bkPxQx)
		.int L(bkP1QG)-L(bkPxQx)
		.int L(bkP2QG)-L(bkPxQx)
		.int L(bkP3QG)-L(bkPxQx)
		.int L(bkP4QG)-L(bkPxQx)
		.int L(bkP5QG)-L(bkPxQx)
		.int L(bkP6QG)-L(bkPxQx)
		.int L(bkP7QG)-L(bkPxQx)

		.int L(bkP0QH)-L(bkPxQx)
		.int L(bkP1QH)-L(bkPxQx)
		.int L(bkP2QH)-L(bkPxQx)
		.int L(bkP3QH)-L(bkPxQx)
		.int L(bkP4QH)-L(bkPxQx)
		.int L(bkP5QH)-L(bkPxQx)
		.int L(bkP6QH)-L(bkPxQx)
		.int L(bkP7QH)-L(bkPxQx)

		.int L(bkP0QI)-L(bkPxQx)
		.int L(bkP1QI)-L(bkPxQx)
		.int L(bkP2QI)-L(bkPxQx)
		.int L(bkP3QI)-L(bkPxQx)
		.int L(bkP4QI)-L(bkPxQx)
		.int L(bkP5QI)-L(bkPxQx)
		.int L(bkP6QI)-L(bkPxQx)
		.int L(bkP7QI)-L(bkPxQx)

	SET_SIZE(memmove)