path: root/vendor/gmp-6.3.0/mpn/x86/pentium/mmx/lshift.asm

dnl  Intel P5 mpn_lshift -- mpn left shift.

dnl  Copyright 2000-2002 Free Software Foundation, Inc.

dnl  This file is part of the GNU MP Library.
dnl
dnl  The GNU MP Library is free software; you can redistribute it and/or modify
dnl  it under the terms of either:
dnl
dnl    * the GNU Lesser General Public License as published by the Free
dnl      Software Foundation; either version 3 of the License, or (at your
dnl      option) any later version.
dnl
dnl  or
dnl
dnl    * the GNU General Public License as published by the Free Software
dnl      Foundation; either version 2 of the License, or (at your option) any
dnl      later version.
dnl
dnl  or both in parallel, as here.
dnl
dnl  The GNU MP Library is distributed in the hope that it will be useful, but
dnl  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
dnl  or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
dnl  for more details.
dnl
dnl  You should have received copies of the GNU General Public License and the
dnl  GNU Lesser General Public License along with the GNU MP Library.  If not,
dnl  see https://www.gnu.org/licenses/.

include(`../config.m4')


C P5: 1.75 cycles/limb.


C mp_limb_t mpn_lshift (mp_ptr dst, mp_srcptr src, mp_size_t size,
C                       unsigned shift);
C
C Shift src,size left by shift many bits and store the result in dst,size.
C Zeros are shifted in at the right.  Return the bits shifted out at the
C left.
C
C The comments in mpn_rshift apply here too.

defframe(PARAM_SHIFT,16)
defframe(PARAM_SIZE, 12)
defframe(PARAM_SRC,  8)
defframe(PARAM_DST,  4)
deflit(`FRAME',0)

dnl  minimum 5, because the unrolled loop can't handle less
deflit(UNROLL_THRESHOLD, 5)

	TEXT
	ALIGN(8)

PROLOGUE(mpn_lshift)

	pushl	%ebx
	pushl	%edi
deflit(`FRAME',8)

	movl	PARAM_SIZE, %eax
	movl	PARAM_DST, %edx

	movl	PARAM_SRC, %ebx
	movl	PARAM_SHIFT, %ecx

	cmp	$UNROLL_THRESHOLD, %eax
	jae	L(unroll)

	movl	-4(%ebx,%eax,4), %edi	C src high limb
	decl	%eax

	jnz	L(simple)

	shldl(	%cl, %edi, %eax)	C eax was decremented to zero

	shll	%cl, %edi

	movl	%edi, (%edx)		C dst low limb
	popl	%edi			C risk of data cache bank clash

	popl	%ebx

	ret


C -----------------------------------------------------------------------------
L(simple):
	C eax	size-1
	C ebx	src
	C ecx	shift
	C edx	dst
	C esi
	C edi
	C ebp
deflit(`FRAME',8)

	movd	(%ebx,%eax,4), %mm5	C src high limb

	movd	%ecx, %mm6		C lshift
	negl	%ecx

	psllq	%mm6, %mm5
	addl	$32, %ecx

	movd	%ecx, %mm7
	psrlq	$32, %mm5		C retval


L(simple_top):
	C eax	counter, limbs, negative
	C ebx	src
	C ecx
	C edx	dst
	C esi
	C edi
	C
	C mm0	scratch
	C mm5	return value
	C mm6	shift
	C mm7	32-shift

	movq	-4(%ebx,%eax,4), %mm0
	decl	%eax

	psrlq	%mm7, %mm0

	C

	movd	%mm0, 4(%edx,%eax,4)
	jnz	L(simple_top)


	movd	(%ebx), %mm0

	movd	%mm5, %eax
	psllq	%mm6, %mm0

	popl	%edi
	popl	%ebx

	movd	%mm0, (%edx)

	emms

	ret


C -----------------------------------------------------------------------------
	ALIGN(8)
L(unroll):
	C eax	size
	C ebx	src
	C ecx	shift
	C edx	dst
	C esi
	C edi
	C ebp
deflit(`FRAME',8)

	movd	-4(%ebx,%eax,4), %mm5	C src high limb
	leal	(%ebx,%eax,4), %edi

	movd	%ecx, %mm6		C lshift
	andl	$4, %edi

	psllq	%mm6, %mm5
	jz	L(start_src_aligned)


	C src isn't aligned, process high limb separately (marked xxx) to
	C make it so.
	C
	C  source     -8(ebx,%eax,4)
	C                  |
	C  +-------+-------+-------+--
	C  |               |
	C  +-------+-------+-------+--
	C        0mod8   4mod8   0mod8
	C
	C  dest
	C     -4(edx,%eax,4)
	C          |
	C  +-------+-------+--
	C  |  xxx  |       |
	C  +-------+-------+--

	movq	-8(%ebx,%eax,4), %mm0	C unaligned load

	psllq	%mm6, %mm0
	decl	%eax

	psrlq	$32, %mm0

	C

	movd	%mm0, (%edx,%eax,4)
L(start_src_aligned):

	movq	-8(%ebx,%eax,4), %mm1	C src high qword
	leal	(%edx,%eax,4), %edi

	andl	$4, %edi
	psrlq	$32, %mm5		C return value

	movq	-16(%ebx,%eax,4), %mm3	C src second highest qword
	jz	L(start_dst_aligned)

	C dst isn't aligned, subtract 4 to make it so, and pretend the shift
	C is 32 bits extra.  High limb of dst (marked xxx) handled here
	C separately.
	C
	C  source     -8(ebx,%eax,4)
	C                  |
	C  +-------+-------+--
	C  |      mm1      |
	C  +-------+-------+--
	C                0mod8   4mod8
	C
	C  dest
	C     -4(edx,%eax,4)
	C          |
	C  +-------+-------+-------+--
	C  |  xxx  |               |
	C  +-------+-------+-------+--
	C        0mod8   4mod8   0mod8

	movq	%mm1, %mm0
	addl	$32, %ecx		C new shift

	psllq	%mm6, %mm0

	movd	%ecx, %mm6
	psrlq	$32, %mm0

	C wasted cycle here waiting for %mm0

	movd	%mm0, -4(%edx,%eax,4)
	subl	$4, %edx
L(start_dst_aligned):


	psllq	%mm6, %mm1
	negl	%ecx			C -shift

	addl	$64, %ecx		C 64-shift
	movq	%mm3, %mm2

	movd	%ecx, %mm7
	subl	$8, %eax		C size-8

	psrlq	%mm7, %mm3

	por	%mm1, %mm3		C mm3 ready to store
	jc	L(finish)


	C The comments in mpn_rshift apply here too.

	ALIGN(8)
L(unroll_loop):
	C eax	counter, limbs
	C ebx	src
	C ecx
	C edx	dst
	C esi
	C edi
	C
	C mm0
	C mm1
	C mm2	src qword from 16(%ebx,%eax,4)
	C mm3	dst qword ready to store to 24(%edx,%eax,4)
	C
	C mm5	return value
	C mm6	lshift
	C mm7	rshift

	movq	8(%ebx,%eax,4), %mm0
	psllq	%mm6, %mm2

	movq	%mm0, %mm1
	psrlq	%mm7, %mm0

	movq	%mm3, 24(%edx,%eax,4)	C prev
	por	%mm2, %mm0

	movq	(%ebx,%eax,4), %mm3	C
	psllq	%mm6, %mm1		C

	movq	%mm0, 16(%edx,%eax,4)
	movq	%mm3, %mm2		C

	psrlq	%mm7, %mm3		C
	subl	$4, %eax

	por	%mm1, %mm3		C
	jnc	L(unroll_loop)



L(finish):
	C eax	-4 to -1 representing respectively 0 to 3 limbs remaining

	testb	$2, %al

	jz	L(finish_no_two)

	movq	8(%ebx,%eax,4), %mm0
	psllq	%mm6, %mm2

	movq	%mm0, %mm1
	psrlq	%mm7, %mm0

	movq	%mm3, 24(%edx,%eax,4)	C prev
	por	%mm2, %mm0

	movq	%mm1, %mm2
	movq	%mm0, %mm3

	subl	$2, %eax
L(finish_no_two):


	C eax	-4 or -3 representing respectively 0 or 1 limbs remaining
	C
	C mm2	src prev qword, from 16(%ebx,%eax,4)
	C mm3	dst qword, for 24(%edx,%eax,4)

	testb	$1, %al
	movd	%mm5, %eax	C retval

	popl	%edi
	jz	L(finish_zero)


	C One extra src limb, destination was aligned.
	C
	C                 source                  ebx
	C                 --+---------------+-------+
	C                   |      mm2      |       |
	C                 --+---------------+-------+
	C
	C dest         edx+12           edx+4     edx
	C --+---------------+---------------+-------+
	C   |      mm3      |               |       |
	C --+---------------+---------------+-------+
	C
	C mm6 = shift
	C mm7 = ecx = 64-shift


	C One extra src limb, destination was unaligned.
	C
	C                 source                  ebx
	C                 --+---------------+-------+
	C                   |      mm2      |       |
	C                 --+---------------+-------+
	C
	C         dest         edx+12           edx+4
	C         --+---------------+---------------+
	C           |      mm3      |               |
	C         --+---------------+---------------+
	C
	C mm6 = shift+32
	C mm7 = ecx = 64-(shift+32)


	C In both cases there's one extra limb of src to fetch and combine
	C with mm2 to make a qword at 4(%edx), and in the aligned case
	C there's an extra limb of dst to be formed from that extra src limb
	C left shifted.


	movd	(%ebx), %mm0
	psllq	%mm6, %mm2

	movq	%mm3, 12(%edx)
	psllq	$32, %mm0

	movq	%mm0, %mm1
	psrlq	%mm7, %mm0

	por	%mm2, %mm0
	psllq	%mm6, %mm1

	movq	%mm0, 4(%edx)
	psrlq	$32, %mm1

	andl	$32, %ecx
	popl	%ebx

	jz	L(finish_one_unaligned)

	movd	%mm1, (%edx)
L(finish_one_unaligned):

	emms

	ret


L(finish_zero):

	C No extra src limbs, destination was aligned.
	C
	C                 source          ebx
	C                 --+---------------+
	C                   |      mm2      |
	C                 --+---------------+
	C
	C dest          edx+8             edx
	C --+---------------+---------------+
	C   |      mm3      |               |
	C --+---------------+---------------+
	C
	C mm6 = shift
	C mm7 = ecx = 64-shift


	C No extra src limbs, destination was unaligned.
	C
	C               source            ebx
	C                 --+---------------+
	C                   |      mm2      |
	C                 --+---------------+
	C
	C         dest          edx+8   edx+4
	C         --+---------------+-------+
	C           |      mm3      |       |
	C         --+---------------+-------+
	C
	C mm6 = shift+32
	C mm7 = ecx = 64-(shift+32)


	C The movd for the unaligned case writes the same data to 4(%edx)
	C that the movq does for the aligned case.


	movq	%mm3, 8(%edx)
	andl	$32, %ecx

	psllq	%mm6, %mm2
	jz	L(finish_zero_unaligned)

	movq	%mm2, (%edx)
L(finish_zero_unaligned):

	psrlq	$32, %mm2
	popl	%ebx

	movd	%mm5, %eax	C retval

	movd	%mm2, 4(%edx)

	emms

	ret

EPILOGUE()