aboutsummaryrefslogtreecommitdiff
path: root/vendor/gmp-6.3.0/mpn/x86/k7/mmx/divrem_1.asm
blob: cf343280bbda65594c4dc7a69b5469d457b2b95d (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
dnl  AMD K7 mpn_divrem_1, mpn_divrem_1c, mpn_preinv_divrem_1 -- mpn by limb
dnl  division.

dnl  Copyright 1999-2002, 2004 Free Software Foundation, Inc.

dnl  This file is part of the GNU MP Library.
dnl
dnl  The GNU MP Library is free software; you can redistribute it and/or modify
dnl  it under the terms of either:
dnl
dnl    * the GNU Lesser General Public License as published by the Free
dnl      Software Foundation; either version 3 of the License, or (at your
dnl      option) any later version.
dnl
dnl  or
dnl
dnl    * the GNU General Public License as published by the Free Software
dnl      Foundation; either version 2 of the License, or (at your option) any
dnl      later version.
dnl
dnl  or both in parallel, as here.
dnl
dnl  The GNU MP Library is distributed in the hope that it will be useful, but
dnl  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
dnl  or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
dnl  for more details.
dnl
dnl  You should have received copies of the GNU General Public License and the
dnl  GNU Lesser General Public License along with the GNU MP Library.  If not,
dnl  see https://www.gnu.org/licenses/.

include(`../config.m4')


C K7: 17.0 cycles/limb integer part, 15.0 cycles/limb fraction part.


C mp_limb_t mpn_divrem_1 (mp_ptr dst, mp_size_t xsize,
C                         mp_srcptr src, mp_size_t size,
C                         mp_limb_t divisor);
C mp_limb_t mpn_divrem_1c (mp_ptr dst, mp_size_t xsize,
C                          mp_srcptr src, mp_size_t size,
C                          mp_limb_t divisor, mp_limb_t carry);
C mp_limb_t mpn_preinv_divrem_1 (mp_ptr dst, mp_size_t xsize,
C                                mp_srcptr src, mp_size_t size,
C                                mp_limb_t divisor, mp_limb_t inverse,
C                                unsigned shift);
C
C Algorithm:
C
C The method and nomenclature follow part 8 of "Division by Invariant
C Integers using Multiplication" by Granlund and Montgomery, reference in
C gmp.texi.
C
C The "and"s shown in the paper are done here with "cmov"s.  "m" is written
C for m', and "d" for d_norm, which won't cause any confusion since it's
C only the normalized divisor that's of any use in the code.  "b" is written
C for 2^N, the size of a limb, N being 32 here.
C
C The step "sdword dr = n - 2^N*d + (2^N-1-q1) * d" is instead done as
C "n-(q1+1)*d"; this rearrangement gives the same two-limb answer.  If
C q1==0xFFFFFFFF, then q1+1 would overflow.  We branch to a special case
C "q1_ff" if this occurs.  Since the true quotient is either q1 or q1+1 then
C if q1==0xFFFFFFFF that must be the right value.
C
C For the last and second last steps q1==0xFFFFFFFF is instead handled by an
C sbbl to go back to 0xFFFFFFFF if an overflow occurs when adding 1.  This
C then goes through as normal, and finding no addback required.  sbbl costs
C an extra cycle over what the main loop code does, but it keeps code size
C and complexity down.
C
C Notes:
C
C mpn_divrem_1 and mpn_preinv_divrem_1 avoid one division if the src high
C limb is less than the divisor.  mpn_divrem_1c doesn't check for a zero
C carry, since in normal circumstances that will be a very rare event.
C
C The test for skipping a division is branch free (once size>=1 is tested).
C The store to the destination high limb is 0 when a divide is skipped, or
C if it's not skipped then a copy of the src high limb is used.  The latter
C is in case src==dst.
C
C There's a small bias towards expecting xsize==0, by having code for
C xsize==0 in a straight line and xsize!=0 under forward jumps.
C
C Alternatives:
C
C If the divisor is normalized (high bit set) then a division step can
C always be skipped, since the high destination limb is always 0 or 1 in
C that case.  It doesn't seem worth checking for this though, since it
C probably occurs infrequently, in particular note that big_base for a
C decimal mpn_get_str is not normalized in a 32-bit limb.


dnl  MUL_THRESHOLD is the value of xsize+size at which the multiply by
dnl  inverse method is used, rather than plain "divl"s.  Minimum value 1.
dnl
dnl  The inverse takes about 50 cycles to calculate, but after that the
dnl  multiply is 17 c/l versus division at 42 c/l.
dnl
dnl  At 3 limbs the mul is a touch faster than div on the integer part, and
dnl  even more so on the fractional part.

deflit(MUL_THRESHOLD, 3)


defframe(PARAM_PREINV_SHIFT,   28)  dnl mpn_preinv_divrem_1
defframe(PARAM_PREINV_INVERSE, 24)  dnl mpn_preinv_divrem_1
defframe(PARAM_CARRY,  24)          dnl mpn_divrem_1c
defframe(PARAM_DIVISOR,20)
defframe(PARAM_SIZE,   16)
defframe(PARAM_SRC,    12)
defframe(PARAM_XSIZE,  8)
defframe(PARAM_DST,    4)

defframe(SAVE_EBX,    -4)
defframe(SAVE_ESI,    -8)
defframe(SAVE_EDI,    -12)
defframe(SAVE_EBP,    -16)

defframe(VAR_NORM,    -20)
defframe(VAR_INVERSE, -24)
defframe(VAR_SRC,     -28)
defframe(VAR_DST,     -32)
defframe(VAR_DST_STOP,-36)

deflit(STACK_SPACE, 36)

	TEXT
	ALIGN(32)

PROLOGUE(mpn_preinv_divrem_1)
deflit(`FRAME',0)
	movl	PARAM_XSIZE, %ecx
	movl	PARAM_DST, %edx
	subl	$STACK_SPACE, %esp	FRAME_subl_esp(STACK_SPACE)

	movl	%esi, SAVE_ESI
	movl	PARAM_SRC, %esi

	movl	%ebx, SAVE_EBX
	movl	PARAM_SIZE, %ebx

	leal	8(%edx,%ecx,4), %edx	C &dst[xsize+2]
	movl	%ebp, SAVE_EBP
	movl	PARAM_DIVISOR, %ebp

	movl	%edx, VAR_DST_STOP	C &dst[xsize+2]
	movl	%edi, SAVE_EDI
	xorl	%edi, %edi		C carry

	movl	-4(%esi,%ebx,4), %eax	C src high limb
	xor	%ecx, %ecx

	C

	C

	cmpl	%ebp, %eax		C high cmp divisor

	cmovc(	%eax, %edi)		C high is carry if high<divisor
	cmovnc(	%eax, %ecx)		C 0 if skip div, src high if not
					C (the latter in case src==dst)

	movl	%ecx, -12(%edx,%ebx,4)	C dst high limb
	sbbl	$0, %ebx		C skip one division if high<divisor
	movl	PARAM_PREINV_SHIFT, %ecx

	leal	-8(%edx,%ebx,4), %edx	C &dst[xsize+size]
	movl	$32, %eax

	movl	%edx, VAR_DST		C &dst[xsize+size]

	shll	%cl, %ebp		C d normalized
	subl	%ecx, %eax
	movl	%ecx, VAR_NORM

	movd	%eax, %mm7		C rshift
	movl	PARAM_PREINV_INVERSE, %eax
	jmp	L(start_preinv)

EPILOGUE()


	ALIGN(16)

PROLOGUE(mpn_divrem_1c)
deflit(`FRAME',0)
	movl	PARAM_CARRY, %edx
	movl	PARAM_SIZE, %ecx
	subl	$STACK_SPACE, %esp
deflit(`FRAME',STACK_SPACE)

	movl	%ebx, SAVE_EBX
	movl	PARAM_XSIZE, %ebx

	movl	%edi, SAVE_EDI
	movl	PARAM_DST, %edi

	movl	%ebp, SAVE_EBP
	movl	PARAM_DIVISOR, %ebp

	movl	%esi, SAVE_ESI
	movl	PARAM_SRC, %esi

	leal	-4(%edi,%ebx,4), %edi	C &dst[xsize-1]
	jmp	L(start_1c)

EPILOGUE()


	C offset 0xa1, close enough to aligned
PROLOGUE(mpn_divrem_1)
deflit(`FRAME',0)

	movl	PARAM_SIZE, %ecx
	movl	$0, %edx		C initial carry (if can't skip a div)
	subl	$STACK_SPACE, %esp
deflit(`FRAME',STACK_SPACE)

	movl	%esi, SAVE_ESI
	movl	PARAM_SRC, %esi

	movl	%ebx, SAVE_EBX
	movl	PARAM_XSIZE, %ebx

	movl	%ebp, SAVE_EBP
	movl	PARAM_DIVISOR, %ebp
	orl	%ecx, %ecx		C size

	movl	%edi, SAVE_EDI
	movl	PARAM_DST, %edi
	leal	-4(%edi,%ebx,4), %edi	C &dst[xsize-1]

	jz	L(no_skip_div)		C if size==0
	movl	-4(%esi,%ecx,4), %eax	C src high limb
	xorl	%esi, %esi

	cmpl	%ebp, %eax		C high cmp divisor

	cmovc(	%eax, %edx)		C high is carry if high<divisor
	cmovnc(	%eax, %esi)		C 0 if skip div, src high if not

	movl	%esi, (%edi,%ecx,4)	C dst high limb
	sbbl	$0, %ecx		C size-1 if high<divisor
	movl	PARAM_SRC, %esi		C reload
L(no_skip_div):


L(start_1c):
	C eax
	C ebx	xsize
	C ecx	size
	C edx	carry
	C esi	src
	C edi	&dst[xsize-1]
	C ebp	divisor

	leal	(%ebx,%ecx), %eax	C size+xsize
	cmpl	$MUL_THRESHOLD, %eax
	jae	L(mul_by_inverse)


C With MUL_THRESHOLD set to 3, the simple loops here only do 0 to 2 limbs.
C It'd be possible to write them out without the looping, but no speedup
C would be expected.
C
C Using PARAM_DIVISOR instead of %ebp measures 1 cycle/loop faster on the
C integer part, but curiously not on the fractional part, where %ebp is a
C (fixed) couple of cycles faster.

	orl	%ecx, %ecx
	jz	L(divide_no_integer)

L(divide_integer):
	C eax	scratch (quotient)
	C ebx	xsize
	C ecx	counter
	C edx	scratch (remainder)
	C esi	src
	C edi	&dst[xsize-1]
	C ebp	divisor

	movl	-4(%esi,%ecx,4), %eax

	divl	PARAM_DIVISOR

	movl	%eax, (%edi,%ecx,4)
	decl	%ecx
	jnz	L(divide_integer)


L(divide_no_integer):
	movl	PARAM_DST, %edi
	orl	%ebx, %ebx
	jnz	L(divide_fraction)

L(divide_done):
	movl	SAVE_ESI, %esi
	movl	SAVE_EDI, %edi
	movl	%edx, %eax

	movl	SAVE_EBX, %ebx
	movl	SAVE_EBP, %ebp
	addl	$STACK_SPACE, %esp

	ret


L(divide_fraction):
	C eax	scratch (quotient)
	C ebx	counter
	C ecx
	C edx	scratch (remainder)
	C esi
	C edi	dst
	C ebp	divisor

	movl	$0, %eax

	divl	%ebp

	movl	%eax, -4(%edi,%ebx,4)
	decl	%ebx
	jnz	L(divide_fraction)

	jmp	L(divide_done)



C -----------------------------------------------------------------------------

L(mul_by_inverse):
	C eax
	C ebx	xsize
	C ecx	size
	C edx	carry
	C esi	src
	C edi	&dst[xsize-1]
	C ebp	divisor

	bsrl	%ebp, %eax		C 31-l

	leal	12(%edi), %ebx		C &dst[xsize+2], loop dst stop
	leal	4(%edi,%ecx,4), %edi	C &dst[xsize+size]

	movl	%edi, VAR_DST
	movl	%ebx, VAR_DST_STOP

	movl	%ecx, %ebx		C size
	movl	$31, %ecx

	movl	%edx, %edi		C carry
	movl	$-1, %edx

	C

	xorl	%eax, %ecx		C l
	incl	%eax			C 32-l

	shll	%cl, %ebp		C d normalized
	movl	%ecx, VAR_NORM

	movd	%eax, %mm7

	movl	$-1, %eax
	subl	%ebp, %edx		C (b-d)-1 giving edx:eax = b*(b-d)-1

	divl	%ebp			C floor (b*(b-d)-1) / d

L(start_preinv):
	C eax	inverse
	C ebx	size
	C ecx	shift
	C edx
	C esi	src
	C edi	carry
	C ebp	divisor
	C
	C mm7	rshift

	orl	%ebx, %ebx		C size
	movl	%eax, VAR_INVERSE
	leal	-12(%esi,%ebx,4), %eax	C &src[size-3]

	jz	L(start_zero)
	movl	%eax, VAR_SRC
	cmpl	$1, %ebx

	movl	8(%eax), %esi		C src high limb
	jz	L(start_one)

L(start_two_or_more):
	movl	4(%eax), %edx		C src second highest limb

	shldl(	%cl, %esi, %edi)	C n2 = carry,high << l

	shldl(	%cl, %edx, %esi)	C n10 = high,second << l

	cmpl	$2, %ebx
	je	L(integer_two_left)
	jmp	L(integer_top)


L(start_one):
	shldl(	%cl, %esi, %edi)	C n2 = carry,high << l

	shll	%cl, %esi		C n10 = high << l
	movl	%eax, VAR_SRC
	jmp	L(integer_one_left)


L(start_zero):
	C Can be here with xsize==0 if mpn_preinv_divrem_1 had size==1 and
	C skipped a division.

	shll	%cl, %edi		C n2 = carry << l
	movl	%edi, %eax		C return value for zero_done
	cmpl	$0, PARAM_XSIZE

	je	L(zero_done)
	jmp	L(fraction_some)



C -----------------------------------------------------------------------------
C
C The multiply by inverse loop is 17 cycles, and relies on some out-of-order
C execution.  The instruction scheduling is important, with various
C apparently equivalent forms running 1 to 5 cycles slower.
C
C A lower bound for the time would seem to be 16 cycles, based on the
C following successive dependencies.
C
C		      cycles
C		n2+n1	1
C		mul	6
C		q1+1	1
C		mul	6
C		sub	1
C		addback	1
C		       ---
C		       16
C
C This chain is what the loop has already, but 16 cycles isn't achieved.
C K7 has enough decode, and probably enough execute (depending maybe on what
C a mul actually consumes), but nothing running under 17 has been found.
C
C In theory n2+n1 could be done in the sub and addback stages (by
C calculating both n2 and n2+n1 there), but lack of registers makes this an
C unlikely proposition.
C
C The jz in the loop keeps the q1+1 stage to 1 cycle.  Handling an overflow
C from q1+1 with an "sbbl $0, %ebx" would add a cycle to the dependent
C chain, and nothing better than 18 cycles has been found when using it.
C The jump is taken only when q1 is 0xFFFFFFFF, and on random data this will
C be an extremely rare event.
C
C Branch mispredictions will hit random occurrences of q1==0xFFFFFFFF, but
C if some special data is coming out with this always, the q1_ff special
C case actually runs at 15 c/l.  0x2FFF...FFFD divided by 3 is a good way to
C induce the q1_ff case, for speed measurements or testing.  Note that
C 0xFFF...FFF divided by 1 or 2 doesn't induce it.
C
C The instruction groupings and empty comments show the cycles for a naive
C in-order view of the code (conveniently ignoring the load latency on
C VAR_INVERSE).  This shows some of where the time is going, but is nonsense
C to the extent that out-of-order execution rearranges it.  In this case
C there's 19 cycles shown, but it executes at 17.

	ALIGN(16)
L(integer_top):
	C eax	scratch
	C ebx	scratch (nadj, q1)
	C ecx	scratch (src, dst)
	C edx	scratch
	C esi	n10
	C edi	n2
	C ebp	divisor
	C
	C mm0	scratch (src qword)
	C mm7	rshift for normalization

	cmpl	$0x80000000, %esi  C n1 as 0=c, 1=nc
	movl	%edi, %eax         C n2
	movl	VAR_SRC, %ecx

	leal	(%ebp,%esi), %ebx
	cmovc(	%esi, %ebx)	   C nadj = n10 + (-n1 & d), ignoring overflow
	sbbl	$-1, %eax          C n2+n1

	mull	VAR_INVERSE        C m*(n2+n1)

	movq	(%ecx), %mm0       C next limb and the one below it
	subl	$4, %ecx

	movl	%ecx, VAR_SRC

	C

	addl	%ebx, %eax         C m*(n2+n1) + nadj, low giving carry flag
	leal	1(%edi), %ebx      C n2+1
	movl	%ebp, %eax	   C d

	C

	adcl	%edx, %ebx         C 1 + high(n2<<32 + m*(n2+n1) + nadj) = q1+1
	jz	L(q1_ff)
	movl	VAR_DST, %ecx

	mull	%ebx		   C (q1+1)*d

	psrlq	%mm7, %mm0

	leal	-4(%ecx), %ecx

	C

	subl	%eax, %esi
	movl	VAR_DST_STOP, %eax

	C

	sbbl	%edx, %edi	   C n - (q1+1)*d
	movl	%esi, %edi	   C remainder -> n2
	leal	(%ebp,%esi), %edx

	movd	%mm0, %esi

	cmovc(	%edx, %edi)	   C n - q1*d if underflow from using q1+1
	sbbl	$0, %ebx	   C q
	cmpl	%eax, %ecx

	movl	%ebx, (%ecx)
	movl	%ecx, VAR_DST
	jne	L(integer_top)


L(integer_loop_done):


C -----------------------------------------------------------------------------
C
C Here, and in integer_one_left below, an sbbl $0 is used rather than a jz
C q1_ff special case.  This make the code a bit smaller and simpler, and
C costs only 1 cycle (each).

L(integer_two_left):
	C eax	scratch
	C ebx	scratch (nadj, q1)
	C ecx	scratch (src, dst)
	C edx	scratch
	C esi	n10
	C edi	n2
	C ebp	divisor
	C
	C mm7	rshift

	cmpl	$0x80000000, %esi  C n1 as 0=c, 1=nc
	movl	%edi, %eax         C n2
	movl	PARAM_SRC, %ecx

	leal	(%ebp,%esi), %ebx
	cmovc(	%esi, %ebx)	   C nadj = n10 + (-n1 & d), ignoring overflow
	sbbl	$-1, %eax          C n2+n1

	mull	VAR_INVERSE        C m*(n2+n1)

	movd	(%ecx), %mm0	   C src low limb

	movl	VAR_DST_STOP, %ecx

	C

	addl	%ebx, %eax         C m*(n2+n1) + nadj, low giving carry flag
	leal	1(%edi), %ebx      C n2+1
	movl	%ebp, %eax	   C d

	adcl	%edx, %ebx         C 1 + high(n2<<32 + m*(n2+n1) + nadj) = q1+1

	sbbl	$0, %ebx

	mull	%ebx		   C (q1+1)*d

	psllq	$32, %mm0

	psrlq	%mm7, %mm0

	C

	subl	%eax, %esi

	C

	sbbl	%edx, %edi	   C n - (q1+1)*d
	movl	%esi, %edi	   C remainder -> n2
	leal	(%ebp,%esi), %edx

	movd	%mm0, %esi

	cmovc(	%edx, %edi)	   C n - q1*d if underflow from using q1+1
	sbbl	$0, %ebx	   C q

	movl	%ebx, -4(%ecx)


C -----------------------------------------------------------------------------
L(integer_one_left):
	C eax	scratch
	C ebx	scratch (nadj, q1)
	C ecx	dst
	C edx	scratch
	C esi	n10
	C edi	n2
	C ebp	divisor
	C
	C mm7	rshift

	movl	VAR_DST_STOP, %ecx
	cmpl	$0x80000000, %esi  C n1 as 0=c, 1=nc
	movl	%edi, %eax         C n2

	leal	(%ebp,%esi), %ebx
	cmovc(	%esi, %ebx)	   C nadj = n10 + (-n1 & d), ignoring overflow
	sbbl	$-1, %eax          C n2+n1

	mull	VAR_INVERSE        C m*(n2+n1)

	C

	C

	C

	addl	%ebx, %eax         C m*(n2+n1) + nadj, low giving carry flag
	leal	1(%edi), %ebx      C n2+1
	movl	%ebp, %eax	   C d

	C

	adcl	%edx, %ebx         C 1 + high(n2<<32 + m*(n2+n1) + nadj) = q1+1

	sbbl	$0, %ebx           C q1 if q1+1 overflowed

	mull	%ebx

	C

	C

	C

	subl	%eax, %esi

	C

	sbbl	%edx, %edi	   C n - (q1+1)*d
	movl	%esi, %edi	   C remainder -> n2
	leal	(%ebp,%esi), %edx

	cmovc(	%edx, %edi)	   C n - q1*d if underflow from using q1+1
	sbbl	$0, %ebx	   C q

	movl	%ebx, -8(%ecx)
	subl	$8, %ecx



L(integer_none):
	cmpl	$0, PARAM_XSIZE
	jne	L(fraction_some)

	movl	%edi, %eax
L(fraction_done):
	movl	VAR_NORM, %ecx
L(zero_done):
	movl	SAVE_EBP, %ebp

	movl	SAVE_EDI, %edi
	movl	SAVE_ESI, %esi

	movl	SAVE_EBX, %ebx
	addl	$STACK_SPACE, %esp

	shrl	%cl, %eax
	emms

	ret


C -----------------------------------------------------------------------------
C
C Special case for q1=0xFFFFFFFF, giving q=0xFFFFFFFF meaning the low dword
C of q*d is simply -d and the remainder n-q*d = n10+d

L(q1_ff):
	C eax	(divisor)
	C ebx	(q1+1 == 0)
	C ecx
	C edx
	C esi	n10
	C edi	n2
	C ebp	divisor

	movl	VAR_DST, %ecx
	movl	VAR_DST_STOP, %edx
	subl	$4, %ecx

	psrlq	%mm7, %mm0
	leal	(%ebp,%esi), %edi	C n-q*d remainder -> next n2
	movl	%ecx, VAR_DST

	movd	%mm0, %esi		C next n10

	movl	$-1, (%ecx)
	cmpl	%ecx, %edx
	jne	L(integer_top)

	jmp	L(integer_loop_done)



C -----------------------------------------------------------------------------
C
C Being the fractional part, the "source" limbs are all zero, meaning
C n10=0, n1=0, and hence nadj=0, leading to many instructions eliminated.
C
C The loop runs at 15 cycles.  The dependent chain is the same as the
C general case above, but without the n2+n1 stage (due to n1==0), so 15
C would seem to be the lower bound.
C
C A not entirely obvious simplification is that q1+1 never overflows a limb,
C and so there's no need for the sbbl $0 or jz q1_ff from the general case.
C q1 is the high word of m*n2+b*n2 and the following shows q1<=b-2 always.
C rnd() means rounding down to a multiple of d.
C
C	m*n2 + b*n2 <= m*(d-1) + b*(d-1)
C		     = m*d + b*d - m - b
C		     = floor((b(b-d)-1)/d)*d + b*d - m - b
C		     = rnd(b(b-d)-1) + b*d - m - b
C		     = rnd(b(b-d)-1 + b*d) - m - b
C		     = rnd(b*b-1) - m - b
C		     <= (b-2)*b
C
C Unchanged from the general case is that the final quotient limb q can be
C either q1 or q1+1, and the q1+1 case occurs often.  This can be seen from
C equation 8.4 of the paper which simplifies as follows when n1==0 and
C n0==0.
C
C	n-q1*d = (n2*k+q0*d)/b <= d + (d*d-2d)/b
C
C As before, the instruction groupings and empty comments show a naive
C in-order view of the code, which is made a nonsense by out of order
C execution.  There's 17 cycles shown, but it executes at 15.
C
C Rotating the store q and remainder->n2 instructions up to the top of the
C loop gets the run time down from 16 to 15.

	ALIGN(16)
L(fraction_some):
	C eax
	C ebx
	C ecx
	C edx
	C esi
	C edi	carry
	C ebp	divisor

	movl	PARAM_DST, %esi
	movl	VAR_DST_STOP, %ecx	C &dst[xsize+2]
	movl	%edi, %eax

	subl	$8, %ecx		C &dst[xsize]
	jmp	L(fraction_entry)


	ALIGN(16)
L(fraction_top):
	C eax	n2 carry, then scratch
	C ebx	scratch (nadj, q1)
	C ecx	dst, decrementing
	C edx	scratch
	C esi	dst stop point
	C edi	(will be n2)
	C ebp	divisor

	movl	%ebx, (%ecx)	C previous q
	movl	%eax, %edi	C remainder->n2

L(fraction_entry):
	mull	VAR_INVERSE	C m*n2

	movl	%ebp, %eax	C d
	subl	$4, %ecx	C dst
	leal	1(%edi), %ebx

	C

	C

	C

	C

	addl	%edx, %ebx	C 1 + high(n2<<32 + m*n2) = q1+1

	mull	%ebx		C (q1+1)*d

	C

	C

	C

	negl	%eax		C low of n - (q1+1)*d

	C

	sbbl	%edx, %edi	C high of n - (q1+1)*d, caring only about carry
	leal	(%ebp,%eax), %edx

	cmovc(	%edx, %eax)	C n - q1*d if underflow from using q1+1
	sbbl	$0, %ebx	C q
	cmpl	%esi, %ecx

	jne	L(fraction_top)


	movl	%ebx, (%ecx)
	jmp	L(fraction_done)

EPILOGUE()