1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
|
dnl AMD K6-2 mpn_copyi -- copy limb vector, incrementing.
dnl
dnl K6-2: 0.56 or 1.0 cycles/limb (at 32 limbs/loop), depending on data
dnl alignment.
dnl Copyright (C) 1999, 2000 Free Software Foundation, Inc.
dnl
dnl This file is part of the GNU MP Library.
dnl
dnl The GNU MP Library is free software; you can redistribute it and/or
dnl modify it under the terms of the GNU Library General Public License as
dnl published by the Free Software Foundation; either version 2 of the
dnl License, or (at your option) any later version.
dnl
dnl The GNU MP Library is distributed in the hope that it will be useful,
dnl but WITHOUT ANY WARRANTY; without even the implied warranty of
dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
dnl Library General Public License for more details.
dnl
dnl You should have received a copy of the GNU Library General Public
dnl License along with the GNU MP Library; see the file COPYING.LIB. If
dnl not, write to the Free Software Foundation, Inc., 59 Temple Place -
dnl Suite 330, Boston, MA 02111-1307, USA.
include(`../config.m4')
dnl K6-2 aligned:
dnl UNROLL_COUNT cycles/limb
dnl 8 0.75
dnl 16 0.625
dnl 32 0.5625
dnl 64 0.53
dnl Maximum possible with the current code is 64, the minimum is 2.
deflit(UNROLL_COUNT, 32)
C void mpn_copyi (mp_ptr dst, mp_srcptr src, mp_size_t size);
C
C The MMX loop is faster than a rep movs when src and dst are both 0mod8.
C With one 0mod8 and one 4mod8 it's 1.056 c/l and the rep movs at 1.0 c/l is
C used instead.
C
C mod8
C src dst
C 0 0 both aligned, use mmx
C 0 4 unaligned, use rep movs
C 4 0 unaligned, use rep movs
C 4 4 do one movs, then both aligned, use mmx
C
C The MMX code on aligned data is 0.5 c/l, plus loop overhead of 2
C cycles/loop, which is 0.0625 c/l at 32 limbs/loop.
C
C A pattern of two movq loads and two movq stores (or four and four) was
C tried, but found to be the same speed as just one of each.
C
C Note that this code only suits K6-2 and K6-3. Plain K6 does only one mmx
C instruction per cycle, so "movq"s are no faster than the simple 1 c/l rep
C movs.
C
C Enhancement:
C
C Addressing modes like disp(%esi,%ecx,4) aren't currently used. They'd
C make it possible to avoid incrementing %esi and %edi in the loop and hence
C get loop overhead down to 1 cycle. Care would be needed to avoid bad
C cache line crossings since the "movq"s would then be 5 code bytes rather
C than 4.
defframe(PARAM_SIZE,12)
defframe(PARAM_SRC, 8)
defframe(PARAM_DST, 4)
deflit(`FRAME',0)
.text
ALIGN(32)
PROLOGUE(mpn_copyi)
movl PARAM_SIZE, %ecx
movl %esi, %eax
movl PARAM_SRC, %esi
movl %edi, %edx
cld
movl PARAM_DST, %edi
cmpl $UNROLL_COUNT, %ecx
ja L(unroll)
L(simple):
rep
movsl
movl %eax, %esi
movl %edx, %edi
ret
L(unroll):
C if src and dst are different alignments mod8, then use rep movs
C if src and dst are both 4mod8 then process one limb to get 0mod8
pushl %ebx
leal (%esi,%edi), %ebx
testb $4, %bl
popl %ebx
jnz L(simple)
testl $4, %esi
leal -UNROLL_COUNT(%ecx), %ecx
jz L(already_aligned)
decl %ecx
movsl
L(already_aligned):
ifelse(UNROLL_BYTES,256,`
addl $128, %esi
addl $128, %edi
')
C this is offset 0x34, no alignment needed
L(top):
C eax saved esi
C ebx
C ecx counter, limbs
C edx saved edi
C esi src, incrementing
C edi dst, incrementing
C ebp
C
C Zdisp gets 0(%esi) left that way to avoid vector decode, and with
C 0(%edi) keeps code aligned to 16 byte boundaries.
deflit(CHUNK_COUNT, 2)
forloop(`i', 0, UNROLL_COUNT/CHUNK_COUNT-1, `
deflit(`disp', eval(i*CHUNK_COUNT*4 ifelse(UNROLL_BYTES,256,-128)))
Zdisp( movq, disp,(%esi), %mm0)
Zdisp( movq, %mm0, disp,(%edi))
')
addl $UNROLL_BYTES, %esi
subl $UNROLL_COUNT, %ecx
leal UNROLL_BYTES(%edi), %edi
jns L(top)
C now %ecx is -UNROLL_COUNT to -1 representing repectively 0 to
C UNROLL_COUNT-1 limbs remaining
testb $eval(UNROLL_COUNT/2), %cl
leal UNROLL_COUNT(%ecx), %ecx
jz L(not_half)
C at an unroll count of 32 this block of code is 16 cycles faster than
C the rep movs, less 3 or 4 to test whether to do it
forloop(`i', 0, UNROLL_COUNT/CHUNK_COUNT/2-1, `
deflit(`disp', eval(i*CHUNK_COUNT*4 ifelse(UNROLL_BYTES,256,-128)))
movq disp(%esi), %mm0
movq %mm0, disp(%edi)
')
addl $eval(UNROLL_BYTES/2), %esi
addl $eval(UNROLL_BYTES/2), %edi
subl $eval(UNROLL_COUNT/2), %ecx
L(not_half):
ifelse(UNROLL_BYTES,256,`
subl $128, %esi
subl $128, %edi
')
rep
movsl
movl %eax, %esi
movl %edx, %edi
femms
ret
EPILOGUE()
|