| OLD | NEW |
|---|
| (Empty) |
| 1 #if defined(__arm__) | |
| 2 #include <openssl/arm_arch.h> | |
| 3 | |
| 4 .text | |
| 5 #if defined(__thumb2__) | |
| 6 .syntax unified | |
| 7 .thumb | |
| 8 #else | |
| 9 .code 32 | |
| 10 #endif | |
| 11 | |
| 12 #if defined(__thumb2__) || defined(__clang__) | |
| 13 #define ldrhsb ldrbhs | |
| 14 #endif | |
| 15 | |
| 16 .align 5 | |
| 17 .Lsigma: | |
| 18 .long 0x61707865,0x3320646e,0x79622d32,0x6b206574 @ endian-neutral | |
| 19 .Lone: | |
| 20 .long 1,0,0,0 | |
| 21 #if __ARM_MAX_ARCH__>=7 | |
| 22 .LOPENSSL_armcap: | |
| 23 .word OPENSSL_armcap_P-.LChaCha20_ctr32 | |
| 24 #else | |
| 25 .word -1 | |
| 26 #endif | |
| 27 | |
| 28 .globl ChaCha20_ctr32 | |
| 29 .hidden ChaCha20_ctr32 | |
| 30 .type ChaCha20_ctr32,%function | |
| 31 .align 5 | |
| 32 ChaCha20_ctr32: | |
| 33 .LChaCha20_ctr32: | |
| 34 ldr r12,[sp,#0] @ pull pointer to counter and nonce | |
| 35 stmdb sp!,{r0,r1,r2,r4-r11,lr} | |
| 36 #if __ARM_ARCH__<7 && !defined(__thumb2__) | |
| 37 sub r14,pc,#16 @ ChaCha20_ctr32 | |
| 38 #else | |
| 39 adr r14,.LChaCha20_ctr32 | |
| 40 #endif | |
| 41 cmp r2,#0 @ len==0? | |
| 42 #ifdef __thumb2__ | |
| 43 itt eq | |
| 44 #endif | |
| 45 addeq sp,sp,#4*3 | |
| 46 beq .Lno_data | |
| 47 #if __ARM_MAX_ARCH__>=7 | |
| 48 cmp r2,#192 @ test len | |
| 49 bls .Lshort | |
| 50 ldr r4,[r14,#-32] | |
| 51 ldr r4,[r14,r4] | |
| 52 # ifdef __APPLE__ | |
| 53 ldr r4,[r4] | |
| 54 # endif | |
| 55 tst r4,#ARMV7_NEON | |
| 56 bne .LChaCha20_neon | |
| 57 .Lshort: | |
| 58 #endif | |
| 59 ldmia r12,{r4,r5,r6,r7} @ load counter and nonce | |
| 60 sub sp,sp,#4*(16) @ off-load area | |
| 61 sub r14,r14,#64 @ .Lsigma | |
| 62 stmdb sp!,{r4,r5,r6,r7} @ copy counter and nonce | |
| 63 ldmia r3,{r4,r5,r6,r7,r8,r9,r10,r11} @ load key | |
| 64 ldmia r14,{r0,r1,r2,r3} @ load sigma | |
| 65 stmdb sp!,{r4,r5,r6,r7,r8,r9,r10,r11} @ copy key | |
| 66 stmdb sp!,{r0,r1,r2,r3} @ copy sigma | |
| 67 str r10,[sp,#4*(16+10)] @ off-load "rx" | |
| 68 str r11,[sp,#4*(16+11)] @ off-load "rx" | |
| 69 b .Loop_outer_enter | |
| 70 | |
| 71 .align 4 | |
| 72 .Loop_outer: | |
| 73 ldmia sp,{r0,r1,r2,r3,r4,r5,r6,r7,r8,r9} @ load key mater
ial | |
| 74 str r11,[sp,#4*(32+2)] @ save len | |
| 75 str r12, [sp,#4*(32+1)] @ save inp | |
| 76 str r14, [sp,#4*(32+0)] @ save out | |
| 77 .Loop_outer_enter: | |
| 78 ldr r11, [sp,#4*(15)] | |
| 79 ldr r12,[sp,#4*(12)] @ modulo-scheduled load | |
| 80 ldr r10, [sp,#4*(13)] | |
| 81 ldr r14,[sp,#4*(14)] | |
| 82 str r11, [sp,#4*(16+15)] | |
| 83 mov r11,#10 | |
| 84 b .Loop | |
| 85 | |
| 86 .align 4 | |
| 87 .Loop: | |
| 88 subs r11,r11,#1 | |
| 89 add r0,r0,r4 | |
| 90 mov r12,r12,ror#16 | |
| 91 add r1,r1,r5 | |
| 92 mov r10,r10,ror#16 | |
| 93 eor r12,r12,r0,ror#16 | |
| 94 eor r10,r10,r1,ror#16 | |
| 95 add r8,r8,r12 | |
| 96 mov r4,r4,ror#20 | |
| 97 add r9,r9,r10 | |
| 98 mov r5,r5,ror#20 | |
| 99 eor r4,r4,r8,ror#20 | |
| 100 eor r5,r5,r9,ror#20 | |
| 101 add r0,r0,r4 | |
| 102 mov r12,r12,ror#24 | |
| 103 add r1,r1,r5 | |
| 104 mov r10,r10,ror#24 | |
| 105 eor r12,r12,r0,ror#24 | |
| 106 eor r10,r10,r1,ror#24 | |
| 107 add r8,r8,r12 | |
| 108 mov r4,r4,ror#25 | |
| 109 add r9,r9,r10 | |
| 110 mov r5,r5,ror#25 | |
| 111 str r10,[sp,#4*(16+13)] | |
| 112 ldr r10,[sp,#4*(16+15)] | |
| 113 eor r4,r4,r8,ror#25 | |
| 114 eor r5,r5,r9,ror#25 | |
| 115 str r8,[sp,#4*(16+8)] | |
| 116 ldr r8,[sp,#4*(16+10)] | |
| 117 add r2,r2,r6 | |
| 118 mov r14,r14,ror#16 | |
| 119 str r9,[sp,#4*(16+9)] | |
| 120 ldr r9,[sp,#4*(16+11)] | |
| 121 add r3,r3,r7 | |
| 122 mov r10,r10,ror#16 | |
| 123 eor r14,r14,r2,ror#16 | |
| 124 eor r10,r10,r3,ror#16 | |
| 125 add r8,r8,r14 | |
| 126 mov r6,r6,ror#20 | |
| 127 add r9,r9,r10 | |
| 128 mov r7,r7,ror#20 | |
| 129 eor r6,r6,r8,ror#20 | |
| 130 eor r7,r7,r9,ror#20 | |
| 131 add r2,r2,r6 | |
| 132 mov r14,r14,ror#24 | |
| 133 add r3,r3,r7 | |
| 134 mov r10,r10,ror#24 | |
| 135 eor r14,r14,r2,ror#24 | |
| 136 eor r10,r10,r3,ror#24 | |
| 137 add r8,r8,r14 | |
| 138 mov r6,r6,ror#25 | |
| 139 add r9,r9,r10 | |
| 140 mov r7,r7,ror#25 | |
| 141 eor r6,r6,r8,ror#25 | |
| 142 eor r7,r7,r9,ror#25 | |
| 143 add r0,r0,r5 | |
| 144 mov r10,r10,ror#16 | |
| 145 add r1,r1,r6 | |
| 146 mov r12,r12,ror#16 | |
| 147 eor r10,r10,r0,ror#16 | |
| 148 eor r12,r12,r1,ror#16 | |
| 149 add r8,r8,r10 | |
| 150 mov r5,r5,ror#20 | |
| 151 add r9,r9,r12 | |
| 152 mov r6,r6,ror#20 | |
| 153 eor r5,r5,r8,ror#20 | |
| 154 eor r6,r6,r9,ror#20 | |
| 155 add r0,r0,r5 | |
| 156 mov r10,r10,ror#24 | |
| 157 add r1,r1,r6 | |
| 158 mov r12,r12,ror#24 | |
| 159 eor r10,r10,r0,ror#24 | |
| 160 eor r12,r12,r1,ror#24 | |
| 161 add r8,r8,r10 | |
| 162 mov r5,r5,ror#25 | |
| 163 str r10,[sp,#4*(16+15)] | |
| 164 ldr r10,[sp,#4*(16+13)] | |
| 165 add r9,r9,r12 | |
| 166 mov r6,r6,ror#25 | |
| 167 eor r5,r5,r8,ror#25 | |
| 168 eor r6,r6,r9,ror#25 | |
| 169 str r8,[sp,#4*(16+10)] | |
| 170 ldr r8,[sp,#4*(16+8)] | |
| 171 add r2,r2,r7 | |
| 172 mov r10,r10,ror#16 | |
| 173 str r9,[sp,#4*(16+11)] | |
| 174 ldr r9,[sp,#4*(16+9)] | |
| 175 add r3,r3,r4 | |
| 176 mov r14,r14,ror#16 | |
| 177 eor r10,r10,r2,ror#16 | |
| 178 eor r14,r14,r3,ror#16 | |
| 179 add r8,r8,r10 | |
| 180 mov r7,r7,ror#20 | |
| 181 add r9,r9,r14 | |
| 182 mov r4,r4,ror#20 | |
| 183 eor r7,r7,r8,ror#20 | |
| 184 eor r4,r4,r9,ror#20 | |
| 185 add r2,r2,r7 | |
| 186 mov r10,r10,ror#24 | |
| 187 add r3,r3,r4 | |
| 188 mov r14,r14,ror#24 | |
| 189 eor r10,r10,r2,ror#24 | |
| 190 eor r14,r14,r3,ror#24 | |
| 191 add r8,r8,r10 | |
| 192 mov r7,r7,ror#25 | |
| 193 add r9,r9,r14 | |
| 194 mov r4,r4,ror#25 | |
| 195 eor r7,r7,r8,ror#25 | |
| 196 eor r4,r4,r9,ror#25 | |
| 197 bne .Loop | |
| 198 | |
| 199 ldr r11,[sp,#4*(32+2)] @ load len | |
| 200 | |
| 201 str r8, [sp,#4*(16+8)] @ modulo-scheduled store | |
| 202 str r9, [sp,#4*(16+9)] | |
| 203 str r12,[sp,#4*(16+12)] | |
| 204 str r10, [sp,#4*(16+13)] | |
| 205 str r14,[sp,#4*(16+14)] | |
| 206 | |
| 207 @ at this point we have first half of 512-bit result in | |
| 208 @ rx and second half at sp+4*(16+8) | |
| 209 | |
| 210 cmp r11,#64 @ done yet? | |
| 211 #ifdef __thumb2__ | |
| 212 itete lo | |
| 213 #endif | |
| 214 addlo r12,sp,#4*(0) @ shortcut or ... | |
| 215 ldrhs r12,[sp,#4*(32+1)] @ ... load inp | |
| 216 addlo r14,sp,#4*(0) @ shortcut or ... | |
| 217 ldrhs r14,[sp,#4*(32+0)] @ ... load out | |
| 218 | |
| 219 ldr r8,[sp,#4*(0)] @ load key material | |
| 220 ldr r9,[sp,#4*(1)] | |
| 221 | |
| 222 #if __ARM_ARCH__>=6 || !defined(__ARMEB__) | |
| 223 # if __ARM_ARCH__<7 | |
| 224 orr r10,r12,r14 | |
| 225 tst r10,#3 @ are input and output aligned? | |
| 226 ldr r10,[sp,#4*(2)] | |
| 227 bne .Lunaligned | |
| 228 cmp r11,#64 @ restore flags | |
| 229 # else | |
| 230 ldr r10,[sp,#4*(2)] | |
| 231 # endif | |
| 232 ldr r11,[sp,#4*(3)] | |
| 233 | |
| 234 add r0,r0,r8 @ accumulate key material | |
| 235 add r1,r1,r9 | |
| 236 # ifdef __thumb2__ | |
| 237 itt hs | |
| 238 # endif | |
| 239 ldrhs r8,[r12],#16 @ load input | |
| 240 ldrhs r9,[r12,#-12] | |
| 241 | |
| 242 add r2,r2,r10 | |
| 243 add r3,r3,r11 | |
| 244 # ifdef __thumb2__ | |
| 245 itt hs | |
| 246 # endif | |
| 247 ldrhs r10,[r12,#-8] | |
| 248 ldrhs r11,[r12,#-4] | |
| 249 # if __ARM_ARCH__>=6 && defined(__ARMEB__) | |
| 250 rev r0,r0 | |
| 251 rev r1,r1 | |
| 252 rev r2,r2 | |
| 253 rev r3,r3 | |
| 254 # endif | |
| 255 # ifdef __thumb2__ | |
| 256 itt hs | |
| 257 # endif | |
| 258 eorhs r0,r0,r8 @ xor with input | |
| 259 eorhs r1,r1,r9 | |
| 260 add r8,sp,#4*(4) | |
| 261 str r0,[r14],#16 @ store output | |
| 262 # ifdef __thumb2__ | |
| 263 itt hs | |
| 264 # endif | |
| 265 eorhs r2,r2,r10 | |
| 266 eorhs r3,r3,r11 | |
| 267 ldmia r8,{r8,r9,r10,r11} @ load key material | |
| 268 str r1,[r14,#-12] | |
| 269 str r2,[r14,#-8] | |
| 270 str r3,[r14,#-4] | |
| 271 | |
| 272 add r4,r4,r8 @ accumulate key material | |
| 273 add r5,r5,r9 | |
| 274 # ifdef __thumb2__ | |
| 275 itt hs | |
| 276 # endif | |
| 277 ldrhs r8,[r12],#16 @ load input | |
| 278 ldrhs r9,[r12,#-12] | |
| 279 add r6,r6,r10 | |
| 280 add r7,r7,r11 | |
| 281 # ifdef __thumb2__ | |
| 282 itt hs | |
| 283 # endif | |
| 284 ldrhs r10,[r12,#-8] | |
| 285 ldrhs r11,[r12,#-4] | |
| 286 # if __ARM_ARCH__>=6 && defined(__ARMEB__) | |
| 287 rev r4,r4 | |
| 288 rev r5,r5 | |
| 289 rev r6,r6 | |
| 290 rev r7,r7 | |
| 291 # endif | |
| 292 # ifdef __thumb2__ | |
| 293 itt hs | |
| 294 # endif | |
| 295 eorhs r4,r4,r8 | |
| 296 eorhs r5,r5,r9 | |
| 297 add r8,sp,#4*(8) | |
| 298 str r4,[r14],#16 @ store output | |
| 299 # ifdef __thumb2__ | |
| 300 itt hs | |
| 301 # endif | |
| 302 eorhs r6,r6,r10 | |
| 303 eorhs r7,r7,r11 | |
| 304 str r5,[r14,#-12] | |
| 305 ldmia r8,{r8,r9,r10,r11} @ load key material | |
| 306 str r6,[r14,#-8] | |
| 307 add r0,sp,#4*(16+8) | |
| 308 str r7,[r14,#-4] | |
| 309 | |
| 310 ldmia r0,{r0,r1,r2,r3,r4,r5,r6,r7} @ load second half | |
| 311 | |
| 312 add r0,r0,r8 @ accumulate key material | |
| 313 add r1,r1,r9 | |
| 314 # ifdef __thumb2__ | |
| 315 itt hs | |
| 316 # endif | |
| 317 ldrhs r8,[r12],#16 @ load input | |
| 318 ldrhs r9,[r12,#-12] | |
| 319 # ifdef __thumb2__ | |
| 320 itt hi | |
| 321 # endif | |
| 322 strhi r10,[sp,#4*(16+10)] @ copy "rx" while at it | |
| 323 strhi r11,[sp,#4*(16+11)] @ copy "rx" while at it | |
| 324 add r2,r2,r10 | |
| 325 add r3,r3,r11 | |
| 326 # ifdef __thumb2__ | |
| 327 itt hs | |
| 328 # endif | |
| 329 ldrhs r10,[r12,#-8] | |
| 330 ldrhs r11,[r12,#-4] | |
| 331 # if __ARM_ARCH__>=6 && defined(__ARMEB__) | |
| 332 rev r0,r0 | |
| 333 rev r1,r1 | |
| 334 rev r2,r2 | |
| 335 rev r3,r3 | |
| 336 # endif | |
| 337 # ifdef __thumb2__ | |
| 338 itt hs | |
| 339 # endif | |
| 340 eorhs r0,r0,r8 | |
| 341 eorhs r1,r1,r9 | |
| 342 add r8,sp,#4*(12) | |
| 343 str r0,[r14],#16 @ store output | |
| 344 # ifdef __thumb2__ | |
| 345 itt hs | |
| 346 # endif | |
| 347 eorhs r2,r2,r10 | |
| 348 eorhs r3,r3,r11 | |
| 349 str r1,[r14,#-12] | |
| 350 ldmia r8,{r8,r9,r10,r11} @ load key material | |
| 351 str r2,[r14,#-8] | |
| 352 str r3,[r14,#-4] | |
| 353 | |
| 354 add r4,r4,r8 @ accumulate key material | |
| 355 add r5,r5,r9 | |
| 356 # ifdef __thumb2__ | |
| 357 itt hi | |
| 358 # endif | |
| 359 addhi r8,r8,#1 @ next counter value | |
| 360 strhi r8,[sp,#4*(12)] @ save next counter value | |
| 361 # ifdef __thumb2__ | |
| 362 itt hs | |
| 363 # endif | |
| 364 ldrhs r8,[r12],#16 @ load input | |
| 365 ldrhs r9,[r12,#-12] | |
| 366 add r6,r6,r10 | |
| 367 add r7,r7,r11 | |
| 368 # ifdef __thumb2__ | |
| 369 itt hs | |
| 370 # endif | |
| 371 ldrhs r10,[r12,#-8] | |
| 372 ldrhs r11,[r12,#-4] | |
| 373 # if __ARM_ARCH__>=6 && defined(__ARMEB__) | |
| 374 rev r4,r4 | |
| 375 rev r5,r5 | |
| 376 rev r6,r6 | |
| 377 rev r7,r7 | |
| 378 # endif | |
| 379 # ifdef __thumb2__ | |
| 380 itt hs | |
| 381 # endif | |
| 382 eorhs r4,r4,r8 | |
| 383 eorhs r5,r5,r9 | |
| 384 # ifdef __thumb2__ | |
| 385 it ne | |
| 386 # endif | |
| 387 ldrne r8,[sp,#4*(32+2)] @ re-load len | |
| 388 # ifdef __thumb2__ | |
| 389 itt hs | |
| 390 # endif | |
| 391 eorhs r6,r6,r10 | |
| 392 eorhs r7,r7,r11 | |
| 393 str r4,[r14],#16 @ store output | |
| 394 str r5,[r14,#-12] | |
| 395 # ifdef __thumb2__ | |
| 396 it hs | |
| 397 # endif | |
| 398 subhs r11,r8,#64 @ len-=64 | |
| 399 str r6,[r14,#-8] | |
| 400 str r7,[r14,#-4] | |
| 401 bhi .Loop_outer | |
| 402 | |
| 403 beq .Ldone | |
| 404 # if __ARM_ARCH__<7 | |
| 405 b .Ltail | |
| 406 | |
| 407 .align 4 | |
| 408 .Lunaligned:@ unaligned endian-neutral path | |
| 409 cmp r11,#64 @ restore flags | |
| 410 # endif | |
| 411 #endif | |
| 412 #if __ARM_ARCH__<7 | |
| 413 ldr r11,[sp,#4*(3)] | |
| 414 add r0,r0,r8 @ accumulate key material | |
| 415 add r1,r1,r9 | |
| 416 add r2,r2,r10 | |
| 417 # ifdef __thumb2__ | |
| 418 itete lo | |
| 419 # endif | |
| 420 eorlo r8,r8,r8 @ zero or ... | |
| 421 ldrhsb r8,[r12],#16 @ ... load input | |
| 422 eorlo r9,r9,r9 | |
| 423 ldrhsb r9,[r12,#-12] | |
| 424 | |
| 425 add r3,r3,r11 | |
| 426 # ifdef __thumb2__ | |
| 427 itete lo | |
| 428 # endif | |
| 429 eorlo r10,r10,r10 | |
| 430 ldrhsb r10,[r12,#-8] | |
| 431 eorlo r11,r11,r11 | |
| 432 ldrhsb r11,[r12,#-4] | |
| 433 | |
| 434 eor r0,r8,r0 @ xor with input (or zero) | |
| 435 eor r1,r9,r1 | |
| 436 # ifdef __thumb2__ | |
| 437 itt hs | |
| 438 # endif | |
| 439 ldrhsb r8,[r12,#-15] @ load more input | |
| 440 ldrhsb r9,[r12,#-11] | |
| 441 eor r2,r10,r2 | |
| 442 strb r0,[r14],#16 @ store output | |
| 443 eor r3,r11,r3 | |
| 444 # ifdef __thumb2__ | |
| 445 itt hs | |
| 446 # endif | |
| 447 ldrhsb r10,[r12,#-7] | |
| 448 ldrhsb r11,[r12,#-3] | |
| 449 strb r1,[r14,#-12] | |
| 450 eor r0,r8,r0,lsr#8 | |
| 451 strb r2,[r14,#-8] | |
| 452 eor r1,r9,r1,lsr#8 | |
| 453 # ifdef __thumb2__ | |
| 454 itt hs | |
| 455 # endif | |
| 456 ldrhsb r8,[r12,#-14] @ load more input | |
| 457 ldrhsb r9,[r12,#-10] | |
| 458 strb r3,[r14,#-4] | |
| 459 eor r2,r10,r2,lsr#8 | |
| 460 strb r0,[r14,#-15] | |
| 461 eor r3,r11,r3,lsr#8 | |
| 462 # ifdef __thumb2__ | |
| 463 itt hs | |
| 464 # endif | |
| 465 ldrhsb r10,[r12,#-6] | |
| 466 ldrhsb r11,[r12,#-2] | |
| 467 strb r1,[r14,#-11] | |
| 468 eor r0,r8,r0,lsr#8 | |
| 469 strb r2,[r14,#-7] | |
| 470 eor r1,r9,r1,lsr#8 | |
| 471 # ifdef __thumb2__ | |
| 472 itt hs | |
| 473 # endif | |
| 474 ldrhsb r8,[r12,#-13] @ load more input | |
| 475 ldrhsb r9,[r12,#-9] | |
| 476 strb r3,[r14,#-3] | |
| 477 eor r2,r10,r2,lsr#8 | |
| 478 strb r0,[r14,#-14] | |
| 479 eor r3,r11,r3,lsr#8 | |
| 480 # ifdef __thumb2__ | |
| 481 itt hs | |
| 482 # endif | |
| 483 ldrhsb r10,[r12,#-5] | |
| 484 ldrhsb r11,[r12,#-1] | |
| 485 strb r1,[r14,#-10] | |
| 486 strb r2,[r14,#-6] | |
| 487 eor r0,r8,r0,lsr#8 | |
| 488 strb r3,[r14,#-2] | |
| 489 eor r1,r9,r1,lsr#8 | |
| 490 strb r0,[r14,#-13] | |
| 491 eor r2,r10,r2,lsr#8 | |
| 492 strb r1,[r14,#-9] | |
| 493 eor r3,r11,r3,lsr#8 | |
| 494 strb r2,[r14,#-5] | |
| 495 strb r3,[r14,#-1] | |
| 496 add r8,sp,#4*(4+0) | |
| 497 ldmia r8,{r8,r9,r10,r11} @ load key material | |
| 498 add r0,sp,#4*(16+8) | |
| 499 add r4,r4,r8 @ accumulate key material | |
| 500 add r5,r5,r9 | |
| 501 add r6,r6,r10 | |
| 502 # ifdef __thumb2__ | |
| 503 itete lo | |
| 504 # endif | |
| 505 eorlo r8,r8,r8 @ zero or ... | |
| 506 ldrhsb r8,[r12],#16 @ ... load input | |
| 507 eorlo r9,r9,r9 | |
| 508 ldrhsb r9,[r12,#-12] | |
| 509 | |
| 510 add r7,r7,r11 | |
| 511 # ifdef __thumb2__ | |
| 512 itete lo | |
| 513 # endif | |
| 514 eorlo r10,r10,r10 | |
| 515 ldrhsb r10,[r12,#-8] | |
| 516 eorlo r11,r11,r11 | |
| 517 ldrhsb r11,[r12,#-4] | |
| 518 | |
| 519 eor r4,r8,r4 @ xor with input (or zero) | |
| 520 eor r5,r9,r5 | |
| 521 # ifdef __thumb2__ | |
| 522 itt hs | |
| 523 # endif | |
| 524 ldrhsb r8,[r12,#-15] @ load more input | |
| 525 ldrhsb r9,[r12,#-11] | |
| 526 eor r6,r10,r6 | |
| 527 strb r4,[r14],#16 @ store output | |
| 528 eor r7,r11,r7 | |
| 529 # ifdef __thumb2__ | |
| 530 itt hs | |
| 531 # endif | |
| 532 ldrhsb r10,[r12,#-7] | |
| 533 ldrhsb r11,[r12,#-3] | |
| 534 strb r5,[r14,#-12] | |
| 535 eor r4,r8,r4,lsr#8 | |
| 536 strb r6,[r14,#-8] | |
| 537 eor r5,r9,r5,lsr#8 | |
| 538 # ifdef __thumb2__ | |
| 539 itt hs | |
| 540 # endif | |
| 541 ldrhsb r8,[r12,#-14] @ load more input | |
| 542 ldrhsb r9,[r12,#-10] | |
| 543 strb r7,[r14,#-4] | |
| 544 eor r6,r10,r6,lsr#8 | |
| 545 strb r4,[r14,#-15] | |
| 546 eor r7,r11,r7,lsr#8 | |
| 547 # ifdef __thumb2__ | |
| 548 itt hs | |
| 549 # endif | |
| 550 ldrhsb r10,[r12,#-6] | |
| 551 ldrhsb r11,[r12,#-2] | |
| 552 strb r5,[r14,#-11] | |
| 553 eor r4,r8,r4,lsr#8 | |
| 554 strb r6,[r14,#-7] | |
| 555 eor r5,r9,r5,lsr#8 | |
| 556 # ifdef __thumb2__ | |
| 557 itt hs | |
| 558 # endif | |
| 559 ldrhsb r8,[r12,#-13] @ load more input | |
| 560 ldrhsb r9,[r12,#-9] | |
| 561 strb r7,[r14,#-3] | |
| 562 eor r6,r10,r6,lsr#8 | |
| 563 strb r4,[r14,#-14] | |
| 564 eor r7,r11,r7,lsr#8 | |
| 565 # ifdef __thumb2__ | |
| 566 itt hs | |
| 567 # endif | |
| 568 ldrhsb r10,[r12,#-5] | |
| 569 ldrhsb r11,[r12,#-1] | |
| 570 strb r5,[r14,#-10] | |
| 571 strb r6,[r14,#-6] | |
| 572 eor r4,r8,r4,lsr#8 | |
| 573 strb r7,[r14,#-2] | |
| 574 eor r5,r9,r5,lsr#8 | |
| 575 strb r4,[r14,#-13] | |
| 576 eor r6,r10,r6,lsr#8 | |
| 577 strb r5,[r14,#-9] | |
| 578 eor r7,r11,r7,lsr#8 | |
| 579 strb r6,[r14,#-5] | |
| 580 strb r7,[r14,#-1] | |
| 581 add r8,sp,#4*(4+4) | |
| 582 ldmia r8,{r8,r9,r10,r11} @ load key material | |
| 583 ldmia r0,{r0,r1,r2,r3,r4,r5,r6,r7} @ load second half | |
| 584 # ifdef __thumb2__ | |
| 585 itt hi | |
| 586 # endif | |
| 587 strhi r10,[sp,#4*(16+10)] @ copy "rx" | |
| 588 strhi r11,[sp,#4*(16+11)] @ copy "rx" | |
| 589 add r0,r0,r8 @ accumulate key material | |
| 590 add r1,r1,r9 | |
| 591 add r2,r2,r10 | |
| 592 # ifdef __thumb2__ | |
| 593 itete lo | |
| 594 # endif | |
| 595 eorlo r8,r8,r8 @ zero or ... | |
| 596 ldrhsb r8,[r12],#16 @ ... load input | |
| 597 eorlo r9,r9,r9 | |
| 598 ldrhsb r9,[r12,#-12] | |
| 599 | |
| 600 add r3,r3,r11 | |
| 601 # ifdef __thumb2__ | |
| 602 itete lo | |
| 603 # endif | |
| 604 eorlo r10,r10,r10 | |
| 605 ldrhsb r10,[r12,#-8] | |
| 606 eorlo r11,r11,r11 | |
| 607 ldrhsb r11,[r12,#-4] | |
| 608 | |
| 609 eor r0,r8,r0 @ xor with input (or zero) | |
| 610 eor r1,r9,r1 | |
| 611 # ifdef __thumb2__ | |
| 612 itt hs | |
| 613 # endif | |
| 614 ldrhsb r8,[r12,#-15] @ load more input | |
| 615 ldrhsb r9,[r12,#-11] | |
| 616 eor r2,r10,r2 | |
| 617 strb r0,[r14],#16 @ store output | |
| 618 eor r3,r11,r3 | |
| 619 # ifdef __thumb2__ | |
| 620 itt hs | |
| 621 # endif | |
| 622 ldrhsb r10,[r12,#-7] | |
| 623 ldrhsb r11,[r12,#-3] | |
| 624 strb r1,[r14,#-12] | |
| 625 eor r0,r8,r0,lsr#8 | |
| 626 strb r2,[r14,#-8] | |
| 627 eor r1,r9,r1,lsr#8 | |
| 628 # ifdef __thumb2__ | |
| 629 itt hs | |
| 630 # endif | |
| 631 ldrhsb r8,[r12,#-14] @ load more input | |
| 632 ldrhsb r9,[r12,#-10] | |
| 633 strb r3,[r14,#-4] | |
| 634 eor r2,r10,r2,lsr#8 | |
| 635 strb r0,[r14,#-15] | |
| 636 eor r3,r11,r3,lsr#8 | |
| 637 # ifdef __thumb2__ | |
| 638 itt hs | |
| 639 # endif | |
| 640 ldrhsb r10,[r12,#-6] | |
| 641 ldrhsb r11,[r12,#-2] | |
| 642 strb r1,[r14,#-11] | |
| 643 eor r0,r8,r0,lsr#8 | |
| 644 strb r2,[r14,#-7] | |
| 645 eor r1,r9,r1,lsr#8 | |
| 646 # ifdef __thumb2__ | |
| 647 itt hs | |
| 648 # endif | |
| 649 ldrhsb r8,[r12,#-13] @ load more input | |
| 650 ldrhsb r9,[r12,#-9] | |
| 651 strb r3,[r14,#-3] | |
| 652 eor r2,r10,r2,lsr#8 | |
| 653 strb r0,[r14,#-14] | |
| 654 eor r3,r11,r3,lsr#8 | |
| 655 # ifdef __thumb2__ | |
| 656 itt hs | |
| 657 # endif | |
| 658 ldrhsb r10,[r12,#-5] | |
| 659 ldrhsb r11,[r12,#-1] | |
| 660 strb r1,[r14,#-10] | |
| 661 strb r2,[r14,#-6] | |
| 662 eor r0,r8,r0,lsr#8 | |
| 663 strb r3,[r14,#-2] | |
| 664 eor r1,r9,r1,lsr#8 | |
| 665 strb r0,[r14,#-13] | |
| 666 eor r2,r10,r2,lsr#8 | |
| 667 strb r1,[r14,#-9] | |
| 668 eor r3,r11,r3,lsr#8 | |
| 669 strb r2,[r14,#-5] | |
| 670 strb r3,[r14,#-1] | |
| 671 add r8,sp,#4*(4+8) | |
| 672 ldmia r8,{r8,r9,r10,r11} @ load key material | |
| 673 add r4,r4,r8 @ accumulate key material | |
| 674 # ifdef __thumb2__ | |
| 675 itt hi | |
| 676 # endif | |
| 677 addhi r8,r8,#1 @ next counter value | |
| 678 strhi r8,[sp,#4*(12)] @ save next counter value | |
| 679 add r5,r5,r9 | |
| 680 add r6,r6,r10 | |
| 681 # ifdef __thumb2__ | |
| 682 itete lo | |
| 683 # endif | |
| 684 eorlo r8,r8,r8 @ zero or ... | |
| 685 ldrhsb r8,[r12],#16 @ ... load input | |
| 686 eorlo r9,r9,r9 | |
| 687 ldrhsb r9,[r12,#-12] | |
| 688 | |
| 689 add r7,r7,r11 | |
| 690 # ifdef __thumb2__ | |
| 691 itete lo | |
| 692 # endif | |
| 693 eorlo r10,r10,r10 | |
| 694 ldrhsb r10,[r12,#-8] | |
| 695 eorlo r11,r11,r11 | |
| 696 ldrhsb r11,[r12,#-4] | |
| 697 | |
| 698 eor r4,r8,r4 @ xor with input (or zero) | |
| 699 eor r5,r9,r5 | |
| 700 # ifdef __thumb2__ | |
| 701 itt hs | |
| 702 # endif | |
| 703 ldrhsb r8,[r12,#-15] @ load more input | |
| 704 ldrhsb r9,[r12,#-11] | |
| 705 eor r6,r10,r6 | |
| 706 strb r4,[r14],#16 @ store output | |
| 707 eor r7,r11,r7 | |
| 708 # ifdef __thumb2__ | |
| 709 itt hs | |
| 710 # endif | |
| 711 ldrhsb r10,[r12,#-7] | |
| 712 ldrhsb r11,[r12,#-3] | |
| 713 strb r5,[r14,#-12] | |
| 714 eor r4,r8,r4,lsr#8 | |
| 715 strb r6,[r14,#-8] | |
| 716 eor r5,r9,r5,lsr#8 | |
| 717 # ifdef __thumb2__ | |
| 718 itt hs | |
| 719 # endif | |
| 720 ldrhsb r8,[r12,#-14] @ load more input | |
| 721 ldrhsb r9,[r12,#-10] | |
| 722 strb r7,[r14,#-4] | |
| 723 eor r6,r10,r6,lsr#8 | |
| 724 strb r4,[r14,#-15] | |
| 725 eor r7,r11,r7,lsr#8 | |
| 726 # ifdef __thumb2__ | |
| 727 itt hs | |
| 728 # endif | |
| 729 ldrhsb r10,[r12,#-6] | |
| 730 ldrhsb r11,[r12,#-2] | |
| 731 strb r5,[r14,#-11] | |
| 732 eor r4,r8,r4,lsr#8 | |
| 733 strb r6,[r14,#-7] | |
| 734 eor r5,r9,r5,lsr#8 | |
| 735 # ifdef __thumb2__ | |
| 736 itt hs | |
| 737 # endif | |
| 738 ldrhsb r8,[r12,#-13] @ load more input | |
| 739 ldrhsb r9,[r12,#-9] | |
| 740 strb r7,[r14,#-3] | |
| 741 eor r6,r10,r6,lsr#8 | |
| 742 strb r4,[r14,#-14] | |
| 743 eor r7,r11,r7,lsr#8 | |
| 744 # ifdef __thumb2__ | |
| 745 itt hs | |
| 746 # endif | |
| 747 ldrhsb r10,[r12,#-5] | |
| 748 ldrhsb r11,[r12,#-1] | |
| 749 strb r5,[r14,#-10] | |
| 750 strb r6,[r14,#-6] | |
| 751 eor r4,r8,r4,lsr#8 | |
| 752 strb r7,[r14,#-2] | |
| 753 eor r5,r9,r5,lsr#8 | |
| 754 strb r4,[r14,#-13] | |
| 755 eor r6,r10,r6,lsr#8 | |
| 756 strb r5,[r14,#-9] | |
| 757 eor r7,r11,r7,lsr#8 | |
| 758 strb r6,[r14,#-5] | |
| 759 strb r7,[r14,#-1] | |
| 760 # ifdef __thumb2__ | |
| 761 it ne | |
| 762 # endif | |
| 763 ldrne r8,[sp,#4*(32+2)] @ re-load len | |
| 764 # ifdef __thumb2__ | |
| 765 it hs | |
| 766 # endif | |
| 767 subhs r11,r8,#64 @ len-=64 | |
| 768 bhi .Loop_outer | |
| 769 | |
| 770 beq .Ldone | |
| 771 #endif | |
| 772 | |
| 773 .Ltail: | |
| 774 ldr r12,[sp,#4*(32+1)] @ load inp | |
| 775 add r9,sp,#4*(0) | |
| 776 ldr r14,[sp,#4*(32+0)] @ load out | |
| 777 | |
| 778 .Loop_tail: | |
| 779 ldrb r10,[r9],#1 @ read buffer on stack | |
| 780 ldrb r11,[r12],#1 @ read input | |
| 781 subs r8,r8,#1 | |
| 782 eor r11,r11,r10 | |
| 783 strb r11,[r14],#1 @ store output | |
| 784 bne .Loop_tail | |
| 785 | |
| 786 .Ldone: | |
| 787 add sp,sp,#4*(32+3) | |
| 788 .Lno_data: | |
| 789 ldmia sp!,{r4,r5,r6,r7,r8,r9,r10,r11,pc} | |
| 790 .size ChaCha20_ctr32,.-ChaCha20_ctr32 | |
| 791 #if __ARM_MAX_ARCH__>=7 | |
| 792 .arch armv7-a | |
| 793 .fpu neon | |
| 794 | |
| 795 .type ChaCha20_neon,%function | |
| 796 .align 5 | |
| 797 ChaCha20_neon: | |
| 798 ldr r12,[sp,#0] @ pull pointer to counter and nonce | |
| 799 stmdb sp!,{r0,r1,r2,r4-r11,lr} | |
| 800 .LChaCha20_neon: | |
| 801 adr r14,.Lsigma | |
| 802 vstmdb sp!,{d8,d9,d10,d11,d12,d13,d14,d15} @ ABI spec says
so | |
| 803 stmdb sp!,{r0,r1,r2,r3} | |
| 804 | |
| 805 vld1.32 {q1,q2},[r3] @ load key | |
| 806 ldmia r3,{r4,r5,r6,r7,r8,r9,r10,r11} @ load key | |
| 807 | |
| 808 sub sp,sp,#4*(16+16) | |
| 809 vld1.32 {q3},[r12] @ load counter and nonce | |
| 810 add r12,sp,#4*8 | |
| 811 ldmia r14,{r0,r1,r2,r3} @ load sigma | |
| 812 vld1.32 {q0},[r14]! @ load sigma | |
| 813 vld1.32 {q12},[r14] @ one | |
| 814 vst1.32 {q2,q3},[r12] @ copy 1/2key|counter|nonce | |
| 815 vst1.32 {q0,q1},[sp] @ copy sigma|1/2key | |
| 816 | |
| 817 str r10,[sp,#4*(16+10)] @ off-load "rx" | |
| 818 str r11,[sp,#4*(16+11)] @ off-load "rx" | |
| 819 vshl.i32 d26,d24,#1 @ two | |
| 820 vstr d24,[sp,#4*(16+0)] | |
| 821 vshl.i32 d28,d24,#2 @ four | |
| 822 vstr d26,[sp,#4*(16+2)] | |
| 823 vmov q4,q0 | |
| 824 vstr d28,[sp,#4*(16+4)] | |
| 825 vmov q8,q0 | |
| 826 vmov q5,q1 | |
| 827 vmov q9,q1 | |
| 828 b .Loop_neon_enter | |
| 829 | |
| 830 .align 4 | |
| 831 .Loop_neon_outer: | |
| 832 ldmia sp,{r0,r1,r2,r3,r4,r5,r6,r7,r8,r9} @ load key mater
ial | |
| 833 cmp r11,#64*2 @ if len<=64*2 | |
| 834 bls .Lbreak_neon @ switch to integer-only | |
| 835 vmov q4,q0 | |
| 836 str r11,[sp,#4*(32+2)] @ save len | |
| 837 vmov q8,q0 | |
| 838 str r12, [sp,#4*(32+1)] @ save inp | |
| 839 vmov q5,q1 | |
| 840 str r14, [sp,#4*(32+0)] @ save out | |
| 841 vmov q9,q1 | |
| 842 .Loop_neon_enter: | |
| 843 ldr r11, [sp,#4*(15)] | |
| 844 vadd.i32 q7,q3,q12 @ counter+1 | |
| 845 ldr r12,[sp,#4*(12)] @ modulo-scheduled load | |
| 846 vmov q6,q2 | |
| 847 ldr r10, [sp,#4*(13)] | |
| 848 vmov q10,q2 | |
| 849 ldr r14,[sp,#4*(14)] | |
| 850 vadd.i32 q11,q7,q12 @ counter+2 | |
| 851 str r11, [sp,#4*(16+15)] | |
| 852 mov r11,#10 | |
| 853 add r12,r12,#3 @ counter+3 | |
| 854 b .Loop_neon | |
| 855 | |
| 856 .align 4 | |
| 857 .Loop_neon: | |
| 858 subs r11,r11,#1 | |
| 859 vadd.i32 q0,q0,q1 | |
| 860 add r0,r0,r4 | |
| 861 vadd.i32 q4,q4,q5 | |
| 862 mov r12,r12,ror#16 | |
| 863 vadd.i32 q8,q8,q9 | |
| 864 add r1,r1,r5 | |
| 865 veor q3,q3,q0 | |
| 866 mov r10,r10,ror#16 | |
| 867 veor q7,q7,q4 | |
| 868 eor r12,r12,r0,ror#16 | |
| 869 veor q11,q11,q8 | |
| 870 eor r10,r10,r1,ror#16 | |
| 871 vrev32.16 q3,q3 | |
| 872 add r8,r8,r12 | |
| 873 vrev32.16 q7,q7 | |
| 874 mov r4,r4,ror#20 | |
| 875 vrev32.16 q11,q11 | |
| 876 add r9,r9,r10 | |
| 877 vadd.i32 q2,q2,q3 | |
| 878 mov r5,r5,ror#20 | |
| 879 vadd.i32 q6,q6,q7 | |
| 880 eor r4,r4,r8,ror#20 | |
| 881 vadd.i32 q10,q10,q11 | |
| 882 eor r5,r5,r9,ror#20 | |
| 883 veor q12,q1,q2 | |
| 884 add r0,r0,r4 | |
| 885 veor q13,q5,q6 | |
| 886 mov r12,r12,ror#24 | |
| 887 veor q14,q9,q10 | |
| 888 add r1,r1,r5 | |
| 889 vshr.u32 q1,q12,#20 | |
| 890 mov r10,r10,ror#24 | |
| 891 vshr.u32 q5,q13,#20 | |
| 892 eor r12,r12,r0,ror#24 | |
| 893 vshr.u32 q9,q14,#20 | |
| 894 eor r10,r10,r1,ror#24 | |
| 895 vsli.32 q1,q12,#12 | |
| 896 add r8,r8,r12 | |
| 897 vsli.32 q5,q13,#12 | |
| 898 mov r4,r4,ror#25 | |
| 899 vsli.32 q9,q14,#12 | |
| 900 add r9,r9,r10 | |
| 901 vadd.i32 q0,q0,q1 | |
| 902 mov r5,r5,ror#25 | |
| 903 vadd.i32 q4,q4,q5 | |
| 904 str r10,[sp,#4*(16+13)] | |
| 905 vadd.i32 q8,q8,q9 | |
| 906 ldr r10,[sp,#4*(16+15)] | |
| 907 veor q12,q3,q0 | |
| 908 eor r4,r4,r8,ror#25 | |
| 909 veor q13,q7,q4 | |
| 910 eor r5,r5,r9,ror#25 | |
| 911 veor q14,q11,q8 | |
| 912 str r8,[sp,#4*(16+8)] | |
| 913 vshr.u32 q3,q12,#24 | |
| 914 ldr r8,[sp,#4*(16+10)] | |
| 915 vshr.u32 q7,q13,#24 | |
| 916 add r2,r2,r6 | |
| 917 vshr.u32 q11,q14,#24 | |
| 918 mov r14,r14,ror#16 | |
| 919 vsli.32 q3,q12,#8 | |
| 920 str r9,[sp,#4*(16+9)] | |
| 921 vsli.32 q7,q13,#8 | |
| 922 ldr r9,[sp,#4*(16+11)] | |
| 923 vsli.32 q11,q14,#8 | |
| 924 add r3,r3,r7 | |
| 925 vadd.i32 q2,q2,q3 | |
| 926 mov r10,r10,ror#16 | |
| 927 vadd.i32 q6,q6,q7 | |
| 928 eor r14,r14,r2,ror#16 | |
| 929 vadd.i32 q10,q10,q11 | |
| 930 eor r10,r10,r3,ror#16 | |
| 931 veor q12,q1,q2 | |
| 932 add r8,r8,r14 | |
| 933 veor q13,q5,q6 | |
| 934 mov r6,r6,ror#20 | |
| 935 veor q14,q9,q10 | |
| 936 add r9,r9,r10 | |
| 937 vshr.u32 q1,q12,#25 | |
| 938 mov r7,r7,ror#20 | |
| 939 vshr.u32 q5,q13,#25 | |
| 940 eor r6,r6,r8,ror#20 | |
| 941 vshr.u32 q9,q14,#25 | |
| 942 eor r7,r7,r9,ror#20 | |
| 943 vsli.32 q1,q12,#7 | |
| 944 add r2,r2,r6 | |
| 945 vsli.32 q5,q13,#7 | |
| 946 mov r14,r14,ror#24 | |
| 947 vsli.32 q9,q14,#7 | |
| 948 add r3,r3,r7 | |
| 949 vext.8 q2,q2,q2,#8 | |
| 950 mov r10,r10,ror#24 | |
| 951 vext.8 q6,q6,q6,#8 | |
| 952 eor r14,r14,r2,ror#24 | |
| 953 vext.8 q10,q10,q10,#8 | |
| 954 eor r10,r10,r3,ror#24 | |
| 955 vext.8 q1,q1,q1,#4 | |
| 956 add r8,r8,r14 | |
| 957 vext.8 q5,q5,q5,#4 | |
| 958 mov r6,r6,ror#25 | |
| 959 vext.8 q9,q9,q9,#4 | |
| 960 add r9,r9,r10 | |
| 961 vext.8 q3,q3,q3,#12 | |
| 962 mov r7,r7,ror#25 | |
| 963 vext.8 q7,q7,q7,#12 | |
| 964 eor r6,r6,r8,ror#25 | |
| 965 vext.8 q11,q11,q11,#12 | |
| 966 eor r7,r7,r9,ror#25 | |
| 967 vadd.i32 q0,q0,q1 | |
| 968 add r0,r0,r5 | |
| 969 vadd.i32 q4,q4,q5 | |
| 970 mov r10,r10,ror#16 | |
| 971 vadd.i32 q8,q8,q9 | |
| 972 add r1,r1,r6 | |
| 973 veor q3,q3,q0 | |
| 974 mov r12,r12,ror#16 | |
| 975 veor q7,q7,q4 | |
| 976 eor r10,r10,r0,ror#16 | |
| 977 veor q11,q11,q8 | |
| 978 eor r12,r12,r1,ror#16 | |
| 979 vrev32.16 q3,q3 | |
| 980 add r8,r8,r10 | |
| 981 vrev32.16 q7,q7 | |
| 982 mov r5,r5,ror#20 | |
| 983 vrev32.16 q11,q11 | |
| 984 add r9,r9,r12 | |
| 985 vadd.i32 q2,q2,q3 | |
| 986 mov r6,r6,ror#20 | |
| 987 vadd.i32 q6,q6,q7 | |
| 988 eor r5,r5,r8,ror#20 | |
| 989 vadd.i32 q10,q10,q11 | |
| 990 eor r6,r6,r9,ror#20 | |
| 991 veor q12,q1,q2 | |
| 992 add r0,r0,r5 | |
| 993 veor q13,q5,q6 | |
| 994 mov r10,r10,ror#24 | |
| 995 veor q14,q9,q10 | |
| 996 add r1,r1,r6 | |
| 997 vshr.u32 q1,q12,#20 | |
| 998 mov r12,r12,ror#24 | |
| 999 vshr.u32 q5,q13,#20 | |
| 1000 eor r10,r10,r0,ror#24 | |
| 1001 vshr.u32 q9,q14,#20 | |
| 1002 eor r12,r12,r1,ror#24 | |
| 1003 vsli.32 q1,q12,#12 | |
| 1004 add r8,r8,r10 | |
| 1005 vsli.32 q5,q13,#12 | |
| 1006 mov r5,r5,ror#25 | |
| 1007 vsli.32 q9,q14,#12 | |
| 1008 str r10,[sp,#4*(16+15)] | |
| 1009 vadd.i32 q0,q0,q1 | |
| 1010 ldr r10,[sp,#4*(16+13)] | |
| 1011 vadd.i32 q4,q4,q5 | |
| 1012 add r9,r9,r12 | |
| 1013 vadd.i32 q8,q8,q9 | |
| 1014 mov r6,r6,ror#25 | |
| 1015 veor q12,q3,q0 | |
| 1016 eor r5,r5,r8,ror#25 | |
| 1017 veor q13,q7,q4 | |
| 1018 eor r6,r6,r9,ror#25 | |
| 1019 veor q14,q11,q8 | |
| 1020 str r8,[sp,#4*(16+10)] | |
| 1021 vshr.u32 q3,q12,#24 | |
| 1022 ldr r8,[sp,#4*(16+8)] | |
| 1023 vshr.u32 q7,q13,#24 | |
| 1024 add r2,r2,r7 | |
| 1025 vshr.u32 q11,q14,#24 | |
| 1026 mov r10,r10,ror#16 | |
| 1027 vsli.32 q3,q12,#8 | |
| 1028 str r9,[sp,#4*(16+11)] | |
| 1029 vsli.32 q7,q13,#8 | |
| 1030 ldr r9,[sp,#4*(16+9)] | |
| 1031 vsli.32 q11,q14,#8 | |
| 1032 add r3,r3,r4 | |
| 1033 vadd.i32 q2,q2,q3 | |
| 1034 mov r14,r14,ror#16 | |
| 1035 vadd.i32 q6,q6,q7 | |
| 1036 eor r10,r10,r2,ror#16 | |
| 1037 vadd.i32 q10,q10,q11 | |
| 1038 eor r14,r14,r3,ror#16 | |
| 1039 veor q12,q1,q2 | |
| 1040 add r8,r8,r10 | |
| 1041 veor q13,q5,q6 | |
| 1042 mov r7,r7,ror#20 | |
| 1043 veor q14,q9,q10 | |
| 1044 add r9,r9,r14 | |
| 1045 vshr.u32 q1,q12,#25 | |
| 1046 mov r4,r4,ror#20 | |
| 1047 vshr.u32 q5,q13,#25 | |
| 1048 eor r7,r7,r8,ror#20 | |
| 1049 vshr.u32 q9,q14,#25 | |
| 1050 eor r4,r4,r9,ror#20 | |
| 1051 vsli.32 q1,q12,#7 | |
| 1052 add r2,r2,r7 | |
| 1053 vsli.32 q5,q13,#7 | |
| 1054 mov r10,r10,ror#24 | |
| 1055 vsli.32 q9,q14,#7 | |
| 1056 add r3,r3,r4 | |
| 1057 vext.8 q2,q2,q2,#8 | |
| 1058 mov r14,r14,ror#24 | |
| 1059 vext.8 q6,q6,q6,#8 | |
| 1060 eor r10,r10,r2,ror#24 | |
| 1061 vext.8 q10,q10,q10,#8 | |
| 1062 eor r14,r14,r3,ror#24 | |
| 1063 vext.8 q1,q1,q1,#12 | |
| 1064 add r8,r8,r10 | |
| 1065 vext.8 q5,q5,q5,#12 | |
| 1066 mov r7,r7,ror#25 | |
| 1067 vext.8 q9,q9,q9,#12 | |
| 1068 add r9,r9,r14 | |
| 1069 vext.8 q3,q3,q3,#4 | |
| 1070 mov r4,r4,ror#25 | |
| 1071 vext.8 q7,q7,q7,#4 | |
| 1072 eor r7,r7,r8,ror#25 | |
| 1073 vext.8 q11,q11,q11,#4 | |
| 1074 eor r4,r4,r9,ror#25 | |
| 1075 bne .Loop_neon | |
| 1076 | |
| 1077 add r11,sp,#32 | |
| 1078 vld1.32 {q12,q13},[sp] @ load key material | |
| 1079 vld1.32 {q14,q15},[r11] | |
| 1080 | |
| 1081 ldr r11,[sp,#4*(32+2)] @ load len | |
| 1082 | |
| 1083 str r8, [sp,#4*(16+8)] @ modulo-scheduled store | |
| 1084 str r9, [sp,#4*(16+9)] | |
| 1085 str r12,[sp,#4*(16+12)] | |
| 1086 str r10, [sp,#4*(16+13)] | |
| 1087 str r14,[sp,#4*(16+14)] | |
| 1088 | |
| 1089 @ at this point we have first half of 512-bit result in | |
| 1090 @ rx and second half at sp+4*(16+8) | |
| 1091 | |
| 1092 ldr r12,[sp,#4*(32+1)] @ load inp | |
| 1093 ldr r14,[sp,#4*(32+0)] @ load out | |
| 1094 | |
| 1095 vadd.i32 q0,q0,q12 @ accumulate key material | |
| 1096 vadd.i32 q4,q4,q12 | |
| 1097 vadd.i32 q8,q8,q12 | |
| 1098 vldr d24,[sp,#4*(16+0)] @ one | |
| 1099 | |
| 1100 vadd.i32 q1,q1,q13 | |
| 1101 vadd.i32 q5,q5,q13 | |
| 1102 vadd.i32 q9,q9,q13 | |
| 1103 vldr d26,[sp,#4*(16+2)] @ two | |
| 1104 | |
| 1105 vadd.i32 q2,q2,q14 | |
| 1106 vadd.i32 q6,q6,q14 | |
| 1107 vadd.i32 q10,q10,q14 | |
| 1108 vadd.i32 d14,d14,d24 @ counter+1 | |
| 1109 vadd.i32 d22,d22,d26 @ counter+2 | |
| 1110 | |
| 1111 vadd.i32 q3,q3,q15 | |
| 1112 vadd.i32 q7,q7,q15 | |
| 1113 vadd.i32 q11,q11,q15 | |
| 1114 | |
| 1115 cmp r11,#64*4 | |
| 1116 blo .Ltail_neon | |
| 1117 | |
| 1118 vld1.8 {q12,q13},[r12]! @ load input | |
| 1119 mov r11,sp | |
| 1120 vld1.8 {q14,q15},[r12]! | |
| 1121 veor q0,q0,q12 @ xor with input | |
| 1122 veor q1,q1,q13 | |
| 1123 vld1.8 {q12,q13},[r12]! | |
| 1124 veor q2,q2,q14 | |
| 1125 veor q3,q3,q15 | |
| 1126 vld1.8 {q14,q15},[r12]! | |
| 1127 | |
| 1128 veor q4,q4,q12 | |
| 1129 vst1.8 {q0,q1},[r14]! @ store output | |
| 1130 veor q5,q5,q13 | |
| 1131 vld1.8 {q12,q13},[r12]! | |
| 1132 veor q6,q6,q14 | |
| 1133 vst1.8 {q2,q3},[r14]! | |
| 1134 veor q7,q7,q15 | |
| 1135 vld1.8 {q14,q15},[r12]! | |
| 1136 | |
| 1137 veor q8,q8,q12 | |
| 1138 vld1.32 {q0,q1},[r11]! @ load for next iteration | |
| 1139 veor d25,d25,d25 | |
| 1140 vldr d24,[sp,#4*(16+4)] @ four | |
| 1141 veor q9,q9,q13 | |
| 1142 vld1.32 {q2,q3},[r11] | |
| 1143 veor q10,q10,q14 | |
| 1144 vst1.8 {q4,q5},[r14]! | |
| 1145 veor q11,q11,q15 | |
| 1146 vst1.8 {q6,q7},[r14]! | |
| 1147 | |
| 1148 vadd.i32 d6,d6,d24 @ next counter value | |
| 1149 vldr d24,[sp,#4*(16+0)] @ one | |
| 1150 | |
| 1151 ldmia sp,{r8,r9,r10,r11} @ load key material | |
| 1152 add r0,r0,r8 @ accumulate key material | |
| 1153 ldr r8,[r12],#16 @ load input | |
| 1154 vst1.8 {q8,q9},[r14]! | |
| 1155 add r1,r1,r9 | |
| 1156 ldr r9,[r12,#-12] | |
| 1157 vst1.8 {q10,q11},[r14]! | |
| 1158 add r2,r2,r10 | |
| 1159 ldr r10,[r12,#-8] | |
| 1160 add r3,r3,r11 | |
| 1161 ldr r11,[r12,#-4] | |
| 1162 # ifdef __ARMEB__ | |
| 1163 rev r0,r0 | |
| 1164 rev r1,r1 | |
| 1165 rev r2,r2 | |
| 1166 rev r3,r3 | |
| 1167 # endif | |
| 1168 eor r0,r0,r8 @ xor with input | |
| 1169 add r8,sp,#4*(4) | |
| 1170 eor r1,r1,r9 | |
| 1171 str r0,[r14],#16 @ store output | |
| 1172 eor r2,r2,r10 | |
| 1173 str r1,[r14,#-12] | |
| 1174 eor r3,r3,r11 | |
| 1175 ldmia r8,{r8,r9,r10,r11} @ load key material | |
| 1176 str r2,[r14,#-8] | |
| 1177 str r3,[r14,#-4] | |
| 1178 | |
| 1179 add r4,r4,r8 @ accumulate key material | |
| 1180 ldr r8,[r12],#16 @ load input | |
| 1181 add r5,r5,r9 | |
| 1182 ldr r9,[r12,#-12] | |
| 1183 add r6,r6,r10 | |
| 1184 ldr r10,[r12,#-8] | |
| 1185 add r7,r7,r11 | |
| 1186 ldr r11,[r12,#-4] | |
| 1187 # ifdef __ARMEB__ | |
| 1188 rev r4,r4 | |
| 1189 rev r5,r5 | |
| 1190 rev r6,r6 | |
| 1191 rev r7,r7 | |
| 1192 # endif | |
| 1193 eor r4,r4,r8 | |
| 1194 add r8,sp,#4*(8) | |
| 1195 eor r5,r5,r9 | |
| 1196 str r4,[r14],#16 @ store output | |
| 1197 eor r6,r6,r10 | |
| 1198 str r5,[r14,#-12] | |
| 1199 eor r7,r7,r11 | |
| 1200 ldmia r8,{r8,r9,r10,r11} @ load key material | |
| 1201 str r6,[r14,#-8] | |
| 1202 add r0,sp,#4*(16+8) | |
| 1203 str r7,[r14,#-4] | |
| 1204 | |
| 1205 ldmia r0,{r0,r1,r2,r3,r4,r5,r6,r7} @ load second half | |
| 1206 | |
| 1207 add r0,r0,r8 @ accumulate key material | |
| 1208 ldr r8,[r12],#16 @ load input | |
| 1209 add r1,r1,r9 | |
| 1210 ldr r9,[r12,#-12] | |
| 1211 # ifdef __thumb2__ | |
| 1212 it hi | |
| 1213 # endif | |
| 1214 strhi r10,[sp,#4*(16+10)] @ copy "rx" while at it | |
| 1215 add r2,r2,r10 | |
| 1216 ldr r10,[r12,#-8] | |
| 1217 # ifdef __thumb2__ | |
| 1218 it hi | |
| 1219 # endif | |
| 1220 strhi r11,[sp,#4*(16+11)] @ copy "rx" while at it | |
| 1221 add r3,r3,r11 | |
| 1222 ldr r11,[r12,#-4] | |
| 1223 # ifdef __ARMEB__ | |
| 1224 rev r0,r0 | |
| 1225 rev r1,r1 | |
| 1226 rev r2,r2 | |
| 1227 rev r3,r3 | |
| 1228 # endif | |
| 1229 eor r0,r0,r8 | |
| 1230 add r8,sp,#4*(12) | |
| 1231 eor r1,r1,r9 | |
| 1232 str r0,[r14],#16 @ store output | |
| 1233 eor r2,r2,r10 | |
| 1234 str r1,[r14,#-12] | |
| 1235 eor r3,r3,r11 | |
| 1236 ldmia r8,{r8,r9,r10,r11} @ load key material | |
| 1237 str r2,[r14,#-8] | |
| 1238 str r3,[r14,#-4] | |
| 1239 | |
| 1240 add r4,r4,r8 @ accumulate key material | |
| 1241 add r8,r8,#4 @ next counter value | |
| 1242 add r5,r5,r9 | |
| 1243 str r8,[sp,#4*(12)] @ save next counter value | |
| 1244 ldr r8,[r12],#16 @ load input | |
| 1245 add r6,r6,r10 | |
| 1246 add r4,r4,#3 @ counter+3 | |
| 1247 ldr r9,[r12,#-12] | |
| 1248 add r7,r7,r11 | |
| 1249 ldr r10,[r12,#-8] | |
| 1250 ldr r11,[r12,#-4] | |
| 1251 # ifdef __ARMEB__ | |
| 1252 rev r4,r4 | |
| 1253 rev r5,r5 | |
| 1254 rev r6,r6 | |
| 1255 rev r7,r7 | |
| 1256 # endif | |
| 1257 eor r4,r4,r8 | |
| 1258 # ifdef __thumb2__ | |
| 1259 it hi | |
| 1260 # endif | |
| 1261 ldrhi r8,[sp,#4*(32+2)] @ re-load len | |
| 1262 eor r5,r5,r9 | |
| 1263 eor r6,r6,r10 | |
| 1264 str r4,[r14],#16 @ store output | |
| 1265 eor r7,r7,r11 | |
| 1266 str r5,[r14,#-12] | |
| 1267 sub r11,r8,#64*4 @ len-=64*4 | |
| 1268 str r6,[r14,#-8] | |
| 1269 str r7,[r14,#-4] | |
| 1270 bhi .Loop_neon_outer | |
| 1271 | |
| 1272 b .Ldone_neon | |
| 1273 | |
| 1274 .align 4 | |
| 1275 .Lbreak_neon: | |
| 1276 @ harmonize NEON and integer-only stack frames: load data | |
| 1277 @ from NEON frame, but save to integer-only one; distance | |
| 1278 @ between the two is 4*(32+4+16-32)=4*(20). | |
| 1279 | |
| 1280 str r11, [sp,#4*(20+32+2)] @ save len | |
| 1281 add r11,sp,#4*(32+4) | |
| 1282 str r12, [sp,#4*(20+32+1)] @ save inp | |
| 1283 str r14, [sp,#4*(20+32+0)] @ save out | |
| 1284 | |
| 1285 ldr r12,[sp,#4*(16+10)] | |
| 1286 ldr r14,[sp,#4*(16+11)] | |
| 1287 vldmia r11,{d8,d9,d10,d11,d12,d13,d14,d15} @ fulfil
l ABI requirement | |
| 1288 str r12,[sp,#4*(20+16+10)] @ copy "rx" | |
| 1289 str r14,[sp,#4*(20+16+11)] @ copy "rx" | |
| 1290 | |
| 1291 ldr r11, [sp,#4*(15)] | |
| 1292 ldr r12,[sp,#4*(12)] @ modulo-scheduled load | |
| 1293 ldr r10, [sp,#4*(13)] | |
| 1294 ldr r14,[sp,#4*(14)] | |
| 1295 str r11, [sp,#4*(20+16+15)] | |
| 1296 add r11,sp,#4*(20) | |
| 1297 vst1.32 {q0,q1},[r11]! @ copy key | |
| 1298 add sp,sp,#4*(20) @ switch frame | |
| 1299 vst1.32 {q2,q3},[r11] | |
| 1300 mov r11,#10 | |
| 1301 b .Loop @ go integer-only | |
| 1302 | |
| 1303 .align 4 | |
| 1304 .Ltail_neon: | |
| 1305 cmp r11,#64*3 | |
| 1306 bhs .L192_or_more_neon | |
| 1307 cmp r11,#64*2 | |
| 1308 bhs .L128_or_more_neon | |
| 1309 cmp r11,#64*1 | |
| 1310 bhs .L64_or_more_neon | |
| 1311 | |
| 1312 add r8,sp,#4*(8) | |
| 1313 vst1.8 {q0,q1},[sp] | |
| 1314 add r10,sp,#4*(0) | |
| 1315 vst1.8 {q2,q3},[r8] | |
| 1316 b .Loop_tail_neon | |
| 1317 | |
| 1318 .align 4 | |
| 1319 .L64_or_more_neon: | |
| 1320 vld1.8 {q12,q13},[r12]! | |
| 1321 vld1.8 {q14,q15},[r12]! | |
| 1322 veor q0,q0,q12 | |
| 1323 veor q1,q1,q13 | |
| 1324 veor q2,q2,q14 | |
| 1325 veor q3,q3,q15 | |
| 1326 vst1.8 {q0,q1},[r14]! | |
| 1327 vst1.8 {q2,q3},[r14]! | |
| 1328 | |
| 1329 beq .Ldone_neon | |
| 1330 | |
| 1331 add r8,sp,#4*(8) | |
| 1332 vst1.8 {q4,q5},[sp] | |
| 1333 add r10,sp,#4*(0) | |
| 1334 vst1.8 {q6,q7},[r8] | |
| 1335 sub r11,r11,#64*1 @ len-=64*1 | |
| 1336 b .Loop_tail_neon | |
| 1337 | |
| 1338 .align 4 | |
| 1339 .L128_or_more_neon: | |
| 1340 vld1.8 {q12,q13},[r12]! | |
| 1341 vld1.8 {q14,q15},[r12]! | |
| 1342 veor q0,q0,q12 | |
| 1343 veor q1,q1,q13 | |
| 1344 vld1.8 {q12,q13},[r12]! | |
| 1345 veor q2,q2,q14 | |
| 1346 veor q3,q3,q15 | |
| 1347 vld1.8 {q14,q15},[r12]! | |
| 1348 | |
| 1349 veor q4,q4,q12 | |
| 1350 veor q5,q5,q13 | |
| 1351 vst1.8 {q0,q1},[r14]! | |
| 1352 veor q6,q6,q14 | |
| 1353 vst1.8 {q2,q3},[r14]! | |
| 1354 veor q7,q7,q15 | |
| 1355 vst1.8 {q4,q5},[r14]! | |
| 1356 vst1.8 {q6,q7},[r14]! | |
| 1357 | |
| 1358 beq .Ldone_neon | |
| 1359 | |
| 1360 add r8,sp,#4*(8) | |
| 1361 vst1.8 {q8,q9},[sp] | |
| 1362 add r10,sp,#4*(0) | |
| 1363 vst1.8 {q10,q11},[r8] | |
| 1364 sub r11,r11,#64*2 @ len-=64*2 | |
| 1365 b .Loop_tail_neon | |
| 1366 | |
| 1367 .align 4 | |
| 1368 .L192_or_more_neon: | |
| 1369 vld1.8 {q12,q13},[r12]! | |
| 1370 vld1.8 {q14,q15},[r12]! | |
| 1371 veor q0,q0,q12 | |
| 1372 veor q1,q1,q13 | |
| 1373 vld1.8 {q12,q13},[r12]! | |
| 1374 veor q2,q2,q14 | |
| 1375 veor q3,q3,q15 | |
| 1376 vld1.8 {q14,q15},[r12]! | |
| 1377 | |
| 1378 veor q4,q4,q12 | |
| 1379 veor q5,q5,q13 | |
| 1380 vld1.8 {q12,q13},[r12]! | |
| 1381 veor q6,q6,q14 | |
| 1382 vst1.8 {q0,q1},[r14]! | |
| 1383 veor q7,q7,q15 | |
| 1384 vld1.8 {q14,q15},[r12]! | |
| 1385 | |
| 1386 veor q8,q8,q12 | |
| 1387 vst1.8 {q2,q3},[r14]! | |
| 1388 veor q9,q9,q13 | |
| 1389 vst1.8 {q4,q5},[r14]! | |
| 1390 veor q10,q10,q14 | |
| 1391 vst1.8 {q6,q7},[r14]! | |
| 1392 veor q11,q11,q15 | |
| 1393 vst1.8 {q8,q9},[r14]! | |
| 1394 vst1.8 {q10,q11},[r14]! | |
| 1395 | |
| 1396 beq .Ldone_neon | |
| 1397 | |
| 1398 ldmia sp,{r8,r9,r10,r11} @ load key material | |
| 1399 add r0,r0,r8 @ accumulate key material | |
| 1400 add r8,sp,#4*(4) | |
| 1401 add r1,r1,r9 | |
| 1402 add r2,r2,r10 | |
| 1403 add r3,r3,r11 | |
| 1404 ldmia r8,{r8,r9,r10,r11} @ load key material | |
| 1405 | |
| 1406 add r4,r4,r8 @ accumulate key material | |
| 1407 add r8,sp,#4*(8) | |
| 1408 add r5,r5,r9 | |
| 1409 add r6,r6,r10 | |
| 1410 add r7,r7,r11 | |
| 1411 ldmia r8,{r8,r9,r10,r11} @ load key material | |
| 1412 # ifdef __ARMEB__ | |
| 1413 rev r0,r0 | |
| 1414 rev r1,r1 | |
| 1415 rev r2,r2 | |
| 1416 rev r3,r3 | |
| 1417 rev r4,r4 | |
| 1418 rev r5,r5 | |
| 1419 rev r6,r6 | |
| 1420 rev r7,r7 | |
| 1421 # endif | |
| 1422 stmia sp,{r0,r1,r2,r3,r4,r5,r6,r7} | |
| 1423 add r0,sp,#4*(16+8) | |
| 1424 | |
| 1425 ldmia r0,{r0,r1,r2,r3,r4,r5,r6,r7} @ load second half | |
| 1426 | |
| 1427 add r0,r0,r8 @ accumulate key material | |
| 1428 add r8,sp,#4*(12) | |
| 1429 add r1,r1,r9 | |
| 1430 add r2,r2,r10 | |
| 1431 add r3,r3,r11 | |
| 1432 ldmia r8,{r8,r9,r10,r11} @ load key material | |
| 1433 | |
| 1434 add r4,r4,r8 @ accumulate key material | |
| 1435 add r8,sp,#4*(8) | |
| 1436 add r5,r5,r9 | |
| 1437 add r4,r4,#3 @ counter+3 | |
| 1438 add r6,r6,r10 | |
| 1439 add r7,r7,r11 | |
| 1440 ldr r11,[sp,#4*(32+2)] @ re-load len | |
| 1441 # ifdef __ARMEB__ | |
| 1442 rev r0,r0 | |
| 1443 rev r1,r1 | |
| 1444 rev r2,r2 | |
| 1445 rev r3,r3 | |
| 1446 rev r4,r4 | |
| 1447 rev r5,r5 | |
| 1448 rev r6,r6 | |
| 1449 rev r7,r7 | |
| 1450 # endif | |
| 1451 stmia r8,{r0,r1,r2,r3,r4,r5,r6,r7} | |
| 1452 add r10,sp,#4*(0) | |
| 1453 sub r11,r11,#64*3 @ len-=64*3 | |
| 1454 | |
| 1455 .Loop_tail_neon: | |
| 1456 ldrb r8,[r10],#1 @ read buffer on stack | |
| 1457 ldrb r9,[r12],#1 @ read input | |
| 1458 subs r11,r11,#1 | |
| 1459 eor r8,r8,r9 | |
| 1460 strb r8,[r14],#1 @ store ouput | |
| 1461 bne .Loop_tail_neon | |
| 1462 | |
| 1463 .Ldone_neon: | |
| 1464 add sp,sp,#4*(32+4) | |
| 1465 vldmia sp,{d8,d9,d10,d11,d12,d13,d14,d15} | |
| 1466 add sp,sp,#4*(16+3) | |
| 1467 ldmia sp!,{r4,r5,r6,r7,r8,r9,r10,r11,pc} | |
| 1468 .size ChaCha20_neon,.-ChaCha20_neon | |
| 1469 .comm OPENSSL_armcap_P,4,4 | |
| 1470 #endif | |
| 1471 #endif | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2354623003:
Pull boringssl generated source from boringssl_gen (Closed)
