Source release 14.1.0

This commit is contained in:
John W. Bruce
2018-06-29 15:59:47 -07:00
parent 3ab70cec4e
commit afa11a48a0
1941 changed files with 557780 additions and 105547 deletions

View File

@@ -327,11 +327,11 @@ _x86_64_AES_encrypt_compact:
.byte 0xf3,0xc3
.size _x86_64_AES_encrypt_compact,.-_x86_64_AES_encrypt_compact
.align 16
.globl asm_AES_encrypt
.hidden asm_AES_encrypt
.type asm_AES_encrypt,@function
.hidden asm_AES_encrypt
asm_AES_encrypt:
.globl aes_nohw_encrypt
.hidden aes_nohw_encrypt
.type aes_nohw_encrypt,@function
.hidden aes_nohw_encrypt
aes_nohw_encrypt:
.cfi_startproc
movq %rsp,%rax
.cfi_def_cfa_register %rax
@@ -409,7 +409,7 @@ asm_AES_encrypt:
.Lenc_epilogue:
.byte 0xf3,0xc3
.cfi_endproc
.size asm_AES_encrypt,.-asm_AES_encrypt
.size aes_nohw_encrypt,.-aes_nohw_encrypt
.type _x86_64_AES_decrypt,@function
.align 16
_x86_64_AES_decrypt:
@@ -791,11 +791,11 @@ _x86_64_AES_decrypt_compact:
.byte 0xf3,0xc3
.size _x86_64_AES_decrypt_compact,.-_x86_64_AES_decrypt_compact
.align 16
.globl asm_AES_decrypt
.hidden asm_AES_decrypt
.type asm_AES_decrypt,@function
.hidden asm_AES_decrypt
asm_AES_decrypt:
.globl aes_nohw_decrypt
.hidden aes_nohw_decrypt
.type aes_nohw_decrypt,@function
.hidden aes_nohw_decrypt
aes_nohw_decrypt:
.cfi_startproc
movq %rsp,%rax
.cfi_def_cfa_register %rax
@@ -875,12 +875,12 @@ asm_AES_decrypt:
.Ldec_epilogue:
.byte 0xf3,0xc3
.cfi_endproc
.size asm_AES_decrypt,.-asm_AES_decrypt
.size aes_nohw_decrypt,.-aes_nohw_decrypt
.align 16
.globl asm_AES_set_encrypt_key
.hidden asm_AES_set_encrypt_key
.type asm_AES_set_encrypt_key,@function
asm_AES_set_encrypt_key:
.globl aes_nohw_set_encrypt_key
.hidden aes_nohw_set_encrypt_key
.type aes_nohw_set_encrypt_key,@function
aes_nohw_set_encrypt_key:
.cfi_startproc
pushq %rbx
.cfi_adjust_cfa_offset 8
@@ -915,7 +915,7 @@ asm_AES_set_encrypt_key:
.Lenc_key_epilogue:
.byte 0xf3,0xc3
.cfi_endproc
.size asm_AES_set_encrypt_key,.-asm_AES_set_encrypt_key
.size aes_nohw_set_encrypt_key,.-aes_nohw_set_encrypt_key
.type _x86_64_AES_set_encrypt_key,@function
.align 16
@@ -1157,10 +1157,10 @@ _x86_64_AES_set_encrypt_key:
.byte 0xf3,0xc3
.size _x86_64_AES_set_encrypt_key,.-_x86_64_AES_set_encrypt_key
.align 16
.globl asm_AES_set_decrypt_key
.hidden asm_AES_set_decrypt_key
.type asm_AES_set_decrypt_key,@function
asm_AES_set_decrypt_key:
.globl aes_nohw_set_decrypt_key
.hidden aes_nohw_set_decrypt_key
.type aes_nohw_set_decrypt_key,@function
aes_nohw_set_decrypt_key:
.cfi_startproc
pushq %rbx
.cfi_adjust_cfa_offset 8
@@ -1365,15 +1365,15 @@ asm_AES_set_decrypt_key:
.Ldec_key_epilogue:
.byte 0xf3,0xc3
.cfi_endproc
.size asm_AES_set_decrypt_key,.-asm_AES_set_decrypt_key
.size aes_nohw_set_decrypt_key,.-aes_nohw_set_decrypt_key
.align 16
.globl asm_AES_cbc_encrypt
.hidden asm_AES_cbc_encrypt
.type asm_AES_cbc_encrypt,@function
.globl aes_nohw_cbc_encrypt
.hidden aes_nohw_cbc_encrypt
.type aes_nohw_cbc_encrypt,@function
.extern OPENSSL_ia32cap_P
.hidden OPENSSL_ia32cap_P
.hidden asm_AES_cbc_encrypt
asm_AES_cbc_encrypt:
.hidden aes_nohw_cbc_encrypt
aes_nohw_cbc_encrypt:
.cfi_startproc
cmpq $0,%rdx
je .Lcbc_epilogue
@@ -1850,7 +1850,7 @@ asm_AES_cbc_encrypt:
.Lcbc_epilogue:
.byte 0xf3,0xc3
.cfi_endproc
.size asm_AES_cbc_encrypt,.-asm_AES_cbc_encrypt
.size aes_nohw_cbc_encrypt,.-aes_nohw_cbc_encrypt
.align 64
.LAES_Te:
.long 0xa56363c6,0xa56363c6

View File

@@ -2,11 +2,11 @@
.text
.extern OPENSSL_ia32cap_P
.hidden OPENSSL_ia32cap_P
.globl aesni_encrypt
.hidden aesni_encrypt
.type aesni_encrypt,@function
.globl aes_hw_encrypt
.hidden aes_hw_encrypt
.type aes_hw_encrypt,@function
.align 16
aesni_encrypt:
aes_hw_encrypt:
movups (%rdi),%xmm2
movl 240(%rdx),%eax
movups (%rdx),%xmm0
@@ -25,13 +25,13 @@ aesni_encrypt:
movups %xmm2,(%rsi)
pxor %xmm2,%xmm2
.byte 0xf3,0xc3
.size aesni_encrypt,.-aesni_encrypt
.size aes_hw_encrypt,.-aes_hw_encrypt
.globl aesni_decrypt
.hidden aesni_decrypt
.type aesni_decrypt,@function
.globl aes_hw_decrypt
.hidden aes_hw_decrypt
.type aes_hw_decrypt,@function
.align 16
aesni_decrypt:
aes_hw_decrypt:
movups (%rdi),%xmm2
movl 240(%rdx),%eax
movups (%rdx),%xmm0
@@ -50,7 +50,7 @@ aesni_decrypt:
movups %xmm2,(%rsi)
pxor %xmm2,%xmm2
.byte 0xf3,0xc3
.size aesni_decrypt, .-aesni_decrypt
.size aes_hw_decrypt, .-aes_hw_decrypt
.type _aesni_encrypt2,@function
.align 16
_aesni_encrypt2:
@@ -493,11 +493,11 @@ _aesni_decrypt8:
.byte 102,68,15,56,223,200
.byte 0xf3,0xc3
.size _aesni_decrypt8,.-_aesni_decrypt8
.globl aesni_ecb_encrypt
.hidden aesni_ecb_encrypt
.type aesni_ecb_encrypt,@function
.globl aes_hw_ecb_encrypt
.hidden aes_hw_ecb_encrypt
.type aes_hw_ecb_encrypt,@function
.align 16
aesni_ecb_encrypt:
aes_hw_ecb_encrypt:
andq $-16,%rdx
jz .Lecb_ret
@@ -835,12 +835,12 @@ aesni_ecb_encrypt:
xorps %xmm0,%xmm0
pxor %xmm1,%xmm1
.byte 0xf3,0xc3
.size aesni_ecb_encrypt,.-aesni_ecb_encrypt
.globl aesni_ccm64_encrypt_blocks
.hidden aesni_ccm64_encrypt_blocks
.type aesni_ccm64_encrypt_blocks,@function
.size aes_hw_ecb_encrypt,.-aes_hw_ecb_encrypt
.globl aes_hw_ccm64_encrypt_blocks
.hidden aes_hw_ccm64_encrypt_blocks
.type aes_hw_ccm64_encrypt_blocks,@function
.align 16
aesni_ccm64_encrypt_blocks:
aes_hw_ccm64_encrypt_blocks:
movl 240(%rcx),%eax
movdqu (%r8),%xmm6
movdqa .Lincrement64(%rip),%xmm9
@@ -899,12 +899,12 @@ aesni_ccm64_encrypt_blocks:
pxor %xmm8,%xmm8
pxor %xmm6,%xmm6
.byte 0xf3,0xc3
.size aesni_ccm64_encrypt_blocks,.-aesni_ccm64_encrypt_blocks
.globl aesni_ccm64_decrypt_blocks
.hidden aesni_ccm64_decrypt_blocks
.type aesni_ccm64_decrypt_blocks,@function
.size aes_hw_ccm64_encrypt_blocks,.-aes_hw_ccm64_encrypt_blocks
.globl aes_hw_ccm64_decrypt_blocks
.hidden aes_hw_ccm64_decrypt_blocks
.type aes_hw_ccm64_decrypt_blocks,@function
.align 16
aesni_ccm64_decrypt_blocks:
aes_hw_ccm64_decrypt_blocks:
movl 240(%rcx),%eax
movups (%r8),%xmm6
movdqu (%r9),%xmm3
@@ -997,12 +997,12 @@ aesni_ccm64_decrypt_blocks:
pxor %xmm8,%xmm8
pxor %xmm6,%xmm6
.byte 0xf3,0xc3
.size aesni_ccm64_decrypt_blocks,.-aesni_ccm64_decrypt_blocks
.globl aesni_ctr32_encrypt_blocks
.hidden aesni_ctr32_encrypt_blocks
.type aesni_ctr32_encrypt_blocks,@function
.size aes_hw_ccm64_decrypt_blocks,.-aes_hw_ccm64_decrypt_blocks
.globl aes_hw_ctr32_encrypt_blocks
.hidden aes_hw_ctr32_encrypt_blocks
.type aes_hw_ctr32_encrypt_blocks,@function
.align 16
aesni_ctr32_encrypt_blocks:
aes_hw_ctr32_encrypt_blocks:
.cfi_startproc
cmpq $1,%rdx
jne .Lctr32_bulk
@@ -1577,12 +1577,12 @@ aesni_ctr32_encrypt_blocks:
.Lctr32_epilogue:
.byte 0xf3,0xc3
.cfi_endproc
.size aesni_ctr32_encrypt_blocks,.-aesni_ctr32_encrypt_blocks
.globl aesni_xts_encrypt
.hidden aesni_xts_encrypt
.type aesni_xts_encrypt,@function
.size aes_hw_ctr32_encrypt_blocks,.-aes_hw_ctr32_encrypt_blocks
.globl aes_hw_xts_encrypt
.hidden aes_hw_xts_encrypt
.type aes_hw_xts_encrypt,@function
.align 16
aesni_xts_encrypt:
aes_hw_xts_encrypt:
.cfi_startproc
leaq (%rsp),%r11
.cfi_def_cfa_register %r11
@@ -2048,12 +2048,12 @@ aesni_xts_encrypt:
.Lxts_enc_epilogue:
.byte 0xf3,0xc3
.cfi_endproc
.size aesni_xts_encrypt,.-aesni_xts_encrypt
.globl aesni_xts_decrypt
.hidden aesni_xts_decrypt
.type aesni_xts_decrypt,@function
.size aes_hw_xts_encrypt,.-aes_hw_xts_encrypt
.globl aes_hw_xts_decrypt
.hidden aes_hw_xts_decrypt
.type aes_hw_xts_decrypt,@function
.align 16
aesni_xts_decrypt:
aes_hw_xts_decrypt:
.cfi_startproc
leaq (%rsp),%r11
.cfi_def_cfa_register %r11
@@ -2556,12 +2556,12 @@ aesni_xts_decrypt:
.Lxts_dec_epilogue:
.byte 0xf3,0xc3
.cfi_endproc
.size aesni_xts_decrypt,.-aesni_xts_decrypt
.globl aesni_ocb_encrypt
.hidden aesni_ocb_encrypt
.type aesni_ocb_encrypt,@function
.size aes_hw_xts_decrypt,.-aes_hw_xts_decrypt
.globl aes_hw_ocb_encrypt
.hidden aes_hw_ocb_encrypt
.type aes_hw_ocb_encrypt,@function
.align 32
aesni_ocb_encrypt:
aes_hw_ocb_encrypt:
.cfi_startproc
leaq (%rsp),%rax
pushq %rbx
@@ -2771,7 +2771,7 @@ aesni_ocb_encrypt:
.Locb_enc_epilogue:
.byte 0xf3,0xc3
.cfi_endproc
.size aesni_ocb_encrypt,.-aesni_ocb_encrypt
.size aes_hw_ocb_encrypt,.-aes_hw_ocb_encrypt
.type __ocb_encrypt6,@function
.align 32
@@ -2979,11 +2979,11 @@ __ocb_encrypt1:
.byte 0xf3,0xc3
.size __ocb_encrypt1,.-__ocb_encrypt1
.globl aesni_ocb_decrypt
.hidden aesni_ocb_decrypt
.type aesni_ocb_decrypt,@function
.globl aes_hw_ocb_decrypt
.hidden aes_hw_ocb_decrypt
.type aes_hw_ocb_decrypt,@function
.align 32
aesni_ocb_decrypt:
aes_hw_ocb_decrypt:
.cfi_startproc
leaq (%rsp),%rax
pushq %rbx
@@ -3215,7 +3215,7 @@ aesni_ocb_decrypt:
.Locb_dec_epilogue:
.byte 0xf3,0xc3
.cfi_endproc
.size aesni_ocb_decrypt,.-aesni_ocb_decrypt
.size aes_hw_ocb_decrypt,.-aes_hw_ocb_decrypt
.type __ocb_decrypt6,@function
.align 32
@@ -3411,11 +3411,11 @@ __ocb_decrypt1:
.byte 102,15,56,223,215
.byte 0xf3,0xc3
.size __ocb_decrypt1,.-__ocb_decrypt1
.globl aesni_cbc_encrypt
.hidden aesni_cbc_encrypt
.type aesni_cbc_encrypt,@function
.globl aes_hw_cbc_encrypt
.hidden aes_hw_cbc_encrypt
.type aes_hw_cbc_encrypt,@function
.align 16
aesni_cbc_encrypt:
aes_hw_cbc_encrypt:
.cfi_startproc
testq %rdx,%rdx
jz .Lcbc_ret
@@ -4003,12 +4003,12 @@ aesni_cbc_encrypt:
.Lcbc_ret:
.byte 0xf3,0xc3
.cfi_endproc
.size aesni_cbc_encrypt,.-aesni_cbc_encrypt
.globl aesni_set_decrypt_key
.hidden aesni_set_decrypt_key
.type aesni_set_decrypt_key,@function
.size aes_hw_cbc_encrypt,.-aes_hw_cbc_encrypt
.globl aes_hw_set_decrypt_key
.hidden aes_hw_set_decrypt_key
.type aes_hw_set_decrypt_key,@function
.align 16
aesni_set_decrypt_key:
aes_hw_set_decrypt_key:
.cfi_startproc
.byte 0x48,0x83,0xEC,0x08
.cfi_adjust_cfa_offset 8
@@ -4048,12 +4048,12 @@ aesni_set_decrypt_key:
.byte 0xf3,0xc3
.cfi_endproc
.LSEH_end_set_decrypt_key:
.size aesni_set_decrypt_key,.-aesni_set_decrypt_key
.globl aesni_set_encrypt_key
.hidden aesni_set_encrypt_key
.type aesni_set_encrypt_key,@function
.size aes_hw_set_decrypt_key,.-aes_hw_set_decrypt_key
.globl aes_hw_set_encrypt_key
.hidden aes_hw_set_encrypt_key
.type aes_hw_set_encrypt_key,@function
.align 16
aesni_set_encrypt_key:
aes_hw_set_encrypt_key:
__aesni_set_encrypt_key:
.cfi_startproc
.byte 0x48,0x83,0xEC,0x08
@@ -4424,7 +4424,7 @@ __aesni_set_encrypt_key:
shufps $170,%xmm1,%xmm1
xorps %xmm1,%xmm2
.byte 0xf3,0xc3
.size aesni_set_encrypt_key,.-aesni_set_encrypt_key
.size aes_hw_set_encrypt_key,.-aes_hw_set_encrypt_key
.size __aesni_set_encrypt_key,.-__aesni_set_encrypt_key
.align 64
.Lbswap_mask:

View File

@@ -1,10 +1,10 @@
#if defined(__x86_64__) && !defined(OPENSSL_NO_ASM)
.text
.extern asm_AES_encrypt
.hidden asm_AES_encrypt
.extern asm_AES_decrypt
.hidden asm_AES_decrypt
.extern aes_nohw_encrypt
.hidden aes_nohw_encrypt
.extern aes_nohw_decrypt
.hidden aes_nohw_decrypt
.type _bsaes_encrypt8,@function
.align 64
@@ -1065,8 +1065,8 @@ _bsaes_key_convert:
.byte 0xf3,0xc3
.size _bsaes_key_convert,.-_bsaes_key_convert
.extern asm_AES_cbc_encrypt
.hidden asm_AES_cbc_encrypt
.extern aes_nohw_cbc_encrypt
.hidden aes_nohw_cbc_encrypt
.globl bsaes_cbc_encrypt
.hidden bsaes_cbc_encrypt
.type bsaes_cbc_encrypt,@function
@@ -1074,9 +1074,9 @@ _bsaes_key_convert:
bsaes_cbc_encrypt:
.cfi_startproc
cmpl $0,%r9d
jne asm_AES_cbc_encrypt
jne aes_nohw_cbc_encrypt
cmpq $128,%rdx
jb asm_AES_cbc_encrypt
jb aes_nohw_cbc_encrypt
movq %rsp,%rax
.Lcbc_dec_prologue:
@@ -1304,7 +1304,7 @@ bsaes_cbc_encrypt:
leaq (%r12),%rdi
leaq 32(%rbp),%rsi
leaq (%r15),%rdx
call asm_AES_decrypt
call aes_nohw_decrypt
pxor 32(%rbp),%xmm14
movdqu %xmm14,(%r13)
movdqa %xmm15,%xmm14
@@ -1520,7 +1520,7 @@ bsaes_ctr32_encrypt_blocks:
leaq 32(%rbp),%rdi
leaq 48(%rbp),%rsi
leaq (%r15),%rdx
call asm_AES_encrypt
call aes_nohw_encrypt
movdqu (%r12),%xmm0
leaq 16(%r12),%r12
movl 44(%rbp),%eax
@@ -1602,7 +1602,7 @@ bsaes_xts_encrypt:
leaq (%r9),%rdi
leaq 32(%rbp),%rsi
leaq (%r8),%rdx
call asm_AES_encrypt
call aes_nohw_encrypt
movl 240(%r15),%eax
movq %r14,%rbx
@@ -1972,7 +1972,7 @@ bsaes_xts_encrypt:
leaq 32(%rbp),%rdi
leaq 32(%rbp),%rsi
leaq (%r15),%rdx
call asm_AES_encrypt
call aes_nohw_encrypt
pxor 32(%rbp),%xmm15
@@ -2005,7 +2005,7 @@ bsaes_xts_encrypt:
leaq 32(%rbp),%rsi
movdqa %xmm15,32(%rbp)
leaq (%r15),%rdx
call asm_AES_encrypt
call aes_nohw_encrypt
pxor 32(%rbp),%xmm6
movdqu %xmm6,-16(%r13)
@@ -2077,7 +2077,7 @@ bsaes_xts_decrypt:
leaq (%r9),%rdi
leaq 32(%rbp),%rsi
leaq (%r8),%rdx
call asm_AES_encrypt
call aes_nohw_encrypt
movl 240(%r15),%eax
movq %r14,%rbx
@@ -2454,7 +2454,7 @@ bsaes_xts_decrypt:
leaq 32(%rbp),%rdi
leaq 32(%rbp),%rsi
leaq (%r15),%rdx
call asm_AES_decrypt
call aes_nohw_decrypt
pxor 32(%rbp),%xmm15
@@ -2485,7 +2485,7 @@ bsaes_xts_decrypt:
leaq 32(%rbp),%rsi
movdqa %xmm15,32(%rbp)
leaq (%r15),%rdx
call asm_AES_decrypt
call aes_nohw_decrypt
pxor 32(%rbp),%xmm6
movq %r13,%rdx
movdqu %xmm6,(%r13)
@@ -2506,7 +2506,7 @@ bsaes_xts_decrypt:
leaq 32(%rbp),%rsi
movdqa %xmm15,32(%rbp)
leaq (%r15),%rdx
call asm_AES_decrypt
call aes_nohw_decrypt
pxor 32(%rbp),%xmm5
movdqu %xmm5,(%r13)

File diff suppressed because it is too large Load Diff

View File

@@ -1724,6 +1724,11 @@ rsaz_1024_gather5_avx2:
rsaz_avx2_eligible:
leaq OPENSSL_ia32cap_P(%rip),%rax
movl 8(%rax),%eax
movl $524544,%ecx
movl $0,%edx
andl %eax,%ecx
cmpl $524544,%ecx
cmovel %edx,%eax
andl $32,%eax
shrl $5,%eax
.byte 0xf3,0xc3

View File

@@ -17,6 +17,8 @@ bn_mul_mont:
jnz .Lmul_enter
cmpl $8,%r9d
jb .Lmul_enter
leaq OPENSSL_ia32cap_P(%rip),%r11
movl 8(%r11),%r11d
cmpq %rsi,%rdx
jne .Lmul4x_enter
testl $7,%r9d
@@ -208,30 +210,30 @@ bn_mul_mont:
xorq %r14,%r14
movq (%rsp),%rax
leaq (%rsp),%rsi
movq %r9,%r15
jmp .Lsub
.align 16
.Lsub: sbbq (%rcx,%r14,8),%rax
movq %rax,(%rdi,%r14,8)
movq 8(%rsi,%r14,8),%rax
movq 8(%rsp,%r14,8),%rax
leaq 1(%r14),%r14
decq %r15
jnz .Lsub
sbbq $0,%rax
movq $-1,%rbx
xorq %rax,%rbx
xorq %r14,%r14
andq %rax,%rsi
notq %rax
movq %rdi,%rcx
andq %rax,%rcx
movq %r9,%r15
orq %rcx,%rsi
.align 16
.Lcopy:
movq (%rsi,%r14,8),%rax
movq %r14,(%rsp,%r14,8)
movq %rax,(%rdi,%r14,8)
movq (%rdi,%r14,8),%rcx
movq (%rsp,%r14,8),%rdx
andq %rbx,%rcx
andq %rax,%rdx
movq %r9,(%rsp,%r14,8)
orq %rcx,%rdx
movq %rdx,(%rdi,%r14,8)
leaq 1(%r14),%r14
subq $1,%r15
jnz .Lcopy
@@ -265,6 +267,9 @@ bn_mul4x_mont:
movq %rsp,%rax
.cfi_def_cfa_register %rax
.Lmul4x_enter:
andl $0x80100,%r11d
cmpl $0x80100,%r11d
je .Lmulx4x_enter
pushq %rbx
.cfi_offset %rbx,-16
pushq %rbp
@@ -602,7 +607,6 @@ bn_mul4x_mont:
movq 16(%rsp,%r9,8),%rdi
leaq -4(%r9),%r15
movq 0(%rsp),%rax
pxor %xmm0,%xmm0
movq 8(%rsp),%rdx
shrq $2,%r15
leaq (%rsp),%rsi
@@ -612,8 +616,7 @@ bn_mul4x_mont:
movq 16(%rsi),%rbx
movq 24(%rsi),%rbp
sbbq 8(%rcx),%rdx
jmp .Lsub4x
.align 16
.Lsub4x:
movq %rax,0(%rdi,%r14,8)
movq %rdx,8(%rdi,%r14,8)
@@ -640,34 +643,35 @@ bn_mul4x_mont:
sbbq $0,%rax
movq %rbp,24(%rdi,%r14,8)
xorq %r14,%r14
andq %rax,%rsi
notq %rax
movq %rdi,%rcx
andq %rax,%rcx
leaq -4(%r9),%r15
orq %rcx,%rsi
pxor %xmm0,%xmm0
.byte 102,72,15,110,224
pcmpeqd %xmm5,%xmm5
pshufd $0,%xmm4,%xmm4
movq %r9,%r15
pxor %xmm4,%xmm5
shrq $2,%r15
xorl %eax,%eax
movdqu (%rsi),%xmm1
movdqa %xmm0,(%rsp)
movdqu %xmm1,(%rdi)
jmp .Lcopy4x
.align 16
.Lcopy4x:
movdqu 16(%rsi,%r14,1),%xmm2
movdqu 32(%rsi,%r14,1),%xmm1
movdqa %xmm0,16(%rsp,%r14,1)
movdqu %xmm2,16(%rdi,%r14,1)
movdqa %xmm0,32(%rsp,%r14,1)
movdqu %xmm1,32(%rdi,%r14,1)
leaq 32(%r14),%r14
movdqa (%rsp,%rax,1),%xmm1
movdqu (%rdi,%rax,1),%xmm2
pand %xmm4,%xmm1
pand %xmm5,%xmm2
movdqa 16(%rsp,%rax,1),%xmm3
movdqa %xmm0,(%rsp,%rax,1)
por %xmm2,%xmm1
movdqu 16(%rdi,%rax,1),%xmm2
movdqu %xmm1,(%rdi,%rax,1)
pand %xmm4,%xmm3
pand %xmm5,%xmm2
movdqa %xmm0,16(%rsp,%rax,1)
por %xmm2,%xmm3
movdqu %xmm3,16(%rdi,%rax,1)
leaq 32(%rax),%rax
decq %r15
jnz .Lcopy4x
movdqu 16(%rsi,%r14,1),%xmm2
movdqa %xmm0,16(%rsp,%r14,1)
movdqu %xmm2,16(%rdi,%r14,1)
movq 8(%rsp,%r9,8),%rsi
.cfi_def_cfa %rsi, 8
movq $1,%rax
@@ -689,6 +693,8 @@ bn_mul4x_mont:
.byte 0xf3,0xc3
.cfi_endproc
.size bn_mul4x_mont,.-bn_mul4x_mont
.extern bn_sqrx8x_internal
.hidden bn_sqrx8x_internal
.extern bn_sqr8x_internal
.hidden bn_sqr8x_internal
@@ -773,6 +779,26 @@ bn_sqr8x_mont:
pxor %xmm0,%xmm0
.byte 102,72,15,110,207
.byte 102,73,15,110,218
leaq OPENSSL_ia32cap_P(%rip),%rax
movl 8(%rax),%eax
andl $0x80100,%eax
cmpl $0x80100,%eax
jne .Lsqr8x_nox
call bn_sqrx8x_internal
leaq (%r8,%rcx,1),%rbx
movq %rcx,%r9
movq %rcx,%rdx
.byte 102,72,15,126,207
sarq $3+2,%rcx
jmp .Lsqr8x_sub
.align 32
.Lsqr8x_nox:
call bn_sqr8x_internal
@@ -860,6 +886,362 @@ bn_sqr8x_mont:
.byte 0xf3,0xc3
.cfi_endproc
.size bn_sqr8x_mont,.-bn_sqr8x_mont
.type bn_mulx4x_mont,@function
.align 32
bn_mulx4x_mont:
.cfi_startproc
movq %rsp,%rax
.cfi_def_cfa_register %rax
.Lmulx4x_enter:
pushq %rbx
.cfi_offset %rbx,-16
pushq %rbp
.cfi_offset %rbp,-24
pushq %r12
.cfi_offset %r12,-32
pushq %r13
.cfi_offset %r13,-40
pushq %r14
.cfi_offset %r14,-48
pushq %r15
.cfi_offset %r15,-56
.Lmulx4x_prologue:
shll $3,%r9d
xorq %r10,%r10
subq %r9,%r10
movq (%r8),%r8
leaq -72(%rsp,%r10,1),%rbp
andq $-128,%rbp
movq %rsp,%r11
subq %rbp,%r11
andq $-4096,%r11
leaq (%r11,%rbp,1),%rsp
movq (%rsp),%r10
cmpq %rbp,%rsp
ja .Lmulx4x_page_walk
jmp .Lmulx4x_page_walk_done
.align 16
.Lmulx4x_page_walk:
leaq -4096(%rsp),%rsp
movq (%rsp),%r10
cmpq %rbp,%rsp
ja .Lmulx4x_page_walk
.Lmulx4x_page_walk_done:
leaq (%rdx,%r9,1),%r10
movq %r9,0(%rsp)
shrq $5,%r9
movq %r10,16(%rsp)
subq $1,%r9
movq %r8,24(%rsp)
movq %rdi,32(%rsp)
movq %rax,40(%rsp)
.cfi_escape 0x0f,0x05,0x77,0x28,0x06,0x23,0x08
movq %r9,48(%rsp)
jmp .Lmulx4x_body
.align 32
.Lmulx4x_body:
leaq 8(%rdx),%rdi
movq (%rdx),%rdx
leaq 64+32(%rsp),%rbx
movq %rdx,%r9
mulxq 0(%rsi),%r8,%rax
mulxq 8(%rsi),%r11,%r14
addq %rax,%r11
movq %rdi,8(%rsp)
mulxq 16(%rsi),%r12,%r13
adcq %r14,%r12
adcq $0,%r13
movq %r8,%rdi
imulq 24(%rsp),%r8
xorq %rbp,%rbp
mulxq 24(%rsi),%rax,%r14
movq %r8,%rdx
leaq 32(%rsi),%rsi
adcxq %rax,%r13
adcxq %rbp,%r14
mulxq 0(%rcx),%rax,%r10
adcxq %rax,%rdi
adoxq %r11,%r10
mulxq 8(%rcx),%rax,%r11
adcxq %rax,%r10
adoxq %r12,%r11
.byte 0xc4,0x62,0xfb,0xf6,0xa1,0x10,0x00,0x00,0x00
movq 48(%rsp),%rdi
movq %r10,-32(%rbx)
adcxq %rax,%r11
adoxq %r13,%r12
mulxq 24(%rcx),%rax,%r15
movq %r9,%rdx
movq %r11,-24(%rbx)
adcxq %rax,%r12
adoxq %rbp,%r15
leaq 32(%rcx),%rcx
movq %r12,-16(%rbx)
jmp .Lmulx4x_1st
.align 32
.Lmulx4x_1st:
adcxq %rbp,%r15
mulxq 0(%rsi),%r10,%rax
adcxq %r14,%r10
mulxq 8(%rsi),%r11,%r14
adcxq %rax,%r11
mulxq 16(%rsi),%r12,%rax
adcxq %r14,%r12
mulxq 24(%rsi),%r13,%r14
.byte 0x67,0x67
movq %r8,%rdx
adcxq %rax,%r13
adcxq %rbp,%r14
leaq 32(%rsi),%rsi
leaq 32(%rbx),%rbx
adoxq %r15,%r10
mulxq 0(%rcx),%rax,%r15
adcxq %rax,%r10
adoxq %r15,%r11
mulxq 8(%rcx),%rax,%r15
adcxq %rax,%r11
adoxq %r15,%r12
mulxq 16(%rcx),%rax,%r15
movq %r10,-40(%rbx)
adcxq %rax,%r12
movq %r11,-32(%rbx)
adoxq %r15,%r13
mulxq 24(%rcx),%rax,%r15
movq %r9,%rdx
movq %r12,-24(%rbx)
adcxq %rax,%r13
adoxq %rbp,%r15
leaq 32(%rcx),%rcx
movq %r13,-16(%rbx)
decq %rdi
jnz .Lmulx4x_1st
movq 0(%rsp),%rax
movq 8(%rsp),%rdi
adcq %rbp,%r15
addq %r15,%r14
sbbq %r15,%r15
movq %r14,-8(%rbx)
jmp .Lmulx4x_outer
.align 32
.Lmulx4x_outer:
movq (%rdi),%rdx
leaq 8(%rdi),%rdi
subq %rax,%rsi
movq %r15,(%rbx)
leaq 64+32(%rsp),%rbx
subq %rax,%rcx
mulxq 0(%rsi),%r8,%r11
xorl %ebp,%ebp
movq %rdx,%r9
mulxq 8(%rsi),%r14,%r12
adoxq -32(%rbx),%r8
adcxq %r14,%r11
mulxq 16(%rsi),%r15,%r13
adoxq -24(%rbx),%r11
adcxq %r15,%r12
adoxq -16(%rbx),%r12
adcxq %rbp,%r13
adoxq %rbp,%r13
movq %rdi,8(%rsp)
movq %r8,%r15
imulq 24(%rsp),%r8
xorl %ebp,%ebp
mulxq 24(%rsi),%rax,%r14
movq %r8,%rdx
adcxq %rax,%r13
adoxq -8(%rbx),%r13
adcxq %rbp,%r14
leaq 32(%rsi),%rsi
adoxq %rbp,%r14
mulxq 0(%rcx),%rax,%r10
adcxq %rax,%r15
adoxq %r11,%r10
mulxq 8(%rcx),%rax,%r11
adcxq %rax,%r10
adoxq %r12,%r11
mulxq 16(%rcx),%rax,%r12
movq %r10,-32(%rbx)
adcxq %rax,%r11
adoxq %r13,%r12
mulxq 24(%rcx),%rax,%r15
movq %r9,%rdx
movq %r11,-24(%rbx)
leaq 32(%rcx),%rcx
adcxq %rax,%r12
adoxq %rbp,%r15
movq 48(%rsp),%rdi
movq %r12,-16(%rbx)
jmp .Lmulx4x_inner
.align 32
.Lmulx4x_inner:
mulxq 0(%rsi),%r10,%rax
adcxq %rbp,%r15
adoxq %r14,%r10
mulxq 8(%rsi),%r11,%r14
adcxq 0(%rbx),%r10
adoxq %rax,%r11
mulxq 16(%rsi),%r12,%rax
adcxq 8(%rbx),%r11
adoxq %r14,%r12
mulxq 24(%rsi),%r13,%r14
movq %r8,%rdx
adcxq 16(%rbx),%r12
adoxq %rax,%r13
adcxq 24(%rbx),%r13
adoxq %rbp,%r14
leaq 32(%rsi),%rsi
leaq 32(%rbx),%rbx
adcxq %rbp,%r14
adoxq %r15,%r10
mulxq 0(%rcx),%rax,%r15
adcxq %rax,%r10
adoxq %r15,%r11
mulxq 8(%rcx),%rax,%r15
adcxq %rax,%r11
adoxq %r15,%r12
mulxq 16(%rcx),%rax,%r15
movq %r10,-40(%rbx)
adcxq %rax,%r12
adoxq %r15,%r13
mulxq 24(%rcx),%rax,%r15
movq %r9,%rdx
movq %r11,-32(%rbx)
movq %r12,-24(%rbx)
adcxq %rax,%r13
adoxq %rbp,%r15
leaq 32(%rcx),%rcx
movq %r13,-16(%rbx)
decq %rdi
jnz .Lmulx4x_inner
movq 0(%rsp),%rax
movq 8(%rsp),%rdi
adcq %rbp,%r15
subq 0(%rbx),%rbp
adcq %r15,%r14
sbbq %r15,%r15
movq %r14,-8(%rbx)
cmpq 16(%rsp),%rdi
jne .Lmulx4x_outer
leaq 64(%rsp),%rbx
subq %rax,%rcx
negq %r15
movq %rax,%rdx
shrq $3+2,%rax
movq 32(%rsp),%rdi
jmp .Lmulx4x_sub
.align 32
.Lmulx4x_sub:
movq 0(%rbx),%r11
movq 8(%rbx),%r12
movq 16(%rbx),%r13
movq 24(%rbx),%r14
leaq 32(%rbx),%rbx
sbbq 0(%rcx),%r11
sbbq 8(%rcx),%r12
sbbq 16(%rcx),%r13
sbbq 24(%rcx),%r14
leaq 32(%rcx),%rcx
movq %r11,0(%rdi)
movq %r12,8(%rdi)
movq %r13,16(%rdi)
movq %r14,24(%rdi)
leaq 32(%rdi),%rdi
decq %rax
jnz .Lmulx4x_sub
sbbq $0,%r15
leaq 64(%rsp),%rbx
subq %rdx,%rdi
.byte 102,73,15,110,207
pxor %xmm0,%xmm0
pshufd $0,%xmm1,%xmm1
movq 40(%rsp),%rsi
.cfi_def_cfa %rsi,8
jmp .Lmulx4x_cond_copy
.align 32
.Lmulx4x_cond_copy:
movdqa 0(%rbx),%xmm2
movdqa 16(%rbx),%xmm3
leaq 32(%rbx),%rbx
movdqu 0(%rdi),%xmm4
movdqu 16(%rdi),%xmm5
leaq 32(%rdi),%rdi
movdqa %xmm0,-32(%rbx)
movdqa %xmm0,-16(%rbx)
pcmpeqd %xmm1,%xmm0
pand %xmm1,%xmm2
pand %xmm1,%xmm3
pand %xmm0,%xmm4
pand %xmm0,%xmm5
pxor %xmm0,%xmm0
por %xmm2,%xmm4
por %xmm3,%xmm5
movdqu %xmm4,-32(%rdi)
movdqu %xmm5,-16(%rdi)
subq $32,%rdx
jnz .Lmulx4x_cond_copy
movq %rdx,(%rbx)
movq $1,%rax
movq -48(%rsi),%r15
.cfi_restore %r15
movq -40(%rsi),%r14
.cfi_restore %r14
movq -32(%rsi),%r13
.cfi_restore %r13
movq -24(%rsi),%r12
.cfi_restore %r12
movq -16(%rsi),%rbp
.cfi_restore %rbp
movq -8(%rsi),%rbx
.cfi_restore %rbx
leaq (%rsi),%rsp
.cfi_def_cfa_register %rsp
.Lmulx4x_epilogue:
.byte 0xf3,0xc3
.cfi_endproc
.size bn_mulx4x_mont,.-bn_mulx4x_mont
.byte 77,111,110,116,103,111,109,101,114,121,32,77,117,108,116,105,112,108,105,99,97,116,105,111,110,32,102,111,114,32,120,56,54,95,54,52,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0
.align 16
#endif

File diff suppressed because it is too large Load Diff