erasure-code/isa/isa-l/erasure_code/gf_vect_dot_prod_avx2.asm.s \
erasure-code/isa/isa-l/erasure_code/gf_vect_dot_prod_avx.asm.s \
erasure-code/isa/isa-l/erasure_code/gf_vect_dot_prod_sse.asm.s \
+ erasure-code/isa/isa-l/erasure_code/gf_2vect_mad_avx2.asm.s \
+ erasure-code/isa/isa-l/erasure_code/gf_2vect_mad_avx.asm.s \
+ erasure-code/isa/isa-l/erasure_code/gf_2vect_mad_sse.asm.s \
+ erasure-code/isa/isa-l/erasure_code/gf_3vect_mad_avx2.asm.s \
+ erasure-code/isa/isa-l/erasure_code/gf_3vect_mad_avx.asm.s \
+ erasure-code/isa/isa-l/erasure_code/gf_3vect_mad_sse.asm.s \
+ erasure-code/isa/isa-l/erasure_code/gf_4vect_mad_avx2.asm.s \
+ erasure-code/isa/isa-l/erasure_code/gf_4vect_mad_avx.asm.s \
+ erasure-code/isa/isa-l/erasure_code/gf_4vect_mad_sse.asm.s \
+ erasure-code/isa/isa-l/erasure_code/gf_5vect_mad_avx2.asm.s \
+ erasure-code/isa/isa-l/erasure_code/gf_5vect_mad_avx.asm.s \
+ erasure-code/isa/isa-l/erasure_code/gf_5vect_mad_sse.asm.s \
+ erasure-code/isa/isa-l/erasure_code/gf_6vect_mad_avx2.asm.s \
+ erasure-code/isa/isa-l/erasure_code/gf_6vect_mad_avx.asm.s \
+ erasure-code/isa/isa-l/erasure_code/gf_6vect_mad_sse.asm.s \
+ erasure-code/isa/isa-l/erasure_code/gf_vect_mad_avx2.asm.s \
+ erasure-code/isa/isa-l/erasure_code/gf_vect_mad_avx.asm.s \
+ erasure-code/isa/isa-l/erasure_code/gf_vect_mad_sse.asm.s \
erasure-code/isa/isa-l/erasure_code/gf_vect_mul_avx.asm.s \
erasure-code/isa/isa-l/erasure_code/gf_vect_mul_sse.asm.s \
erasure-code/isa/ErasureCodeIsa.cc \
libec_isa_la_CCASFLAGS = ${AM_CCASFLAGS} -I $(abs_srcdir)/erasure-code/isa/isa-l/include/
libec_isa_la_LIBADD = $(LIBCRUSH) $(PTHREAD_LIBS) $(EXTRALIBS)
-libec_isa_la_LDFLAGS = ${AM_LDFLAGS} -version-info 2:10:0
+libec_isa_la_LDFLAGS = ${AM_LDFLAGS} -version-info 2:13:0
if LINUX
libec_isa_la_LDFLAGS += -export-symbols-regex '.*__erasure_code_.*'
endif
/**********************************************************************
- Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+ Copyright(c) 2011-2015 Intel Corporation All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
}
}
+void gf_vect_mad_base(int len, int vec, int vec_i,
+ unsigned char *v, unsigned char *src, unsigned char *dest)
+{
+ int i;
+ unsigned char s;
+ for (i = 0; i < len; i++) {
+ s = dest[i];
+ s ^= gf_mul(src[i], v[vec_i * 32 + 1]);
+ dest[i] = s;
+ }
+}
+
void ec_encode_data_base(int len, int srcs, int dests, unsigned char *v,
unsigned char **src, unsigned char **dest)
{
}
}
+void ec_encode_data_update_base(int len, int k, int rows, int vec_i, unsigned char *v,
+ unsigned char *data, unsigned char **dest)
+{
+ int i, l;
+ unsigned char s;
+
+ for (l = 0; l < rows; l++) {
+ for (i = 0; i < len; i++) {
+ s = dest[l][i];
+ s ^= gf_mul(data[i], v[vec_i * 32 + l * k * 32 + 1]);
+
+ dest[l][i] = s;
+ }
+ }
+}
+
void gf_vect_mul_base(int len, unsigned char *a, unsigned char *src, unsigned char *dest)
{
//2nd element of table array is ref value used to fill it in
/**********************************************************************
- Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+ Copyright(c) 2011-2015 Intel Corporation All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
/**********************************************************************
- Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+ Copyright(c) 2011-2015 Intel Corporation All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
}
}
-#if __WORDSIZE == 64 || _WIN64 || __x86_64__
void ec_encode_data_sse(int len, int k, int rows, unsigned char *g_tbls, unsigned char **data,
unsigned char **coding)
{
void ec_encode_data_avx(int len, int k, int rows, unsigned char *g_tbls, unsigned char **data,
unsigned char **coding)
{
-
if (len < 16) {
ec_encode_data_base(len, k, rows, g_tbls, data, coding);
return;
}
+#if __WORDSIZE == 64 || _WIN64 || __x86_64__
+
+void ec_encode_data_update_sse(int len, int k, int rows, int vec_i, unsigned char *g_tbls,
+ unsigned char *data, unsigned char **coding)
+{
+ if (len < 16) {
+ ec_encode_data_update_base(len, k, rows, vec_i, g_tbls, data, coding);
+ return;
+ }
+
+ while (rows > 6) {
+ gf_6vect_mad_sse(len, k, vec_i, g_tbls, data, coding);
+ g_tbls += 6 * k * 32;
+ coding += 6;
+ rows -= 6;
+ }
+ switch (rows) {
+ case 6:
+ gf_6vect_mad_sse(len, k, vec_i, g_tbls, data, coding);
+ break;
+ case 5:
+ gf_5vect_mad_sse(len, k, vec_i, g_tbls, data, coding);
+ break;
+ case 4:
+ gf_4vect_mad_sse(len, k, vec_i, g_tbls, data, coding);
+ break;
+ case 3:
+ gf_3vect_mad_sse(len, k, vec_i, g_tbls, data, coding);
+ break;
+ case 2:
+ gf_2vect_mad_sse(len, k, vec_i, g_tbls, data, coding);
+ break;
+ case 1:
+ gf_vect_mad_sse(len, k, vec_i, g_tbls, data, *coding);
+ break;
+ case 0:
+ break;
+ }
+
+}
+
+void ec_encode_data_update_avx(int len, int k, int rows, int vec_i, unsigned char *g_tbls,
+ unsigned char *data, unsigned char **coding)
+{
+ if (len < 16) {
+ ec_encode_data_update_base(len, k, rows, vec_i, g_tbls, data, coding);
+ return;
+ }
+ while (rows > 6) {
+ gf_6vect_mad_avx(len, k, vec_i, g_tbls, data, coding);
+ g_tbls += 6 * k * 32;
+ coding += 6;
+ rows -= 6;
+ }
+ switch (rows) {
+ case 6:
+ gf_6vect_mad_avx(len, k, vec_i, g_tbls, data, coding);
+ break;
+ case 5:
+ gf_5vect_mad_avx(len, k, vec_i, g_tbls, data, coding);
+ break;
+ case 4:
+ gf_4vect_mad_avx(len, k, vec_i, g_tbls, data, coding);
+ break;
+ case 3:
+ gf_3vect_mad_avx(len, k, vec_i, g_tbls, data, coding);
+ break;
+ case 2:
+ gf_2vect_mad_avx(len, k, vec_i, g_tbls, data, coding);
+ break;
+ case 1:
+ gf_vect_mad_avx(len, k, vec_i, g_tbls, data, *coding);
+ break;
+ case 0:
+ break;
+ }
+
+}
+
+void ec_encode_data_update_avx2(int len, int k, int rows, int vec_i, unsigned char *g_tbls,
+ unsigned char *data, unsigned char **coding)
+{
+ if (len < 32) {
+ ec_encode_data_update_base(len, k, rows, vec_i, g_tbls, data, coding);
+ return;
+ }
+ while (rows > 6) {
+ gf_6vect_mad_avx2(len, k, vec_i, g_tbls, data, coding);
+ g_tbls += 6 * k * 32;
+ coding += 6;
+ rows -= 6;
+ }
+ switch (rows) {
+ case 6:
+ gf_6vect_mad_avx2(len, k, vec_i, g_tbls, data, coding);
+ break;
+ case 5:
+ gf_5vect_mad_avx2(len, k, vec_i, g_tbls, data, coding);
+ break;
+ case 4:
+ gf_4vect_mad_avx2(len, k, vec_i, g_tbls, data, coding);
+ break;
+ case 3:
+ gf_3vect_mad_avx2(len, k, vec_i, g_tbls, data, coding);
+ break;
+ case 2:
+ gf_2vect_mad_avx2(len, k, vec_i, g_tbls, data, coding);
+ break;
+ case 1:
+ gf_vect_mad_avx2(len, k, vec_i, g_tbls, data, *coding);
+ break;
+ case 0:
+ break;
+ }
+
+}
+
#endif //__WORDSIZE == 64 || _WIN64 || __x86_64__
struct slver {
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
%ifidn __OUTPUT_FORMAT__, elf64
-%define WRT_OPT wrt ..plt
+ %define WRT_OPT wrt ..plt
%else
-%define WRT_OPT
+ %define WRT_OPT
%endif
+%include "reg_sizes.asm"
+
%ifidn __OUTPUT_FORMAT__, elf32
[bits 32]
-%define def_wrd dd
-%define wrd_sz dword
-%define arg1 esi
+ %define def_wrd dd
+ %define wrd_sz dword
+ %define arg1 esi
+ %define arg2 eax
+ %define arg3 ebx
+ %define arg4 ecx
+ %define arg5 edx
%else
-%include "reg_sizes.asm"
-default rel
-[bits 64]
+ default rel
+ [bits 64]
-%define def_wrd dq
-%define wrd_sz qword
-%define arg1 rsi
+ %define def_wrd dq
+ %define wrd_sz qword
+ %define arg1 rsi
+ %define arg2 rax
+ %define arg3 rbx
+ %define arg4 rcx
+ %define arg5 rdx
-extern ec_encode_data_sse
-extern ec_encode_data_avx
-extern ec_encode_data_avx2
-extern gf_vect_mul_sse
-extern gf_vect_mul_avx
-extern gf_vect_dot_prod_sse
-extern gf_vect_dot_prod_avx
-extern gf_vect_dot_prod_avx2
+
+ extern ec_encode_data_update_sse
+ extern ec_encode_data_update_avx
+ extern ec_encode_data_update_avx2
+ extern gf_vect_mul_sse
+ extern gf_vect_mul_avx
+
+ extern gf_vect_mad_sse
+ extern gf_vect_mad_avx
+ extern gf_vect_mad_avx2
%endif
extern gf_vect_mul_base
extern ec_encode_data_base
+extern ec_encode_data_update_base
extern gf_vect_dot_prod_base
+extern gf_vect_mad_base
+
+extern gf_vect_dot_prod_sse
+extern gf_vect_dot_prod_avx
+extern gf_vect_dot_prod_avx2
+extern ec_encode_data_sse
+extern ec_encode_data_avx
+extern ec_encode_data_avx2
+
section .data
;;; *_mbinit are initial values for *_dispatched; is updated on first call.
gf_vect_dot_prod_dispatched:
def_wrd gf_vect_dot_prod_mbinit
+ec_encode_data_update_dispatched:
+ def_wrd ec_encode_data_update_mbinit
+
+gf_vect_mad_dispatched:
+ def_wrd gf_vect_mad_mbinit
+
section .text
;;;;
; ec_encode_data multibinary function
ec_encode_data_dispatch_init:
push arg1
-%ifidn __OUTPUT_FORMAT__, elf32 ;; 32-bit check
- lea arg1, [ec_encode_data_base]
-%else
- push rax
- push rbx
- push rcx
- push rdx
+ push arg2
+ push arg3
+ push arg4
+ push arg5
lea arg1, [ec_encode_data_base WRT_OPT] ; Default
mov eax, 1
cpuid
- lea rbx, [ec_encode_data_sse WRT_OPT]
+ lea arg3, [ec_encode_data_sse WRT_OPT]
test ecx, FLAG_CPUID1_ECX_SSE4_1
- cmovne arg1, rbx
+ cmovne arg1, arg3
and ecx, (FLAG_CPUID1_ECX_AVX | FLAG_CPUID1_ECX_OSXSAVE)
cmp ecx, (FLAG_CPUID1_ECX_AVX | FLAG_CPUID1_ECX_OSXSAVE)
- lea rbx, [ec_encode_data_avx WRT_OPT]
+ lea arg3, [ec_encode_data_avx WRT_OPT]
jne _done_ec_encode_data_init
- mov rsi, rbx
+ mov arg1, arg3
;; Try for AVX2
xor ecx, ecx
mov eax, 7
cpuid
test ebx, FLAG_CPUID1_EBX_AVX2
- lea rbx, [ec_encode_data_avx2 WRT_OPT]
- cmovne rsi, rbx
-
+ lea arg3, [ec_encode_data_avx2 WRT_OPT]
+ cmovne arg1, arg3
;; Does it have xmm and ymm support
xor ecx, ecx
xgetbv
and eax, FLAG_XGETBV_EAX_XMM_YMM
cmp eax, FLAG_XGETBV_EAX_XMM_YMM
je _done_ec_encode_data_init
- lea rsi, [ec_encode_data_sse WRT_OPT]
+ lea arg1, [ec_encode_data_sse WRT_OPT]
_done_ec_encode_data_init:
- pop rdx
- pop rcx
- pop rbx
- pop rax
-%endif ;; END 32-bit check
+ pop arg5
+ pop arg4
+ pop arg3
+ pop arg2
mov [ec_encode_data_dispatched], arg1
pop arg1
ret
pop arg1
ret
+;;;;
+; ec_encode_data_update multibinary function
+;;;;
+global ec_encode_data_update:function
+ec_encode_data_update_mbinit:
+ call ec_encode_data_update_dispatch_init
+
+ec_encode_data_update:
+ jmp wrd_sz [ec_encode_data_update_dispatched]
+
+ec_encode_data_update_dispatch_init:
+ push arg1
+%ifidn __OUTPUT_FORMAT__, elf32 ;; 32-bit check
+ lea arg1, [ec_encode_data_update_base]
+%else
+ push rax
+ push rbx
+ push rcx
+ push rdx
+ lea arg1, [ec_encode_data_update_base WRT_OPT] ; Default
+
+ mov eax, 1
+ cpuid
+ lea rbx, [ec_encode_data_update_sse WRT_OPT]
+ test ecx, FLAG_CPUID1_ECX_SSE4_1
+ cmovne arg1, rbx
+
+ and ecx, (FLAG_CPUID1_ECX_AVX | FLAG_CPUID1_ECX_OSXSAVE)
+ cmp ecx, (FLAG_CPUID1_ECX_AVX | FLAG_CPUID1_ECX_OSXSAVE)
+ lea rbx, [ec_encode_data_update_avx WRT_OPT]
+
+ jne _done_ec_encode_data_update_init
+ mov rsi, rbx
+
+ ;; Try for AVX2
+ xor ecx, ecx
+ mov eax, 7
+ cpuid
+ test ebx, FLAG_CPUID1_EBX_AVX2
+ lea rbx, [ec_encode_data_update_avx2 WRT_OPT]
+ cmovne rsi, rbx
+
+ ;; Does it have xmm and ymm support
+ xor ecx, ecx
+ xgetbv
+ and eax, FLAG_XGETBV_EAX_XMM_YMM
+ cmp eax, FLAG_XGETBV_EAX_XMM_YMM
+ je _done_ec_encode_data_update_init
+ lea rsi, [ec_encode_data_update_sse WRT_OPT]
+
+_done_ec_encode_data_update_init:
+ pop rdx
+ pop rcx
+ pop rbx
+ pop rax
+%endif ;; END 32-bit check
+ mov [ec_encode_data_update_dispatched], arg1
+ pop arg1
+ ret
;;;;
; gf_vect_dot_prod multibinary function
gf_vect_dot_prod_dispatch_init:
push arg1
+ push arg2
+ push arg3
+ push arg4
+ push arg5
+ lea arg1, [gf_vect_dot_prod_base WRT_OPT] ; Default
+
+ mov eax, 1
+ cpuid
+ lea arg3, [gf_vect_dot_prod_sse WRT_OPT]
+ test ecx, FLAG_CPUID1_ECX_SSE4_1
+ cmovne arg1, arg3
+
+ and ecx, (FLAG_CPUID1_ECX_AVX | FLAG_CPUID1_ECX_OSXSAVE)
+ cmp ecx, (FLAG_CPUID1_ECX_AVX | FLAG_CPUID1_ECX_OSXSAVE)
+ lea arg3, [gf_vect_dot_prod_avx WRT_OPT]
+
+ jne _done_gf_vect_dot_prod_init
+ mov arg1, arg3
+
+ ;; Try for AVX2
+ xor ecx, ecx
+ mov eax, 7
+ cpuid
+ test ebx, FLAG_CPUID1_EBX_AVX2
+ lea arg3, [gf_vect_dot_prod_avx2 WRT_OPT]
+ cmovne arg1, arg3
+ ;; Does it have xmm and ymm support
+ xor ecx, ecx
+ xgetbv
+ and eax, FLAG_XGETBV_EAX_XMM_YMM
+ cmp eax, FLAG_XGETBV_EAX_XMM_YMM
+ je _done_gf_vect_dot_prod_init
+ lea arg1, [gf_vect_dot_prod_sse WRT_OPT]
+
+_done_gf_vect_dot_prod_init:
+ pop arg5
+ pop arg4
+ pop arg3
+ pop arg2
+ mov [gf_vect_dot_prod_dispatched], arg1
+ pop arg1
+ ret
+
+;;;;
+; gf_vect_mad multibinary function
+;;;;
+global gf_vect_mad:function
+gf_vect_mad_mbinit:
+ call gf_vect_mad_dispatch_init
+
+gf_vect_mad:
+ jmp wrd_sz [gf_vect_mad_dispatched]
+
+gf_vect_mad_dispatch_init:
+ push arg1
%ifidn __OUTPUT_FORMAT__, elf32 ;; 32-bit check
- lea arg1, [gf_vect_dot_prod_base]
+ lea arg1, [gf_vect_mad_base]
%else
push rax
push rbx
push rcx
push rdx
- lea arg1, [gf_vect_dot_prod_base WRT_OPT] ; Default
+ lea arg1, [gf_vect_mad_base WRT_OPT] ; Default
mov eax, 1
cpuid
- lea rbx, [gf_vect_dot_prod_sse WRT_OPT]
+ lea rbx, [gf_vect_mad_sse WRT_OPT]
test ecx, FLAG_CPUID1_ECX_SSE4_1
cmovne arg1, rbx
and ecx, (FLAG_CPUID1_ECX_AVX | FLAG_CPUID1_ECX_OSXSAVE)
cmp ecx, (FLAG_CPUID1_ECX_AVX | FLAG_CPUID1_ECX_OSXSAVE)
- lea rbx, [gf_vect_dot_prod_avx WRT_OPT]
+ lea rbx, [gf_vect_mad_avx WRT_OPT]
- jne _done_gf_vect_dot_prod_init
+ jne _done_gf_vect_mad_init
mov rsi, rbx
;; Try for AVX2
mov eax, 7
cpuid
test ebx, FLAG_CPUID1_EBX_AVX2
- lea rbx, [gf_vect_dot_prod_avx2 WRT_OPT]
+ lea rbx, [gf_vect_mad_avx2 WRT_OPT]
cmovne rsi, rbx
;; Does it have xmm and ymm support
xgetbv
and eax, FLAG_XGETBV_EAX_XMM_YMM
cmp eax, FLAG_XGETBV_EAX_XMM_YMM
- je _done_gf_vect_dot_prod_init
- lea rsi, [gf_vect_dot_prod_sse WRT_OPT]
+ je _done_gf_vect_mad_init
+ lea rsi, [gf_vect_mad_sse WRT_OPT]
-_done_gf_vect_dot_prod_init:
+_done_gf_vect_mad_init:
pop rdx
pop rcx
pop rbx
pop rax
%endif ;; END 32-bit check
- mov [gf_vect_dot_prod_dispatched], arg1
+ mov [gf_vect_mad_dispatched], arg1
pop arg1
ret
db 0x%3, 0x%2
%endmacro
-;;; func core, ver, snum
-slversion ec_encode_data, 00, 02, 0133
-slversion gf_vect_mul, 00, 02, 0134
-slversion gf_vect_dot_prod, 00, 01, 0138
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
+;;; func core, ver, snum
+slversion ec_encode_data, 00, 03, 0133
+slversion gf_vect_mul, 00, 02, 0134
+slversion ec_encode_data_update, 00, 02, 0212
+slversion gf_vect_dot_prod, 00, 02, 0138
+slversion gf_vect_mad, 00, 01, 0213
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_2vect_dot_prod_avx(len, vec, *g_tbls, **buffs, **dests);
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
%define tmp3 r9
%define tmp4 r12 ; must be saved and restored
%define return rax
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
%define PS 8
%define LOG_PS 3
%define tmp3 r13 ; must be saved and restored
%define tmp4 r14 ; must be saved and restored
%define return rax
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
%define PS 8
%define LOG_PS 3
%define stack_size 3*16 + 3*8 ; must be an odd multiple of 8
%endmacro
%endif
+%ifidn __OUTPUT_FORMAT__, elf32
+
+;;;================== High Address;
+;;; arg4
+;;; arg3
+;;; arg2
+;;; arg1
+;;; arg0
+;;; return
+;;;<================= esp of caller
+;;; ebp
+;;;<================= ebp = esp
+;;; var0
+;;; esi
+;;; edi
+;;; ebx
+;;;<================= esp of callee
+;;;
+;;;================== Low Address;
+
+ %define PS 4
+ %define LOG_PS 2
+ %define func(x) x:
+ %define arg(x) [ebp + PS*2 + PS*x]
+ %define var(x) [ebp - PS - PS*x]
+
+ %define trans ecx
+ %define trans2 esi
+ %define arg0 trans ;trans and trans2 are for the variables in stack
+ %define arg0_m arg(0)
+ %define arg1 ebx
+ %define arg2 arg2_m
+ %define arg2_m arg(2)
+ %define arg3 trans
+ %define arg3_m arg(3)
+ %define arg4 trans
+ %define arg4_m arg(4)
+ %define tmp edx
+ %define tmp2 edi
+ %define tmp3 trans2
+ %define tmp4 trans2
+ %define tmp4_m var(0)
+ %define return eax
+ %macro SLDR 2 ;; stack load/restore
+ mov %1, %2
+ %endmacro
+ %define SSTR SLDR
+
+ %macro FUNC_SAVE 0
+ push ebp
+ mov ebp, esp
+ sub esp, PS*1 ;1 local variable
+ push esi
+ push edi
+ push ebx
+ mov arg1, arg(1)
+ %endmacro
+
+ %macro FUNC_RESTORE 0
+ pop ebx
+ pop edi
+ pop esi
+ add esp, PS*1 ;1 local variable
+ pop ebp
+ %endmacro
+
+%endif ; output formats
+
%define len arg0
%define vec arg1
%define mul_array arg2
%define src arg3
-%define dest1 arg4
+%define dest1 arg4
%define vec_i tmp2
%define ptr tmp3
%define dest2 tmp4
%define pos return
+ %ifidn PS,4 ;32-bit code
+ %define len_m arg0_m
+ %define src_m arg3_m
+ %define dest1_m arg4_m
+ %define dest2_m tmp4_m
+ %endif
+
%ifndef EC_ALIGNED_ADDR
;;; Use Un-aligned load/store
%define XLDR vmovdqu
%endif
%endif
+%ifidn PS,8 ; 64-bit code
+ default rel
+ [bits 64]
+%endif
-default rel
-
-[bits 64]
section .text
-%define xmask0f xmm8
-%define xgft1_lo xmm7
-%define xgft1_hi xmm6
-%define xgft2_lo xmm5
-%define xgft2_hi xmm4
-
-%define x0 xmm0
-%define xtmpa xmm1
-%define xp1 xmm2
-%define xp2 xmm3
+%ifidn PS,8 ;64-bit code
+ %define xmask0f xmm8
+ %define xgft1_lo xmm7
+ %define xgft1_hi xmm6
+ %define xgft2_lo xmm5
+ %define xgft2_hi xmm4
+
+ %define x0 xmm0
+ %define xtmpa xmm1
+ %define xp1 xmm2
+ %define xp2 xmm3
+%else ;32-bit code
+ %define xmask0f xmm4
+ %define xgft1_lo xmm7
+ %define xgft1_hi xmm6
+ %define xgft2_lo xgft1_lo
+ %define xgft2_hi xgft1_hi
+
+ %define x0 xmm0
+ %define xtmpa xmm1
+ %define xp1 xmm2
+ %define xp2 xmm3
+%endif
align 16
global gf_2vect_dot_prod_avx:function
func(gf_2vect_dot_prod_avx)
FUNC_SAVE
+ SLDR len, len_m
sub len, 16
+ SSTR len_m, len
jl .return_fail
xor pos, pos
vmovdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
sal vec, LOG_PS ;vec *= PS. Make vec_i count by PS
+ SLDR dest1, dest1_m
mov dest2, [dest1+PS]
+ SSTR dest2_m, dest2
mov dest1, [dest1]
+ SSTR dest1_m, dest1
.loop16
vpxor xp1, xp1
xor vec_i, vec_i
.next_vect
+ SLDR src, src_m
mov ptr, [src+vec_i]
vmovdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, ..., Ax{0f}
vmovdqu xgft1_hi, [tmp+16] ; " Ax{00}, Ax{10}, ..., Ax{f0}
+ %ifidn PS,8 ; 64-bit code
vmovdqu xgft2_lo, [tmp+vec*(32/PS)] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
vmovdqu xgft2_hi, [tmp+vec*(32/PS)+16] ; " Bx{00}, Bx{10}, ..., Bx{f0}
-
- XLDR x0, [ptr+pos] ;Get next source vector
add tmp, 32
add vec_i, PS
+ %endif
+ XLDR x0, [ptr+pos] ;Get next source vector
vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
vpxor xgft1_hi, xgft1_lo ;GF add high and low partials
vpxor xp1, xgft1_hi ;xp1 += partial
+ %ifidn PS,4 ; 32-bit code
+ vmovdqu xgft2_lo, [tmp+vec*(32/PS)] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
+ vmovdqu xgft2_hi, [tmp+vec*(32/PS)+16] ; " Bx{00}, Bx{10}, ..., Bx{f0}
+ add tmp, 32
+ add vec_i, PS
+ %endif
vpshufb xgft2_hi, x0 ;Lookup mul table of high nibble
vpshufb xgft2_lo, xtmpa ;Lookup mul table of low nibble
vpxor xgft2_hi, xgft2_lo ;GF add high and low partials
cmp vec_i, vec
jl .next_vect
+ SLDR dest1, dest1_m
+ SLDR dest2, dest2_m
XSTR [dest1+pos], xp1
XSTR [dest2+pos], xp2
+ SLDR len, len_m
add pos, 16 ;Loop on 16 bytes at a time
cmp pos, len
jle .loop16
db 0x%3, 0x%2
%endmacro
;;; func core, ver, snum
-slversion gf_2vect_dot_prod_avx, 02, 03, 0191
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
+slversion gf_2vect_dot_prod_avx, 02, 04, 0191
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_2vect_dot_prod_avx2(len, vec, *g_tbls, **buffs, **dests);
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
%define tmp3 r9
%define tmp4 r12 ; must be saved and restored
%define return rax
- %define PS 8
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
+ %define PS 8
%define LOG_PS 3
%define func(x) x:
%define tmp3 r13 ; must be saved and restored
%define tmp4 r14 ; must be saved and restored
%define return rax
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
%define PS 8
%define LOG_PS 3
%define stack_size 3*16 + 3*8 ; must be an odd multiple of 8
%endmacro
%endif
+%ifidn __OUTPUT_FORMAT__, elf32
+
+;;;================== High Address;
+;;; arg4
+;;; arg3
+;;; arg2
+;;; arg1
+;;; arg0
+;;; return
+;;;<================= esp of caller
+;;; ebp
+;;;<================= ebp = esp
+;;; var0
+;;; esi
+;;; edi
+;;; ebx
+;;;<================= esp of callee
+;;;
+;;;================== Low Address;
+
+ %define PS 4
+ %define LOG_PS 2
+ %define func(x) x:
+ %define arg(x) [ebp + PS*2 + PS*x]
+ %define var(x) [ebp - PS - PS*x]
+
+ %define trans ecx
+ %define trans2 esi
+ %define arg0 trans ;trans and trans2 are for the variables in stack
+ %define arg0_m arg(0)
+ %define arg1 ebx
+ %define arg2 arg2_m
+ %define arg2_m arg(2)
+ %define arg3 trans
+ %define arg3_m arg(3)
+ %define arg4 trans
+ %define arg4_m arg(4)
+ %define tmp edx
+ %define tmp.w edx
+ %define tmp.b dl
+ %define tmp2 edi
+ %define tmp3 trans2
+ %define tmp4 trans2
+ %define tmp4_m var(0)
+ %define return eax
+ %macro SLDR 2 ;stack load/restore
+ mov %1, %2
+ %endmacro
+ %define SSTR SLDR
+
+ %macro FUNC_SAVE 0
+ push ebp
+ mov ebp, esp
+ sub esp, PS*1 ;1 local variable
+ push esi
+ push edi
+ push ebx
+ mov arg1, arg(1)
+ %endmacro
+
+ %macro FUNC_RESTORE 0
+ pop ebx
+ pop edi
+ pop esi
+ add esp, PS*1 ;1 local variable
+ pop ebp
+ %endmacro
+
+%endif ; output formats
+
%define len arg0
%define vec arg1
%define mul_array arg2
%define dest2 tmp4
%define pos return
+%ifidn PS,4 ;32-bit code
+ %define len_m arg0_m
+ %define src_m arg3_m
+ %define dest1_m arg4_m
+ %define dest2_m tmp4_m
+%endif
+
%ifndef EC_ALIGNED_ADDR
;;; Use Un-aligned load/store
%define XLDR vmovdqu
%endif
%endif
+%ifidn PS,8 ;64-bit code
+ default rel
+ [bits 64]
+%endif
-default rel
-
-[bits 64]
section .text
-%define xmask0f ymm8
-%define xmask0fx xmm8
-%define xgft1_lo ymm7
-%define xgft1_hi ymm6
-%define xgft2_lo ymm5
-%define xgft2_hi ymm4
+%ifidn PS,8 ;64-bit code
+ %define xmask0f ymm8
+ %define xmask0fx xmm8
+ %define xgft1_lo ymm7
+ %define xgft1_hi ymm6
+ %define xgft2_lo ymm5
+ %define xgft2_hi ymm4
+
+ %define x0 ymm0
+ %define xtmpa ymm1
+ %define xp1 ymm2
+ %define xp2 ymm3
+%else ;32-bit code
+ %define xmask0f ymm7
+ %define xmask0fx xmm7
+ %define xgft1_lo ymm5
+ %define xgft1_hi ymm4
+ %define xgft2_lo xgft1_lo
+ %define xgft2_hi xgft1_hi
+
+ %define x0 ymm0
+ %define xtmpa ymm1
+ %define xp1 ymm2
+ %define xp2 ymm3
-%define x0 ymm0
-%define xtmpa ymm1
-%define xp1 ymm2
-%define xp2 ymm3
+%endif
align 16
global gf_2vect_dot_prod_avx2:function
func(gf_2vect_dot_prod_avx2)
FUNC_SAVE
+ SLDR len, len_m
sub len, 32
+ SSTR len_m, len
jl .return_fail
xor pos, pos
mov tmp.b, 0x0f
vpbroadcastb xmask0f, xmask0fx ;Construct mask 0x0f0f0f...
sal vec, LOG_PS ;vec *= PS. Make vec_i count by PS
+ SLDR dest1, dest1_m
mov dest2, [dest1+PS]
+ SSTR dest2_m, dest2
mov dest1, [dest1]
+ SSTR dest1_m, dest1
.loop32
vpxor xp1, xp1
xor vec_i, vec_i
.next_vect
+ SLDR src, src_m
mov ptr, [src+vec_i]
vmovdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, ..., Ax{0f}
; " Ax{00}, Ax{10}, ..., Ax{f0}
vperm2i128 xgft1_hi, xgft1_lo, xgft1_lo, 0x11 ; swapped to hi | hi
vperm2i128 xgft1_lo, xgft1_lo, xgft1_lo, 0x00 ; swapped to lo | lo
-
+ %ifidn PS,8 ; 64-bit code
vmovdqu xgft2_lo, [tmp+vec*(32/PS)] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
; " Bx{00}, Bx{10}, ..., Bx{f0}
vperm2i128 xgft2_hi, xgft2_lo, xgft2_lo, 0x11 ; swapped to hi | hi
vperm2i128 xgft2_lo, xgft2_lo, xgft2_lo, 0x00 ; swapped to lo | lo
-
XLDR x0, [ptr+pos] ;Get next source vector
add tmp, 32
add vec_i, PS
+ %else
+ XLDR x0, [ptr+pos] ;Get next source vector
+ %endif
vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
vpxor xgft1_hi, xgft1_lo ;GF add high and low partials
vpxor xp1, xgft1_hi ;xp1 += partial
+ %ifidn PS,4 ; 32-bit code
+ vmovdqu xgft2_lo, [tmp+vec*(32/PS)] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
+ ; " Bx{00}, Bx{10}, ..., Bx{f0}
+ vperm2i128 xgft2_hi, xgft2_lo, xgft2_lo, 0x11 ; swapped to hi | hi
+ vperm2i128 xgft2_lo, xgft2_lo, xgft2_lo, 0x00 ; swapped to lo | lo
+ add tmp, 32
+ add vec_i, PS
+ %endif
vpshufb xgft2_hi, x0 ;Lookup mul table of high nibble
vpshufb xgft2_lo, xtmpa ;Lookup mul table of low nibble
vpxor xgft2_hi, xgft2_lo ;GF add high and low partials
cmp vec_i, vec
jl .next_vect
+ SLDR dest1, dest1_m
+ SLDR dest2, dest2_m
XSTR [dest1+pos], xp1
XSTR [dest2+pos], xp2
+ SLDR len, len_m
add pos, 32 ;Loop on 32 bytes at a time
cmp pos, len
jle .loop32
db 0x%3, 0x%2
%endmacro
;;; func core, ver, snum
-slversion gf_2vect_dot_prod_avx2, 04, 03, 0196
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
+slversion gf_2vect_dot_prod_avx2, 04, 04, 0196
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_2vect_dot_prod_sse(len, vec, *g_tbls, **buffs, **dests);
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
%define tmp3 r9
%define tmp4 r12 ; must be saved and restored
%define return rax
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
%define PS 8
%define LOG_PS 3
%define tmp3 r13 ; must be saved and restored
%define tmp4 r14 ; must be saved and restored
%define return rax
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
%define PS 8
%define LOG_PS 3
%define stack_size 3*16 + 3*8 ; must be an odd multiple of 8
%endmacro
%endif
+%ifidn __OUTPUT_FORMAT__, elf32
+
+;;;================== High Address;
+;;; arg4
+;;; arg3
+;;; arg2
+;;; arg1
+;;; arg0
+;;; return
+;;;<================= esp of caller
+;;; ebp
+;;;<================= ebp = esp
+;;; var0
+;;; esi
+;;; edi
+;;; ebx
+;;;<================= esp of callee
+;;;
+;;;================== Low Address;
+
+ %define PS 4
+ %define LOG_PS 2
+ %define func(x) x:
+ %define arg(x) [ebp + PS*2 + PS*x]
+ %define var(x) [ebp - PS - PS*x]
+
+ %define trans ecx
+ %define trans2 esi
+ %define arg0 trans ;trans and trans2 are for the variables in stack
+ %define arg0_m arg(0)
+ %define arg1 ebx
+ %define arg2 arg2_m
+ %define arg2_m arg(2)
+ %define arg3 trans
+ %define arg3_m arg(3)
+ %define arg4 trans
+ %define arg4_m arg(4)
+ %define tmp edx
+ %define tmp2 edi
+ %define tmp3 trans2
+ %define tmp4 trans2
+ %define tmp4_m var(0)
+ %define return eax
+ %macro SLDR 2 ;; stack load/restore
+ mov %1, %2
+ %endmacro
+ %define SSTR SLDR
+
+ %macro FUNC_SAVE 0
+ push ebp
+ mov ebp, esp
+ sub esp, PS*1 ;1 local variable
+ push esi
+ push edi
+ push ebx
+ mov arg1, arg(1)
+ %endmacro
+
+ %macro FUNC_RESTORE 0
+ pop ebx
+ pop edi
+ pop esi
+ add esp, PS*1 ;1 local variable
+ pop ebp
+ %endmacro
+
+%endif ; output formats
+
%define len arg0
%define vec arg1
%define mul_array arg2
%define src arg3
-%define dest1 arg4
+%define dest1 arg4
%define vec_i tmp2
%define ptr tmp3
%define dest2 tmp4
%define pos return
+ %ifidn PS,4 ;32-bit code
+ %define len_m arg0_m
+ %define src_m arg3_m
+ %define dest1_m arg4_m
+ %define dest2_m tmp4_m
+ %endif
+
%ifndef EC_ALIGNED_ADDR
;;; Use Un-aligned load/store
%define XLDR movdqu
%define XSTR movdqu
%else
-
;;; Use Non-temporal load/stor
%ifdef NO_NT_LDST
%define XLDR movdqa
%endif
%endif
+%ifidn PS,8 ;64-bit code
+ default rel
+ [bits 64]
+%endif
-default rel
-
-[bits 64]
section .text
-%define xmask0f xmm8
-%define xgft1_lo xmm7
-%define xgft1_hi xmm6
-%define xgft2_lo xmm5
-%define xgft2_hi xmm4
-
-%define x0 xmm0
-%define xtmpa xmm1
-%define xp1 xmm2
-%define xp2 xmm3
+%ifidn PS,8 ;64-bit code
+ %define xmask0f xmm8
+ %define xgft1_lo xmm7
+ %define xgft1_hi xmm6
+ %define xgft2_lo xmm5
+ %define xgft2_hi xmm4
+
+ %define x0 xmm0
+ %define xtmpa xmm1
+ %define xp1 xmm2
+ %define xp2 xmm3
+%else ;32-bit code
+ %define xmask0f xmm4
+ %define xgft1_lo xmm7
+ %define xgft1_hi xmm6
+ %define xgft2_lo xgft1_lo
+ %define xgft2_hi xgft1_hi
+
+ %define x0 xmm0
+ %define xtmpa xmm1
+ %define xp1 xmm2
+ %define xp2 xmm3
+%endif
align 16
global gf_2vect_dot_prod_sse:function
func(gf_2vect_dot_prod_sse)
FUNC_SAVE
+ SLDR len, len_m
sub len, 16
+ SSTR len_m, len
jl .return_fail
xor pos, pos
movdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
sal vec, LOG_PS ;vec *= PS. Make vec_i count by PS
+ SLDR dest1, dest1_m
mov dest2, [dest1+PS]
+ SSTR dest2_m, dest2
mov dest1, [dest1]
+ SSTR dest1_m, dest1
.loop16
pxor xp1, xp1
xor vec_i, vec_i
.next_vect
+ SLDR src, src_m
mov ptr, [src+vec_i]
movdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, ..., Ax{0f}
movdqu xgft1_hi, [tmp+16] ; " Ax{00}, Ax{10}, ..., Ax{f0}
+ %ifidn PS,8 ;64-bit code
movdqu xgft2_lo, [tmp+vec*(32/PS)] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
movdqu xgft2_hi, [tmp+vec*(32/PS)+16] ; " Bx{00}, Bx{10}, ..., Bx{f0}
-
- XLDR x0, [ptr+pos] ;Get next source vector
add tmp, 32
add vec_i, PS
+ %endif
+ XLDR x0, [ptr+pos] ;Get next source vector
movdqa xtmpa, x0 ;Keep unshifted copy of src
psraw x0, 4 ;Shift to put high nibble into bits 4-0
pxor xgft1_hi, xgft1_lo ;GF add high and low partials
pxor xp1, xgft1_hi ;xp1 += partial
+ %ifidn PS,4 ;32-bit code
+ movdqu xgft2_lo, [tmp+vec*(32/PS)] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
+ movdqu xgft2_hi, [tmp+vec*(32/PS)+16] ; " Bx{00}, Bx{10}, ..., Bx{f0}
+
+ add tmp, 32
+ add vec_i, PS
+ %endif
pshufb xgft2_hi, x0 ;Lookup mul table of high nibble
pshufb xgft2_lo, xtmpa ;Lookup mul table of low nibble
pxor xgft2_hi, xgft2_lo ;GF add high and low partials
cmp vec_i, vec
jl .next_vect
+ SLDR dest1, dest1_m
+ SLDR dest2, dest2_m
XSTR [dest1+pos], xp1
XSTR [dest2+pos], xp2
+ SLDR len, len_m
add pos, 16 ;Loop on 16 bytes at a time
cmp pos, len
jle .loop16
db 0x%3, 0x%2
%endmacro
;;; func core, ver, snum
-slversion gf_2vect_dot_prod_sse, 00, 02, 0062
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
+slversion gf_2vect_dot_prod_sse, 00, 03, 0062
--- /dev/null
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+; * Redistributions of source code must retain the above copyright
+; notice, this list of conditions and the following disclaimer.
+; * Redistributions in binary form must reproduce the above copyright
+; notice, this list of conditions and the following disclaimer in
+; the documentation and/or other materials provided with the
+; distribution.
+; * Neither the name of Intel Corporation nor the names of its
+; contributors may be used to endorse or promote products derived
+; from this software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;
+;;; gf_2vect_mad_avx(len, vec, vec_i, mul_array, src, dest);
+;;;
+
+%define PS 8
+
+%ifidn __OUTPUT_FORMAT__, win64
+ %define arg0 rcx
+ %define arg0.w ecx
+ %define arg1 rdx
+ %define arg2 r8
+ %define arg3 r9
+ %define arg4 r12
+ %define arg5 r15
+ %define tmp r11
+ %define tmp2 r10
+ %define return rax
+ %define return.w eax
+ %define stack_size 16*9 + 3*8
+ %define arg(x) [rsp + stack_size + PS + PS*x]
+ %define func(x) proc_frame x
+
+%macro FUNC_SAVE 0
+ sub rsp, stack_size
+ movdqa [rsp+16*0],xmm6
+ movdqa [rsp+16*1],xmm7
+ movdqa [rsp+16*2],xmm8
+ movdqa [rsp+16*3],xmm9
+ movdqa [rsp+16*4],xmm10
+ movdqa [rsp+16*5],xmm11
+ movdqa [rsp+16*6],xmm12
+ movdqa [rsp+16*7],xmm13
+ movdqa [rsp+16*8],xmm14
+ save_reg r12, 9*16 + 0*8
+ save_reg r15, 9*16 + 1*8
+ end_prolog
+ mov arg4, arg(4)
+ mov arg5, arg(5)
+%endmacro
+
+%macro FUNC_RESTORE 0
+ movdqa xmm6, [rsp+16*0]
+ movdqa xmm7, [rsp+16*1]
+ movdqa xmm8, [rsp+16*2]
+ movdqa xmm9, [rsp+16*3]
+ movdqa xmm10, [rsp+16*4]
+ movdqa xmm11, [rsp+16*5]
+ movdqa xmm12, [rsp+16*6]
+ movdqa xmm13, [rsp+16*7]
+ movdqa xmm14, [rsp+16*8]
+ mov r12, [rsp + 9*16 + 0*8]
+ mov r15, [rsp + 9*16 + 1*8]
+ add rsp, stack_size
+%endmacro
+
+%elifidn __OUTPUT_FORMAT__, elf64
+ %define arg0 rdi
+ %define arg0.w edi
+ %define arg1 rsi
+ %define arg2 rdx
+ %define arg3 rcx
+ %define arg4 r8
+ %define arg5 r9
+ %define tmp r11
+ %define tmp2 r10
+ %define return rax
+ %define return.w eax
+
+ %define func(x) x:
+ %define FUNC_SAVE
+ %define FUNC_RESTORE
+%endif
+
+;;; gf_2vect_mad_avx(len, vec, vec_i, mul_array, src, dest)
+%define len arg0
+%define len.w arg0.w
+%define vec arg1
+%define vec_i arg2
+%define mul_array arg3
+%define src arg4
+%define dest1 arg5
+%define pos return
+%define pos.w return.w
+
+%define dest2 tmp2
+
+%ifndef EC_ALIGNED_ADDR
+;;; Use Un-aligned load/store
+ %define XLDR vmovdqu
+ %define XSTR vmovdqu
+%else
+;;; Use Non-temporal load/stor
+ %ifdef NO_NT_LDST
+ %define XLDR vmovdqa
+ %define XSTR vmovdqa
+ %else
+ %define XLDR vmovntdqa
+ %define XSTR vmovntdq
+ %endif
+%endif
+
+
+default rel
+
+[bits 64]
+section .text
+
+%define xmask0f xmm14
+%define xgft1_lo xmm13
+%define xgft1_hi xmm12
+%define xgft2_lo xmm11
+%define xgft2_hi xmm10
+
+%define x0 xmm0
+%define xtmpa xmm1
+%define xtmph1 xmm2
+%define xtmpl1 xmm3
+%define xtmph2 xmm4
+%define xtmpl2 xmm5
+%define xd1 xmm6
+%define xd2 xmm7
+%define xtmpd1 xmm8
+%define xtmpd2 xmm9
+
+
+align 16
+global gf_2vect_mad_avx:function
+
+func(gf_2vect_mad_avx)
+ FUNC_SAVE
+ sub len, 16
+ jl .return_fail
+
+ xor pos, pos
+ vmovdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
+ sal vec_i, 5 ;Multiply by 32
+ sal vec, 5
+ lea tmp, [mul_array + vec_i]
+ vmovdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, Ax{02}, ...
+ vmovdqu xgft1_hi, [tmp+16] ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
+ vmovdqu xgft2_lo, [tmp+vec] ;Load array Bx{00}, Bx{01}, Bx{02}, ...
+ vmovdqu xgft2_hi, [tmp+vec+16] ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
+
+ mov dest2, [dest1+PS]
+ mov dest1, [dest1]
+
+ XLDR xtmpd1, [dest1+len] ;backup the last 16 bytes in dest
+ XLDR xtmpd2, [dest2+len] ;backup the last 16 bytes in dest
+
+.loop16
+ XLDR xd1, [dest1+pos] ;Get next dest vector
+ XLDR xd2, [dest2+pos] ;Get next dest vector
+.loop16_overlap:
+ XLDR x0, [src+pos] ;Get next source vector
+
+ vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+
+ vpshufb xtmph1, xgft1_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl1, xgft1_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph1, xtmph1, xtmpl1 ;GF add high and low partials
+ vpxor xd1, xd1, xtmph1 ;xd1 += partial
+
+ vpshufb xtmph2, xgft2_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl2, xgft2_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph2, xtmph2, xtmpl2 ;GF add high and low partials
+ vpxor xd2, xd2, xtmph2 ;xd2 += partial
+
+ XSTR [dest1+pos], xd1
+ XSTR [dest2+pos], xd2
+
+ add pos, 16 ;Loop on 16 bytes at a time
+ cmp pos, len
+ jle .loop16
+
+ lea tmp, [len + 16]
+ cmp pos, tmp
+ je .return_pass
+
+ ;; Tail len
+ mov pos, len ;Overlapped offset length-16
+ vmovdqa xd1, xtmpd1 ;Restore xd1
+ vmovdqa xd2, xtmpd2 ;Restore xd2
+ jmp .loop16_overlap ;Do one more overlap pass
+
+.return_pass:
+ mov return, 0
+ FUNC_RESTORE
+ ret
+
+.return_fail:
+ mov return, 1
+ FUNC_RESTORE
+ ret
+
+endproc_frame
+
+section .data
+
+align 16
+mask0f: ddq 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
+
+%macro slversion 4
+global %1_slver_%2%3%4
+global %1_slver
+%1_slver:
+%1_slver_%2%3%4:
+ dw 0x%4
+ db 0x%3, 0x%2
+%endmacro
+;;; func core, ver, snum
+slversion gf_2vect_mad_avx, 02, 00, 0204
--- /dev/null
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+; * Redistributions of source code must retain the above copyright
+; notice, this list of conditions and the following disclaimer.
+; * Redistributions in binary form must reproduce the above copyright
+; notice, this list of conditions and the following disclaimer in
+; the documentation and/or other materials provided with the
+; distribution.
+; * Neither the name of Intel Corporation nor the names of its
+; contributors may be used to endorse or promote products derived
+; from this software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;
+;;; gf_2vect_mad_avx2(len, vec, vec_i, mul_array, src, dest);
+;;;
+
+%define PS 8
+
+%ifidn __OUTPUT_FORMAT__, win64
+ %define arg0 rcx
+ %define arg0.w ecx
+ %define arg1 rdx
+ %define arg2 r8
+ %define arg3 r9
+ %define arg4 r12
+ %define arg5 r15
+
+ %define tmp r11
+ %define tmp.w r11d
+ %define tmp.b r11b
+ %define tmp2 r10
+ %define return rax
+ %define return.w eax
+ %define stack_size 16*9 + 3*8 ; must be an odd multiple of 8
+ %define arg(x) [rsp + stack_size + PS + PS*x]
+
+ %define func(x) proc_frame x
+ %macro FUNC_SAVE 0
+ sub rsp, stack_size
+ vmovdqa [rsp+16*0],xmm6
+ vmovdqa [rsp+16*1],xmm7
+ vmovdqa [rsp+16*2],xmm8
+ vmovdqa [rsp+16*3],xmm9
+ vmovdqa [rsp+16*4],xmm10
+ vmovdqa [rsp+16*5],xmm11
+ vmovdqa [rsp+16*6],xmm12
+ vmovdqa [rsp+16*7],xmm13
+ vmovdqa [rsp+16*8],xmm14
+ save_reg r12, 9*16 + 0*8
+ save_reg r15, 9*16 + 1*8
+ end_prolog
+ mov arg4, arg(4)
+ mov arg5, arg(5)
+ %endmacro
+
+ %macro FUNC_RESTORE 0
+ vmovdqa xmm6, [rsp+16*0]
+ vmovdqa xmm7, [rsp+16*1]
+ vmovdqa xmm8, [rsp+16*2]
+ vmovdqa xmm9, [rsp+16*3]
+ vmovdqa xmm10, [rsp+16*4]
+ vmovdqa xmm11, [rsp+16*5]
+ vmovdqa xmm12, [rsp+16*6]
+ vmovdqa xmm13, [rsp+16*7]
+ vmovdqa xmm14, [rsp+16*8]
+ mov r12, [rsp + 9*16 + 0*8]
+ mov r15, [rsp + 9*16 + 1*8]
+ add rsp, stack_size
+ %endmacro
+%endif
+
+%ifidn __OUTPUT_FORMAT__, elf64
+ %define arg0 rdi
+ %define arg0.w edi
+ %define arg1 rsi
+ %define arg2 rdx
+ %define arg3 rcx
+ %define arg4 r8
+ %define arg5 r9
+
+ %define tmp r11
+ %define tmp.w r11d
+ %define tmp.b r11b
+ %define tmp2 r10
+ %define return rax
+ %define return.w eax
+
+ %define func(x) x:
+ %define FUNC_SAVE
+ %define FUNC_RESTORE
+%endif
+
+;;; gf_2vect_mad_avx2(len, vec, vec_i, mul_array, src, dest)
+%define len arg0
+%define len.w arg0.w
+%define vec arg1
+%define vec_i arg2
+%define mul_array arg3
+%define src arg4
+%define dest1 arg5
+%define pos return
+%define pos.w return.w
+
+%define dest2 tmp2
+
+%ifndef EC_ALIGNED_ADDR
+;;; Use Un-aligned load/store
+ %define XLDR vmovdqu
+ %define XSTR vmovdqu
+%else
+
+;;; Use Non-temporal load/stor
+ %ifdef NO_NT_LDST
+ %define XLDR vmovdqa
+ %define XSTR vmovdqa
+ %else
+ %define XLDR vmovntdqa
+ %define XSTR vmovntdq
+ %endif
+%endif
+
+
+default rel
+
+[bits 64]
+section .text
+
+%define xmask0f ymm14
+%define xmask0fx xmm14
+%define xgft1_lo ymm13
+%define xgft1_hi ymm12
+%define xgft2_lo ymm11
+%define xgft2_hi ymm10
+
+%define x0 ymm0
+%define xtmpa ymm1
+%define xtmph1 ymm2
+%define xtmpl1 ymm3
+%define xtmph2 ymm4
+%define xtmpl2 ymm5
+%define xd1 ymm6
+%define xd2 ymm7
+%define xtmpd1 ymm8
+%define xtmpd2 ymm9
+
+align 16
+global gf_2vect_mad_avx2:function
+
+func(gf_2vect_mad_avx2)
+ FUNC_SAVE
+ sub len, 32
+ jl .return_fail
+ xor pos, pos
+ mov tmp.b, 0x0f
+ vpinsrb xmask0fx, xmask0fx, tmp.w, 0
+ vpbroadcastb xmask0f, xmask0fx ;Construct mask 0x0f0f0f...
+
+ sal vec_i, 5 ;Multiply by 32
+ sal vec, 5
+ lea tmp, [mul_array + vec_i]
+ vmovdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, ..., Ax{0f}
+ ; " Ax{00}, Ax{10}, ..., Ax{f0}
+ vmovdqu xgft2_lo, [tmp+vec] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
+ ; " Bx{00}, Bx{10}, ..., Bx{f0}
+
+ vperm2i128 xgft1_hi, xgft1_lo, xgft1_lo, 0x11 ; swapped to hi | hi
+ vperm2i128 xgft1_lo, xgft1_lo, xgft1_lo, 0x00 ; swapped to lo | lo
+ vperm2i128 xgft2_hi, xgft2_lo, xgft2_lo, 0x11 ; swapped to hi | hi
+ vperm2i128 xgft2_lo, xgft2_lo, xgft2_lo, 0x00 ; swapped to lo | lo
+ mov dest2, [dest1+PS] ; reuse mul_array
+ mov dest1, [dest1]
+
+ XLDR xtmpd1, [dest1+len] ;backup the last 16 bytes in dest
+ XLDR xtmpd2, [dest2+len] ;backup the last 16 bytes in dest
+
+.loop32
+ XLDR xd1, [dest1+pos] ;Get next dest vector
+ XLDR xd2, [dest2+pos] ;Get next dest vector
+.loop32_overlap:
+ XLDR x0, [src+pos] ;Get next source vector
+
+ vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+
+ vpshufb xtmph1, xgft1_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl1, xgft1_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph1, xtmph1, xtmpl1 ;GF add high and low partials
+ vpxor xd1, xd1, xtmph1 ;xd1 += partial
+
+ vpshufb xtmph2, xgft2_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl2, xgft2_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph2, xtmph2, xtmpl2 ;GF add high and low partials
+ vpxor xd2, xd2, xtmph2 ;xd2 += partial
+
+ XSTR [dest1+pos], xd1
+ XSTR [dest2+pos], xd2
+
+ add pos, 32 ;Loop on 32 bytes at a time
+ cmp pos, len
+ jle .loop32
+
+ lea tmp, [len + 32]
+ cmp pos, tmp
+ je .return_pass
+
+ ;; Tail len
+ mov pos, len ;Overlapped offset length-32
+ vmovdqa xd1, xtmpd1 ;Restore xd1
+ vmovdqa xd2, xtmpd2 ;Restore xd2
+ jmp .loop32_overlap ;Do one more overlap pass
+
+.return_pass:
+ mov return, 0
+ FUNC_RESTORE
+ ret
+
+.return_fail:
+ mov return, 1
+ FUNC_RESTORE
+ ret
+
+endproc_frame
+
+section .data
+
+%macro slversion 4
+global %1_slver_%2%3%4
+global %1_slver
+%1_slver:
+%1_slver_%2%3%4:
+ dw 0x%4
+ db 0x%3, 0x%2
+%endmacro
+;;; func core, ver, snum
+slversion gf_2vect_mad_avx2, 04, 00, 0205
--- /dev/null
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+; * Redistributions of source code must retain the above copyright
+; notice, this list of conditions and the following disclaimer.
+; * Redistributions in binary form must reproduce the above copyright
+; notice, this list of conditions and the following disclaimer in
+; the documentation and/or other materials provided with the
+; distribution.
+; * Neither the name of Intel Corporation nor the names of its
+; contributors may be used to endorse or promote products derived
+; from this software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;
+;;; gf_2vect_mad_sse(len, vec, vec_i, mul_array, src, dest);
+;;;
+
+%define PS 8
+
+%ifidn __OUTPUT_FORMAT__, win64
+ %define arg0 rcx
+ %define arg0.w ecx
+ %define arg1 rdx
+ %define arg2 r8
+ %define arg3 r9
+ %define arg4 r12
+ %define arg5 r15
+ %define tmp r11
+ %define tmp2 r10
+ %define return rax
+ %define return.w eax
+ %define stack_size 16*9 + 3*8
+ %define arg(x) [rsp + stack_size + PS + PS*x]
+ %define func(x) proc_frame x
+
+%macro FUNC_SAVE 0
+ sub rsp, stack_size
+ movdqa [rsp+16*0],xmm6
+ movdqa [rsp+16*1],xmm7
+ movdqa [rsp+16*2],xmm8
+ movdqa [rsp+16*3],xmm9
+ movdqa [rsp+16*4],xmm10
+ movdqa [rsp+16*5],xmm11
+ movdqa [rsp+16*6],xmm12
+ movdqa [rsp+16*7],xmm13
+ movdqa [rsp+16*8],xmm14
+ save_reg r12, 9*16 + 0*8
+ save_reg r15, 9*16 + 1*8
+ end_prolog
+ mov arg4, arg(4)
+ mov arg5, arg(5)
+%endmacro
+
+%macro FUNC_RESTORE 0
+ movdqa xmm6, [rsp+16*0]
+ movdqa xmm7, [rsp+16*1]
+ movdqa xmm8, [rsp+16*2]
+ movdqa xmm9, [rsp+16*3]
+ movdqa xmm10, [rsp+16*4]
+ movdqa xmm11, [rsp+16*5]
+ movdqa xmm12, [rsp+16*6]
+ movdqa xmm13, [rsp+16*7]
+ movdqa xmm14, [rsp+16*8]
+ mov r12, [rsp + 9*16 + 0*8]
+ mov r15, [rsp + 9*16 + 1*8]
+ add rsp, stack_size
+%endmacro
+
+%elifidn __OUTPUT_FORMAT__, elf64
+ %define arg0 rdi
+ %define arg0.w edi
+ %define arg1 rsi
+ %define arg2 rdx
+ %define arg3 rcx
+ %define arg4 r8
+ %define arg5 r9
+ %define tmp r11
+ %define tmp2 r10
+ %define return rax
+ %define return.w eax
+
+ %define func(x) x:
+ %define FUNC_SAVE
+ %define FUNC_RESTORE
+%endif
+
+;;; gf_2vect_mad_sse(len, vec, vec_i, mul_array, src, dest)
+%define len arg0
+%define len.w arg0.w
+%define vec arg1
+%define vec_i arg2
+%define mul_array arg3
+%define src arg4
+%define dest1 arg5
+%define pos return
+%define pos.w return.w
+
+%define dest2 tmp2
+
+%ifndef EC_ALIGNED_ADDR
+;;; Use Un-aligned load/store
+ %define XLDR movdqu
+ %define XSTR movdqu
+%else
+;;; Use Non-temporal load/stor
+ %ifdef NO_NT_LDST
+ %define XLDR movdqa
+ %define XSTR movdqa
+ %else
+ %define XLDR movntdqa
+ %define XSTR movntdq
+ %endif
+%endif
+
+default rel
+
+[bits 64]
+section .text
+
+%define xmask0f xmm14
+%define xgft1_lo xmm13
+%define xgft1_hi xmm12
+%define xgft2_lo xmm11
+%define xgft2_hi xmm10
+
+%define x0 xmm0
+%define xtmpa xmm1
+%define xtmph1 xmm2
+%define xtmpl1 xmm3
+%define xtmph2 xmm4
+%define xtmpl2 xmm5
+%define xd1 xmm6
+%define xd2 xmm7
+%define xtmpd1 xmm8
+%define xtmpd2 xmm9
+
+
+align 16
+global gf_2vect_mad_sse:function
+func(gf_2vect_mad_sse)
+ FUNC_SAVE
+ sub len, 16
+ jl .return_fail
+
+ xor pos, pos
+ movdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
+ sal vec_i, 5 ;Multiply by 32
+ sal vec, 5
+ lea tmp, [mul_array + vec_i]
+ movdqu xgft1_lo,[tmp] ;Load array Ax{00}, Ax{01}, Ax{02}, ...
+ movdqu xgft1_hi, [tmp+16] ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
+ movdqu xgft2_lo, [tmp+vec] ;Load array Bx{00}, Bx{01}, Bx{02}, ...
+ movdqu xgft2_hi, [tmp+vec+16] ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
+ mov dest2, [dest1+PS]
+ mov dest1, [dest1]
+
+ XLDR xtmpd1, [dest1+len] ;backup the last 16 bytes in dest
+ XLDR xtmpd2, [dest2+len] ;backup the last 16 bytes in dest
+
+.loop16:
+ XLDR xd1, [dest1+pos] ;Get next dest vector
+ XLDR xd2, [dest2+pos] ;Get next dest vector
+.loop16_overlap:
+ XLDR x0, [src+pos] ;Get next source vector
+ movdqa xtmph1, xgft1_hi ;Reload const array registers
+ movdqa xtmpl1, xgft1_lo
+ movdqa xtmph2, xgft2_hi ;Reload const array registers
+ movdqa xtmpl2, xgft2_lo
+ movdqa xtmpa, x0 ;Keep unshifted copy of src
+ psraw x0, 4 ;Shift to put high nibble into bits 4-0
+ pand x0, xmask0f ;Mask high src nibble in bits 4-0
+ pand xtmpa, xmask0f ;Mask low src nibble in bits 4-0
+
+ pshufb xtmph1, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl1, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph1, xtmpl1 ;GF add high and low partials
+ pxor xd1, xtmph1
+
+ pshufb xtmph2, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl2, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph2, xtmpl2 ;GF add high and low partials
+ pxor xd2, xtmph2
+
+ XSTR [dest1+pos], xd1 ;Store result
+ XSTR [dest2+pos], xd2 ;Store result
+
+ add pos, 16 ;Loop on 16 bytes at a time
+ cmp pos, len
+ jle .loop16
+
+ lea tmp, [len + 16]
+ cmp pos, tmp
+ je .return_pass
+
+ ;; Tail len
+ mov pos, len ;Overlapped offset length-16
+ movdqa xd1, xtmpd1 ;Restore xd1
+ movdqa xd2, xtmpd2 ;Restore xd2
+ jmp .loop16_overlap ;Do one more overlap pass
+
+.return_pass:
+ FUNC_RESTORE
+ mov return, 0
+ ret
+
+.return_fail:
+ FUNC_RESTORE
+ mov return, 1
+ ret
+
+endproc_frame
+
+section .data
+
+align 16
+
+mask0f:
+ ddq 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
+
+%macro slversion 4
+global %1_slver_%2%3%4
+global %1_slver
+%1_slver:
+%1_slver_%2%3%4:
+ dw 0x%4
+ db 0x%3, 0x%2
+%endmacro
+;;; func core, ver, snum
+slversion gf_2vect_mad_sse, 00, 00, 0203
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_3vect_dot_prod_avx(len, vec, *g_tbls, **buffs, **dests);
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
%define tmp3 r13 ; must be saved and restored
%define tmp4 r12 ; must be saved and restored
%define return rax
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
%define PS 8
%define LOG_PS 3
%define tmp3 r13 ; must be saved and restored
%define tmp4 r14 ; must be saved and restored
%define return rax
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
%define PS 8
%define LOG_PS 3
%define stack_size 6*16 + 5*8 ; must be an odd multiple of 8
%endmacro
%endif
+%ifidn __OUTPUT_FORMAT__, elf32
+
+;;;================== High Address;
+;;; arg4
+;;; arg3
+;;; arg2
+;;; arg1
+;;; arg0
+;;; return
+;;;<================= esp of caller
+;;; ebp
+;;;<================= ebp = esp
+;;; var0
+;;; var1
+;;; esi
+;;; edi
+;;; ebx
+;;;<================= esp of callee
+;;;
+;;;================== Low Address;
+
+ %define PS 4
+ %define LOG_PS 2
+ %define func(x) x:
+ %define arg(x) [ebp + PS*2 + PS*x]
+ %define var(x) [ebp - PS - PS*x]
+
+ %define trans ecx
+ %define trans2 esi
+ %define arg0 trans ;trans and trans2 are for the variables in stack
+ %define arg0_m arg(0)
+ %define arg1 ebx
+ %define arg2 arg2_m
+ %define arg2_m arg(2)
+ %define arg3 trans
+ %define arg3_m arg(3)
+ %define arg4 trans
+ %define arg4_m arg(4)
+ %define arg5 trans2
+ %define tmp edx
+ %define tmp2 edi
+ %define tmp3 trans2
+ %define tmp3_m var(0)
+ %define tmp4 trans2
+ %define tmp4_m var(1)
+ %define return eax
+ %macro SLDR 2 ;; stack load/restore
+ mov %1, %2
+ %endmacro
+ %define SSTR SLDR
+
+ %macro FUNC_SAVE 0
+ push ebp
+ mov ebp, esp
+ sub esp, PS*2 ;2 local variables
+ push esi
+ push edi
+ push ebx
+ mov arg1, arg(1)
+ %endmacro
+
+ %macro FUNC_RESTORE 0
+ pop ebx
+ pop edi
+ pop esi
+ add esp, PS*2 ;2 local variables
+ pop ebp
+ %endmacro
+
+%endif ; output formats
+
%define len arg0
%define vec arg1
%define mul_array arg2
%define src arg3
-%define dest1 arg4
+%define dest1 arg4
%define ptr arg5
+
%define vec_i tmp2
%define dest2 tmp3
%define dest3 tmp4
%define pos return
+ %ifidn PS,4 ;32-bit code
+ %define len_m arg0_m
+ %define src_m arg3_m
+ %define dest1_m arg4_m
+ %define dest2_m tmp3_m
+ %define dest3_m tmp4_m
+ %endif
+
%ifndef EC_ALIGNED_ADDR
;;; Use Un-aligned load/store
%define XLDR vmovdqu
%endif
%endif
+%ifidn PS,8 ; 64-bit code
+ default rel
+ [bits 64]
+%endif
-default rel
-[bits 64]
section .text
-%define xmask0f xmm11
-%define xgft1_lo xmm10
-%define xgft1_hi xmm9
-%define xgft2_lo xmm8
-%define xgft2_hi xmm7
-%define xgft3_lo xmm6
-%define xgft3_hi xmm5
-
-%define x0 xmm0
-%define xtmpa xmm1
-%define xp1 xmm2
-%define xp2 xmm3
-%define xp3 xmm4
+%ifidn PS,8 ;64-bit code
+ %define xmask0f xmm11
+ %define xgft1_lo xmm10
+ %define xgft1_hi xmm9
+ %define xgft2_lo xmm8
+ %define xgft2_hi xmm7
+ %define xgft3_lo xmm6
+ %define xgft3_hi xmm5
+
+ %define x0 xmm0
+ %define xtmpa xmm1
+ %define xp1 xmm2
+ %define xp2 xmm3
+ %define xp3 xmm4
+%else
+ %define xmask0f xmm7
+ %define xgft1_lo xmm6
+ %define xgft1_hi xmm5
+ %define xgft2_lo xgft1_lo
+ %define xgft2_hi xgft1_hi
+ %define xgft3_lo xgft1_lo
+ %define xgft3_hi xgft1_hi
+
+ %define x0 xmm0
+ %define xtmpa xmm1
+ %define xp1 xmm2
+ %define xp2 xmm3
+ %define xp3 xmm4
+%endif
align 16
global gf_3vect_dot_prod_avx:function
func(gf_3vect_dot_prod_avx)
FUNC_SAVE
+ SLDR len, len_m
sub len, 16
+ SSTR len_m, len
jl .return_fail
xor pos, pos
vmovdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
sal vec, LOG_PS ;vec *= PS. Make vec_i count by PS
+ SLDR dest1, dest1_m
mov dest2, [dest1+PS]
+ SSTR dest2_m, dest2
mov dest3, [dest1+2*PS]
+ SSTR dest3_m, dest3
mov dest1, [dest1]
-
+ SSTR dest1_m, dest1
.loop16:
vpxor xp1, xp1
xor vec_i, vec_i
.next_vect:
+ SLDR src, src_m
mov ptr, [src+vec_i]
vmovdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, ..., Ax{0f}
vmovdqu xgft1_hi, [tmp+16] ; " Ax{00}, Ax{10}, ..., Ax{f0}
+ %ifidn PS,8 ; 64-bit code
vmovdqu xgft2_lo, [tmp+vec*(32/PS)] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
vmovdqu xgft2_hi, [tmp+vec*(32/PS)+16] ; " Bx{00}, Bx{10}, ..., Bx{f0}
vmovdqu xgft3_lo, [tmp+vec*(64/PS)] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
vmovdqu xgft3_hi, [tmp+vec*(64/PS)+16] ; " Cx{00}, Cx{10}, ..., Cx{f0}
-
add tmp, 32
add vec_i, PS
+ %endif
XLDR x0, [ptr+pos] ;Get next source vector
vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
vpxor xgft1_hi, xgft1_lo ;GF add high and low partials
vpxor xp1, xgft1_hi ;xp1 += partial
+ %ifidn PS,4 ; 32-bit code
+ vmovdqu xgft2_lo, [tmp+vec*(32/PS)] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
+ vmovdqu xgft2_hi, [tmp+vec*(32/PS)+16] ; " Bx{00}, Bx{10}, ..., Bx{f0}
+ %endif
vpshufb xgft2_hi, x0 ;Lookup mul table of high nibble
vpshufb xgft2_lo, xtmpa ;Lookup mul table of low nibble
vpxor xgft2_hi, xgft2_lo ;GF add high and low partials
vpxor xp2, xgft2_hi ;xp2 += partial
+ %ifidn PS,4 ; 32-bit code
+ sal vec, 1
+ vmovdqu xgft3_lo, [tmp+vec*(32/PS)] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
+ vmovdqu xgft3_hi, [tmp+vec*(32/PS)+16] ; " Cx{00}, Cx{10}, ..., Cx{f0}
+ sar vec, 1
+ add tmp, 32
+ add vec_i, PS
+ %endif
vpshufb xgft3_hi, x0 ;Lookup mul table of high nibble
vpshufb xgft3_lo, xtmpa ;Lookup mul table of low nibble
vpxor xgft3_hi, xgft3_lo ;GF add high and low partials
cmp vec_i, vec
jl .next_vect
+ SLDR dest1, dest1_m
+ SLDR dest2, dest2_m
XSTR [dest1+pos], xp1
XSTR [dest2+pos], xp2
+ SLDR dest3, dest3_m
XSTR [dest3+pos], xp3
+ SLDR len, len_m
add pos, 16 ;Loop on 16 bytes at a time
cmp pos, len
jle .loop16
db 0x%3, 0x%2
%endmacro
;;; func core, ver, snum
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
-slversion gf_3vect_dot_prod_avx, 02, 03, 0192
+slversion gf_3vect_dot_prod_avx, 02, 04, 0192
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_3vect_dot_prod_avx2(len, vec, *g_tbls, **buffs, **dests);
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
%define tmp3 r13 ; must be saved and restored
%define tmp4 r12 ; must be saved and restored
%define return rax
- %define PS 8
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
+ %define PS 8
%define LOG_PS 3
%define func(x) x:
%define tmp3 r13 ; must be saved and restored
%define tmp4 r14 ; must be saved and restored
%define return rax
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
%define PS 8
%define LOG_PS 3
%define stack_size 6*16 + 5*8 ; must be an odd multiple of 8
%endmacro
%endif
+%ifidn __OUTPUT_FORMAT__, elf32
+
+;;;================== High Address;
+;;; arg4
+;;; arg3
+;;; arg2
+;;; arg1
+;;; arg0
+;;; return
+;;;<================= esp of caller
+;;; ebp
+;;;<================= ebp = esp
+;;; var0
+;;; var1
+;;; esi
+;;; edi
+;;; ebx
+;;;<================= esp of callee
+;;;
+;;;================== Low Address;
+
+ %define PS 4
+ %define LOG_PS 2
+ %define func(x) x:
+ %define arg(x) [ebp + PS*2 + PS*x]
+ %define var(x) [ebp - PS - PS*x]
+
+ %define trans ecx
+ %define trans2 esi
+ %define arg0 trans ;trans and trans2 are for the variables in stack
+ %define arg0_m arg(0)
+ %define arg1 ebx
+ %define arg2 arg2_m
+ %define arg2_m arg(2)
+ %define arg3 trans
+ %define arg3_m arg(3)
+ %define arg4 trans
+ %define arg4_m arg(4)
+ %define arg5 trans2
+ %define tmp edx
+ %define tmp.w edx
+ %define tmp.b dl
+ %define tmp2 edi
+ %define tmp3 trans2
+ %define tmp3_m var(0)
+ %define tmp4 trans2
+ %define tmp4_m var(1)
+ %define return eax
+ %macro SLDR 2 ;stack load/restore
+ mov %1, %2
+ %endmacro
+ %define SSTR SLDR
+
+ %macro FUNC_SAVE 0
+ push ebp
+ mov ebp, esp
+ sub esp, PS*2 ;2 local variables
+ push esi
+ push edi
+ push ebx
+ mov arg1, arg(1)
+ %endmacro
+
+ %macro FUNC_RESTORE 0
+ pop ebx
+ pop edi
+ pop esi
+ add esp, PS*2 ;2 local variables
+ pop ebp
+ %endmacro
+
+%endif ; output formats
+
%define len arg0
%define vec arg1
%define mul_array arg2
%define src arg3
%define dest1 arg4
%define ptr arg5
+
%define vec_i tmp2
%define dest2 tmp3
%define dest3 tmp4
%define pos return
+%ifidn PS,4 ;32-bit code
+ %define len_m arg0_m
+ %define src_m arg3_m
+ %define dest1_m arg4_m
+ %define dest2_m tmp3_m
+ %define dest3_m tmp4_m
+%endif
+
%ifndef EC_ALIGNED_ADDR
;;; Use Un-aligned load/store
%define XLDR vmovdqu
%endif
%endif
+%ifidn PS,8 ;64-bit code
+ default rel
+ [bits 64]
+%endif
-default rel
-
-[bits 64]
section .text
-%define xmask0f ymm11
-%define xmask0fx xmm11
-%define xgft1_lo ymm10
-%define xgft1_hi ymm9
-%define xgft2_lo ymm8
-%define xgft2_hi ymm7
-%define xgft3_lo ymm6
-%define xgft3_hi ymm5
-
-%define x0 ymm0
-%define xtmpa ymm1
-%define xp1 ymm2
-%define xp2 ymm3
-%define xp3 ymm4
+%ifidn PS,8 ;64-bit code
+ %define xmask0f ymm11
+ %define xmask0fx xmm11
+ %define xgft1_lo ymm10
+ %define xgft1_hi ymm9
+ %define xgft2_lo ymm8
+ %define xgft2_hi ymm7
+ %define xgft3_lo ymm6
+ %define xgft3_hi ymm5
+
+ %define x0 ymm0
+ %define xtmpa ymm1
+ %define xp1 ymm2
+ %define xp2 ymm3
+ %define xp3 ymm4
+%else
+ %define xmask0f ymm7
+ %define xmask0fx xmm7
+ %define xgft1_lo ymm6
+ %define xgft1_hi ymm5
+ %define xgft2_lo xgft1_lo
+ %define xgft2_hi xgft1_hi
+ %define xgft3_lo xgft1_lo
+ %define xgft3_hi xgft1_hi
+
+ %define x0 ymm0
+ %define xtmpa ymm1
+ %define xp1 ymm2
+ %define xp2 ymm3
+ %define xp3 ymm4
+
+%endif
align 16
global gf_3vect_dot_prod_avx2:function
func(gf_3vect_dot_prod_avx2)
FUNC_SAVE
+ SLDR len, len_m
sub len, 32
+ SSTR len_m, len
jl .return_fail
xor pos, pos
mov tmp.b, 0x0f
vpbroadcastb xmask0f, xmask0fx ;Construct mask 0x0f0f0f...
sal vec, LOG_PS ;vec *= PS. Make vec_i count by PS
+ SLDR dest1, dest1_m
mov dest2, [dest1+PS]
+ SSTR dest2_m, dest2
mov dest3, [dest1+2*PS]
+ SSTR dest3_m, dest3
mov dest1, [dest1]
-
+ SSTR dest1_m, dest1
.loop32:
vpxor xp1, xp1
xor vec_i, vec_i
.next_vect:
+ SLDR src, src_m
mov ptr, [src+vec_i]
vmovdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, ..., Ax{0f}
; " Ax{00}, Ax{10}, ..., Ax{f0}
vperm2i128 xgft1_hi, xgft1_lo, xgft1_lo, 0x11 ; swapped to hi | hi
vperm2i128 xgft1_lo, xgft1_lo, xgft1_lo, 0x00 ; swapped to lo | lo
-
- vmovdqu xgft2_lo, [tmp+vec*(32/PS)] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
+ %ifidn PS,8 ; 64-bit code
+ vmovdqu xgft2_lo, [tmp+vec*(32/PS)] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
; " Bx{00}, Bx{10}, ..., Bx{f0}
vperm2i128 xgft2_hi, xgft2_lo, xgft2_lo, 0x11 ; swapped to hi | hi
vperm2i128 xgft2_lo, xgft2_lo, xgft2_lo, 0x00 ; swapped to lo | lo
- vmovdqu xgft3_lo, [tmp+vec*(64/PS)] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
+ vmovdqu xgft3_lo, [tmp+vec*(64/PS)] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
; " Cx{00}, Cx{10}, ..., Cx{f0}
vperm2i128 xgft3_hi, xgft3_lo, xgft3_lo, 0x11 ; swapped to hi | hi
vperm2i128 xgft3_lo, xgft3_lo, xgft3_lo, 0x00 ; swapped to lo | lo
add tmp, 32
add vec_i, PS
+ %endif
XLDR x0, [ptr+pos] ;Get next source vector
vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
vpxor xgft1_hi, xgft1_lo ;GF add high and low partials
vpxor xp1, xgft1_hi ;xp1 += partial
- vpshufb xgft2_hi, x0 ;Lookup mul table of high nibble
- vpshufb xgft2_lo, xtmpa ;Lookup mul table of low nibble
- vpxor xgft2_hi, xgft2_lo ;GF add high and low partials
- vpxor xp2, xgft2_hi ;xp2 += partial
-
+ %ifidn PS,4 ; 32-bit code
+ vmovdqu xgft2_lo, [tmp+vec*(32/PS)] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
+ ; " Bx{00}, Bx{10}, ..., Bx{f0}
+ vperm2i128 xgft2_hi, xgft2_lo, xgft2_lo, 0x11 ; swapped to hi | hi
+ vperm2i128 xgft2_lo, xgft2_lo, xgft2_lo, 0x00 ; swapped to lo | lo
+ %endif
+ vpshufb xgft2_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xgft2_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xgft2_hi, xgft2_lo ;GF add high and low partials
+ vpxor xp2, xgft2_hi ;xp2 += partial
+
+ %ifidn PS,4 ; 32-bit code
+ sal vec, 1
+ vmovdqu xgft3_lo, [tmp+vec*(32/PS)] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
+ ; " Cx{00}, Cx{10}, ..., Cx{f0}
+ vperm2i128 xgft3_hi, xgft3_lo, xgft3_lo, 0x11 ; swapped to hi | hi
+ vperm2i128 xgft3_lo, xgft3_lo, xgft3_lo, 0x00 ; swapped to lo | lo
+ sar vec, 1
+ add tmp, 32
+ add vec_i, PS
+ %endif
vpshufb xgft3_hi, x0 ;Lookup mul table of high nibble
vpshufb xgft3_lo, xtmpa ;Lookup mul table of low nibble
vpxor xgft3_hi, xgft3_lo ;GF add high and low partials
cmp vec_i, vec
jl .next_vect
+ SLDR dest1, dest1_m
+ SLDR dest2, dest2_m
XSTR [dest1+pos], xp1
XSTR [dest2+pos], xp2
+ SLDR dest3, dest3_m
XSTR [dest3+pos], xp3
+ SLDR len, len_m
add pos, 32 ;Loop on 32 bytes at a time
cmp pos, len
jle .loop32
db 0x%3, 0x%2
%endmacro
;;; func core, ver, snum
-slversion gf_3vect_dot_prod_avx2, 04, 03, 0197
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
+slversion gf_3vect_dot_prod_avx2, 04, 04, 0197
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_3vect_dot_prod_sse(len, vec, *g_tbls, **buffs, **dests);
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
%define tmp3 r13 ; must be saved and restored
%define tmp4 r12 ; must be saved and restored
%define return rax
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
%define PS 8
%define LOG_PS 3
%define tmp3 r13 ; must be saved and restored
%define tmp4 r14 ; must be saved and restored
%define return rax
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
%define PS 8
%define LOG_PS 3
%define stack_size 6*16 + 5*8 ; must be an odd multiple of 8
%endmacro
%endif
+%ifidn __OUTPUT_FORMAT__, elf32
+
+;;;================== High Address;
+;;; arg4
+;;; arg3
+;;; arg2
+;;; arg1
+;;; arg0
+;;; return
+;;;<================= esp of caller
+;;; ebp
+;;;<================= ebp = esp
+;;; var0
+;;; var1
+;;; esi
+;;; edi
+;;; ebx
+;;;<================= esp of callee
+;;;
+;;;================== Low Address;
+
+ %define PS 4
+ %define LOG_PS 2
+ %define func(x) x:
+ %define arg(x) [ebp + PS*2 + PS*x]
+ %define var(x) [ebp - PS - PS*x]
+
+ %define trans ecx
+ %define trans2 esi
+ %define arg0 trans ;trans and trans2 are for the variables in stack
+ %define arg0_m arg(0)
+ %define arg1 ebx
+ %define arg2 arg2_m
+ %define arg2_m arg(2)
+ %define arg3 trans
+ %define arg3_m arg(3)
+ %define arg4 trans
+ %define arg4_m arg(4)
+ %define arg5 trans2
+ %define tmp edx
+ %define tmp2 edi
+ %define tmp3 trans2
+ %define tmp3_m var(0)
+ %define tmp4 trans2
+ %define tmp4_m var(1)
+ %define return eax
+ %macro SLDR 2 ;; stack load/restore
+ mov %1, %2
+ %endmacro
+ %define SSTR SLDR
+
+ %macro FUNC_SAVE 0
+ push ebp
+ mov ebp, esp
+ sub esp, PS*2 ;2 local variables
+ push esi
+ push edi
+ push ebx
+ mov arg1, arg(1)
+ %endmacro
+
+ %macro FUNC_RESTORE 0
+ pop ebx
+ pop edi
+ pop esi
+ add esp, PS*2 ;2 local variables
+ pop ebp
+ %endmacro
+
+%endif ; output formats
+
%define len arg0
%define vec arg1
%define mul_array arg2
%define src arg3
-%define dest1 arg4
+%define dest1 arg4
%define ptr arg5
+
%define vec_i tmp2
%define dest2 tmp3
%define dest3 tmp4
%define pos return
+ %ifidn PS,4 ;32-bit code
+ %define len_m arg0_m
+ %define src_m arg3_m
+ %define dest1_m arg4_m
+ %define dest2_m tmp3_m
+ %define dest3_m tmp4_m
+ %endif
+
%ifndef EC_ALIGNED_ADDR
;;; Use Un-aligned load/store
%define XLDR movdqu
%endif
%endif
+%ifidn PS,8 ; 64-bit code
+ default rel
+ [bits 64]
+%endif
-default rel
-[bits 64]
section .text
-%define xmask0f xmm11
-%define xgft1_lo xmm10
-%define xgft1_hi xmm9
-%define xgft2_lo xmm8
-%define xgft2_hi xmm7
-%define xgft3_lo xmm6
-%define xgft3_hi xmm5
-
-%define x0 xmm0
-%define xtmpa xmm1
-%define xp1 xmm2
-%define xp2 xmm3
-%define xp3 xmm4
+%ifidn PS,8 ;64-bit code
+ %define xmask0f xmm11
+ %define xgft1_lo xmm2
+ %define xgft1_hi xmm3
+ %define xgft2_lo xmm4
+ %define xgft2_hi xmm7
+ %define xgft3_lo xmm6
+ %define xgft3_hi xmm5
+
+ %define x0 xmm0
+ %define xtmpa xmm1
+ %define xp1 xmm10
+ %define xp2 xmm9
+ %define xp3 xmm8
+%else
+ %define xmask0f xmm7
+ %define xgft1_lo xmm6
+ %define xgft1_hi xmm5
+ %define xgft2_lo xgft1_lo
+ %define xgft2_hi xgft1_hi
+ %define xgft3_lo xgft1_lo
+ %define xgft3_hi xgft1_hi
+
+ %define x0 xmm0
+ %define xtmpa xmm1
+ %define xp1 xmm2
+ %define xp2 xmm3
+ %define xp3 xmm4
+%endif
align 16
global gf_3vect_dot_prod_sse:function
func(gf_3vect_dot_prod_sse)
FUNC_SAVE
+ SLDR len, len_m
sub len, 16
+ SSTR len_m, len
jl .return_fail
xor pos, pos
movdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
sal vec, LOG_PS ;vec *= PS. Make vec_i count by PS
+ SLDR dest1, dest1_m
mov dest2, [dest1+PS]
+ SSTR dest2_m, dest2
mov dest3, [dest1+2*PS]
+ SSTR dest3_m, dest3
mov dest1, [dest1]
-
+ SSTR dest1_m, dest1
.loop16:
pxor xp1, xp1
xor vec_i, vec_i
.next_vect:
+ SLDR src, src_m
mov ptr, [src+vec_i]
movdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, ..., Ax{0f}
movdqu xgft1_hi, [tmp+16] ; " Ax{00}, Ax{10}, ..., Ax{f0}
+ %ifidn PS,8 ;64-bit code
movdqu xgft2_lo, [tmp+vec*(32/PS)] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
movdqu xgft2_hi, [tmp+vec*(32/PS)+16] ; " Bx{00}, Bx{10}, ..., Bx{f0}
movdqu xgft3_lo, [tmp+vec*(64/PS)] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
movdqu xgft3_hi, [tmp+vec*(64/PS)+16] ; " Cx{00}, Cx{10}, ..., Cx{f0}
-
add tmp, 32
add vec_i, PS
+ %endif
XLDR x0, [ptr+pos] ;Get next source vector
movdqa xtmpa, x0 ;Keep unshifted copy of src
pxor xgft1_hi, xgft1_lo ;GF add high and low partials
pxor xp1, xgft1_hi ;xp1 += partial
+ %ifidn PS,4 ;32-bit code
+ movdqu xgft2_lo, [tmp+vec*(32/PS)] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
+ movdqu xgft2_hi, [tmp+vec*(32/PS)+16] ; " Bx{00}, Bx{10}, ..., Bx{f0}
+ %endif
pshufb xgft2_hi, x0 ;Lookup mul table of high nibble
pshufb xgft2_lo, xtmpa ;Lookup mul table of low nibble
pxor xgft2_hi, xgft2_lo ;GF add high and low partials
pxor xp2, xgft2_hi ;xp2 += partial
+ %ifidn PS,4 ;32-bit code
+ sal vec, 1
+ movdqu xgft3_lo, [tmp+vec*(32/PS)] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
+ movdqu xgft3_hi, [tmp+vec*(32/PS)+16] ; " Cx{00}, Cx{10}, ..., Cx{f0}
+ sar vec, 1
+ add tmp, 32
+ add vec_i, PS
+ %endif
pshufb xgft3_hi, x0 ;Lookup mul table of high nibble
pshufb xgft3_lo, xtmpa ;Lookup mul table of low nibble
pxor xgft3_hi, xgft3_lo ;GF add high and low partials
cmp vec_i, vec
jl .next_vect
+ SLDR dest1, dest1_m
+ SLDR dest2, dest2_m
XSTR [dest1+pos], xp1
XSTR [dest2+pos], xp2
+ SLDR dest3, dest3_m
XSTR [dest3+pos], xp3
+ SLDR len, len_m
add pos, 16 ;Loop on 16 bytes at a time
cmp pos, len
jle .loop16
db 0x%3, 0x%2
%endmacro
;;; func core, ver, snum
-slversion gf_3vect_dot_prod_sse, 00, 03, 0063
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
+slversion gf_3vect_dot_prod_sse, 00, 05, 0063
--- /dev/null
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+; * Redistributions of source code must retain the above copyright
+; notice, this list of conditions and the following disclaimer.
+; * Redistributions in binary form must reproduce the above copyright
+; notice, this list of conditions and the following disclaimer in
+; the documentation and/or other materials provided with the
+; distribution.
+; * Neither the name of Intel Corporation nor the names of its
+; contributors may be used to endorse or promote products derived
+; from this software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;
+;;; gf_3vect_mad_avx(len, vec, vec_i, mul_array, src, dest);
+;;;
+
+%define PS 8
+
+%ifidn __OUTPUT_FORMAT__, win64
+ %define arg0 rcx
+ %define arg0.w ecx
+ %define arg1 rdx
+ %define arg2 r8
+ %define arg3 r9
+ %define arg4 r12
+ %define arg5 r15
+ %define tmp r11
+ %define return rax
+ %define return.w eax
+ %define stack_size 16*10 + 3*8
+ %define arg(x) [rsp + stack_size + PS + PS*x]
+ %define func(x) proc_frame x
+
+%macro FUNC_SAVE 0
+ sub rsp, stack_size
+ vmovdqa [rsp+16*0],xmm6
+ vmovdqa [rsp+16*1],xmm7
+ vmovdqa [rsp+16*2],xmm8
+ vmovdqa [rsp+16*3],xmm9
+ vmovdqa [rsp+16*4],xmm10
+ vmovdqa [rsp+16*5],xmm11
+ vmovdqa [rsp+16*6],xmm12
+ vmovdqa [rsp+16*7],xmm13
+ vmovdqa [rsp+16*8],xmm14
+ vmovdqa [rsp+16*9],xmm15
+ save_reg r12, 10*16 + 0*8
+ save_reg r15, 10*16 + 1*8
+ end_prolog
+ mov arg4, arg(4)
+ mov arg5, arg(5)
+%endmacro
+
+%macro FUNC_RESTORE 0
+ vmovdqa xmm6, [rsp+16*0]
+ vmovdqa xmm7, [rsp+16*1]
+ vmovdqa xmm8, [rsp+16*2]
+ vmovdqa xmm9, [rsp+16*3]
+ vmovdqa xmm10, [rsp+16*4]
+ vmovdqa xmm11, [rsp+16*5]
+ vmovdqa xmm12, [rsp+16*6]
+ vmovdqa xmm13, [rsp+16*7]
+ vmovdqa xmm14, [rsp+16*8]
+ vmovdqa xmm15, [rsp+16*9]
+ mov r12, [rsp + 10*16 + 0*8]
+ mov r15, [rsp + 10*16 + 1*8]
+ add rsp, stack_size
+%endmacro
+
+%elifidn __OUTPUT_FORMAT__, elf64
+ %define arg0 rdi
+ %define arg0.w edi
+ %define arg1 rsi
+ %define arg2 rdx
+ %define arg3 rcx
+ %define arg4 r8
+ %define arg5 r9
+ %define tmp r11
+ %define return rax
+ %define return.w eax
+
+ %define func(x) x:
+ %define FUNC_SAVE
+ %define FUNC_RESTORE
+%endif
+
+;;; gf_3vect_mad_avx(len, vec, vec_i, mul_array, src, dest)
+%define len arg0
+%define len.w arg0.w
+%define vec arg1
+%define vec_i arg2
+%define mul_array arg3
+%define src arg4
+%define dest1 arg5
+%define pos return
+%define pos.w return.w
+
+%define dest2 mul_array
+%define dest3 vec_i
+
+%ifndef EC_ALIGNED_ADDR
+;;; Use Un-aligned load/store
+ %define XLDR vmovdqu
+ %define XSTR vmovdqu
+%else
+;;; Use Non-temporal load/stor
+ %ifdef NO_NT_LDST
+ %define XLDR vmovdqa
+ %define XSTR vmovdqa
+ %else
+ %define XLDR vmovntdqa
+ %define XSTR vmovntdq
+ %endif
+%endif
+
+
+default rel
+
+[bits 64]
+section .text
+
+%define xmask0f xmm15
+%define xgft1_lo xmm14
+%define xgft1_hi xmm13
+%define xgft2_lo xmm12
+%define xgft2_hi xmm11
+%define xgft3_lo xmm10
+%define xgft3_hi xmm9
+
+%define x0 xmm0
+%define xtmpa xmm1
+%define xtmph1 xmm2
+%define xtmpl1 xmm3
+%define xtmph2 xmm4
+%define xtmpl2 xmm5
+%define xtmph3 xmm6
+%define xtmpl3 xmm7
+%define xd1 xmm8
+%define xd2 xtmpl1
+%define xd3 xtmph1
+
+align 16
+global gf_3vect_mad_avx:function
+func(gf_3vect_mad_avx)
+ FUNC_SAVE
+ sub len, 16
+ jl .return_fail
+ xor pos, pos
+ vmovdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
+
+ sal vec_i, 5 ;Multiply by 32
+ sal vec, 5
+ lea tmp, [mul_array + vec_i]
+ vmovdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, Ax{02}, ...
+ vmovdqu xgft1_hi, [tmp+16] ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
+ vmovdqu xgft2_lo, [tmp+vec] ;Load array Bx{00}, Bx{01}, Bx{02}, ...
+ vmovdqu xgft2_hi, [tmp+vec+16] ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
+ vmovdqu xgft3_lo, [tmp+2*vec] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
+ vmovdqu xgft3_hi, [tmp+2*vec+16]; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
+ mov dest2, [dest1+PS] ; reuse mul_array
+ mov dest3, [dest1+2*PS] ; reuse vec_i
+ mov dest1, [dest1]
+
+.loop16:
+ XLDR x0, [src+pos] ;Get next source vector
+ XLDR xd1, [dest1+pos] ;Get next dest vector
+
+ vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+
+ ; dest1
+ vpshufb xtmph1, xgft1_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl1, xgft1_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph1, xtmph1, xtmpl1 ;GF add high and low partials
+ vpxor xd1, xd1, xtmph1 ;xd1 += partial
+
+ XLDR xd2, [dest2+pos] ;reuse xtmpl1. Get next dest vector
+ XLDR xd3, [dest3+pos] ;reuse xtmph1. Get next dest vector
+
+ ; dest2
+ vpshufb xtmph2, xgft2_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl2, xgft2_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph2, xtmph2, xtmpl2 ;GF add high and low partials
+ vpxor xd2, xd2, xtmph2 ;xd2 += partial
+
+ ; dest3
+ vpshufb xtmph3, xgft3_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl3, xgft3_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph3, xtmph3, xtmpl3 ;GF add high and low partials
+ vpxor xd3, xd3, xtmph3 ;xd3 += partial
+
+ XSTR [dest1+pos], xd1
+ XSTR [dest2+pos], xd2
+ XSTR [dest3+pos], xd3
+
+ add pos, 16 ;Loop on 16 bytes at a time
+ cmp pos, len
+ jle .loop16
+
+ lea tmp, [len + 16]
+ cmp pos, tmp
+ je .return_pass
+
+.lessthan16:
+ ;; Tail len
+ ;; Do one more overlap pass
+ mov tmp, len ;Overlapped offset length-16
+ XLDR x0, [src+tmp] ;Get next source vector
+ XLDR xd1, [dest1+tmp] ;Get next dest vector
+ XLDR xd2, [dest2+tmp] ;reuse xtmpl1. Get next dest vector
+ XLDR xd3, [dest3+tmp] ;reuse xtmph1. Get next dest vector
+
+ sub len, pos
+
+ movdqa xtmph3, [constip16] ;Load const of i + 16
+ vpinsrb xtmpl3, xtmpl3, len.w, 15
+ vpshufb xtmpl3, xtmpl3, xmask0f ;Broadcast len to all bytes
+ vpcmpgtb xtmpl3, xtmpl3, xtmph3
+
+ vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+
+ ; dest1
+ vpshufb xgft1_hi, xgft1_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xgft1_lo, xgft1_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xgft1_hi, xgft1_hi, xgft1_lo ;GF add high and low partials
+ vpand xgft1_hi, xgft1_hi, xtmpl3
+ vpxor xd1, xd1, xgft1_hi
+
+ ; dest2
+ vpshufb xgft2_hi, xgft2_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xgft2_lo, xgft2_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xgft2_hi, xgft2_hi, xgft2_lo ;GF add high and low partials
+ vpand xgft2_hi, xgft2_hi, xtmpl3
+ vpxor xd2, xd2, xgft2_hi
+
+ ; dest3
+ vpshufb xgft3_hi, xgft3_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xgft3_lo, xgft3_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xgft3_hi, xgft3_hi, xgft3_lo ;GF add high and low partials
+ vpand xgft3_hi, xgft3_hi, xtmpl3
+ vpxor xd3, xd3, xgft3_hi
+
+ XSTR [dest1+tmp], xd1
+ XSTR [dest2+tmp], xd2
+ XSTR [dest3+tmp], xd3
+
+.return_pass:
+ mov return, 0
+ FUNC_RESTORE
+ ret
+
+.return_fail:
+ mov return, 1
+ FUNC_RESTORE
+ ret
+
+endproc_frame
+
+section .data
+
+align 16
+mask0f: ddq 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
+constip16:
+ ddq 0xf0f1f2f3f4f5f6f7f8f9fafbfcfdfeff
+
+%macro slversion 4
+global %1_slver_%2%3%4
+global %1_slver
+%1_slver:
+%1_slver_%2%3%4:
+ dw 0x%4
+ db 0x%3, 0x%2
+%endmacro
+;;; func core, ver, snum
+slversion gf_3vect_mad_avx, 02, 00, 0207
--- /dev/null
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+; * Redistributions of source code must retain the above copyright
+; notice, this list of conditions and the following disclaimer.
+; * Redistributions in binary form must reproduce the above copyright
+; notice, this list of conditions and the following disclaimer in
+; the documentation and/or other materials provided with the
+; distribution.
+; * Neither the name of Intel Corporation nor the names of its
+; contributors may be used to endorse or promote products derived
+; from this software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;
+;;; gf_3vect_mad_avx2(len, vec, vec_i, mul_array, src, dest);
+;;;
+
+%define PS 8
+
+%ifidn __OUTPUT_FORMAT__, win64
+ %define arg0 rcx
+ %define arg0.w ecx
+ %define arg1 rdx
+ %define arg2 r8
+ %define arg3 r9
+ %define arg4 r12 ; must be saved, loaded and restored
+ %define arg5 r15 ; must be saved and restored
+
+ %define tmp r11
+ %define tmp.w r11d
+ %define tmp.b r11b
+ %define return rax
+ %define return.w eax
+ %define stack_size 16*10 + 3*8
+ %define arg(x) [rsp + stack_size + PS + PS*x]
+ %define func(x) proc_frame x
+
+ %macro FUNC_SAVE 0
+ sub rsp, stack_size
+ vmovdqa [rsp+16*0],xmm6
+ vmovdqa [rsp+16*1],xmm7
+ vmovdqa [rsp+16*2],xmm8
+ vmovdqa [rsp+16*3],xmm9
+ vmovdqa [rsp+16*4],xmm10
+ vmovdqa [rsp+16*5],xmm11
+ vmovdqa [rsp+16*6],xmm12
+ vmovdqa [rsp+16*7],xmm13
+ vmovdqa [rsp+16*8],xmm14
+ vmovdqa [rsp+16*9],xmm15
+ save_reg r12, 10*16 + 0*8
+ save_reg r15, 10*16 + 1*8
+ end_prolog
+ mov arg4, arg(4)
+ mov arg5, arg(5)
+ %endmacro
+
+ %macro FUNC_RESTORE 0
+ vmovdqa xmm6, [rsp+16*0]
+ vmovdqa xmm7, [rsp+16*1]
+ vmovdqa xmm8, [rsp+16*2]
+ vmovdqa xmm9, [rsp+16*3]
+ vmovdqa xmm10, [rsp+16*4]
+ vmovdqa xmm11, [rsp+16*5]
+ vmovdqa xmm12, [rsp+16*6]
+ vmovdqa xmm13, [rsp+16*7]
+ vmovdqa xmm14, [rsp+16*8]
+ vmovdqa xmm15, [rsp+16*9]
+ mov r12, [rsp + 10*16 + 0*8]
+ mov r15, [rsp + 10*16 + 1*8]
+ add rsp, stack_size
+ %endmacro
+
+%elifidn __OUTPUT_FORMAT__, elf64
+ %define arg0 rdi
+ %define arg0.w edi
+ %define arg1 rsi
+ %define arg2 rdx
+ %define arg3 rcx
+ %define arg4 r8
+ %define arg5 r9
+
+ %define tmp r11
+ %define tmp.w r11d
+ %define tmp.b r11b
+ %define return rax
+ %define return.w eax
+
+ %define func(x) x:
+ %define FUNC_SAVE
+ %define FUNC_RESTORE
+%endif
+
+;;; gf_3vect_mad_avx2(len, vec, vec_i, mul_array, src, dest)
+%define len arg0
+%define len.w arg0.w
+%define vec arg1
+%define vec_i arg2
+%define mul_array arg3
+%define src arg4
+%define dest1 arg5
+%define pos return
+%define pos.w return.w
+
+%define dest2 mul_array
+%define dest3 vec_i
+
+%ifndef EC_ALIGNED_ADDR
+;;; Use Un-aligned load/store
+ %define XLDR vmovdqu
+ %define XSTR vmovdqu
+%else
+;;; Use Non-temporal load/stor
+ %ifdef NO_NT_LDST
+ %define XLDR vmovdqa
+ %define XSTR vmovdqa
+ %else
+ %define XLDR vmovntdqa
+ %define XSTR vmovntdq
+ %endif
+%endif
+
+
+default rel
+
+[bits 64]
+section .text
+
+%define xmask0f ymm15
+%define xmask0fx xmm15
+%define xgft1_lo ymm14
+%define xgft1_hi ymm13
+%define xgft2_lo ymm12
+%define xgft3_lo ymm11
+
+%define x0 ymm0
+%define xtmpa ymm1
+%define xtmph1 ymm2
+%define xtmpl1 ymm3
+%define xtmph2 ymm4
+%define xtmpl2 ymm5
+%define xtmpl2x xmm5
+%define xtmph3 ymm6
+%define xtmpl3 ymm7
+%define xtmpl3x xmm7
+%define xd1 ymm8
+%define xd2 ymm9
+%define xd3 ymm10
+
+align 16
+global gf_3vect_mad_avx2:function
+func(gf_3vect_mad_avx2)
+ FUNC_SAVE
+ sub len, 32
+ jl .return_fail
+ xor pos, pos
+ mov tmp.b, 0x0f
+ vpinsrb xmask0fx, xmask0fx, tmp.w, 0
+ vpbroadcastb xmask0f, xmask0fx ;Construct mask 0x0f0f0f...
+
+ sal vec_i, 5 ;Multiply by 32
+ sal vec, 5
+ lea tmp, [mul_array + vec_i]
+
+ vmovdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, ..., Ax{0f}
+ ; " Ax{00}, Ax{10}, ..., Ax{f0}
+ vperm2i128 xgft1_hi, xgft1_lo, xgft1_lo, 0x11 ; swapped to hi | hi
+ vperm2i128 xgft1_lo, xgft1_lo, xgft1_lo, 0x00 ; swapped to lo | lo
+
+ vmovdqu xgft2_lo, [tmp+vec] ;Load array Bx{00}, Bx{01}, Bx{02}, ...
+ ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
+ vmovdqu xgft3_lo, [tmp+2*vec] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
+ ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
+ mov dest2, [dest1+PS] ; reuse mul_array
+ mov dest3, [dest1+2*PS] ; reuse vec_i
+ mov dest1, [dest1]
+
+.loop32:
+ XLDR x0, [src+pos] ;Get next source vector
+ XLDR xd1, [dest1+pos] ;Get next dest vector
+ XLDR xd2, [dest2+pos] ;Get next dest vector
+ XLDR xd3, [dest3+pos] ;Get next dest vector
+ vperm2i128 xtmph2, xgft2_lo, xgft2_lo, 0x11 ; swapped to hi | hi
+ vperm2i128 xtmpl2, xgft2_lo, xgft2_lo, 0x00 ; swapped to lo | lo
+
+ vperm2i128 xtmph3, xgft3_lo, xgft3_lo, 0x11 ; swapped to hi | hi
+ vperm2i128 xtmpl3, xgft3_lo, xgft3_lo, 0x00 ; swapped to lo | lo
+
+ vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+
+ ; dest1
+ vpshufb xtmph1, xgft1_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl1, xgft1_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph1, xtmph1, xtmpl1 ;GF add high and low partials
+ vpxor xd1, xd1, xtmph1 ;xd1 += partial
+
+ ; dest2
+ vpshufb xtmph2, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl2, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph2, xtmpl2 ;GF add high and low partials
+ vpxor xd2, xtmph2 ;xd2 += partial
+
+ ; dest3
+ vpshufb xtmph3, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl3, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph3, xtmpl3 ;GF add high and low partials
+ vpxor xd3, xtmph3 ;xd3 += partial
+
+ XSTR [dest1+pos], xd1
+ XSTR [dest2+pos], xd2
+ XSTR [dest3+pos], xd3
+
+ add pos, 32 ;Loop on 32 bytes at a time
+ cmp pos, len
+ jle .loop32
+
+ lea tmp, [len + 32]
+ cmp pos, tmp
+ je .return_pass
+
+.lessthan32:
+ ;; Tail len
+ ;; Do one more overlap pass
+ mov tmp.b, 0x1f
+ vpinsrb xtmpl2x, xtmpl2x, tmp.w, 0
+ vpbroadcastb xtmpl2, xtmpl2x ;Construct mask 0x1f1f1f...
+
+ mov tmp, len ;Overlapped offset length-32
+
+ XLDR x0, [src+tmp] ;Get next source vector
+ XLDR xd1, [dest1+tmp] ;Get next dest vector
+ XLDR xd2, [dest2+tmp] ;Get next dest vector
+ XLDR xd3, [dest3+tmp] ;Get next dest vector
+
+ sub len, pos
+
+ vmovdqa xtmph3, [constip32] ;Load const of i + 32
+ vpinsrb xtmpl3x, xtmpl3x, len.w, 15
+ vinserti128 xtmpl3, xtmpl3, xtmpl3x, 1 ;swapped to xtmpl3x | xtmpl3x
+ vpshufb xtmpl3, xtmpl3, xtmpl2 ;Broadcast len to all bytes. xtmpl2=0x1f1f1f...
+ vpcmpgtb xtmpl3, xtmpl3, xtmph3
+
+ vperm2i128 xtmph2, xgft2_lo, xgft2_lo, 0x11 ; swapped to hi | hi
+ vperm2i128 xgft2_lo, xgft2_lo, xgft2_lo, 0x00 ; swapped to lo | lo
+
+ vperm2i128 xtmph3, xgft3_lo, xgft3_lo, 0x11 ; swapped to hi | hi
+ vperm2i128 xgft3_lo, xgft3_lo, xgft3_lo, 0x00 ; swapped to lo | lo
+
+ vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+
+ ; dest1
+ vpshufb xtmph1, xgft1_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl1, xgft1_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph1, xtmph1, xtmpl1 ;GF add high and low partials
+ vpand xtmph1, xtmph1, xtmpl3
+ vpxor xd1, xd1, xtmph1 ;xd1 += partial
+
+ ; dest2
+ vpshufb xtmph2, xtmph2, x0 ;Lookup mul table of high nibble
+ vpshufb xgft2_lo, xgft2_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph2, xtmph2, xgft2_lo ;GF add high and low partials
+ vpand xtmph2, xtmph2, xtmpl3
+ vpxor xd2, xd2, xtmph2 ;xd2 += partial
+
+ ; dest3
+ vpshufb xtmph3, xtmph3, x0 ;Lookup mul table of high nibble
+ vpshufb xgft3_lo, xgft3_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph3, xtmph3, xgft3_lo ;GF add high and low partials
+ vpand xtmph3, xtmph3, xtmpl3
+ vpxor xd3, xd3, xtmph3 ;xd3 += partial
+
+ XSTR [dest1+tmp], xd1
+ XSTR [dest2+tmp], xd2
+ XSTR [dest3+tmp], xd3
+
+.return_pass:
+ mov return, 0
+ FUNC_RESTORE
+ ret
+
+.return_fail:
+ mov return, 1
+ FUNC_RESTORE
+ ret
+
+endproc_frame
+
+section .data
+
+align 32
+constip32:
+ ddq 0xf0f1f2f3f4f5f6f7f8f9fafbfcfdfeff
+ ddq 0xe0e1e2e3e4e5e6e7e8e9eaebecedeeef
+
+%macro slversion 4
+global %1_slver_%2%3%4
+global %1_slver
+%1_slver:
+%1_slver_%2%3%4:
+ dw 0x%4
+ db 0x%3, 0x%2
+%endmacro
+;;; func core, ver, snum
+slversion gf_3vect_mad_avx2, 04, 00, 0208
--- /dev/null
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+; * Redistributions of source code must retain the above copyright
+; notice, this list of conditions and the following disclaimer.
+; * Redistributions in binary form must reproduce the above copyright
+; notice, this list of conditions and the following disclaimer in
+; the documentation and/or other materials provided with the
+; distribution.
+; * Neither the name of Intel Corporation nor the names of its
+; contributors may be used to endorse or promote products derived
+; from this software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;
+;;; gf_3vect_mad_sse(len, vec, vec_i, mul_array, src, dest);
+;;;
+
+%define PS 8
+%ifidn __OUTPUT_FORMAT__, win64
+ %define arg0 rcx
+ %define arg0.w ecx
+ %define arg1 rdx
+ %define arg2 r8
+ %define arg3 r9
+ %define arg4 r12
+ %define arg5 r15
+ %define tmp r11
+ %define return rax
+ %define return.w eax
+ %define stack_size 16*10 + 3*8
+ %define arg(x) [rsp + stack_size + PS + PS*x]
+ %define func(x) proc_frame x
+
+%macro FUNC_SAVE 0
+ sub rsp, stack_size
+ movdqa [rsp+16*0],xmm6
+ movdqa [rsp+16*1],xmm7
+ movdqa [rsp+16*2],xmm8
+ movdqa [rsp+16*3],xmm9
+ movdqa [rsp+16*4],xmm10
+ movdqa [rsp+16*5],xmm11
+ movdqa [rsp+16*6],xmm12
+ movdqa [rsp+16*7],xmm13
+ movdqa [rsp+16*8],xmm14
+ movdqa [rsp+16*9],xmm15
+ save_reg r12, 10*16 + 0*8
+ save_reg r15, 10*16 + 1*8
+ end_prolog
+ mov arg4, arg(4)
+ mov arg5, arg(5)
+%endmacro
+
+%macro FUNC_RESTORE 0
+ movdqa xmm6, [rsp+16*0]
+ movdqa xmm7, [rsp+16*1]
+ movdqa xmm8, [rsp+16*2]
+ movdqa xmm9, [rsp+16*3]
+ movdqa xmm10, [rsp+16*4]
+ movdqa xmm11, [rsp+16*5]
+ movdqa xmm12, [rsp+16*6]
+ movdqa xmm13, [rsp+16*7]
+ movdqa xmm14, [rsp+16*8]
+ movdqa xmm15, [rsp+16*9]
+ mov r12, [rsp + 10*16 + 0*8]
+ mov r15, [rsp + 10*16 + 1*8]
+ add rsp, stack_size
+%endmacro
+
+%elifidn __OUTPUT_FORMAT__, elf64
+ %define arg0 rdi
+ %define arg0.w edi
+ %define arg1 rsi
+ %define arg2 rdx
+ %define arg3 rcx
+ %define arg4 r8
+ %define arg5 r9
+ %define tmp r11
+ %define return rax
+ %define return.w eax
+
+ %define func(x) x:
+ %define FUNC_SAVE
+ %define FUNC_RESTORE
+%endif
+
+;;; gf_3vect_mad_sse(len, vec, vec_i, mul_array, src, dest)
+%define len arg0
+%define len.w arg0.w
+%define vec arg1
+%define vec_i arg2
+%define mul_array arg3
+%define src arg4
+%define dest1 arg5
+%define pos return
+%define pos.w return.w
+
+%define dest2 mul_array
+%define dest3 vec_i
+
+%ifndef EC_ALIGNED_ADDR
+;;; Use Un-aligned load/store
+ %define XLDR movdqu
+ %define XSTR movdqu
+%else
+;;; Use Non-temporal load/stor
+ %ifdef NO_NT_LDST
+ %define XLDR movdqa
+ %define XSTR movdqa
+ %else
+ %define XLDR movntdqa
+ %define XSTR movntdq
+ %endif
+%endif
+
+default rel
+
+[bits 64]
+section .text
+
+%define xmask0f xmm15
+%define xgft1_lo xmm14
+%define xgft1_hi xmm13
+%define xgft2_lo xmm12
+%define xgft2_hi xmm11
+%define xgft3_lo xmm10
+%define xgft3_hi xmm9
+
+%define x0 xmm0
+%define xtmpa xmm1
+%define xtmph1 xmm2
+%define xtmpl1 xmm3
+%define xtmph2 xmm4
+%define xtmpl2 xmm5
+%define xtmph3 xmm6
+%define xtmpl3 xmm7
+%define xd1 xmm8
+%define xd2 xtmpl1
+%define xd3 xtmph1
+
+align 16
+global gf_3vect_mad_sse:function
+func(gf_3vect_mad_sse)
+ FUNC_SAVE
+ sub len, 16
+ jl .return_fail
+ xor pos, pos
+ movdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
+ sal vec_i, 5 ;Multiply by 32
+ sal vec, 5
+ lea tmp, [mul_array + vec_i]
+
+ movdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, Ax{02}, ...
+ movdqu xgft1_hi, [tmp+16] ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
+ movdqu xgft2_lo, [tmp+vec] ;Load array Bx{00}, Bx{01}, Bx{02}, ...
+ movdqu xgft2_hi, [tmp+vec+16] ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
+ movdqu xgft3_lo, [tmp+2*vec] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
+ movdqu xgft3_hi, [tmp+2*vec+16] ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
+ mov dest2, [dest1+PS] ; reuse mul_array
+ mov dest3, [dest1+2*PS] ; reuse vec_i
+ mov dest1, [dest1]
+
+.loop16:
+ XLDR x0, [src+pos] ;Get next source vector
+ movdqa xtmph1, xgft1_hi ;Reload const array registers
+ movdqa xtmpl1, xgft1_lo
+ movdqa xtmph2, xgft2_hi ;Reload const array registers
+ movdqa xtmpl2, xgft2_lo
+ movdqa xtmph3, xgft3_hi ;Reload const array registers
+ movdqa xtmpl3, xgft3_lo
+
+ XLDR xd1, [dest1+pos] ;Get next dest vector
+
+ movdqa xtmpa, x0 ;Keep unshifted copy of src
+ psraw x0, 4 ;Shift to put high nibble into bits 4-0
+ pand x0, xmask0f ;Mask high src nibble in bits 4-0
+ pand xtmpa, xmask0f ;Mask low src nibble in bits 4-0
+
+ ; dest1
+ pshufb xtmph1, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl1, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph1, xtmpl1 ;GF add high and low partials
+ pxor xd1, xtmph1
+
+ XLDR xd2, [dest2+pos] ;reuse xtmpl1. Get next dest vector
+ XLDR xd3, [dest3+pos] ;reuse xtmph1. Get next dest vector
+
+ ; dest2
+ pshufb xtmph2, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl2, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph2, xtmpl2 ;GF add high and low partials
+ pxor xd2, xtmph2
+
+ ; dest3
+ pshufb xtmph3, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl3, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph3, xtmpl3 ;GF add high and low partials
+ pxor xd3, xtmph3
+
+ XSTR [dest1+pos], xd1 ;Store result
+ XSTR [dest2+pos], xd2 ;Store result
+ XSTR [dest3+pos], xd3 ;Store result
+
+ add pos, 16 ;Loop on 16 bytes at a time
+ cmp pos, len
+ jle .loop16
+
+ lea tmp, [len + 16]
+ cmp pos, tmp
+ je .return_pass
+
+.lessthan16:
+ ;; Tail len
+ ;; Do one more overlap pass
+ mov tmp, len ;Overlapped offset length-16
+
+ XLDR x0, [src+tmp] ;Get next source vector
+ XLDR xd1, [dest1+tmp] ;Get next dest vector
+ XLDR xd2, [dest2+tmp] ;reuse xtmpl1. Get next dest vector
+ XLDR xd3, [dest3+tmp] ;reuse xtmph1. Get next dest vector
+
+ sub len, pos
+
+ movdqa xtmph3, [constip16] ;Load const of i + 16
+ pinsrb xtmpl3, len.w, 15
+ pshufb xtmpl3, xmask0f ;Broadcast len to all bytes
+ pcmpgtb xtmpl3, xtmph3
+
+ movdqa xtmpa, x0 ;Keep unshifted copy of src
+ psraw x0, 4 ;Shift to put high nibble into bits 4-0
+ pand x0, xmask0f ;Mask high src nibble in bits 4-0
+ pand xtmpa, xmask0f ;Mask low src nibble in bits 4-0
+
+ ; dest1
+ pshufb xgft1_hi, x0 ;Lookup mul table of high nibble
+ pshufb xgft1_lo, xtmpa ;Lookup mul table of low nibble
+ pxor xgft1_hi, xgft1_lo ;GF add high and low partials
+ pand xgft1_hi, xtmpl3
+ pxor xd1, xgft1_hi
+
+ ; dest2
+ pshufb xgft2_hi, x0 ;Lookup mul table of high nibble
+ pshufb xgft2_lo, xtmpa ;Lookup mul table of low nibble
+ pxor xgft2_hi, xgft2_lo ;GF add high and low partials
+ pand xgft2_hi, xtmpl3
+ pxor xd2, xgft2_hi
+
+ ; dest3
+ pshufb xgft3_hi, x0 ;Lookup mul table of high nibble
+ pshufb xgft3_lo, xtmpa ;Lookup mul table of low nibble
+ pxor xgft3_hi, xgft3_lo ;GF add high and low partials
+ pand xgft3_hi, xtmpl3
+ pxor xd3, xgft3_hi
+
+ XSTR [dest1+tmp], xd1 ;Store result
+ XSTR [dest2+tmp], xd2 ;Store result
+ XSTR [dest3+tmp], xd3 ;Store result
+
+.return_pass:
+ FUNC_RESTORE
+ mov return, 0
+ ret
+
+.return_fail:
+ FUNC_RESTORE
+ mov return, 1
+ ret
+
+endproc_frame
+
+section .data
+
+align 16
+
+mask0f:
+ ddq 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
+constip16:
+ ddq 0xf0f1f2f3f4f5f6f7f8f9fafbfcfdfeff
+
+%macro slversion 4
+global %1_slver_%2%3%4
+global %1_slver
+%1_slver:
+%1_slver_%2%3%4:
+ dw 0x%4
+ db 0x%3, 0x%2
+%endmacro
+;;; func core, ver, snum
+slversion gf_3vect_mad_sse, 00, 00, 0206
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_4vect_dot_prod_avx(len, vec, *g_tbls, **buffs, **dests);
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
%define tmp5 r14 ; must be saved and restored
%define tmp6 r15 ; must be saved and restored
%define return rax
- %define PS 8
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
+ %define PS 8
%define LOG_PS 3
%define func(x) x:
%define tmp5 rdi ; must be saved and restored
%define tmp6 rsi ; must be saved and restored
%define return rax
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
%define PS 8
%define LOG_PS 3
%define stack_size 9*16 + 7*8 ; must be an odd multiple of 8
%endmacro
%endif
+%ifidn __OUTPUT_FORMAT__, elf32
+
+;;;================== High Address;
+;;; arg4
+;;; arg3
+;;; arg2
+;;; arg1
+;;; arg0
+;;; return
+;;;<================= esp of caller
+;;; ebp
+;;;<================= ebp = esp
+;;; var0
+;;; var1
+;;; var2
+;;; var3
+;;; esi
+;;; edi
+;;; ebx
+;;;<================= esp of callee
+;;;
+;;;================== Low Address;
+
+ %define PS 4
+ %define LOG_PS 2
+ %define func(x) x:
+ %define arg(x) [ebp + PS*2 + PS*x]
+ %define var(x) [ebp - PS - PS*x]
+
+ %define trans ecx
+ %define trans2 esi
+ %define arg0 trans ;trans and trans2 are for the variables in stack
+ %define arg0_m arg(0)
+ %define arg1 ebx
+ %define arg2 arg2_m
+ %define arg2_m arg(2)
+ %define arg3 trans
+ %define arg3_m arg(3)
+ %define arg4 trans
+ %define arg4_m arg(4)
+ %define arg5 trans2
+ %define tmp edx
+ %define tmp2 edi
+ %define tmp3 trans2
+ %define tmp3_m var(0)
+ %define tmp4 trans2
+ %define tmp4_m var(1)
+ %define tmp5 trans2
+ %define tmp5_m var(2)
+ %define tmp6 trans2
+ %define tmp6_m var(3)
+ %define return eax
+ %macro SLDR 2 ;stack load/restore
+ mov %1, %2
+ %endmacro
+ %define SSTR SLDR
+
+ %macro FUNC_SAVE 0
+ push ebp
+ mov ebp, esp
+ sub esp, PS*4 ;4 local variables
+ push esi
+ push edi
+ push ebx
+ mov arg1, arg(1)
+ %endmacro
+
+ %macro FUNC_RESTORE 0
+ pop ebx
+ pop edi
+ pop esi
+ add esp, PS*4 ;4 local variables
+ pop ebp
+ %endmacro
+
+%endif ; output formats
%define len arg0
%define vec arg1
%define dest3 tmp4
%define dest4 tmp5
%define vskip3 tmp6
-%define pos return
+%define pos return
+
+ %ifidn PS,4 ;32-bit code
+ %define len_m arg0_m
+ %define src_m arg3_m
+ %define dest1_m arg4_m
+ %define dest2_m tmp3_m
+ %define dest3_m tmp4_m
+ %define dest4_m tmp5_m
+ %define vskip3_m tmp6_m
+ %endif
%ifndef EC_ALIGNED_ADDR
;;; Use Un-aligned load/store
%endif
%endif
+%ifidn PS,8 ; 64-bit code
+ default rel
+ [bits 64]
+%endif
-default rel
-[bits 64]
section .text
-%define xmask0f xmm14
-%define xgft1_lo xmm13
-%define xgft1_hi xmm12
-%define xgft2_lo xmm11
-%define xgft2_hi xmm10
-%define xgft3_lo xmm9
-%define xgft3_hi xmm8
-%define xgft4_lo xmm7
-%define xgft4_hi xmm6
-
-
-%define x0 xmm0
-%define xtmpa xmm1
-%define xp1 xmm2
-%define xp2 xmm3
-%define xp3 xmm4
-%define xp4 xmm5
-
+%ifidn PS,8 ;64-bit code
+ %define xmask0f xmm14
+ %define xgft1_lo xmm13
+ %define xgft1_hi xmm12
+ %define xgft2_lo xmm11
+ %define xgft2_hi xmm10
+ %define xgft3_lo xmm9
+ %define xgft3_hi xmm8
+ %define xgft4_lo xmm7
+ %define xgft4_hi xmm6
+
+ %define x0 xmm0
+ %define xtmpa xmm1
+ %define xp1 xmm2
+ %define xp2 xmm3
+ %define xp3 xmm4
+ %define xp4 xmm5
+%else
+ %define xmm_trans xmm7 ;reuse xmask0f and xgft1_lo
+ %define xmask0f xmm_trans
+ %define xgft1_lo xmm_trans
+ %define xgft1_hi xmm6
+ %define xgft2_lo xgft1_lo
+ %define xgft2_hi xgft1_hi
+ %define xgft3_lo xgft1_lo
+ %define xgft3_hi xgft1_hi
+ %define xgft4_lo xgft1_lo
+ %define xgft4_hi xgft1_hi
+
+ %define x0 xmm0
+ %define xtmpa xmm1
+ %define xp1 xmm2
+ %define xp2 xmm3
+ %define xp3 xmm4
+ %define xp4 xmm5
+%endif
align 16
global gf_4vect_dot_prod_avx:function
func(gf_4vect_dot_prod_avx)
FUNC_SAVE
+ SLDR len, len_m
sub len, 16
+ SSTR len_m, len
jl .return_fail
xor pos, pos
vmovdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
mov vskip3, vec
imul vskip3, 96
+ SSTR vskip3_m, vskip3
sal vec, LOG_PS ;vec *= PS. Make vec_i count by PS
+ SLDR dest1, dest1_m
mov dest2, [dest1+PS]
+ SSTR dest2_m, dest2
mov dest3, [dest1+2*PS]
+ SSTR dest3_m, dest3
mov dest4, [dest1+3*PS]
+ SSTR dest4_m, dest4
mov dest1, [dest1]
-
+ SSTR dest1_m, dest1
.loop16:
vpxor xp1, xp1
xor vec_i, vec_i
.next_vect:
+ SLDR src, src_m
mov ptr, [src+vec_i]
+ %ifidn PS,8 ;64-bit code
vmovdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, ..., Ax{0f}
vmovdqu xgft1_hi, [tmp+16] ; " Ax{00}, Ax{10}, ..., Ax{f0}
vmovdqu xgft2_lo, [tmp+vec*(32/PS)] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
vmovdqu xgft2_hi, [tmp+vec*(32/PS)+16] ; " Bx{00}, Bx{10}, ..., Bx{f0}
vmovdqu xgft3_lo, [tmp+vec*(64/PS)] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
vmovdqu xgft3_hi, [tmp+vec*(64/PS)+16] ; " Cx{00}, Cx{10}, ..., Cx{f0}
- vmovdqu xgft4_lo, [tmp+vskip3] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
- vmovdqu xgft4_hi, [tmp+vskip3+16] ; " Cx{00}, Cx{10}, ..., Cx{f0}
+ vmovdqu xgft4_lo, [tmp+vskip3] ;Load array Dx{00}, Dx{01}, ..., Dx{0f}
+ vmovdqu xgft4_hi, [tmp+vskip3+16] ; " Dx{00}, Dx{10}, ..., Dx{f0}
- XLDR x0, [ptr+pos] ;Get next source vector
+ XLDR x0, [ptr+pos] ;Get next source vector
add tmp, 32
add vec_i, PS
vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+ %else ;32-bit code
+ XLDR x0, [ptr+pos] ;Get next source vector
+ vmovdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
+
+ vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+ vmovdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, ..., Ax{0f}
+ vmovdqu xgft1_hi, [tmp+16] ; " Ax{00}, Ax{10}, ..., Ax{f0}
+ %endif
vpshufb xgft1_hi, x0 ;Lookup mul table of high nibble
vpshufb xgft1_lo, xtmpa ;Lookup mul table of low nibble
vpxor xgft1_hi, xgft1_lo ;GF add high and low partials
vpxor xp1, xgft1_hi ;xp1 += partial
+ %ifidn PS,4 ;32-bit code
+ vmovdqu xgft2_lo, [tmp+vec*(32/PS)] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
+ vmovdqu xgft2_hi, [tmp+vec*(32/PS)+16] ; " Bx{00}, Bx{10}, ..., Bx{f0}
+ %endif
vpshufb xgft2_hi, x0 ;Lookup mul table of high nibble
vpshufb xgft2_lo, xtmpa ;Lookup mul table of low nibble
vpxor xgft2_hi, xgft2_lo ;GF add high and low partials
vpxor xp2, xgft2_hi ;xp2 += partial
+ %ifidn PS,4 ;32-bit code
+ sal vec, 1
+ vmovdqu xgft3_lo, [tmp+vec*(32/PS)] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
+ vmovdqu xgft3_hi, [tmp+vec*(32/PS)+16] ; " Cx{00}, Cx{10}, ..., Cx{f0}
+ sar vec, 1
+ %endif
vpshufb xgft3_hi, x0 ;Lookup mul table of high nibble
vpshufb xgft3_lo, xtmpa ;Lookup mul table of low nibble
vpxor xgft3_hi, xgft3_lo ;GF add high and low partials
vpxor xp3, xgft3_hi ;xp3 += partial
+ %ifidn PS,4 ;32-bit code
+ SLDR vskip3, vskip3_m
+ vmovdqu xgft4_lo, [tmp+vskip3] ;Load array Dx{00}, Dx{01}, ..., Dx{0f}
+ vmovdqu xgft4_hi, [tmp+vskip3+16] ; " Dx{00}, Dx{10}, ..., Dx{f0}
+ add tmp, 32
+ add vec_i, PS
+ %endif
vpshufb xgft4_hi, x0 ;Lookup mul table of high nibble
vpshufb xgft4_lo, xtmpa ;Lookup mul table of low nibble
vpxor xgft4_hi, xgft4_lo ;GF add high and low partials
cmp vec_i, vec
jl .next_vect
+ SLDR dest1, dest1_m
+ SLDR dest2, dest2_m
XSTR [dest1+pos], xp1
XSTR [dest2+pos], xp2
+ SLDR dest3, dest3_m
XSTR [dest3+pos], xp3
+ SLDR dest4, dest4_m
XSTR [dest4+pos], xp4
+ SLDR len, len_m
add pos, 16 ;Loop on 16 bytes at a time
cmp pos, len
jle .loop16
db 0x%3, 0x%2
%endmacro
;;; func core, ver, snum
-slversion gf_4vect_dot_prod_avx, 00, 02, 0064
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
+slversion gf_4vect_dot_prod_avx, 02, 04, 0193
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_4vect_dot_prod_avx2(len, vec, *g_tbls, **buffs, **dests);
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
%define tmp5 r14 ; must be saved and restored
%define tmp6 r15 ; must be saved and restored
%define return rax
- %define PS 8
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
+ %define PS 8
%define LOG_PS 3
%define func(x) x:
%define tmp5 rdi ; must be saved and restored
%define tmp6 rsi ; must be saved and restored
%define return rax
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
%define PS 8
%define LOG_PS 3
%define stack_size 9*16 + 7*8 ; must be an odd multiple of 8
%endmacro
%endif
+%ifidn __OUTPUT_FORMAT__, elf32
+
+;;;================== High Address;
+;;; arg4
+;;; arg3
+;;; arg2
+;;; arg1
+;;; arg0
+;;; return
+;;;<================= esp of caller
+;;; ebp
+;;;<================= ebp = esp
+;;; var0
+;;; var1
+;;; var2
+;;; var3
+;;; esi
+;;; edi
+;;; ebx
+;;;<================= esp of callee
+;;;
+;;;================== Low Address;
+
+ %define PS 4
+ %define LOG_PS 2
+ %define func(x) x:
+ %define arg(x) [ebp + PS*2 + PS*x]
+ %define var(x) [ebp - PS - PS*x]
+
+ %define trans ecx
+ %define trans2 esi
+ %define arg0 trans ;trans and trans2 are for the variables in stack
+ %define arg0_m arg(0)
+ %define arg1 ebx
+ %define arg2 arg2_m
+ %define arg2_m arg(2)
+ %define arg3 trans
+ %define arg3_m arg(3)
+ %define arg4 trans
+ %define arg4_m arg(4)
+ %define arg5 trans2
+ %define tmp edx
+ %define tmp.w edx
+ %define tmp.b dl
+ %define tmp2 edi
+ %define tmp3 trans2
+ %define tmp3_m var(0)
+ %define tmp4 trans2
+ %define tmp4_m var(1)
+ %define tmp5 trans2
+ %define tmp5_m var(2)
+ %define tmp6 trans2
+ %define tmp6_m var(3)
+ %define return eax
+ %macro SLDR 2 ;stack load/restore
+ mov %1, %2
+ %endmacro
+ %define SSTR SLDR
+
+ %macro FUNC_SAVE 0
+ push ebp
+ mov ebp, esp
+ sub esp, PS*4 ;4 local variables
+ push esi
+ push edi
+ push ebx
+ mov arg1, arg(1)
+ %endmacro
+
+ %macro FUNC_RESTORE 0
+ pop ebx
+ pop edi
+ pop esi
+ add esp, PS*4 ;4 local variables
+ pop ebp
+ %endmacro
+
+%endif ; output formats
%define len arg0
%define vec arg1
%define dest3 tmp4
%define dest4 tmp5
%define vskip3 tmp6
-%define pos return
+%define pos return
+
+ %ifidn PS,4 ;32-bit code
+ %define len_m arg0_m
+ %define src_m arg3_m
+ %define dest1_m arg4_m
+ %define dest2_m tmp3_m
+ %define dest3_m tmp4_m
+ %define dest4_m tmp5_m
+ %define vskip3_m tmp6_m
+ %endif
%ifndef EC_ALIGNED_ADDR
;;; Use Un-aligned load/store
%endif
%endif
+%ifidn PS,8 ;64-bit code
+ default rel
+ [bits 64]
+%endif
-default rel
-[bits 64]
section .text
-%define xmask0f ymm14
-%define xmask0fx xmm14
-%define xgft1_lo ymm13
-%define xgft1_hi ymm12
-%define xgft2_lo ymm11
-%define xgft2_hi ymm10
-%define xgft3_lo ymm9
-%define xgft3_hi ymm8
-%define xgft4_lo ymm7
-%define xgft4_hi ymm6
-
-
-%define x0 ymm0
-%define xtmpa ymm1
-%define xp1 ymm2
-%define xp2 ymm3
-%define xp3 ymm4
-%define xp4 ymm5
-
+%ifidn PS,8 ;64-bit code
+ %define xmask0f ymm14
+ %define xmask0fx xmm14
+ %define xgft1_lo ymm13
+ %define xgft1_hi ymm12
+ %define xgft2_lo ymm11
+ %define xgft2_hi ymm10
+ %define xgft3_lo ymm9
+ %define xgft3_hi ymm8
+ %define xgft4_lo ymm7
+ %define xgft4_hi ymm6
+
+ %define x0 ymm0
+ %define xtmpa ymm1
+ %define xp1 ymm2
+ %define xp2 ymm3
+ %define xp3 ymm4
+ %define xp4 ymm5
+%else
+ %define ymm_trans ymm7 ;reuse xmask0f and xgft1_hi
+ %define xmask0f ymm_trans
+ %define xmask0fx xmm7
+ %define xgft1_lo ymm6
+ %define xgft1_hi ymm_trans
+ %define xgft2_lo xgft1_lo
+ %define xgft2_hi xgft1_hi
+ %define xgft3_lo xgft1_lo
+ %define xgft3_hi xgft1_hi
+ %define xgft4_lo xgft1_lo
+ %define xgft4_hi xgft1_hi
+
+ %define x0 ymm0
+ %define xtmpa ymm1
+ %define xp1 ymm2
+ %define xp2 ymm3
+ %define xp3 ymm4
+ %define xp4 ymm5
+%endif
align 16
global gf_4vect_dot_prod_avx2:function
func(gf_4vect_dot_prod_avx2)
FUNC_SAVE
+ SLDR len, len_m
sub len, 32
+ SSTR len_m, len
jl .return_fail
xor pos, pos
mov tmp.b, 0x0f
vpbroadcastb xmask0f, xmask0fx ;Construct mask 0x0f0f0f...
mov vskip3, vec
imul vskip3, 96
+ SSTR vskip3_m, vskip3
sal vec, LOG_PS ;vec *= PS. Make vec_i count by PS
+ SLDR dest1, dest1_m
mov dest2, [dest1+PS]
+ SSTR dest2_m, dest2
mov dest3, [dest1+2*PS]
+ SSTR dest3_m, dest3
mov dest4, [dest1+3*PS]
+ SSTR dest4_m, dest4
mov dest1, [dest1]
-
+ SSTR dest1_m, dest1
.loop32:
vpxor xp1, xp1
xor vec_i, vec_i
.next_vect:
+ SLDR src, src_m
mov ptr, [src+vec_i]
XLDR x0, [ptr+pos] ;Get next source vector
- add vec_i, PS
+ add vec_i, PS
+ %ifidn PS,8 ;64-bit code
vpand xgft4_lo, x0, xmask0f ;Mask low src nibble in bits 4-0
vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
; " Bx{00}, Bx{10}, ..., Bx{f0}
vmovdqu xgft3_lo, [tmp+vec*(64/PS)] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
; " Cx{00}, Cx{10}, ..., Cx{f0}
- vmovdqu xgft4_lo, [tmp+vskip3] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
- ; " Cx{00}, Cx{10}, ..., Cx{f0}
+ vmovdqu xgft4_lo, [tmp+vskip3] ;Load array Dx{00}, Dx{01}, ..., Dx{0f}
+ ; " Dx{00}, Dx{10}, ..., Dx{f0}
vperm2i128 xgft1_hi, xgft1_lo, xgft1_lo, 0x01 ; swapped to hi | lo
vperm2i128 xgft2_hi, xgft2_lo, xgft2_lo, 0x01 ; swapped to hi | lo
vperm2i128 xgft3_hi, xgft3_lo, xgft3_lo, 0x01 ; swapped to hi | lo
vperm2i128 xgft4_hi, xgft4_lo, xgft4_lo, 0x01 ; swapped to hi | lo
add tmp, 32
+ %else ;32-bit code
+ mov cl, 0x0f ;use ecx as a temp variable
+ vpinsrb xmask0fx, xmask0fx, ecx, 0
+ vpbroadcastb xmask0f, xmask0fx ;Construct mask 0x0f0f0f...
+
+ vpand xgft4_lo, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+ vperm2i128 xtmpa, xgft4_lo, x0, 0x30 ;swap xtmpa from 1lo|2lo to 1lo|2hi
+ vperm2i128 x0, xgft4_lo, x0, 0x12 ;swap x0 from 1hi|2hi to 1hi|2lo
+
+ vmovdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, ..., Ax{0f}
+ ; " Ax{00}, Ax{10}, ..., Ax{f0}
+ vperm2i128 xgft1_hi, xgft1_lo, xgft1_lo, 0x01 ; swapped to hi | lo
+ %endif
vpshufb xgft1_hi, x0 ;Lookup mul table of high nibble
vpshufb xgft1_lo, xtmpa ;Lookup mul table of low nibble
vpxor xgft1_hi, xgft1_lo ;GF add high and low partials
vpxor xp1, xgft1_hi ;xp1 += partial
+ %ifidn PS,4 ; 32-bit code
+ vmovdqu xgft2_lo, [tmp+vec*(32/PS)] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
+ ; " Bx{00}, Bx{10}, ..., Bx{f0}
+ vperm2i128 xgft2_hi, xgft2_lo, xgft2_lo, 0x01 ; swapped to hi | lo
+ %endif
vpshufb xgft2_hi, x0 ;Lookup mul table of high nibble
vpshufb xgft2_lo, xtmpa ;Lookup mul table of low nibble
vpxor xgft2_hi, xgft2_lo ;GF add high and low partials
vpxor xp2, xgft2_hi ;xp2 += partial
+ %ifidn PS,4 ; 32-bit code
+ sal vec, 1
+ vmovdqu xgft3_lo, [tmp+vec*(32/PS)] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
+ ; " Cx{00}, Cx{10}, ..., Cx{f0}
+ vperm2i128 xgft3_hi, xgft3_lo, xgft3_lo, 0x01 ; swapped to hi | lo
+ sar vec, 1
+ %endif
vpshufb xgft3_hi, x0 ;Lookup mul table of high nibble
vpshufb xgft3_lo, xtmpa ;Lookup mul table of low nibble
vpxor xgft3_hi, xgft3_lo ;GF add high and low partials
vpxor xp3, xgft3_hi ;xp3 += partial
+ %ifidn PS,4 ; 32-bit code
+ SLDR vskip3, vskip3_m
+ vmovdqu xgft4_lo, [tmp+vskip3] ;Load array Dx{00}, Dx{01}, ..., Dx{0f}
+ ; " DX{00}, Dx{10}, ..., Dx{f0}
+ vperm2i128 xgft4_hi, xgft4_lo, xgft4_lo, 0x01 ; swapped to hi | lo
+ add tmp, 32
+ %endif
vpshufb xgft4_hi, x0 ;Lookup mul table of high nibble
vpshufb xgft4_lo, xtmpa ;Lookup mul table of low nibble
vpxor xgft4_hi, xgft4_lo ;GF add high and low partials
cmp vec_i, vec
jl .next_vect
+ SLDR dest1, dest1_m
+ SLDR dest2, dest2_m
XSTR [dest1+pos], xp1
XSTR [dest2+pos], xp2
+ SLDR dest3, dest3_m
XSTR [dest3+pos], xp3
+ SLDR dest4, dest4_m
XSTR [dest4+pos], xp4
+ SLDR len, len_m
add pos, 32 ;Loop on 32 bytes at a time
cmp pos, len
jle .loop32
db 0x%3, 0x%2
%endmacro
;;; func core, ver, snum
-slversion gf_4vect_dot_prod_avx2, 04, 03, 0064
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
+slversion gf_4vect_dot_prod_avx2, 04, 04, 0198
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_4vect_dot_prod_sse(len, vec, *g_tbls, **buffs, **dests);
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
%define tmp5 r14 ; must be saved and restored
%define tmp6 r15 ; must be saved and restored
%define return rax
- %define PS 8
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
+ %define PS 8
%define LOG_PS 3
%define func(x) x:
%define tmp5 rdi ; must be saved and restored
%define tmp6 rsi ; must be saved and restored
%define return rax
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
%define PS 8
%define LOG_PS 3
%define stack_size 9*16 + 7*8 ; must be an odd multiple of 8
%endmacro
%endif
+%ifidn __OUTPUT_FORMAT__, elf32
+
+;;;================== High Address;
+;;; arg4
+;;; arg3
+;;; arg2
+;;; arg1
+;;; arg0
+;;; return
+;;;<================= esp of caller
+;;; ebp
+;;;<================= ebp = esp
+;;; var0
+;;; var1
+;;; var2
+;;; var3
+;;; esi
+;;; edi
+;;; ebx
+;;;<================= esp of callee
+;;;
+;;;================== Low Address;
+
+ %define PS 4
+ %define LOG_PS 2
+ %define func(x) x:
+ %define arg(x) [ebp + PS*2 + PS*x]
+ %define var(x) [ebp - PS - PS*x]
+
+ %define trans ecx
+ %define trans2 esi
+ %define arg0 trans ;trans and trans2 are for the variables in stack
+ %define arg0_m arg(0)
+ %define arg1 ebx
+ %define arg2 arg2_m
+ %define arg2_m arg(2)
+ %define arg3 trans
+ %define arg3_m arg(3)
+ %define arg4 trans
+ %define arg4_m arg(4)
+ %define arg5 trans2
+ %define tmp edx
+ %define tmp2 edi
+ %define tmp3 trans2
+ %define tmp3_m var(0)
+ %define tmp4 trans2
+ %define tmp4_m var(1)
+ %define tmp5 trans2
+ %define tmp5_m var(2)
+ %define tmp6 trans2
+ %define tmp6_m var(3)
+ %define return eax
+ %macro SLDR 2 ;stack load/restore
+ mov %1, %2
+ %endmacro
+ %define SSTR SLDR
+
+ %macro FUNC_SAVE 0
+ push ebp
+ mov ebp, esp
+ sub esp, PS*4 ;4 local variables
+ push esi
+ push edi
+ push ebx
+ mov arg1, arg(1)
+ %endmacro
+
+ %macro FUNC_RESTORE 0
+ pop ebx
+ pop edi
+ pop esi
+ add esp, PS*4 ;4 local variables
+ pop ebp
+ %endmacro
+
+%endif ; output formats
%define len arg0
%define vec arg1
%define dest3 tmp4
%define dest4 tmp5
%define vskip3 tmp6
-%define pos return
+%define pos return
+
+ %ifidn PS,4 ;32-bit code
+ %define len_m arg0_m
+ %define src_m arg3_m
+ %define dest1_m arg4_m
+ %define dest2_m tmp3_m
+ %define dest3_m tmp4_m
+ %define dest4_m tmp5_m
+ %define vskip3_m tmp6_m
+ %endif
%ifndef EC_ALIGNED_ADDR
;;; Use Un-aligned load/store
%endif
%endif
+%ifidn PS,8 ; 64-bit code
+ default rel
+ [bits 64]
+%endif
-default rel
-[bits 64]
section .text
-%define xmask0f xmm14
-%define xgft1_lo xmm13
-%define xgft1_hi xmm12
-%define xgft2_lo xmm11
-%define xgft2_hi xmm10
-%define xgft3_lo xmm9
-%define xgft3_hi xmm8
-%define xgft4_lo xmm7
-%define xgft4_hi xmm6
-
-
-%define x0 xmm0
-%define xtmpa xmm1
-%define xp1 xmm2
-%define xp2 xmm3
-%define xp3 xmm4
-%define xp4 xmm5
-
+%ifidn PS,8 ;64-bit code
+ %define xmask0f xmm14
+ %define xgft1_lo xmm2
+ %define xgft1_hi xmm3
+ %define xgft2_lo xmm11
+ %define xgft2_hi xmm4
+ %define xgft3_lo xmm9
+ %define xgft3_hi xmm5
+ %define xgft4_lo xmm7
+ %define xgft4_hi xmm6
+
+ %define x0 xmm0
+ %define xtmpa xmm1
+ %define xp1 xmm8
+ %define xp2 xmm10
+ %define xp3 xmm12
+ %define xp4 xmm13
+%else
+ %define xmm_trans xmm7 ;reuse xmask0f and xgft1_lo
+ %define xmask0f xmm_trans
+ %define xgft1_lo xmm_trans
+ %define xgft1_hi xmm6
+ %define xgft2_lo xgft1_lo
+ %define xgft2_hi xgft1_hi
+ %define xgft3_lo xgft1_lo
+ %define xgft3_hi xgft1_hi
+ %define xgft4_lo xgft1_lo
+ %define xgft4_hi xgft1_hi
+
+ %define x0 xmm0
+ %define xtmpa xmm1
+ %define xp1 xmm2
+ %define xp2 xmm3
+ %define xp3 xmm4
+ %define xp4 xmm5
+%endif
align 16
global gf_4vect_dot_prod_sse:function
func(gf_4vect_dot_prod_sse)
FUNC_SAVE
+ SLDR len, len_m
sub len, 16
+ SSTR len_m, len
jl .return_fail
xor pos, pos
movdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
- mov vskip3, vec
- imul vskip3, 96
- sal vec, LOG_PS ;vec *= PS. Make vec_i count by PS
- mov dest2, [dest1+PS]
- mov dest3, [dest1+2*PS]
- mov dest4, [dest1+3*PS]
- mov dest1, [dest1]
-
+ mov vskip3, vec
+ imul vskip3, 96
+ SSTR vskip3_m, vskip3
+ sal vec, LOG_PS ;vec *= PS. Make vec_i count by PS
+ SLDR dest1, dest1_m
+ mov dest2, [dest1+PS]
+ SSTR dest2_m, dest2
+ mov dest3, [dest1+2*PS]
+ SSTR dest3_m, dest3
+ mov dest4, [dest1+3*PS]
+ SSTR dest4_m, dest4
+ mov dest1, [dest1]
+ SSTR dest1_m, dest1
.loop16:
pxor xp1, xp1
xor vec_i, vec_i
.next_vect:
+ SLDR src, src_m
mov ptr, [src+vec_i]
+ %ifidn PS,8 ;64-bit code
movdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, ..., Ax{0f}
movdqu xgft1_hi, [tmp+16] ; " Ax{00}, Ax{10}, ..., Ax{f0}
movdqu xgft2_lo, [tmp+vec*(32/PS)] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
movdqu xgft2_hi, [tmp+vec*(32/PS)+16] ; " Bx{00}, Bx{10}, ..., Bx{f0}
movdqu xgft3_lo, [tmp+vec*(64/PS)] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
movdqu xgft3_hi, [tmp+vec*(64/PS)+16] ; " Cx{00}, Cx{10}, ..., Cx{f0}
- movdqu xgft4_lo, [tmp+vskip3] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
- movdqu xgft4_hi, [tmp+vskip3+16] ; " Cx{00}, Cx{10}, ..., Cx{f0}
+ movdqu xgft4_lo, [tmp+vskip3] ;Load array Dx{00}, Dx{01}, ..., Dx{0f}
+ movdqu xgft4_hi, [tmp+vskip3+16] ; " Dx{00}, Dx{10}, ..., Dx{f0}
- XLDR x0, [ptr+pos] ;Get next source vector
- add tmp, 32
- add vec_i, PS
+ XLDR x0, [ptr+pos] ;Get next source vector
+ add tmp, 32
+ add vec_i, PS
movdqa xtmpa, x0 ;Keep unshifted copy of src
psraw x0, 4 ;Shift to put high nibble into bits 4-0
pand x0, xmask0f ;Mask high src nibble in bits 4-0
+ pand xtmpa, xmask0f ;Mask low src nibble in bits 4-0
+ %else ;32-bit code
+ XLDR x0, [ptr+pos] ;Get next source vector
+ movdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
+
+ movdqa xtmpa, x0 ;Keep unshifted copy of src
+ psraw x0, 4 ;Shift to put high nibble into bits 4-0
+ pand x0, xmask0f ;Mask high src nibble in bits 4-0
pand xtmpa, xmask0f ;Mask low src nibble in bits 4-0
+ movdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, ..., Ax{0f}
+ movdqu xgft1_hi, [tmp+16] ; " Ax{00}, Ax{10}, ..., Ax{f0}
+ %endif
+
pshufb xgft1_hi, x0 ;Lookup mul table of high nibble
pshufb xgft1_lo, xtmpa ;Lookup mul table of low nibble
pxor xgft1_hi, xgft1_lo ;GF add high and low partials
pxor xp1, xgft1_hi ;xp1 += partial
+ %ifidn PS,4 ;32-bit code
+ movdqu xgft2_lo, [tmp+vec*(32/PS)] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
+ movdqu xgft2_hi, [tmp+vec*(32/PS)+16] ; " Bx{00}, Bx{10}, ..., Bx{f0}
+ %endif
pshufb xgft2_hi, x0 ;Lookup mul table of high nibble
pshufb xgft2_lo, xtmpa ;Lookup mul table of low nibble
pxor xgft2_hi, xgft2_lo ;GF add high and low partials
pxor xp2, xgft2_hi ;xp2 += partial
+ %ifidn PS,4 ;32-bit code
+ sal vec, 1
+ movdqu xgft3_lo, [tmp+vec*(32/PS)] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
+ movdqu xgft3_hi, [tmp+vec*(32/PS)+16] ; " Cx{00}, Cx{10}, ..., Cx{f0}
+ sar vec, 1
+ %endif
pshufb xgft3_hi, x0 ;Lookup mul table of high nibble
pshufb xgft3_lo, xtmpa ;Lookup mul table of low nibble
pxor xgft3_hi, xgft3_lo ;GF add high and low partials
pxor xp3, xgft3_hi ;xp3 += partial
+ %ifidn PS,4 ;32-bit code
+ SLDR vskip3, vskip3_m
+ movdqu xgft4_lo, [tmp+vskip3] ;Load array Dx{00}, Dx{01}, ..., Dx{0f}
+ movdqu xgft4_hi, [tmp+vskip3+16] ; " Dx{00}, Dx{10}, ..., Dx{f0}
+ add tmp, 32
+ add vec_i, PS
+ %endif
pshufb xgft4_hi, x0 ;Lookup mul table of high nibble
pshufb xgft4_lo, xtmpa ;Lookup mul table of low nibble
pxor xgft4_hi, xgft4_lo ;GF add high and low partials
cmp vec_i, vec
jl .next_vect
+ SLDR dest1, dest1_m
+ SLDR dest2, dest2_m
XSTR [dest1+pos], xp1
XSTR [dest2+pos], xp2
+ SLDR dest3, dest3_m
XSTR [dest3+pos], xp3
+ SLDR dest4, dest4_m
XSTR [dest4+pos], xp4
+ SLDR len, len_m
add pos, 16 ;Loop on 16 bytes at a time
cmp pos, len
jle .loop16
db 0x%3, 0x%2
%endmacro
;;; func core, ver, snum
-slversion gf_4vect_dot_prod_sse, 00, 03, 0064
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
+slversion gf_4vect_dot_prod_sse, 00, 05, 0064
--- /dev/null
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+; * Redistributions of source code must retain the above copyright
+; notice, this list of conditions and the following disclaimer.
+; * Redistributions in binary form must reproduce the above copyright
+; notice, this list of conditions and the following disclaimer in
+; the documentation and/or other materials provided with the
+; distribution.
+; * Neither the name of Intel Corporation nor the names of its
+; contributors may be used to endorse or promote products derived
+; from this software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;
+;;; gf_4vect_mad_avx(len, vec, vec_i, mul_array, src, dest);
+;;;
+
+%define PS 8
+
+%ifidn __OUTPUT_FORMAT__, win64
+ %define arg0 rcx
+ %define arg0.w ecx
+ %define arg1 rdx
+ %define arg2 r8
+ %define arg3 r9
+ %define arg4 r12
+ %define arg5 r15
+ %define tmp r11
+ %define tmp2 r10
+ %define tmp3 r13
+ %define return rax
+ %define return.w eax
+ %define stack_size 16*10 + 3*8
+ %define arg(x) [rsp + stack_size + PS + PS*x]
+ %define func(x) proc_frame x
+
+%macro FUNC_SAVE 0
+ sub rsp, stack_size
+ movdqa [rsp+16*0],xmm6
+ movdqa [rsp+16*1],xmm7
+ movdqa [rsp+16*2],xmm8
+ movdqa [rsp+16*3],xmm9
+ movdqa [rsp+16*4],xmm10
+ movdqa [rsp+16*5],xmm11
+ movdqa [rsp+16*6],xmm12
+ movdqa [rsp+16*7],xmm13
+ movdqa [rsp+16*8],xmm14
+ movdqa [rsp+16*9],xmm15
+ save_reg r12, 10*16 + 0*8
+ save_reg r13, 10*16 + 1*8
+ save_reg r15, 10*16 + 2*8
+ end_prolog
+ mov arg4, arg(4)
+ mov arg5, arg(5)
+%endmacro
+
+%macro FUNC_RESTORE 0
+ movdqa xmm6, [rsp+16*0]
+ movdqa xmm7, [rsp+16*1]
+ movdqa xmm8, [rsp+16*2]
+ movdqa xmm9, [rsp+16*3]
+ movdqa xmm10, [rsp+16*4]
+ movdqa xmm11, [rsp+16*5]
+ movdqa xmm12, [rsp+16*6]
+ movdqa xmm13, [rsp+16*7]
+ movdqa xmm14, [rsp+16*8]
+ movdqa xmm15, [rsp+16*9]
+ mov r12, [rsp + 10*16 + 0*8]
+ mov r13, [rsp + 10*16 + 1*8]
+ mov r15, [rsp + 10*16 + 2*8]
+ add rsp, stack_size
+%endmacro
+
+%elifidn __OUTPUT_FORMAT__, elf64
+ %define arg0 rdi
+ %define arg0.w edi
+ %define arg1 rsi
+ %define arg2 rdx
+ %define arg3 rcx
+ %define arg4 r8
+ %define arg5 r9
+ %define tmp r11
+ %define tmp2 r10
+ %define tmp3 r12
+ %define return rax
+ %define return.w eax
+
+ %define func(x) x:
+ %macro FUNC_SAVE 0
+ push r12
+ %endmacro
+ %macro FUNC_RESTORE 0
+ pop r12
+ %endmacro
+%endif
+
+;;; gf_4vect_mad_avx(len, vec, vec_i, mul_array, src, dest)
+%define len arg0
+%define len.w arg0.w
+%define vec arg1
+%define vec_i arg2
+%define mul_array arg3
+%define src arg4
+%define dest1 arg5
+%define pos return
+%define pos.w return.w
+
+%define dest2 mul_array
+%define dest3 tmp2
+%define dest4 vec_i
+
+%ifndef EC_ALIGNED_ADDR
+;;; Use Un-aligned load/store
+ %define XLDR vmovdqu
+ %define XSTR vmovdqu
+%else
+;;; Use Non-temporal load/stor
+ %ifdef NO_NT_LDST
+ %define XLDR vmovdqa
+ %define XSTR vmovdqa
+ %else
+ %define XLDR vmovntdqa
+ %define XSTR vmovntdq
+ %endif
+%endif
+
+
+default rel
+
+[bits 64]
+section .text
+
+%define xmask0f xmm15
+%define xgft3_hi xmm14
+%define xgft4_hi xmm13
+%define xgft4_lo xmm12
+
+%define x0 xmm0
+%define xtmpa xmm1
+%define xtmph1 xmm2
+%define xtmpl1 xmm3
+%define xtmph2 xmm4
+%define xtmpl2 xmm5
+%define xtmph3 xmm6
+%define xtmpl3 xmm7
+%define xtmph4 xmm8
+%define xtmpl4 xmm9
+%define xd1 xmm10
+%define xd2 xmm11
+%define xd3 xtmph1
+%define xd4 xtmpl1
+
+align 16
+global gf_4vect_mad_avx:function
+func(gf_4vect_mad_avx)
+ FUNC_SAVE
+ sub len, 16
+ jl .return_fail
+ xor pos, pos
+ vmovdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
+
+ mov tmp, vec
+
+ sal vec_i, 5 ;Multiply by 32
+ lea tmp3, [mul_array + vec_i]
+
+ sal tmp, 6 ;Multiply by 64
+ vmovdqu xgft3_hi, [tmp3+tmp+16] ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
+ sal vec, 5 ;Multiply by 32
+ add tmp, vec
+ vmovdqu xgft4_lo, [tmp3+tmp] ;Load array Dx{00}, Dx{01}, Dx{02}, ...
+ vmovdqu xgft4_hi, [tmp3+tmp+16] ; " Dx{00}, Dx{10}, Dx{20}, ... , Dx{f0}
+
+ mov dest2, [dest1+PS] ; reuse mul_array
+ mov dest3, [dest1+2*PS]
+ mov dest4, [dest1+3*PS] ; reuse vec_i
+ mov dest1, [dest1]
+
+.loop16:
+ XLDR x0, [src+pos] ;Get next source vector
+ vmovdqu xtmph1, [tmp3+16] ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
+ vmovdqu xtmpl1, [tmp3] ;Load array Ax{00}, Ax{01}, Ax{02}, ...
+ vmovdqu xtmph2, [tmp3+vec+16] ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
+ vmovdqu xtmpl2, [tmp3+vec] ;Load array Bx{00}, Bx{01}, Bx{02}, ...
+ vmovdqu xtmpl3, [tmp3+2*vec] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
+
+ XLDR xd1, [dest1+pos] ;Get next dest vector
+ XLDR xd2, [dest2+pos] ;Get next dest vector
+
+ vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+
+ ; dest1
+ vpshufb xtmph1, xtmph1, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl1, xtmpl1, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph1, xtmph1, xtmpl1 ;GF add high and low partials
+ vpxor xd1, xd1, xtmph1
+
+ XLDR xd3, [dest3+pos] ;Reuse xtmph1, Get next dest vector
+ XLDR xd4, [dest4+pos] ;Reuse xtmpl1, Get next dest vector
+
+ ; dest2
+ vpshufb xtmph2, xtmph2, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl2, xtmpl2, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph2, xtmph2, xtmpl2 ;GF add high and low partials
+ vpxor xd2, xd2, xtmph2
+
+ ; dest3
+ vpshufb xtmph3, xgft3_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl3, xtmpl3, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph3, xtmph3, xtmpl3 ;GF add high and low partials
+ vpxor xd3, xd3, xtmph3
+
+ ; dest4
+ vpshufb xtmph4, xgft4_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl4, xgft4_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph4, xtmph4, xtmpl4 ;GF add high and low partials
+ vpxor xd4, xd4, xtmph4
+
+ XSTR [dest1+pos], xd1 ;Store result
+ XSTR [dest2+pos], xd2 ;Store result
+ XSTR [dest3+pos], xd3 ;Store result
+ XSTR [dest4+pos], xd4 ;Store result
+
+ add pos, 16 ;Loop on 16 bytes at a time
+ cmp pos, len
+ jle .loop16
+
+ lea tmp, [len + 16]
+ cmp pos, tmp
+ je .return_pass
+
+.lessthan16:
+ ;; Tail len
+ ;; Do one more overlap pass
+
+ mov tmp, len ;Overlapped offset length-16
+
+ XLDR x0, [src+tmp] ;Get next source vector
+
+ vmovdqu xtmph1, [tmp3+16] ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
+ vmovdqu xtmpl1, [tmp3] ;Load array Ax{00}, Ax{01}, Ax{02}, ...
+ vmovdqu xtmph2, [tmp3+vec+16] ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
+ vmovdqu xtmpl2, [tmp3+vec] ;Load array Bx{00}, Bx{01}, Bx{02}, ...
+ vmovdqu xtmpl3, [tmp3+2*vec] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
+
+ XLDR xd1, [dest1+tmp] ;Get next dest vector
+ XLDR xd2, [dest2+tmp] ;Get next dest vector
+ XLDR xtmph4, [dest3+tmp] ;Get next dest vector
+
+ sub len, pos
+
+ vmovdqa xtmpl4, [constip16] ;Load const of i + 16
+ vpinsrb xtmph3, xtmph3, len.w, 15
+ vpshufb xtmph3, xtmph3, xmask0f ;Broadcast len to all bytes
+ vpcmpgtb xtmph3, xtmph3, xtmpl4
+
+ XLDR xtmpl4, [dest4+tmp] ;Get next dest vector
+
+ vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+
+ ; dest1
+ vpshufb xtmph1, xtmph1, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl1, xtmpl1, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph1, xtmph1, xtmpl1 ;GF add high and low partials
+ vpand xtmph1, xtmph1, xtmph3
+ vpxor xd1, xd1, xtmph1
+
+ ; dest2
+ vpshufb xtmph2, xtmph2, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl2, xtmpl2, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph2, xtmph2, xtmpl2 ;GF add high and low partials
+ vpand xtmph2, xtmph2, xtmph3
+ vpxor xd2, xd2, xtmph2
+
+ ; dest3
+ vpshufb xgft3_hi, xgft3_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl3, xtmpl3, xtmpa ;Lookup mul table of low nibble
+ vpxor xgft3_hi, xgft3_hi, xtmpl3 ;GF add high and low partials
+ vpand xgft3_hi, xgft3_hi, xtmph3
+ vpxor xtmph4, xtmph4, xgft3_hi
+
+ ; dest4
+ vpshufb xgft4_hi, xgft4_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xgft4_lo, xgft4_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xgft4_hi, xgft4_hi, xgft4_lo ;GF add high and low partials
+ vpand xgft4_hi, xgft4_hi, xtmph3
+ vpxor xtmpl4, xtmpl4, xgft4_hi
+
+ XSTR [dest1+tmp], xd1 ;Store result
+ XSTR [dest2+tmp], xd2 ;Store result
+ XSTR [dest3+tmp], xtmph4 ;Store result
+ XSTR [dest4+tmp], xtmpl4 ;Store result
+
+.return_pass:
+ mov return, 0
+ FUNC_RESTORE
+ ret
+
+.return_fail:
+ mov return, 1
+ FUNC_RESTORE
+ ret
+
+endproc_frame
+
+section .data
+
+align 16
+mask0f: ddq 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
+constip16:
+ ddq 0xf0f1f2f3f4f5f6f7f8f9fafbfcfdfeff
+
+
+%macro slversion 4
+global %1_slver_%2%3%4
+global %1_slver
+%1_slver:
+%1_slver_%2%3%4:
+ dw 0x%4
+ db 0x%3, 0x%2
+%endmacro
+;;; func core, ver, snum
+slversion gf_4vect_mad_avx, 02, 00, 020a
--- /dev/null
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+; * Redistributions of source code must retain the above copyright
+; notice, this list of conditions and the following disclaimer.
+; * Redistributions in binary form must reproduce the above copyright
+; notice, this list of conditions and the following disclaimer in
+; the documentation and/or other materials provided with the
+; distribution.
+; * Neither the name of Intel Corporation nor the names of its
+; contributors may be used to endorse or promote products derived
+; from this software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;
+;;; gf_4vect_mad_avx2(len, vec, vec_i, mul_array, src, dest);
+;;;
+
+%define PS 8
+
+%ifidn __OUTPUT_FORMAT__, win64
+ %define arg0 rcx
+ %define arg0.w ecx
+ %define arg1 rdx
+ %define arg2 r8
+ %define arg3 r9
+ %define arg4 r12
+ %define arg5 r15
+ %define tmp r11
+ %define tmp.w r11d
+ %define tmp.b r11b
+ %define return rax
+ %define return.w eax
+ %define stack_size 16*10 + 3*8
+ %define arg(x) [rsp + stack_size + PS + PS*x]
+ %define func(x) proc_frame x
+
+%macro FUNC_SAVE 0
+ sub rsp, stack_size
+ movdqa [rsp+16*0],xmm6
+ movdqa [rsp+16*1],xmm7
+ movdqa [rsp+16*2],xmm8
+ movdqa [rsp+16*3],xmm9
+ movdqa [rsp+16*4],xmm10
+ movdqa [rsp+16*5],xmm11
+ movdqa [rsp+16*6],xmm12
+ movdqa [rsp+16*7],xmm13
+ movdqa [rsp+16*8],xmm14
+ movdqa [rsp+16*9],xmm15
+ save_reg r12, 10*16 + 0*8
+ save_reg r15, 10*16 + 1*8
+ end_prolog
+ mov arg4, arg(4)
+ mov arg5, arg(5)
+%endmacro
+
+%macro FUNC_RESTORE 0
+ movdqa xmm6, [rsp+16*0]
+ movdqa xmm7, [rsp+16*1]
+ movdqa xmm8, [rsp+16*2]
+ movdqa xmm9, [rsp+16*3]
+ movdqa xmm10, [rsp+16*4]
+ movdqa xmm11, [rsp+16*5]
+ movdqa xmm12, [rsp+16*6]
+ movdqa xmm13, [rsp+16*7]
+ movdqa xmm14, [rsp+16*8]
+ movdqa xmm15, [rsp+16*9]
+ mov r12, [rsp + 10*16 + 0*8]
+ mov r15, [rsp + 10*16 + 1*8]
+ add rsp, stack_size
+%endmacro
+
+%elifidn __OUTPUT_FORMAT__, elf64
+ %define arg0 rdi
+ %define arg0.w edi
+ %define arg1 rsi
+ %define arg2 rdx
+ %define arg3 rcx
+ %define arg4 r8
+ %define arg5 r9
+ %define tmp r11
+ %define tmp.w r11d
+ %define tmp.b r11b
+ %define return rax
+ %define return.w eax
+
+ %define func(x) x:
+ %define FUNC_SAVE
+ %define FUNC_RESTORE
+%endif
+
+
+;;; gf_4vect_mad_avx2(len, vec, vec_i, mul_array, src, dest)
+%define len arg0
+%define len.w arg0.w
+%define vec arg1
+%define vec_i arg2
+%define mul_array arg3
+%define src arg4
+%define dest1 arg5
+%define pos return
+%define pos.w return.w
+
+%define dest2 mul_array
+%define dest3 vec
+%define dest4 vec_i
+
+%ifndef EC_ALIGNED_ADDR
+;;; Use Un-aligned load/store
+ %define XLDR vmovdqu
+ %define XSTR vmovdqu
+%else
+;;; Use Non-temporal load/stor
+ %ifdef NO_NT_LDST
+ %define XLDR vmovdqa
+ %define XSTR vmovdqa
+ %else
+ %define XLDR vmovntdqa
+ %define XSTR vmovntdq
+ %endif
+%endif
+
+
+default rel
+
+[bits 64]
+section .text
+
+%define xmask0f ymm15
+%define xmask0fx xmm15
+%define xgft1_lo ymm14
+%define xgft2_lo ymm13
+%define xgft3_lo ymm12
+%define xgft4_lo ymm11
+
+%define x0 ymm0
+%define xtmpa ymm1
+%define xtmpl ymm2
+%define xtmplx xmm2
+%define xtmph1 ymm3
+%define xtmph1x xmm3
+%define xtmph2 ymm4
+%define xtmph3 ymm5
+%define xtmph4 ymm6
+%define xd1 ymm7
+%define xd2 ymm8
+%define xd3 ymm9
+%define xd4 ymm10
+
+align 16
+global gf_4vect_mad_avx2:function
+func(gf_4vect_mad_avx2)
+ FUNC_SAVE
+ sub len, 32
+ jl .return_fail
+ xor pos, pos
+ mov tmp.b, 0x0f
+ vpinsrb xmask0fx, xmask0fx, tmp.w, 0
+ vpbroadcastb xmask0f, xmask0fx ;Construct mask 0x0f0f0f...
+
+ sal vec_i, 5 ;Multiply by 32
+ sal vec, 5 ;Multiply by 32
+ lea tmp, [mul_array + vec_i]
+
+ vmovdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, Ax{02}, ...
+ ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
+ vmovdqu xgft2_lo, [tmp+vec] ;Load array Bx{00}, Bx{01}, Bx{02}, ...
+ ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
+ vmovdqu xgft3_lo, [tmp+2*vec] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
+ ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
+ add tmp, vec
+ vmovdqu xgft4_lo, [tmp+2*vec] ;Load array Dx{00}, Dx{01}, Dx{02}, ...
+ ; " Dx{00}, Dx{10}, Dx{20}, ... , Dx{f0}
+
+ mov dest2, [dest1+PS] ; reuse mul_array
+ mov dest3, [dest1+2*PS] ; reuse vec
+ mov dest4, [dest1+3*PS] ; reuse vec_i
+ mov dest1, [dest1]
+
+.loop32:
+ XLDR x0, [src+pos] ;Get next source vector
+
+ XLDR xd1, [dest1+pos] ;Get next dest vector
+ XLDR xd2, [dest2+pos] ;Get next dest vector
+ XLDR xd3, [dest3+pos] ;Get next dest vector
+ XLDR xd4, [dest4+pos] ;reuse xtmpl1. Get next dest vector
+
+ vpand xtmpl, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+
+ vperm2i128 xtmpa, xtmpl, x0, 0x30 ;swap xtmpa from 1lo|2lo to 1lo|2hi
+ vperm2i128 x0, xtmpl, x0, 0x12 ;swap x0 from 1hi|2hi to 1hi|2lo
+
+ vperm2i128 xtmph1, xgft1_lo, xgft1_lo, 0x01 ; swapped to hi | lo
+ vperm2i128 xtmph2, xgft2_lo, xgft2_lo, 0x01 ; swapped to hi | lo
+ vperm2i128 xtmph3, xgft3_lo, xgft3_lo, 0x01 ; swapped to hi | lo
+ vperm2i128 xtmph4, xgft4_lo, xgft4_lo, 0x01 ; swapped to hi | lo
+
+ ; dest1
+ vpshufb xtmph1, xtmph1, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl, xgft1_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph1, xtmph1, xtmpl ;GF add high and low partials
+ vpxor xd1, xd1, xtmph1 ;xd1 += partial
+
+ ; dest2
+ vpshufb xtmph2, xtmph2, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl, xgft2_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph2, xtmph2, xtmpl ;GF add high and low partials
+ vpxor xd2, xd2, xtmph2 ;xd2 += partial
+
+ ; dest3
+ vpshufb xtmph3, xtmph3, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl, xgft3_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph3, xtmph3, xtmpl ;GF add high and low partials
+ vpxor xd3, xd3, xtmph3 ;xd3 += partial
+
+ ; dest4
+ vpshufb xtmph4, xtmph4, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl, xgft4_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph4, xtmph4, xtmpl ;GF add high and low partials
+ vpxor xd4, xd4, xtmph4 ;xd4 += partial
+
+ XSTR [dest1+pos], xd1
+ XSTR [dest2+pos], xd2
+ XSTR [dest3+pos], xd3
+ XSTR [dest4+pos], xd4
+
+ add pos, 32 ;Loop on 32 bytes at a time
+ cmp pos, len
+ jle .loop32
+
+ lea tmp, [len + 32]
+ cmp pos, tmp
+ je .return_pass
+
+.lessthan32:
+ ;; Tail len
+ ;; Do one more overlap pass
+ mov tmp.b, 0x1f
+ vpinsrb xtmph1x, xtmph1x, tmp.w, 0
+ vpbroadcastb xtmph1, xtmph1x ;Construct mask 0x1f1f1f...
+
+ mov tmp, len ;Overlapped offset length-32
+
+ XLDR x0, [src+tmp] ;Get next source vector
+
+ XLDR xd1, [dest1+tmp] ;Get next dest vector
+ XLDR xd2, [dest2+tmp] ;Get next dest vector
+ XLDR xd3, [dest3+tmp] ;Get next dest vector
+ XLDR xd4, [dest4+tmp] ;Get next dest vector
+
+ sub len, pos
+
+ vmovdqa xtmph2, [constip32] ;Load const of i + 32
+ vpinsrb xtmplx, xtmplx, len.w, 15
+ vinserti128 xtmpl, xtmpl, xtmplx, 1 ;swapped to xtmplx | xtmplx
+ vpshufb xtmpl, xtmpl, xtmph1 ;Broadcast len to all bytes. xtmph1=0x1f1f1f...
+ vpcmpgtb xtmpl, xtmpl, xtmph2
+
+ vpand xtmph1, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+
+ vperm2i128 xtmpa, xtmph1, x0, 0x30 ;swap xtmpa from 1lo|2lo to 1lo|2hi
+ vperm2i128 x0, xtmph1, x0, 0x12 ;swap x0 from 1hi|2hi to 1hi|2lo
+
+ vperm2i128 xtmph1, xgft1_lo, xgft1_lo, 0x01 ; swapped to hi | lo
+ vperm2i128 xtmph2, xgft2_lo, xgft2_lo, 0x01 ; swapped to hi | lo
+ vperm2i128 xtmph3, xgft3_lo, xgft3_lo, 0x01 ; swapped to hi | lo
+ vperm2i128 xtmph4, xgft4_lo, xgft4_lo, 0x01 ; swapped to hi | lo
+
+ ; dest1
+ vpshufb xtmph1, xtmph1, x0 ;Lookup mul table of high nibble
+ vpshufb xgft1_lo, xgft1_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph1, xtmph1, xgft1_lo ;GF add high and low partials
+ vpand xtmph1, xtmph1, xtmpl
+ vpxor xd1, xd1, xtmph1 ;xd1 += partial
+
+ ; dest2
+ vpshufb xtmph2, xtmph2, x0 ;Lookup mul table of high nibble
+ vpshufb xgft2_lo, xgft2_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph2, xtmph2, xgft2_lo ;GF add high and low partials
+ vpand xtmph2, xtmph2, xtmpl
+ vpxor xd2, xd2, xtmph2 ;xd2 += partial
+
+ ; dest3
+ vpshufb xtmph3, xtmph3, x0 ;Lookup mul table of high nibble
+ vpshufb xgft3_lo, xgft3_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph3, xtmph3, xgft3_lo ;GF add high and low partials
+ vpand xtmph3, xtmph3, xtmpl
+ vpxor xd3, xd3, xtmph3 ;xd3 += partial
+
+ ; dest4
+ vpshufb xtmph4, xtmph4, x0 ;Lookup mul table of high nibble
+ vpshufb xgft4_lo, xgft4_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph4, xtmph4, xgft4_lo ;GF add high and low partials
+ vpand xtmph4, xtmph4, xtmpl
+ vpxor xd4, xd4, xtmph4 ;xd4 += partial
+
+ XSTR [dest1+tmp], xd1
+ XSTR [dest2+tmp], xd2
+ XSTR [dest3+tmp], xd3
+ XSTR [dest4+tmp], xd4
+
+.return_pass:
+ mov return, 0
+ FUNC_RESTORE
+ ret
+
+.return_fail:
+ mov return, 1
+ FUNC_RESTORE
+ ret
+
+endproc_frame
+
+section .data
+align 32
+constip32:
+ ddq 0xf0f1f2f3f4f5f6f7f8f9fafbfcfdfeff
+ ddq 0xe0e1e2e3e4e5e6e7e8e9eaebecedeeef
+
+%macro slversion 4
+global %1_slver_%2%3%4
+global %1_slver
+%1_slver:
+%1_slver_%2%3%4:
+ dw 0x%4
+ db 0x%3, 0x%2
+%endmacro
+;;; func core, ver, snum
+slversion gf_4vect_mad_avx2, 04, 00, 020b
--- /dev/null
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+; * Redistributions of source code must retain the above copyright
+; notice, this list of conditions and the following disclaimer.
+; * Redistributions in binary form must reproduce the above copyright
+; notice, this list of conditions and the following disclaimer in
+; the documentation and/or other materials provided with the
+; distribution.
+; * Neither the name of Intel Corporation nor the names of its
+; contributors may be used to endorse or promote products derived
+; from this software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;
+;;; gf_4vect_mad_sse(len, vec, vec_i, mul_array, src, dest);
+;;;
+
+%define PS 8
+
+%ifidn __OUTPUT_FORMAT__, win64
+ %define arg0 rcx
+ %define arg0.w ecx
+ %define arg1 rdx
+ %define arg2 r8
+ %define arg3 r9
+ %define arg4 r12
+ %define arg5 r15
+ %define tmp r11
+ %define tmp2 r10
+ %define tmp3 r13
+ %define return rax
+ %define return.w eax
+ %define stack_size 16*10 + 3*8
+ %define arg(x) [rsp + stack_size + PS + PS*x]
+ %define func(x) proc_frame x
+
+%macro FUNC_SAVE 0
+ sub rsp, stack_size
+ movdqa [rsp+16*0],xmm6
+ movdqa [rsp+16*1],xmm7
+ movdqa [rsp+16*2],xmm8
+ movdqa [rsp+16*3],xmm9
+ movdqa [rsp+16*4],xmm10
+ movdqa [rsp+16*5],xmm11
+ movdqa [rsp+16*6],xmm12
+ movdqa [rsp+16*7],xmm13
+ movdqa [rsp+16*8],xmm14
+ movdqa [rsp+16*9],xmm15
+ save_reg r12, 10*16 + 0*8
+ save_reg r13, 10*16 + 1*8
+ save_reg r15, 10*16 + 2*8
+ end_prolog
+ mov arg4, arg(4)
+ mov arg5, arg(5)
+%endmacro
+
+%macro FUNC_RESTORE 0
+ movdqa xmm6, [rsp+16*0]
+ movdqa xmm7, [rsp+16*1]
+ movdqa xmm8, [rsp+16*2]
+ movdqa xmm9, [rsp+16*3]
+ movdqa xmm10, [rsp+16*4]
+ movdqa xmm11, [rsp+16*5]
+ movdqa xmm12, [rsp+16*6]
+ movdqa xmm13, [rsp+16*7]
+ movdqa xmm14, [rsp+16*8]
+ movdqa xmm15, [rsp+16*9]
+ mov r12, [rsp + 10*16 + 0*8]
+ mov r13, [rsp + 10*16 + 1*8]
+ mov r15, [rsp + 10*16 + 2*8]
+ add rsp, stack_size
+%endmacro
+
+%elifidn __OUTPUT_FORMAT__, elf64
+ %define arg0 rdi
+ %define arg0.w edi
+ %define arg1 rsi
+ %define arg2 rdx
+ %define arg3 rcx
+ %define arg4 r8
+ %define arg5 r9
+ %define tmp r11
+ %define tmp2 r10
+ %define tmp3 r12
+ %define return rax
+ %define return.w eax
+
+ %define func(x) x:
+ %macro FUNC_SAVE 0
+ push r12
+ %endmacro
+ %macro FUNC_RESTORE 0
+ pop r12
+ %endmacro
+%endif
+
+;;; gf_4vect_mad_sse(len, vec, vec_i, mul_array, src, dest)
+%define len arg0
+%define len.w arg0.w
+%define vec arg1
+%define vec_i arg2
+%define mul_array arg3
+%define src arg4
+%define dest1 arg5
+%define pos return
+%define pos.w return.w
+
+%define dest2 mul_array
+%define dest3 tmp2
+%define dest4 vec_i
+
+%ifndef EC_ALIGNED_ADDR
+;;; Use Un-aligned load/store
+ %define XLDR movdqu
+ %define XSTR movdqu
+%else
+;;; Use Non-temporal load/stor
+ %ifdef NO_NT_LDST
+ %define XLDR movdqa
+ %define XSTR movdqa
+ %else
+ %define XLDR movntdqa
+ %define XSTR movntdq
+ %endif
+%endif
+
+default rel
+
+[bits 64]
+section .text
+
+%define xmask0f xmm15
+%define xgft3_hi xmm14
+%define xgft4_hi xmm13
+%define xgft4_lo xmm12
+
+%define x0 xmm0
+%define xtmpa xmm1
+%define xtmph1 xmm2
+%define xtmpl1 xmm3
+%define xtmph2 xmm4
+%define xtmpl2 xmm5
+%define xtmph3 xmm6
+%define xtmpl3 xmm7
+%define xtmph4 xmm8
+%define xtmpl4 xmm9
+%define xd1 xmm10
+%define xd2 xmm11
+%define xd3 xtmph1
+%define xd4 xtmpl1
+
+align 16
+global gf_4vect_mad_sse:function
+func(gf_4vect_mad_sse)
+ FUNC_SAVE
+ sub len, 16
+ jl .return_fail
+ xor pos, pos
+ movdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
+ mov tmp, vec
+
+ sal vec_i, 5 ;Multiply by 32
+ lea tmp3, [mul_array + vec_i]
+
+ sal tmp, 6 ;Multiply by 64
+
+ movdqu xgft3_hi, [tmp3+tmp+16] ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
+ sal vec, 5 ;Multiply by 32
+ add tmp, vec
+ movdqu xgft4_lo, [tmp3+tmp] ;Load array Dx{00}, Dx{01}, Dx{02}, ...
+ movdqu xgft4_hi, [tmp3+tmp+16] ; " Dx{00}, Dx{10}, Dx{20}, ... , Dx{f0}
+
+ mov dest2, [dest1+PS] ; reuse mul_array
+ mov dest3, [dest1+2*PS]
+ mov dest4, [dest1+3*PS] ; reuse vec_i
+ mov dest1, [dest1]
+
+.loop16:
+ XLDR x0, [src+pos] ;Get next source vector
+ movdqu xtmph1, [tmp3+16] ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
+ movdqu xtmpl1, [tmp3] ;Load array Ax{00}, Ax{01}, Ax{02}, ...
+ movdqu xtmph2, [tmp3+vec+16] ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
+ movdqu xtmpl2, [tmp3+vec] ;Load array Bx{00}, Bx{01}, Bx{02}, ...
+ movdqu xtmpl3, [tmp3+2*vec] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
+
+ movdqa xtmph3, xgft3_hi
+ movdqa xtmpl4, xgft4_lo
+ movdqa xtmph4, xgft4_hi
+
+ XLDR xd1, [dest1+pos] ;Get next dest vector
+ XLDR xd2, [dest2+pos] ;Get next dest vector
+
+ movdqa xtmpa, x0 ;Keep unshifted copy of src
+ psraw x0, 4 ;Shift to put high nibble into bits 4-0
+ pand x0, xmask0f ;Mask high src nibble in bits 4-0
+ pand xtmpa, xmask0f ;Mask low src nibble in bits 4-0
+
+ ; dest1
+ pshufb xtmph1, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl1, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph1, xtmpl1 ;GF add high and low partials
+ pxor xd1, xtmph1
+
+ XLDR xd3, [dest3+pos] ;Reuse xtmph1, Get next dest vector
+ XLDR xd4, [dest4+pos] ;Reuse xtmpl1, Get next dest vector
+
+ ; dest2
+ pshufb xtmph2, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl2, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph2, xtmpl2 ;GF add high and low partials
+ pxor xd2, xtmph2
+
+ ; dest3
+ pshufb xtmph3, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl3, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph3, xtmpl3 ;GF add high and low partials
+ pxor xd3, xtmph3
+
+ ; dest4
+ pshufb xtmph4, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl4, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph4, xtmpl4 ;GF add high and low partials
+ pxor xd4, xtmph4
+
+ XSTR [dest1+pos], xd1 ;Store result
+ XSTR [dest2+pos], xd2 ;Store result
+ XSTR [dest3+pos], xd3 ;Store result
+ XSTR [dest4+pos], xd4 ;Store result
+
+ add pos, 16 ;Loop on 16 bytes at a time
+ cmp pos, len
+ jle .loop16
+
+ lea tmp, [len + 16]
+ cmp pos, tmp
+ je .return_pass
+
+.lessthan16:
+ ;; Tail len
+ ;; Do one more overlap pass
+ mov tmp, len ;Overlapped offset length-16
+
+ XLDR x0, [src+tmp] ;Get next source vector
+
+ movdqu xtmph1, [tmp3+16] ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
+ movdqu xtmpl1, [tmp3] ;Load array Ax{00}, Ax{01}, Ax{02}, ...
+ movdqu xtmph2, [tmp3+vec+16] ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
+ movdqu xtmpl2, [tmp3+vec] ;Load array Bx{00}, Bx{01}, Bx{02}, ...
+ movdqu xtmpl3, [tmp3+2*vec] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
+
+ XLDR xd1, [dest1+tmp] ;Get next dest vector
+ XLDR xd2, [dest2+tmp] ;Get next dest vector
+ XLDR xtmph4, [dest3+tmp] ;Reuse xtmph1. Get next dest vector
+
+ sub len, pos
+
+ movdqa xtmpl4, [constip16] ;Load const of i + 16
+ pinsrb xtmph3, len.w, 15
+ pshufb xtmph3, xmask0f ;Broadcast len to all bytes
+ pcmpgtb xtmph3, xtmpl4
+
+ XLDR xtmpl4, [dest4+tmp] ;Get next dest vector
+
+ movdqa xtmpa, x0 ;Keep unshifted copy of src
+ psraw x0, 4 ;Shift to put high nibble into bits 4-0
+ pand x0, xmask0f ;Mask high src nibble in bits 4-0
+ pand xtmpa, xmask0f ;Mask low src nibble in bits 4-0
+
+ ; dest1
+ pshufb xtmph1, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl1, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph1, xtmpl1 ;GF add high and low partials
+ pand xtmph1, xtmph3
+ pxor xd1, xtmph1
+
+ ; dest2
+ pshufb xtmph2, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl2, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph2, xtmpl2 ;GF add high and low partials
+ pand xtmph2, xtmph3
+ pxor xd2, xtmph2
+
+ ; dest3
+ pshufb xgft3_hi, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl3, xtmpa ;Lookup mul table of low nibble
+ pxor xgft3_hi, xtmpl3 ;GF add high and low partials
+ pand xgft3_hi, xtmph3
+ pxor xtmph4, xgft3_hi
+
+ ; dest4
+ pshufb xgft4_hi, x0 ;Lookup mul table of high nibble
+ pshufb xgft4_lo, xtmpa ;Lookup mul table of low nibble
+ pxor xgft4_hi, xgft4_lo ;GF add high and low partials
+ pand xgft4_hi, xtmph3
+ pxor xtmpl4, xgft4_hi
+
+ XSTR [dest1+tmp], xd1 ;Store result
+ XSTR [dest2+tmp], xd2 ;Store result
+ XSTR [dest3+tmp], xtmph4 ;Store result
+ XSTR [dest4+tmp], xtmpl4 ;Store result
+
+.return_pass:
+ FUNC_RESTORE
+ mov return, 0
+ ret
+
+.return_fail:
+ FUNC_RESTORE
+ mov return, 1
+ ret
+
+endproc_frame
+
+section .data
+
+align 16
+
+mask0f:
+ ddq 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
+constip16:
+ ddq 0xf0f1f2f3f4f5f6f7f8f9fafbfcfdfeff
+
+%macro slversion 4
+global %1_slver_%2%3%4
+global %1_slver
+%1_slver:
+%1_slver_%2%3%4:
+ dw 0x%4
+ db 0x%3, 0x%2
+%endmacro
+;;; func core, ver, snum
+slversion gf_4vect_mad_sse, 00, 00, 0209
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_5vect_dot_prod_avx(len, vec, *g_tbls, **buffs, **dests);
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
%endmacro
;;; func core, ver, snum
slversion gf_5vect_dot_prod_avx, 02, 03, 0194
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_5vect_dot_prod_avx2(len, vec, *g_tbls, **buffs, **dests);
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
%endmacro
;;; func core, ver, snum
slversion gf_5vect_dot_prod_avx2, 04, 03, 0199
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_5vect_dot_prod_sse(len, vec, *g_tbls, **buffs, **dests);
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
section .text
%define xmask0f xmm15
-%define xgft1_lo xmm14
-%define xgft1_hi xmm13
-%define xgft2_lo xmm12
-%define xgft2_hi xmm11
+%define xgft1_lo xmm2
+%define xgft1_hi xmm3
+%define xgft2_lo xmm4
+%define xgft2_hi xmm5
%define xgft3_lo xmm10
-%define xgft3_hi xmm9
+%define xgft3_hi xmm6
%define xgft4_lo xmm8
%define xgft4_hi xmm7
%define x0 xmm0
%define xtmpa xmm1
-%define xp1 xmm2
-%define xp2 xmm3
-%define xp3 xmm4
-%define xp4 xmm5
-%define xp5 xmm6
+%define xp1 xmm9
+%define xp2 xmm11
+%define xp3 xmm12
+%define xp4 xmm13
+%define xp5 xmm14
align 16
global gf_5vect_dot_prod_sse:function
db 0x%3, 0x%2
%endmacro
;;; func core, ver, snum
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
-slversion gf_5vect_dot_prod_sse, 00, 03, 0065
+slversion gf_5vect_dot_prod_sse, 00, 04, 0065
--- /dev/null
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+; * Redistributions of source code must retain the above copyright
+; notice, this list of conditions and the following disclaimer.
+; * Redistributions in binary form must reproduce the above copyright
+; notice, this list of conditions and the following disclaimer in
+; the documentation and/or other materials provided with the
+; distribution.
+; * Neither the name of Intel Corporation nor the names of its
+; contributors may be used to endorse or promote products derived
+; from this software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;
+;;; gf_5vect_mad_avx(len, vec, vec_i, mul_array, src, dest);
+;;;
+
+%define PS 8
+
+%ifidn __OUTPUT_FORMAT__, win64
+ %define arg0 rcx
+ %define arg0.w ecx
+ %define arg1 rdx
+ %define arg2 r8
+ %define arg3 r9
+ %define arg4 r12
+ %define arg5 r15
+ %define tmp r11
+ %define tmp2 r10
+ %define tmp3 r13
+ %define tmp4 r14
+ %define return rax
+ %define return.w eax
+ %define stack_size 16*10 + 5*8
+ %define arg(x) [rsp + stack_size + PS + PS*x]
+ %define func(x) proc_frame x
+
+%macro FUNC_SAVE 0
+ sub rsp, stack_size
+ movdqa [rsp+16*0],xmm6
+ movdqa [rsp+16*1],xmm7
+ movdqa [rsp+16*2],xmm8
+ movdqa [rsp+16*3],xmm9
+ movdqa [rsp+16*4],xmm10
+ movdqa [rsp+16*5],xmm11
+ movdqa [rsp+16*6],xmm12
+ movdqa [rsp+16*7],xmm13
+ movdqa [rsp+16*8],xmm14
+ movdqa [rsp+16*9],xmm15
+ save_reg r12, 10*16 + 0*8
+ save_reg r13, 10*16 + 1*8
+ save_reg r14, 10*16 + 2*8
+ save_reg r15, 10*16 + 3*8
+ end_prolog
+ mov arg4, arg(4)
+ mov arg5, arg(5)
+%endmacro
+
+%macro FUNC_RESTORE 0
+ movdqa xmm6, [rsp+16*0]
+ movdqa xmm7, [rsp+16*1]
+ movdqa xmm8, [rsp+16*2]
+ movdqa xmm9, [rsp+16*3]
+ movdqa xmm10, [rsp+16*4]
+ movdqa xmm11, [rsp+16*5]
+ movdqa xmm12, [rsp+16*6]
+ movdqa xmm13, [rsp+16*7]
+ movdqa xmm14, [rsp+16*8]
+ movdqa xmm15, [rsp+16*9]
+ mov r12, [rsp + 10*16 + 0*8]
+ mov r13, [rsp + 10*16 + 1*8]
+ mov r14, [rsp + 10*16 + 2*8]
+ mov r15, [rsp + 10*16 + 3*8]
+ add rsp, stack_size
+%endmacro
+
+%elifidn __OUTPUT_FORMAT__, elf64
+ %define arg0 rdi
+ %define arg0.w edi
+ %define arg1 rsi
+ %define arg2 rdx
+ %define arg3 rcx
+ %define arg4 r8
+ %define arg5 r9
+ %define tmp r11
+ %define tmp2 r10
+ %define tmp3 r12
+ %define tmp4 r13
+ %define return rax
+ %define return.w eax
+
+ %define func(x) x:
+ %macro FUNC_SAVE 0
+ push r12
+ push r13
+ %endmacro
+ %macro FUNC_RESTORE 0
+ pop r13
+ pop r12
+ %endmacro
+%endif
+
+;;; gf_5vect_mad_avx(len, vec, vec_i, mul_array, src, dest)
+%define len arg0
+%define len.w arg0.w
+%define vec arg1
+%define vec_i arg2
+%define mul_array arg3
+%define src arg4
+%define dest1 arg5
+%define pos return
+%define pos.w return.w
+
+%define dest2 tmp4
+%define dest3 mul_array
+%define dest4 tmp2
+%define dest5 vec_i
+
+
+%ifndef EC_ALIGNED_ADDR
+;;; Use Un-aligned load/store
+ %define XLDR vmovdqu
+ %define XSTR vmovdqu
+%else
+;;; Use Non-temporal load/stor
+ %ifdef NO_NT_LDST
+ %define XLDR vmovdqa
+ %define XSTR vmovdqa
+ %else
+ %define XLDR vmovntdqa
+ %define XSTR vmovntdq
+ %endif
+%endif
+
+default rel
+
+[bits 64]
+section .text
+
+%define xmask0f xmm15
+%define xgft5_hi xmm14
+%define xgft4_lo xmm13
+%define xgft4_hi xmm12
+
+%define x0 xmm0
+%define xtmpa xmm1
+%define xtmph1 xmm2
+%define xtmpl1 xmm3
+%define xtmph2 xmm4
+%define xtmpl2 xmm5
+%define xtmph3 xmm6
+%define xtmpl3 xmm7
+%define xtmph5 xmm8
+%define xtmpl5 xmm9
+%define xd1 xmm10
+%define xd2 xmm11
+%define xd3 xtmpl1
+%define xd4 xtmph1
+%define xd5 xtmpl2
+
+
+align 16
+global gf_5vect_mad_avx:function
+func(gf_5vect_mad_avx)
+ FUNC_SAVE
+ sub len, 16
+ jl .return_fail
+ xor pos, pos
+ vmovdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
+ mov tmp, vec
+ sal vec_i, 5 ;Multiply by 32
+ lea tmp3, [mul_array + vec_i]
+ sal tmp, 6 ;Multiply by 64
+ vmovdqu xgft5_hi, [tmp3+2*tmp+16] ; " Ex{00}, Ex{10}, ..., Ex{f0}
+ sal vec, 5 ;Multiply by 32
+ add tmp, vec
+ vmovdqu xgft4_hi, [tmp3+tmp+16] ; " Dx{00}, Dx{10}, Dx{20}, ... , Dx{f0}
+ vmovdqu xgft4_lo, [tmp3+tmp] ;Load array Dx{00}, Dx{01}, Dx{02}, ...
+
+ mov dest3, [dest1+2*PS] ; reuse mul_array
+ mov dest4, [dest1+3*PS]
+ mov dest5, [dest1+4*PS] ; reuse vec_i
+ mov dest2, [dest1+PS]
+ mov dest1, [dest1]
+
+.loop16:
+ XLDR x0, [src+pos] ;Get next source vector
+
+ vmovdqu xtmph1, [tmp3+16] ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
+ vmovdqu xtmpl1, [tmp3] ;Load array Ax{00}, Ax{01}, Ax{02}, ...
+ vmovdqu xtmph2, [tmp3+vec+16] ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
+ vmovdqu xtmpl2, [tmp3+vec] ;Load array Bx{00}, Bx{01}, Bx{02}, ...
+ vmovdqu xtmph3, [tmp3+2*vec+16] ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
+ vmovdqu xtmpl3, [tmp3+2*vec] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
+ vmovdqu xtmpl5, [tmp3+4*vec] ;Load array Ex{00}, Ex{01}, ..., Ex{0f}
+
+ XLDR xd1, [dest1+pos] ;Get next dest vector
+ XLDR xd2, [dest2+pos] ;Get next dest vector
+
+ vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+
+ ; dest1
+ vpshufb xtmph1, xtmph1, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl1, xtmpl1, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph1, xtmph1, xtmpl1 ;GF add high and low partials
+ vpxor xd1, xd1, xtmph1
+
+ XLDR xd3, [dest3+pos] ;Reuse xtmpl1, Get next dest vector
+ XLDR xd4, [dest4+pos] ;Reuse xtmph1, Get next dest vector
+
+ ; dest2
+ vpshufb xtmph2, xtmph2, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl2, xtmpl2, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph2, xtmph2, xtmpl2 ;GF add high and low partials
+ vpxor xd2, xd2, xtmph2
+
+ XLDR xd5, [dest5+pos] ;Reuse xtmpl2. Get next dest vector
+
+ ; dest3
+ vpshufb xtmph3, xtmph3, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl3, xtmpl3, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph3, xtmph3, xtmpl3 ;GF add high and low partials
+ vpxor xd3, xd3, xtmph3
+
+ ; dest4
+ vpshufb xtmph2, xgft4_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl3, xgft4_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph2, xtmph2, xtmpl3 ;GF add high and low partials
+ vpxor xd4, xd4, xtmph2
+
+ ; dest5
+ vpshufb xtmph5, xgft5_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl5, xtmpl5, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph5, xtmph5, xtmpl5 ;GF add high and low partials
+ vpxor xd5, xd5, xtmph5
+
+ XSTR [dest1+pos], xd1 ;Store result into dest1
+ XSTR [dest2+pos], xd2 ;Store result into dest2
+ XSTR [dest3+pos], xd3 ;Store result into dest3
+ XSTR [dest4+pos], xd4 ;Store result into dest4
+ XSTR [dest5+pos], xd5 ;Store result into dest5
+
+ add pos, 16 ;Loop on 16 bytes at a time
+ cmp pos, len
+ jle .loop16
+
+ lea tmp, [len + 16]
+ cmp pos, tmp
+ je .return_pass
+
+.lessthan16:
+ ;; Tail len
+ ;; Do one more overlap pass
+ mov tmp, len ;Overlapped offset length-16
+ XLDR x0, [src+tmp] ;Get next source vector
+
+ sub len, pos
+
+ vmovdqa xtmph1, [constip16] ;Load const of i + 16
+ vpinsrb xtmph5, len.w, 15
+ vpshufb xtmph5, xmask0f ;Broadcast len to all bytes
+ vpcmpgtb xtmph5, xtmph5, xtmph1
+
+ vmovdqu xtmph1, [tmp3+16] ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
+ vmovdqu xtmpl1, [tmp3] ;Load array Ax{00}, Ax{01}, Ax{02}, ...
+ vmovdqu xtmph2, [tmp3+vec+16] ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
+ vmovdqu xtmpl2, [tmp3+vec] ;Load array Bx{00}, Bx{01}, Bx{02}, ...
+ vmovdqu xtmph3, [tmp3+2*vec+16] ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
+ vmovdqu xtmpl3, [tmp3+2*vec] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
+ vmovdqu xtmpl5, [tmp3+4*vec] ;Load array Ex{00}, Ex{01}, ..., Ex{0f}
+
+ XLDR xd1, [dest1+tmp] ;Get next dest vector
+ XLDR xd2, [dest2+tmp] ;Get next dest vector
+
+ vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+
+ ; dest1
+ vpshufb xtmph1, xtmph1, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl1, xtmpl1, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph1, xtmph1, xtmpl1 ;GF add high and low partials
+ vpand xtmph1, xtmph1, xtmph5
+ vpxor xd1, xd1, xtmph1
+
+ XLDR xd3, [dest3+tmp] ;Reuse xtmpl1, Get next dest vector
+ XLDR xd4, [dest4+tmp] ;Reuse xtmph1, Get next dest vector
+
+ ; dest2
+ vpshufb xtmph2, xtmph2, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl2, xtmpl2, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph2, xtmph2, xtmpl2 ;GF add high and low partials
+ vpand xtmph2, xtmph2, xtmph5
+ vpxor xd2, xd2, xtmph2
+
+ XLDR xd5, [dest5+tmp] ;Reuse xtmpl2. Get next dest vector
+
+ ; dest3
+ vpshufb xtmph3, xtmph3, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl3, xtmpl3, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph3, xtmph3, xtmpl3 ;GF add high and low partials
+ vpand xtmph3, xtmph3, xtmph5
+ vpxor xd3, xd3, xtmph3
+
+ ; dest4
+ vpshufb xgft4_hi, xgft4_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xgft4_lo, xgft4_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xgft4_hi, xgft4_hi, xgft4_lo ;GF add high and low partials
+ vpand xgft4_hi, xgft4_hi, xtmph5
+ vpxor xd4, xd4, xgft4_hi
+
+ ; dest5
+ vpshufb xgft5_hi, xgft5_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl5, xtmpl5, xtmpa ;Lookup mul table of low nibble
+ vpxor xgft5_hi, xgft5_hi, xtmpl5 ;GF add high and low partials
+ vpand xgft5_hi, xgft5_hi, xtmph5
+ vpxor xd5, xd5, xgft5_hi
+
+ XSTR [dest1+tmp], xd1 ;Store result into dest1
+ XSTR [dest2+tmp], xd2 ;Store result into dest2
+ XSTR [dest3+tmp], xd3 ;Store result into dest3
+ XSTR [dest4+tmp], xd4 ;Store result into dest4
+ XSTR [dest5+tmp], xd5 ;Store result into dest5
+
+.return_pass:
+ FUNC_RESTORE
+ mov return, 0
+ ret
+
+.return_fail:
+ FUNC_RESTORE
+ mov return, 1
+ ret
+
+endproc_frame
+
+section .data
+
+align 16
+mask0f: ddq 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
+constip16:
+ ddq 0xf0f1f2f3f4f5f6f7f8f9fafbfcfdfeff
+
+%macro slversion 4
+global %1_slver_%2%3%4
+global %1_slver
+%1_slver:
+%1_slver_%2%3%4:
+ dw 0x%4
+ db 0x%3, 0x%2
+%endmacro
+;;; func core, ver, snum
+slversion gf_5vect_mad_avx, 02, 00, 020d
--- /dev/null
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+; * Redistributions of source code must retain the above copyright
+; notice, this list of conditions and the following disclaimer.
+; * Redistributions in binary form must reproduce the above copyright
+; notice, this list of conditions and the following disclaimer in
+; the documentation and/or other materials provided with the
+; distribution.
+; * Neither the name of Intel Corporation nor the names of its
+; contributors may be used to endorse or promote products derived
+; from this software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;
+;;; gf_5vect_mad_avx2(len, vec, vec_i, mul_array, src, dest);
+;;;
+
+%define PS 8
+
+%ifidn __OUTPUT_FORMAT__, win64
+ %define arg0 rcx
+ %define arg0.w ecx
+ %define arg1 rdx
+ %define arg2 r8
+ %define arg3 r9
+ %define arg4 r12
+ %define arg5 r15
+ %define tmp r11
+ %define tmp.w r11d
+ %define tmp.b r11b
+ %define tmp2 r10
+ %define return rax
+ %define return.w eax
+ %define stack_size 16*10 + 3*8
+ %define arg(x) [rsp + stack_size + PS + PS*x]
+ %define func(x) proc_frame x
+
+%macro FUNC_SAVE 0
+ sub rsp, stack_size
+ movdqa [rsp+16*0],xmm6
+ movdqa [rsp+16*1],xmm7
+ movdqa [rsp+16*2],xmm8
+ movdqa [rsp+16*3],xmm9
+ movdqa [rsp+16*4],xmm10
+ movdqa [rsp+16*5],xmm11
+ movdqa [rsp+16*6],xmm12
+ movdqa [rsp+16*7],xmm13
+ movdqa [rsp+16*8],xmm14
+ movdqa [rsp+16*9],xmm15
+ save_reg r12, 10*16 + 0*8
+ save_reg r15, 10*16 + 1*8
+ end_prolog
+ mov arg4, arg(4)
+ mov arg5, arg(5)
+%endmacro
+
+%macro FUNC_RESTORE 0
+ movdqa xmm6, [rsp+16*0]
+ movdqa xmm7, [rsp+16*1]
+ movdqa xmm8, [rsp+16*2]
+ movdqa xmm9, [rsp+16*3]
+ movdqa xmm10, [rsp+16*4]
+ movdqa xmm11, [rsp+16*5]
+ movdqa xmm12, [rsp+16*6]
+ movdqa xmm13, [rsp+16*7]
+ movdqa xmm14, [rsp+16*8]
+ movdqa xmm15, [rsp+16*9]
+ mov r12, [rsp + 10*16 + 0*8]
+ mov r15, [rsp + 10*16 + 1*8]
+ add rsp, stack_size
+%endmacro
+
+%elifidn __OUTPUT_FORMAT__, elf64
+ %define arg0 rdi
+ %define arg0.w edi
+ %define arg1 rsi
+ %define arg2 rdx
+ %define arg3 rcx
+ %define arg4 r8
+ %define arg5 r9
+ %define tmp r11
+ %define tmp.w r11d
+ %define tmp.b r11b
+ %define tmp2 r10
+ %define return rax
+ %define return.w eax
+
+ %define func(x) x:
+ %define FUNC_SAVE
+ %define FUNC_RESTORE
+%endif
+
+;;; gf_5vect_mad_avx2(len, vec, vec_i, mul_array, src, dest)
+%define len arg0
+%define len.w arg0.w
+%define vec arg1
+%define vec_i arg2
+%define mul_array arg3
+%define src arg4
+%define dest1 arg5
+%define pos return
+%define pos.w return.w
+
+%define dest2 tmp2
+%define dest3 mul_array
+%define dest4 vec
+%define dest5 vec_i
+
+%ifndef EC_ALIGNED_ADDR
+;;; Use Un-aligned load/store
+ %define XLDR vmovdqu
+ %define XSTR vmovdqu
+%else
+;;; Use Non-temporal load/stor
+ %ifdef NO_NT_LDST
+ %define XLDR vmovdqa
+ %define XSTR vmovdqa
+ %else
+ %define XLDR vmovntdqa
+ %define XSTR vmovntdq
+ %endif
+%endif
+
+default rel
+
+[bits 64]
+section .text
+
+%define xmask0f ymm15
+%define xmask0fx xmm15
+%define xgft1_lo ymm14
+%define xgft2_lo ymm13
+%define xgft3_lo ymm12
+%define xgft4_lo ymm11
+%define xgft5_lo ymm10
+
+%define x0 ymm0
+%define xtmpa ymm1
+%define xtmpl ymm2
+%define xtmplx xmm2
+%define xtmph1 ymm3
+%define xtmph1x xmm3
+%define xtmph2 ymm4
+%define xd1 ymm5
+%define xd2 ymm6
+%define xd3 ymm7
+%define xd4 ymm8
+%define xd5 ymm9
+
+align 16
+global gf_5vect_mad_avx2:function
+func(gf_5vect_mad_avx2)
+ FUNC_SAVE
+ sub len, 32
+ jl .return_fail
+ xor pos, pos
+ mov tmp.b, 0x0f
+ vpinsrb xmask0fx, xmask0fx, tmp.w, 0
+ vpbroadcastb xmask0f, xmask0fx ;Construct mask 0x0f0f0f...
+
+ sal vec_i, 5 ;Multiply by 32
+ sal vec, 5 ;Multiply by 32
+ lea tmp, [mul_array + vec_i]
+
+ vmovdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, ..., Ax{0f}
+ ; " Ax{00}, Ax{10}, ..., Ax{f0}
+ vmovdqu xgft2_lo, [tmp+vec] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
+ ; " Bx{00}, Bx{10}, ..., Bx{f0}
+ vmovdqu xgft3_lo, [tmp+2*vec] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
+ ; " Cx{00}, Cx{10}, ..., Cx{f0}
+ vmovdqu xgft5_lo, [tmp+4*vec] ;Load array Ex{00}, Ex{01}, ..., Ex{0f}
+ ; " Ex{00}, Ex{10}, ..., Ex{f0}
+ add tmp, vec
+ vmovdqu xgft4_lo, [tmp+2*vec] ;Load array Dx{00}, Dx{01}, ..., Dx{0f}
+ ; " Dx{00}, Dx{10}, ..., Dx{f0}
+
+ mov dest3, [dest1+2*PS] ; reuse mul_array
+ mov dest4, [dest1+3*PS] ; reuse vec
+ mov dest5, [dest1+4*PS] ; reuse vec_i
+ mov dest2, [dest1+PS]
+ mov dest1, [dest1]
+
+.loop32:
+ XLDR x0, [src+pos] ;Get next source vector
+
+ XLDR xd1, [dest1+pos] ;Get next dest vector
+ XLDR xd2, [dest2+pos] ;Get next dest vector
+ XLDR xd3, [dest3+pos] ;Get next dest vector
+ XLDR xd4, [dest4+pos] ;Get next dest vector
+ XLDR xd5, [dest5+pos] ;Get next dest vector
+
+ vpand xtmpl, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+ vperm2i128 xtmpa, xtmpl, x0, 0x30 ;swap xtmpa from 1lo|2lo to 1lo|2hi
+ vperm2i128 x0, xtmpl, x0, 0x12 ;swap x0 from 1hi|2hi to 1hi|2lo
+
+ vperm2i128 xtmph1, xgft1_lo, xgft1_lo, 0x01 ; swapped to hi | lo
+ vperm2i128 xtmph2, xgft2_lo, xgft2_lo, 0x01 ; swapped to hi | lo
+
+ ; dest1
+ vpshufb xtmph1, xtmph1, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl, xgft1_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph1, xtmph1, xtmpl ;GF add high and low partials
+ vpxor xd1, xd1, xtmph1 ;xd1 += partial
+
+ vperm2i128 xtmph1, xgft3_lo, xgft3_lo, 0x01 ; swapped to hi | lo
+ ; dest2
+ vpshufb xtmph2, xtmph2, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl, xgft2_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph2, xtmph2, xtmpl ;GF add high and low partials
+ vpxor xd2, xd2, xtmph2 ;xd2 += partial
+
+ vperm2i128 xtmph2, xgft4_lo, xgft4_lo, 0x01 ; swapped to hi | lo
+ ; dest3
+ vpshufb xtmph1, xtmph1, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl, xgft3_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph1, xtmph1, xtmpl ;GF add high and low partials
+ vpxor xd3, xd3, xtmph1 ;xd3 += partial
+
+ vperm2i128 xtmph1, xgft5_lo, xgft5_lo, 0x01 ; swapped to hi | lo
+ ; dest4
+ vpshufb xtmph2, xtmph2, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl, xgft4_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph2, xtmph2, xtmpl ;GF add high and low partials
+ vpxor xd4, xd4, xtmph2 ;xd4 += partial
+
+ ; dest5
+ vpshufb xtmph1, xtmph1, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl, xgft5_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph1, xtmph1, xtmpl ;GF add high and low partials
+ vpxor xd5, xd5, xtmph1 ;xd5 += partial
+
+ XSTR [dest1+pos], xd1
+ XSTR [dest2+pos], xd2
+ XSTR [dest3+pos], xd3
+ XSTR [dest4+pos], xd4
+ XSTR [dest5+pos], xd5
+
+ add pos, 32 ;Loop on 32 bytes at a time
+ cmp pos, len
+ jle .loop32
+
+ lea tmp, [len + 32]
+ cmp pos, tmp
+ je .return_pass
+
+.lessthan32:
+ ;; Tail len
+ ;; Do one more overlap pass
+ mov tmp.b, 0x1f
+ vpinsrb xtmph1x, xtmph1x, tmp.w, 0
+ vpbroadcastb xtmph1, xtmph1x ;Construct mask 0x1f1f1f...
+
+ mov tmp, len ;Overlapped offset length-32
+
+ XLDR x0, [src+tmp] ;Get next source vector
+
+ XLDR xd1, [dest1+tmp] ;Get next dest vector
+ XLDR xd2, [dest2+tmp] ;Get next dest vector
+ XLDR xd3, [dest3+tmp] ;Get next dest vector
+ XLDR xd4, [dest4+tmp] ;Get next dest vector
+ XLDR xd5, [dest5+tmp] ;Get next dest vector
+
+ sub len, pos
+
+ vmovdqa xtmph2, [constip32] ;Load const of i + 32
+ vpinsrb xtmplx, xtmplx, len.w, 15
+ vinserti128 xtmpl, xtmpl, xtmplx, 1 ;swapped to xtmplx | xtmplx
+ vpshufb xtmpl, xtmpl, xtmph1 ;Broadcast len to all bytes. xtmph1=0x1f1f1f...
+ vpcmpgtb xtmpl, xtmpl, xtmph2
+
+ vpand xtmph1, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+ vperm2i128 xtmpa, xtmph1, x0, 0x30 ;swap xtmpa from 1lo|2lo to 1lo|2hi
+ vperm2i128 x0, xtmph1, x0, 0x12 ;swap x0 from 1hi|2hi to 1hi|2lo
+
+ vperm2i128 xtmph1, xgft1_lo, xgft1_lo, 0x01 ; swapped to hi | lo
+ vperm2i128 xtmph2, xgft2_lo, xgft2_lo, 0x01 ; swapped to hi | lo
+
+ ; dest1
+ vpshufb xtmph1, xtmph1, x0 ;Lookup mul table of high nibble
+ vpshufb xgft1_lo, xgft1_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph1, xtmph1, xgft1_lo ;GF add high and low partials
+ vpand xtmph1, xtmph1, xtmpl
+ vpxor xd1, xd1, xtmph1 ;xd1 += partial
+
+ vperm2i128 xtmph1, xgft3_lo, xgft3_lo, 0x01 ; swapped to hi | lo
+ ; dest2
+ vpshufb xtmph2, xtmph2, x0 ;Lookup mul table of high nibble
+ vpshufb xgft2_lo, xgft2_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph2, xtmph2, xgft2_lo ;GF add high and low partials
+ vpand xtmph2, xtmph2, xtmpl
+ vpxor xd2, xd2, xtmph2 ;xd2 += partial
+
+ vperm2i128 xtmph2, xgft4_lo, xgft4_lo, 0x01 ; swapped to hi | lo
+ ; dest3
+ vpshufb xtmph1, xtmph1, x0 ;Lookup mul table of high nibble
+ vpshufb xgft3_lo, xgft3_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph1, xtmph1, xgft3_lo ;GF add high and low partials
+ vpand xtmph1, xtmph1, xtmpl
+ vpxor xd3, xd3, xtmph1 ;xd3 += partial
+
+ vperm2i128 xtmph1, xgft5_lo, xgft5_lo, 0x01 ; swapped to hi | lo
+ ; dest4
+ vpshufb xtmph2, xtmph2, x0 ;Lookup mul table of high nibble
+ vpshufb xgft4_lo, xgft4_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph2, xtmph2, xgft4_lo ;GF add high and low partials
+ vpand xtmph2, xtmph2, xtmpl
+ vpxor xd4, xd4, xtmph2 ;xd4 += partial
+
+ ; dest5
+ vpshufb xtmph1, xtmph1, x0 ;Lookup mul table of high nibble
+ vpshufb xgft5_lo, xgft5_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph1, xtmph1, xgft5_lo ;GF add high and low partials
+ vpand xtmph1, xtmph1, xtmpl
+ vpxor xd5, xd5, xtmph1 ;xd5 += partial
+
+ XSTR [dest1+tmp], xd1
+ XSTR [dest2+tmp], xd2
+ XSTR [dest3+tmp], xd3
+ XSTR [dest4+tmp], xd4
+ XSTR [dest5+tmp], xd5
+
+.return_pass:
+ FUNC_RESTORE
+ mov return, 0
+ ret
+
+.return_fail:
+ FUNC_RESTORE
+ mov return, 1
+ ret
+
+endproc_frame
+
+section .data
+align 32
+constip32:
+ ddq 0xf0f1f2f3f4f5f6f7f8f9fafbfcfdfeff
+ ddq 0xe0e1e2e3e4e5e6e7e8e9eaebecedeeef
+
+%macro slversion 4
+global %1_slver_%2%3%4
+global %1_slver
+%1_slver:
+%1_slver_%2%3%4:
+ dw 0x%4
+ db 0x%3, 0x%2
+%endmacro
+;;; func core, ver, snum
+slversion gf_5vect_mad_avx2, 04, 00, 020e
--- /dev/null
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+; * Redistributions of source code must retain the above copyright
+; notice, this list of conditions and the following disclaimer.
+; * Redistributions in binary form must reproduce the above copyright
+; notice, this list of conditions and the following disclaimer in
+; the documentation and/or other materials provided with the
+; distribution.
+; * Neither the name of Intel Corporation nor the names of its
+; contributors may be used to endorse or promote products derived
+; from this software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;
+;;; gf_5vect_mad_sse(len, vec, vec_i, mul_array, src, dest);
+;;;
+
+%define PS 8
+
+%ifidn __OUTPUT_FORMAT__, win64
+ %define arg0 rcx
+ %define arg0.w ecx
+ %define arg1 rdx
+ %define arg2 r8
+ %define arg3 r9
+ %define arg4 r12
+ %define arg5 r15
+ %define tmp r11
+ %define tmp2 r10
+ %define tmp3 r13
+ %define tmp4 r14
+ %define return rax
+ %define return.w eax
+ %define stack_size 16*10 + 5*8
+ %define arg(x) [rsp + stack_size + PS + PS*x]
+ %define func(x) proc_frame x
+
+%macro FUNC_SAVE 0
+ sub rsp, stack_size
+ movdqa [rsp+16*0],xmm6
+ movdqa [rsp+16*1],xmm7
+ movdqa [rsp+16*2],xmm8
+ movdqa [rsp+16*3],xmm9
+ movdqa [rsp+16*4],xmm10
+ movdqa [rsp+16*5],xmm11
+ movdqa [rsp+16*6],xmm12
+ movdqa [rsp+16*7],xmm13
+ movdqa [rsp+16*8],xmm14
+ movdqa [rsp+16*9],xmm15
+ save_reg r12, 10*16 + 0*8
+ save_reg r13, 10*16 + 1*8
+ save_reg r14, 10*16 + 2*8
+ save_reg r15, 10*16 + 3*8
+ end_prolog
+ mov arg4, arg(4)
+ mov arg5, arg(5)
+%endmacro
+
+%macro FUNC_RESTORE 0
+ movdqa xmm6, [rsp+16*0]
+ movdqa xmm7, [rsp+16*1]
+ movdqa xmm8, [rsp+16*2]
+ movdqa xmm9, [rsp+16*3]
+ movdqa xmm10, [rsp+16*4]
+ movdqa xmm11, [rsp+16*5]
+ movdqa xmm12, [rsp+16*6]
+ movdqa xmm13, [rsp+16*7]
+ movdqa xmm14, [rsp+16*8]
+ movdqa xmm15, [rsp+16*9]
+ mov r12, [rsp + 10*16 + 0*8]
+ mov r13, [rsp + 10*16 + 1*8]
+ mov r14, [rsp + 10*16 + 2*8]
+ mov r15, [rsp + 10*16 + 3*8]
+ add rsp, stack_size
+%endmacro
+
+%elifidn __OUTPUT_FORMAT__, elf64
+ %define arg0 rdi
+ %define arg0.w edi
+ %define arg1 rsi
+ %define arg2 rdx
+ %define arg3 rcx
+ %define arg4 r8
+ %define arg5 r9
+ %define tmp r11
+ %define tmp2 r10
+ %define tmp3 r12
+ %define tmp4 r13
+ %define return rax
+ %define return.w eax
+
+ %define func(x) x:
+ %macro FUNC_SAVE 0
+ push r12
+ push r13
+ %endmacro
+ %macro FUNC_RESTORE 0
+ pop r13
+ pop r12
+ %endmacro
+%endif
+
+;;; gf_5vect_mad_sse(len, vec, vec_i, mul_array, src, dest)
+%define len arg0
+%define len.w arg0.w
+%define vec arg1
+%define vec_i arg2
+%define mul_array arg3
+%define src arg4
+%define dest1 arg5
+%define pos return
+%define pos.w return.w
+
+%define dest2 tmp4
+%define dest3 mul_array
+%define dest4 tmp2
+%define dest5 vec_i
+
+%ifndef EC_ALIGNED_ADDR
+;;; Use Un-aligned load/store
+ %define XLDR movdqu
+ %define XSTR movdqu
+%else
+;;; Use Non-temporal load/stor
+ %ifdef NO_NT_LDST
+ %define XLDR movdqa
+ %define XSTR movdqa
+ %else
+ %define XLDR movntdqa
+ %define XSTR movntdq
+ %endif
+%endif
+
+default rel
+
+[bits 64]
+section .text
+
+%define xmask0f xmm15
+%define xgft5_hi xmm14
+%define xgft4_lo xmm13
+%define xgft4_hi xmm12
+
+%define x0 xmm0
+%define xtmpa xmm1
+%define xtmph1 xmm2
+%define xtmpl1 xmm3
+%define xtmph2 xmm4
+%define xtmpl2 xmm5
+%define xtmph3 xmm6
+%define xtmpl3 xmm7
+%define xtmph5 xmm8
+%define xtmpl5 xmm9
+%define xd1 xmm10
+%define xd2 xmm11
+%define xd3 xtmpl1
+%define xd4 xtmph1
+%define xd5 xtmpl2
+
+
+align 16
+global gf_5vect_mad_sse:function
+func(gf_5vect_mad_sse)
+ FUNC_SAVE
+ sub len, 16
+ jl .return_fail
+ xor pos, pos
+
+ movdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
+ mov tmp, vec
+ sal vec_i, 5 ;Multiply by 32
+ lea tmp3, [mul_array + vec_i]
+ sal tmp, 6 ;Multiply by 64
+ movdqu xgft5_hi, [tmp3+2*tmp+16] ; " Ex{00}, Ex{10}, ..., Ex{f0}
+ sal vec, 5 ;Multiply by 32
+ add tmp, vec
+ movdqu xgft4_hi, [tmp3+tmp+16] ; " Dx{00}, Dx{10}, Dx{20}, ... , Dx{f0}
+ movdqu xgft4_lo, [tmp3+tmp] ;Load array Dx{00}, Dx{01}, Dx{02}, ...
+
+ mov dest3, [dest1+2*PS] ; reuse mul_array
+ mov dest4, [dest1+3*PS]
+ mov dest5, [dest1+4*PS] ; reuse vec_i
+ mov dest2, [dest1+PS]
+ mov dest1, [dest1]
+
+.loop16:
+ XLDR x0, [src+pos] ;Get next source vector
+
+ movdqu xtmph1, [tmp3+16] ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
+ movdqu xtmpl1, [tmp3] ;Load array Ax{00}, Ax{01}, Ax{02}, ...
+ movdqu xtmph2, [tmp3+vec+16] ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
+ movdqu xtmpl2, [tmp3+vec] ;Load array Bx{00}, Bx{01}, Bx{02}, ...
+ movdqu xtmph3, [tmp3+2*vec+16] ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
+ movdqu xtmpl3, [tmp3+2*vec] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
+ movdqu xtmpl5, [tmp3+4*vec] ;Load array Ex{00}, Ex{01}, ..., Ex{0f}
+ movdqa xtmph5, xgft5_hi ;Reload const array registers
+
+ XLDR xd1, [dest1+pos] ;Get next dest vector
+ XLDR xd2, [dest2+pos] ;Get next dest vector
+
+ movdqa xtmpa, x0 ;Keep unshifted copy of src
+ psraw x0, 4 ;Shift to put high nibble into bits 4-0
+ pand x0, xmask0f ;Mask high src nibble in bits 4-0
+ pand xtmpa, xmask0f ;Mask low src nibble in bits 4-0
+
+ ; dest1
+ pshufb xtmph1, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl1, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph1, xtmpl1 ;GF add high and low partials
+ pxor xd1, xtmph1
+
+ XLDR xd3, [dest3+pos] ;Reuse xtmpl1, Get next dest vector
+ XLDR xd4, [dest4+pos] ;Reuse xtmph1. Get next dest vector
+
+ ; dest2
+ pshufb xtmph2, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl2, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph2, xtmpl2 ;GF add high and low partials
+ pxor xd2, xtmph2
+
+ XLDR xd5, [dest5+pos] ;Reuse xtmpl2. Get next dest vector
+
+ ; dest3
+ pshufb xtmph3, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl3, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph3, xtmpl3 ;GF add high and low partials
+ pxor xd3, xtmph3
+
+ movdqa xtmph2, xgft4_hi ;Reload const array registers
+ movdqa xtmpl3, xgft4_lo ;Reload const array registers
+
+ ; dest5
+ pshufb xtmph5, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl5, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph5, xtmpl5 ;GF add high and low partials
+ pxor xd5, xtmph5
+
+ ; dest4
+ pshufb xtmph2, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl3, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph2, xtmpl3 ;GF add high and low partials
+ pxor xd4, xtmph2
+
+ XSTR [dest1+pos], xd1 ;Store result into dest1
+ XSTR [dest2+pos], xd2 ;Store result into dest2
+ XSTR [dest3+pos], xd3 ;Store result into dest3
+ XSTR [dest4+pos], xd4 ;Store result into dest4
+ XSTR [dest5+pos], xd5 ;Store result into dest5
+
+ add pos, 16 ;Loop on 16 bytes at a time
+ cmp pos, len
+ jle .loop16
+
+ lea tmp, [len + 16]
+ cmp pos, tmp
+ je .return_pass
+
+.lessthan16:
+ ;; Tail len
+ ;; Do one more overlap pass
+ mov tmp, len ;Overlapped offset length-16
+ XLDR x0, [src+tmp] ;Get next source vector
+
+ sub len, pos
+
+ movdqa xtmpl1, [constip16] ;Load const of i + 16
+ pinsrb xtmph5, len.w, 15
+ pshufb xtmph5, xmask0f ;Broadcast len to all bytes
+ pcmpgtb xtmph5, xtmpl1
+
+ movdqu xtmph1, [tmp3+16] ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
+ movdqu xtmpl1, [tmp3] ;Load array Ax{00}, Ax{01}, Ax{02}, ...
+ movdqu xtmph2, [tmp3+vec+16] ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
+ movdqu xtmpl2, [tmp3+vec] ;Load array Bx{00}, Bx{01}, Bx{02}, ...
+ movdqu xtmph3, [tmp3+2*vec+16] ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
+ movdqu xtmpl3, [tmp3+2*vec] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
+ movdqu xtmpl5, [tmp3+4*vec] ;Load array Ex{00}, Ex{01}, ..., Ex{0f}
+
+ XLDR xd1, [dest1+tmp] ;Get next dest vector
+ XLDR xd2, [dest2+tmp] ;Get next dest vector
+
+ movdqa xtmpa, x0 ;Keep unshifted copy of src
+ psraw x0, 4 ;Shift to put high nibble into bits 4-0
+ pand x0, xmask0f ;Mask high src nibble in bits 4-0
+ pand xtmpa, xmask0f ;Mask low src nibble in bits 4-0
+
+ ; dest1
+ pshufb xtmph1, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl1, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph1, xtmpl1 ;GF add high and low partials
+ pand xtmph1, xtmph5
+ pxor xd1, xtmph1
+
+ XLDR xd3, [dest3+tmp] ;Reuse xtmpl1, Get next dest vector
+ XLDR xd4, [dest4+tmp] ;Reuse xtmph1. Get next dest vector
+
+ ; dest2
+ pshufb xtmph2, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl2, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph2, xtmpl2 ;GF add high and low partials
+ pand xtmph2, xtmph5
+ pxor xd2, xtmph2
+
+ XLDR xd5, [dest5+tmp] ;Reuse xtmpl2. Get next dest vector
+
+ ; dest3
+ pshufb xtmph3, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl3, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph3, xtmpl3 ;GF add high and low partials
+ pand xtmph3, xtmph5
+ pxor xd3, xtmph3
+
+ ; dest4
+ pshufb xgft4_hi, x0 ;Lookup mul table of high nibble
+ pshufb xgft4_lo, xtmpa ;Lookup mul table of low nibble
+ pxor xgft4_hi, xgft4_lo ;GF add high and low partials
+ pand xgft4_hi, xtmph5
+ pxor xd4, xgft4_hi
+
+ ; dest5
+ pshufb xgft5_hi, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl5, xtmpa ;Lookup mul table of low nibble
+ pxor xgft5_hi, xtmpl5 ;GF add high and low partials
+ pand xgft5_hi, xtmph5
+ pxor xd5, xgft5_hi
+
+ XSTR [dest1+tmp], xd1 ;Store result into dest1
+ XSTR [dest2+tmp], xd2 ;Store result into dest2
+ XSTR [dest3+tmp], xd3 ;Store result into dest3
+ XSTR [dest4+tmp], xd4 ;Store result into dest4
+ XSTR [dest5+tmp], xd5 ;Store result into dest5
+
+.return_pass:
+ FUNC_RESTORE
+ mov return, 0
+ ret
+
+.return_fail:
+ FUNC_RESTORE
+ mov return, 1
+ ret
+
+endproc_frame
+
+section .data
+
+align 16
+
+mask0f:
+ ddq 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
+constip16:
+ ddq 0xf0f1f2f3f4f5f6f7f8f9fafbfcfdfeff
+
+%macro slversion 4
+global %1_slver_%2%3%4
+global %1_slver
+%1_slver:
+%1_slver_%2%3%4:
+ dw 0x%4
+ db 0x%3, 0x%2
+%endmacro
+;;; func core, ver, snum
+slversion gf_5vect_mad_sse, 00, 00, 020c
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_6vect_dot_prod_avx(len, vec, *g_tbls, **buffs, **dests);
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
%endmacro
;;; func core, ver, snum
slversion gf_6vect_dot_prod_avx, 02, 03, 0195
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_6vect_dot_prod_avx2(len, vec, *g_tbls, **buffs, **dests);
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
%endmacro
;;; func core, ver, snum
slversion gf_6vect_dot_prod_avx2, 04, 03, 019a
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_6vect_dot_prod_sse(len, vec, *g_tbls, **buffs, **dests);
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
section .text
%define xmask0f xmm15
-%define xgft1_lo xmm14
-%define xgft1_hi xmm13
-%define xgft2_lo xmm12
-%define xgft2_hi xmm11
-%define xgft3_lo xmm10
-%define xgft3_hi xmm9
+%define xgft1_lo xmm2
+%define xgft1_hi xmm3
+%define xgft2_lo xmm4
+%define xgft2_hi xmm5
+%define xgft3_lo xmm6
+%define xgft3_hi xmm7
%define x0 xmm0
%define xtmpa xmm1
-%define xp1 xmm2
-%define xp2 xmm3
-%define xp3 xmm4
-%define xp4 xmm5
-%define xp5 xmm6
-%define xp6 xmm7
+%define xp1 xmm8
+%define xp2 xmm9
+%define xp3 xmm10
+%define xp4 xmm11
+%define xp5 xmm12
+%define xp6 xmm13
align 16
global gf_6vect_dot_prod_sse:function
db 0x%3, 0x%2
%endmacro
;;; func core, ver, snum
-slversion gf_6vect_dot_prod_sse, 00, 03, 0066
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
+slversion gf_6vect_dot_prod_sse, 00, 04, 0066
--- /dev/null
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+; * Redistributions of source code must retain the above copyright
+; notice, this list of conditions and the following disclaimer.
+; * Redistributions in binary form must reproduce the above copyright
+; notice, this list of conditions and the following disclaimer in
+; the documentation and/or other materials provided with the
+; distribution.
+; * Neither the name of Intel Corporation nor the names of its
+; contributors may be used to endorse or promote products derived
+; from this software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;
+;;; gf_6vect_mad_avx(len, vec, vec_i, mul_array, src, dest);
+;;;
+
+%define PS 8
+
+%ifidn __OUTPUT_FORMAT__, win64
+ %define arg0 rcx
+ %define arg0.w ecx
+ %define arg1 rdx
+ %define arg2 r8
+ %define arg3 r9
+ %define arg4 r12
+ %define arg5 r15
+ %define tmp r11
+ %define tmp2 r10
+ %define tmp3 r13
+ %define tmp4 r14
+ %define tmp5 rdi
+ %define return rax
+ %define return.w eax
+ %define stack_size 16*10 + 5*8
+ %define arg(x) [rsp + stack_size + PS + PS*x]
+ %define func(x) proc_frame x
+
+%macro FUNC_SAVE 0
+ sub rsp, stack_size
+ movdqa [rsp+16*0],xmm6
+ movdqa [rsp+16*1],xmm7
+ movdqa [rsp+16*2],xmm8
+ movdqa [rsp+16*3],xmm9
+ movdqa [rsp+16*4],xmm10
+ movdqa [rsp+16*5],xmm11
+ movdqa [rsp+16*6],xmm12
+ movdqa [rsp+16*7],xmm13
+ movdqa [rsp+16*8],xmm14
+ movdqa [rsp+16*9],xmm15
+ save_reg r12, 10*16 + 0*8
+ save_reg r13, 10*16 + 1*8
+ save_reg r14, 10*16 + 2*8
+ save_reg r15, 10*16 + 3*8
+ save_reg rdi, 10*16 + 4*8
+ end_prolog
+ mov arg4, arg(4)
+ mov arg5, arg(5)
+%endmacro
+
+%macro FUNC_RESTORE 0
+ movdqa xmm6, [rsp+16*0]
+ movdqa xmm7, [rsp+16*1]
+ movdqa xmm8, [rsp+16*2]
+ movdqa xmm9, [rsp+16*3]
+ movdqa xmm10, [rsp+16*4]
+ movdqa xmm11, [rsp+16*5]
+ movdqa xmm12, [rsp+16*6]
+ movdqa xmm13, [rsp+16*7]
+ movdqa xmm14, [rsp+16*8]
+ movdqa xmm15, [rsp+16*9]
+ mov r12, [rsp + 10*16 + 0*8]
+ mov r13, [rsp + 10*16 + 1*8]
+ mov r14, [rsp + 10*16 + 2*8]
+ mov r15, [rsp + 10*16 + 3*8]
+ mov rdi, [rsp + 10*16 + 4*8]
+ add rsp, stack_size
+%endmacro
+
+%elifidn __OUTPUT_FORMAT__, elf64
+ %define arg0 rdi
+ %define arg0.w edi
+ %define arg1 rsi
+ %define arg2 rdx
+ %define arg3 rcx
+ %define arg4 r8
+ %define arg5 r9
+ %define tmp r11
+ %define tmp2 r10
+ %define tmp3 r12
+ %define tmp4 r13
+ %define tmp5 r14
+ %define return rax
+ %define return.w eax
+
+ %define func(x) x:
+ %macro FUNC_SAVE 0
+ push r12
+ push r13
+ push r14
+ %endmacro
+ %macro FUNC_RESTORE 0
+ pop r14
+ pop r13
+ pop r12
+ %endmacro
+%endif
+
+;;; gf_6vect_mad_avx(len, vec, vec_i, mul_array, src, dest)
+%define len arg0
+%define len.w arg0.w
+%define vec arg1
+%define vec_i arg2
+%define mul_array arg3
+%define src arg4
+%define dest1 arg5
+%define pos return
+%define pos.w return.w
+
+%define dest2 tmp4
+%define dest3 tmp2
+%define dest4 mul_array
+%define dest5 tmp5
+%define dest6 vec_i
+
+%ifndef EC_ALIGNED_ADDR
+;;; Use Un-aligned load/store
+ %define XLDR vmovdqu
+ %define XSTR vmovdqu
+%else
+;;; Use Non-temporal load/stor
+ %ifdef NO_NT_LDST
+ %define XLDR vmovdqa
+ %define XSTR vmovdqa
+ %else
+ %define XLDR vmovntdqa
+ %define XSTR vmovntdq
+ %endif
+%endif
+
+
+default rel
+
+[bits 64]
+section .text
+
+%define xmask0f xmm15
+%define xgft4_lo xmm14
+%define xgft4_hi xmm13
+%define xgft5_lo xmm12
+%define xgft5_hi xmm11
+%define xgft6_lo xmm10
+%define xgft6_hi xmm9
+
+%define x0 xmm0
+%define xtmpa xmm1
+%define xtmph1 xmm2
+%define xtmpl1 xmm3
+%define xtmph2 xmm4
+%define xtmpl2 xmm5
+%define xtmph3 xmm6
+%define xtmpl3 xmm7
+%define xd1 xmm8
+%define xd2 xtmpl1
+%define xd3 xtmph1
+
+
+align 16
+global gf_6vect_mad_avx:function
+func(gf_6vect_mad_avx)
+ FUNC_SAVE
+ sub len, 16
+ jl .return_fail
+ xor pos, pos
+ vmovdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
+ mov tmp, vec
+ sal vec_i, 5 ;Multiply by 32
+ lea tmp3, [mul_array + vec_i]
+ sal tmp, 6 ;Multiply by 64
+
+ sal vec, 5 ;Multiply by 32
+ lea vec_i, [tmp + vec] ;vec_i = vec*96
+ lea mul_array, [tmp + vec_i] ;mul_array = vec*160
+
+ vmovdqu xgft5_lo, [tmp3+2*tmp] ;Load array Ex{00}, Ex{01}, ..., Ex{0f}
+ vmovdqu xgft5_hi, [tmp3+2*tmp+16] ; " Ex{00}, Ex{10}, ..., Ex{f0}
+ vmovdqu xgft4_lo, [tmp3+vec_i] ;Load array Dx{00}, Dx{01}, Dx{02}, ...
+ vmovdqu xgft4_hi, [tmp3+vec_i+16] ; " Dx{00}, Dx{10}, Dx{20}, ... , Dx{f0}
+ vmovdqu xgft6_lo, [tmp3+mul_array] ;Load array Fx{00}, Fx{01}, ..., Fx{0f}
+ vmovdqu xgft6_hi, [tmp3+mul_array+16] ; " Fx{00}, Fx{10}, ..., Fx{f0}
+
+ mov dest2, [dest1+PS]
+ mov dest3, [dest1+2*PS]
+ mov dest4, [dest1+3*PS] ; reuse mul_array
+ mov dest5, [dest1+4*PS]
+ mov dest6, [dest1+5*PS] ; reuse vec_i
+ mov dest1, [dest1]
+
+.loop16:
+ XLDR x0, [src+pos] ;Get next source vector
+
+ vmovdqu xtmpl1, [tmp3] ;Load array Ax{00}, Ax{01}, Ax{02}, ...
+ vmovdqu xtmph1, [tmp3+16] ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
+ vmovdqu xtmpl2, [tmp3+vec] ;Load array Bx{00}, Bx{01}, Bx{02}, ...
+ vmovdqu xtmph2, [tmp3+vec+16] ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
+ vmovdqu xtmpl3, [tmp3+2*vec] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
+ vmovdqu xtmph3, [tmp3+2*vec+16] ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
+ XLDR xd1, [dest1+pos] ;Get next dest vector
+
+ vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+
+
+ ;dest1
+ vpshufb xtmph1, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl1, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph1, xtmpl1 ;GF add high and low partials
+ vpxor xd1, xtmph1
+
+ XLDR xd2, [dest2+pos] ;reuse xtmpl1. Get next dest vector
+ XLDR xd3, [dest3+pos] ;reuse xtmph1. Get next dest vector
+
+ ;dest2
+ vpshufb xtmph2, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl2, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph2, xtmpl2 ;GF add high and low partials
+ vpxor xd2, xtmph2
+
+ ;dest3
+ vpshufb xtmph3, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl3, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph3, xtmpl3 ;GF add high and low partials
+ vpxor xd3, xtmph3
+
+ XSTR [dest1+pos], xd1 ;Store result into dest1
+ XSTR [dest2+pos], xd2 ;Store result into dest2
+ XSTR [dest3+pos], xd3 ;Store result into dest3
+
+ ;dest4
+ XLDR xd1, [dest4+pos] ;Get next dest vector
+ vpshufb xtmph1, xgft4_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl1, xgft4_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph1, xtmph1, xtmpl1 ;GF add high and low partials
+ vpxor xd1, xd1, xtmph1
+
+ XLDR xd2, [dest5+pos] ;reuse xtmpl1. Get next dest vector
+ XLDR xd3, [dest6+pos] ;reuse xtmph1. Get next dest vector
+
+ ;dest5
+ vpshufb xtmph2, xgft5_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl2, xgft5_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph2, xtmph2, xtmpl2 ;GF add high and low partials
+ vpxor xd2, xd2, xtmph2
+
+ ;dest6
+ vpshufb xtmph3, xgft6_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl3, xgft6_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph3, xtmph3, xtmpl3 ;GF add high and low partials
+ vpxor xd3, xd3, xtmph3
+
+ XSTR [dest4+pos], xd1 ;Store result into dest4
+ XSTR [dest5+pos], xd2 ;Store result into dest5
+ XSTR [dest6+pos], xd3 ;Store result into dest6
+
+ add pos, 16 ;Loop on 16 bytes at a time
+ cmp pos, len
+ jle .loop16
+
+ lea tmp, [len + 16]
+ cmp pos, tmp
+ je .return_pass
+
+.lessthan16:
+ ;; Tail len
+ ;; Do one more overlap pass
+ ;; Overlapped offset length-16
+ mov tmp, len ;Backup len as len=rdi
+
+ XLDR x0, [src+tmp] ;Get next source vector
+ XLDR xd1, [dest4+tmp] ;Get next dest vector
+ XLDR xd2, [dest5+tmp] ;reuse xtmpl1. Get next dest vector
+ XLDR xd3, [dest6+tmp] ;reuse xtmph1. Get next dest vector
+
+ sub len, pos
+
+ vmovdqa xtmph3, [constip16] ;Load const of i + 16
+ vpinsrb xtmpl3, len.w, 15
+ vpshufb xtmpl3, xmask0f ;Broadcast len to all bytes
+ vpcmpgtb xtmpl3, xtmpl3, xtmph3
+
+ vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+
+ ;dest4
+ vpshufb xgft4_hi, xgft4_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xgft4_lo, xgft4_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xgft4_hi, xgft4_hi, xgft4_lo ;GF add high and low partials
+ vpand xgft4_hi, xgft4_hi, xtmpl3
+ vpxor xd1, xd1, xgft4_hi
+
+ ;dest5
+ vpshufb xgft5_hi, xgft5_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xgft5_lo, xgft5_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xgft5_hi, xgft5_hi, xgft5_lo ;GF add high and low partials
+ vpand xgft5_hi, xgft5_hi, xtmpl3
+ vpxor xd2, xd2, xgft5_hi
+
+ ;dest6
+ vpshufb xgft6_hi, xgft6_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xgft6_lo, xgft6_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xgft6_hi, xgft6_hi, xgft6_lo ;GF add high and low partials
+ vpand xgft6_hi, xgft6_hi, xtmpl3
+ vpxor xd3, xd3, xgft6_hi
+
+ XSTR [dest4+tmp], xd1 ;Store result into dest4
+ XSTR [dest5+tmp], xd2 ;Store result into dest5
+ XSTR [dest6+tmp], xd3 ;Store result into dest6
+
+ vmovdqu xgft4_lo, [tmp3] ;Load array Ax{00}, Ax{01}, Ax{02}, ...
+ vmovdqu xgft4_hi, [tmp3+16] ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
+ vmovdqu xgft5_lo, [tmp3+vec] ;Load array Bx{00}, Bx{01}, Bx{02}, ...
+ vmovdqu xgft5_hi, [tmp3+vec+16] ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
+ vmovdqu xgft6_lo, [tmp3+2*vec] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
+ vmovdqu xgft6_hi, [tmp3+2*vec+16] ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
+ XLDR xd1, [dest1+tmp] ;Get next dest vector
+ XLDR xd2, [dest2+tmp] ;reuse xtmpl1. Get next dest vector
+ XLDR xd3, [dest3+tmp] ;reuse xtmph1. Get next dest3 vector
+
+ ;dest1
+ vpshufb xgft4_hi, xgft4_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xgft4_lo, xgft4_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xgft4_hi, xgft4_hi, xgft4_lo ;GF add high and low partials
+ vpand xgft4_hi, xgft4_hi, xtmpl3
+ vpxor xd1, xd1, xgft4_hi
+
+ ;dest2
+ vpshufb xgft5_hi, xgft5_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xgft5_lo, xgft5_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xgft5_hi, xgft5_hi, xgft5_lo ;GF add high and low partials
+ vpand xgft5_hi, xgft5_hi, xtmpl3
+ vpxor xd2, xd2, xgft5_hi
+
+ ;dest3
+ vpshufb xgft6_hi, xgft6_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xgft6_lo, xgft6_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xgft6_hi, xgft6_hi, xgft6_lo ;GF add high and low partials
+ vpand xgft6_hi, xgft6_hi, xtmpl3
+ vpxor xd3, xd3, xgft6_hi
+
+ XSTR [dest1+tmp], xd1 ;Store result into dest1
+ XSTR [dest2+tmp], xd2 ;Store result into dest2
+ XSTR [dest3+tmp], xd3 ;Store result into dest3
+
+.return_pass:
+ FUNC_RESTORE
+ mov return, 0
+ ret
+
+.return_fail:
+ FUNC_RESTORE
+ mov return, 1
+ ret
+
+endproc_frame
+
+section .data
+
+align 16
+mask0f: ddq 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
+constip16:
+ ddq 0xf0f1f2f3f4f5f6f7f8f9fafbfcfdfeff
+
+%macro slversion 4
+global %1_slver_%2%3%4
+global %1_slver
+%1_slver:
+%1_slver_%2%3%4:
+ dw 0x%4
+ db 0x%3, 0x%2
+%endmacro
+;;; func core, ver, snum
+slversion gf_6vect_mad_avx, 02, 00, 0210
--- /dev/null
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+; * Redistributions of source code must retain the above copyright
+; notice, this list of conditions and the following disclaimer.
+; * Redistributions in binary form must reproduce the above copyright
+; notice, this list of conditions and the following disclaimer in
+; the documentation and/or other materials provided with the
+; distribution.
+; * Neither the name of Intel Corporation nor the names of its
+; contributors may be used to endorse or promote products derived
+; from this software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;
+;;; gf_6vect_mad_avx2(len, vec, vec_i, mul_array, src, dest);
+;;;
+
+
+%define PS 8
+
+%ifidn __OUTPUT_FORMAT__, win64
+ %define arg0 rcx
+ %define arg0.w ecx
+ %define arg1 rdx
+ %define arg2 r8
+ %define arg3 r9
+ %define arg4 r12
+ %define arg5 r15
+ %define tmp r11
+ %define tmp.w r11d
+ %define tmp.b r11b
+ %define tmp2 r10
+ %define tmp3 r13
+ %define return rax
+ %define return.w eax
+ %define stack_size 16*10 + 3*8
+ %define arg(x) [rsp + stack_size + PS + PS*x]
+ %define func(x) proc_frame x
+
+%macro FUNC_SAVE 0
+ sub rsp, stack_size
+ movdqa [rsp+16*0],xmm6
+ movdqa [rsp+16*1],xmm7
+ movdqa [rsp+16*2],xmm8
+ movdqa [rsp+16*3],xmm9
+ movdqa [rsp+16*4],xmm10
+ movdqa [rsp+16*5],xmm11
+ movdqa [rsp+16*6],xmm12
+ movdqa [rsp+16*7],xmm13
+ movdqa [rsp+16*8],xmm14
+ movdqa [rsp+16*9],xmm15
+ save_reg r12, 10*16 + 0*8
+ save_reg r13, 10*16 + 1*8
+ save_reg r15, 10*16 + 2*8
+ end_prolog
+ mov arg4, arg(4)
+ mov arg5, arg(5)
+%endmacro
+
+%macro FUNC_RESTORE 0
+ movdqa xmm6, [rsp+16*0]
+ movdqa xmm7, [rsp+16*1]
+ movdqa xmm8, [rsp+16*2]
+ movdqa xmm9, [rsp+16*3]
+ movdqa xmm10, [rsp+16*4]
+ movdqa xmm11, [rsp+16*5]
+ movdqa xmm12, [rsp+16*6]
+ movdqa xmm13, [rsp+16*7]
+ movdqa xmm14, [rsp+16*8]
+ movdqa xmm15, [rsp+16*9]
+ mov r12, [rsp + 10*16 + 0*8]
+ mov r13, [rsp + 10*16 + 1*8]
+ mov r15, [rsp + 10*16 + 3*8]
+ add rsp, stack_size
+%endmacro
+
+%elifidn __OUTPUT_FORMAT__, elf64
+ %define arg0 rdi
+ %define arg0.w edi
+ %define arg1 rsi
+ %define arg2 rdx
+ %define arg3 rcx
+ %define arg4 r8
+ %define arg5 r9
+ %define tmp r11
+ %define tmp.w r11d
+ %define tmp.b r11b
+ %define tmp2 r10
+ %define tmp3 r12
+ %define return rax
+ %define return.w eax
+
+ %define func(x) x:
+ %macro FUNC_SAVE 0
+ push r12
+ %endmacro
+ %macro FUNC_RESTORE 0
+ pop r12
+ %endmacro
+%endif
+
+;;; gf_6vect_mad_avx2(len, vec, vec_i, mul_array, src, dest)
+%define len arg0
+%define len.w arg0.w
+%define vec arg1
+%define vec_i arg2
+%define mul_array arg3
+%define src arg4
+%define dest1 arg5
+%define pos return
+%define pos.w return.w
+
+%define dest2 tmp3
+%define dest3 tmp2
+%define dest4 mul_array
+%define dest5 vec
+%define dest6 vec_i
+
+%ifndef EC_ALIGNED_ADDR
+;;; Use Un-aligned load/store
+ %define XLDR vmovdqu
+ %define XSTR vmovdqu
+%else
+;;; Use Non-temporal load/stor
+ %ifdef NO_NT_LDST
+ %define XLDR vmovdqa
+ %define XSTR vmovdqa
+ %else
+ %define XLDR vmovntdqa
+ %define XSTR vmovntdq
+ %endif
+%endif
+
+
+default rel
+
+[bits 64]
+section .text
+
+%define xmask0f ymm15
+%define xmask0fx xmm15
+%define xgft1_lo ymm14
+%define xgft2_lo ymm13
+%define xgft3_lo ymm12
+%define xgft4_lo ymm11
+%define xgft5_lo ymm10
+%define xgft6_lo ymm9
+
+%define x0 ymm0
+%define xtmpa ymm1
+%define xtmpl ymm2
+%define xtmplx xmm2
+%define xtmph ymm3
+%define xtmphx xmm3
+%define xd1 ymm4
+%define xd2 ymm5
+%define xd3 ymm6
+%define xd4 ymm7
+%define xd5 ymm8
+%define xd6 xd1
+
+align 16
+global gf_6vect_mad_avx2:function
+func(gf_6vect_mad_avx2)
+ FUNC_SAVE
+ sub len, 32
+ jl .return_fail
+ xor pos, pos
+ mov tmp.b, 0x0f
+ vpinsrb xmask0fx, xmask0fx, tmp.w, 0
+ vpbroadcastb xmask0f, xmask0fx ;Construct mask 0x0f0f0f...
+
+ sal vec_i, 5 ;Multiply by 32
+ sal vec, 5 ;Multiply by 32
+ lea tmp, [mul_array + vec_i]
+ mov vec_i, vec
+ mov mul_array, vec
+ sal vec_i, 1
+ sal mul_array, 1
+ add vec_i, vec ;vec_i=vec*96
+ add mul_array, vec_i ;vec_i=vec*160
+
+ vmovdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, ..., Ax{0f}
+ ; " Ax{00}, Ax{10}, ..., Ax{f0}
+ vmovdqu xgft2_lo, [tmp+vec] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
+ ; " Bx{00}, Bx{10}, ..., Bx{f0}
+ vmovdqu xgft3_lo, [tmp+2*vec] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
+ ; " Cx{00}, Cx{10}, ..., Cx{f0}
+ vmovdqu xgft4_lo, [tmp+vec_i] ;Load array Fx{00}, Fx{01}, ..., Fx{0f}
+ ; " Fx{00}, Fx{10}, ..., Fx{f0}
+ vmovdqu xgft5_lo, [tmp+4*vec] ;Load array Ex{00}, Ex{01}, ..., Ex{0f}
+ ; " Ex{00}, Ex{10}, ..., Ex{f0}
+ vmovdqu xgft6_lo, [tmp+mul_array] ;Load array Dx{00}, Dx{01}, ..., Dx{0f}
+ ; " Dx{00}, Dx{10}, ..., Dx{f0}
+
+ mov dest2, [dest1+PS] ; reuse tmp3
+ mov dest3, [dest1+2*PS] ; reuse tmp2
+ mov dest4, [dest1+3*PS] ; reuse mul_array
+ mov dest5, [dest1+4*PS] ; reuse vec
+ mov dest6, [dest1+5*PS] ; reuse vec_i
+ mov dest1, [dest1]
+
+.loop32:
+ XLDR x0, [src+pos] ;Get next source vector
+ XLDR xd1, [dest1+pos] ;Get next dest vector
+ XLDR xd2, [dest2+pos] ;Get next dest vector
+ XLDR xd3, [dest3+pos] ;Get next dest vector
+ XLDR xd4, [dest4+pos] ;Get next dest vector
+ XLDR xd5, [dest5+pos] ;Get next dest vector
+
+ vpand xtmpl, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+ vperm2i128 xtmpa, xtmpl, x0, 0x30 ;swap xtmpa from 1lo|2lo to 1lo|2hi
+ vperm2i128 x0, xtmpl, x0, 0x12 ;swap x0 from 1hi|2hi to 1hi|2lo
+
+ ;dest1
+ vperm2i128 xtmph, xgft1_lo, xgft1_lo, 0x01 ; swapped to hi | lo
+ vpshufb xtmph, xtmph, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl, xgft1_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph, xtmph, xtmpl ;GF add high and low partials
+ vpxor xd1, xd1, xtmph ;xd1 += partial
+
+ XSTR [dest1+pos], xd1 ;Store result into dest1
+
+ ;dest2
+ vperm2i128 xtmph, xgft2_lo, xgft2_lo, 0x01 ; swapped to hi | lo
+ vpshufb xtmph, xtmph, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl, xgft2_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph, xtmph, xtmpl ;GF add high and low partials
+ vpxor xd2, xd2, xtmph ;xd2 += partial
+
+ ;dest3
+ vperm2i128 xtmph, xgft3_lo, xgft3_lo, 0x01 ; swapped to hi | lo
+ vpshufb xtmph, xtmph, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl, xgft3_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph, xtmph, xtmpl ;GF add high and low partials
+ vpxor xd3, xd3, xtmph ;xd3 += partial
+
+ XLDR xd6, [dest6+pos] ;reuse xd1. Get next dest vector
+
+ ;dest4
+ vperm2i128 xtmph, xgft4_lo, xgft4_lo, 0x01 ; swapped to hi | lo
+ vpshufb xtmph, xtmph, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl, xgft4_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph, xtmph, xtmpl ;GF add high and low partials
+ vpxor xd4, xd4, xtmph ;xd4 += partial
+
+ ;dest5
+ vperm2i128 xtmph, xgft5_lo, xgft5_lo, 0x01 ; swapped to hi | lo
+ vpshufb xtmph, xtmph, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl, xgft5_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph, xtmph, xtmpl ;GF add high and low partials
+ vpxor xd5, xd5, xtmph ;xd5 += partial
+
+ ;dest6
+ vperm2i128 xtmph, xgft6_lo, xgft6_lo, 0x01 ; swapped to hi | lo
+ vpshufb xtmph, xtmph, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl, xgft6_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph, xtmph, xtmpl ;GF add high and low partials
+ vpxor xd6, xd6, xtmph ;xd6 += partial
+
+ XSTR [dest2+pos], xd2 ;Store result into dest2
+ XSTR [dest3+pos], xd3 ;Store result into dest3
+ XSTR [dest4+pos], xd4 ;Store result into dest4
+ XSTR [dest5+pos], xd5 ;Store result into dest5
+ XSTR [dest6+pos], xd6 ;Store result into dest6
+
+ add pos, 32 ;Loop on 32 bytes at a time
+ cmp pos, len
+ jle .loop32
+
+ lea tmp, [len + 32]
+ cmp pos, tmp
+ je .return_pass
+
+.lessthan32:
+ ;; Tail len
+ ;; Do one more overlap pass
+ mov tmp.b, 0x1f
+ vpinsrb xtmphx, xtmphx, tmp.w, 0
+ vpbroadcastb xtmph, xtmphx ;Construct mask 0x1f1f1f...
+
+ mov tmp, len ;Overlapped offset length-32
+
+ XLDR x0, [src+tmp] ;Get next source vector
+ XLDR xd1, [dest1+tmp] ;Get next dest vector
+ XLDR xd2, [dest2+tmp] ;Get next dest vector
+ XLDR xd3, [dest3+tmp] ;Get next dest vector
+ XLDR xd4, [dest4+tmp] ;Get next dest vector
+ XLDR xd5, [dest5+tmp] ;Get next dest vector
+
+ sub len, pos
+
+ vpinsrb xtmplx, xtmplx, len.w, 15
+ vinserti128 xtmpl, xtmpl, xtmplx, 1 ;swapped to xtmplx | xtmplx
+ vpshufb xtmpl, xtmpl, xtmph ;Broadcast len to all bytes. xtmph=0x1f1f1f...
+ vpcmpgtb xtmpl, xtmpl, [constip32]
+
+ vpand xtmph, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+ vperm2i128 xtmpa, xtmph, x0, 0x30 ;swap xtmpa from 1lo|2lo to 1lo|2hi
+ vperm2i128 x0, xtmph, x0, 0x12 ;swap x0 from 1hi|2hi to 1hi|2lo
+
+ ;dest1
+ vperm2i128 xtmph, xgft1_lo, xgft1_lo, 0x01 ; swapped to hi | lo
+ vpshufb xtmph, xtmph, x0 ;Lookup mul table of high nibble
+ vpshufb xgft1_lo, xgft1_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph, xtmph, xgft1_lo ;GF add high and low partials
+ vpand xtmph, xtmph, xtmpl
+ vpxor xd1, xd1, xtmph ;xd1 += partial
+
+ XSTR [dest1+tmp], xd1 ;Store result into dest1
+
+ ;dest2
+ vperm2i128 xtmph, xgft2_lo, xgft2_lo, 0x01 ; swapped to hi | lo
+ vpshufb xtmph, xtmph, x0 ;Lookup mul table of high nibble
+ vpshufb xgft2_lo, xgft2_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph, xtmph, xgft2_lo ;GF add high and low partials
+ vpand xtmph, xtmph, xtmpl
+ vpxor xd2, xd2, xtmph ;xd2 += partial
+
+ ;dest3
+ vperm2i128 xtmph, xgft3_lo, xgft3_lo, 0x01 ; swapped to hi | lo
+ vpshufb xtmph, xtmph, x0 ;Lookup mul table of high nibble
+ vpshufb xgft3_lo, xgft3_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph, xtmph, xgft3_lo ;GF add high and low partials
+ vpand xtmph, xtmph, xtmpl
+ vpxor xd3, xd3, xtmph ;xd3 += partial
+
+ XLDR xd6, [dest6+tmp] ;reuse xd1. Get next dest vector
+
+ ;dest4
+ vperm2i128 xtmph, xgft4_lo, xgft4_lo, 0x01 ; swapped to hi | lo
+ vpshufb xtmph, xtmph, x0 ;Lookup mul table of high nibble
+ vpshufb xgft4_lo, xgft4_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph, xtmph, xgft4_lo ;GF add high and low partials
+ vpand xtmph, xtmph, xtmpl
+ vpxor xd4, xd4, xtmph ;xd4 += partial
+
+ ;dest5
+ vperm2i128 xtmph, xgft5_lo, xgft5_lo, 0x01 ; swapped to hi | lo
+ vpshufb xtmph, xtmph, x0 ;Lookup mul table of high nibble
+ vpshufb xgft5_lo, xgft5_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph, xtmph, xgft5_lo ;GF add high and low partials
+ vpand xtmph, xtmph, xtmpl
+ vpxor xd5, xd5, xtmph ;xd5 += partial
+
+ ;dest6
+ vperm2i128 xtmph, xgft6_lo, xgft6_lo, 0x01 ; swapped to hi | lo
+ vpshufb xtmph, xtmph, x0 ;Lookup mul table of high nibble
+ vpshufb xgft6_lo, xgft6_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph, xtmph, xgft6_lo ;GF add high and low partials
+ vpand xtmph, xtmph, xtmpl
+ vpxor xd6, xd6, xtmph ;xd6 += partial
+
+ XSTR [dest2+tmp], xd2 ;Store result into dest2
+ XSTR [dest3+tmp], xd3 ;Store result into dest3
+ XSTR [dest4+tmp], xd4 ;Store result into dest4
+ XSTR [dest5+tmp], xd5 ;Store result into dest5
+ XSTR [dest6+tmp], xd6 ;Store result into dest6
+
+.return_pass:
+ FUNC_RESTORE
+ mov return, 0
+ ret
+
+.return_fail:
+ FUNC_RESTORE
+ mov return, 1
+ ret
+
+endproc_frame
+
+section .data
+align 32
+constip32:
+ ddq 0xf0f1f2f3f4f5f6f7f8f9fafbfcfdfeff
+ ddq 0xe0e1e2e3e4e5e6e7e8e9eaebecedeeef
+
+%macro slversion 4
+global %1_slver_%2%3%4
+global %1_slver
+%1_slver:
+%1_slver_%2%3%4:
+ dw 0x%4
+ db 0x%3, 0x%2
+%endmacro
+;;; func core, ver, snum
+slversion gf_6vect_mad_avx2, 04, 00, 0211
--- /dev/null
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+; * Redistributions of source code must retain the above copyright
+; notice, this list of conditions and the following disclaimer.
+; * Redistributions in binary form must reproduce the above copyright
+; notice, this list of conditions and the following disclaimer in
+; the documentation and/or other materials provided with the
+; distribution.
+; * Neither the name of Intel Corporation nor the names of its
+; contributors may be used to endorse or promote products derived
+; from this software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;
+;;; gf_6vect_mad_sse(len, vec, vec_i, mul_array, src, dest);
+;;;
+
+%define PS 8
+
+%ifidn __OUTPUT_FORMAT__, win64
+ %define arg0 rcx
+ %define arg0.w ecx
+ %define arg1 rdx
+ %define arg2 r8
+ %define arg3 r9
+ %define arg4 r12
+ %define arg5 r15
+ %define tmp r11
+ %define tmp.w r11d
+ %define tmp2 r10
+ %define tmp3 r13
+ %define tmp4 r14
+ %define tmp5 rdi
+ %define return rax
+ %define return.w eax
+ %define stack_size 16*10 + 5*8
+ %define arg(x) [rsp + stack_size + PS + PS*x]
+ %define func(x) proc_frame x
+
+%macro FUNC_SAVE 0
+ sub rsp, stack_size
+ movdqa [rsp+16*0],xmm6
+ movdqa [rsp+16*1],xmm7
+ movdqa [rsp+16*2],xmm8
+ movdqa [rsp+16*3],xmm9
+ movdqa [rsp+16*4],xmm10
+ movdqa [rsp+16*5],xmm11
+ movdqa [rsp+16*6],xmm12
+ movdqa [rsp+16*7],xmm13
+ movdqa [rsp+16*8],xmm14
+ movdqa [rsp+16*9],xmm15
+ save_reg r12, 10*16 + 0*8
+ save_reg r13, 10*16 + 1*8
+ save_reg r14, 10*16 + 2*8
+ save_reg r15, 10*16 + 3*8
+ save_reg rdi, 10*16 + 4*8
+ end_prolog
+ mov arg4, arg(4)
+ mov arg5, arg(5)
+%endmacro
+
+%macro FUNC_RESTORE 0
+ movdqa xmm6, [rsp+16*0]
+ movdqa xmm7, [rsp+16*1]
+ movdqa xmm8, [rsp+16*2]
+ movdqa xmm9, [rsp+16*3]
+ movdqa xmm10, [rsp+16*4]
+ movdqa xmm11, [rsp+16*5]
+ movdqa xmm12, [rsp+16*6]
+ movdqa xmm13, [rsp+16*7]
+ movdqa xmm14, [rsp+16*8]
+ movdqa xmm15, [rsp+16*9]
+ mov r12, [rsp + 10*16 + 0*8]
+ mov r13, [rsp + 10*16 + 1*8]
+ mov r14, [rsp + 10*16 + 2*8]
+ mov r15, [rsp + 10*16 + 3*8]
+ mov rdi, [rsp + 10*16 + 4*8]
+ add rsp, stack_size
+%endmacro
+
+%elifidn __OUTPUT_FORMAT__, elf64
+ %define arg0 rdi
+ %define arg0.w edi
+ %define arg1 rsi
+ %define arg2 rdx
+ %define arg3 rcx
+ %define arg4 r8
+ %define arg5 r9
+ %define tmp r11
+ %define tmp.w r11d
+ %define tmp2 r10
+ %define tmp3 r12
+ %define tmp4 r13
+ %define tmp5 r14
+ %define return rax
+ %define return.w eax
+
+ %define func(x) x:
+ %macro FUNC_SAVE 0
+ push r12
+ push r13
+ push r14
+ %endmacro
+ %macro FUNC_RESTORE 0
+ pop r14
+ pop r13
+ pop r12
+ %endmacro
+%endif
+
+;;; gf_6vect_mad_sse(len, vec, vec_i, mul_array, src, dest)
+%define len arg0
+%define len.w arg0.w
+%define vec arg1
+%define vec_i arg2
+%define mul_array arg3
+%define src arg4
+%define dest1 arg5
+%define pos return
+%define pos.w return.w
+
+%define dest2 mul_array
+%define dest3 tmp2
+%define dest4 tmp4
+%define dest5 tmp5
+%define dest6 vec_i
+
+%ifndef EC_ALIGNED_ADDR
+;;; Use Un-aligned load/store
+ %define XLDR movdqu
+ %define XSTR movdqu
+%else
+;;; Use Non-temporal load/stor
+ %ifdef NO_NT_LDST
+ %define XLDR movdqa
+ %define XSTR movdqa
+ %else
+ %define XLDR movntdqa
+ %define XSTR movntdq
+ %endif
+%endif
+
+default rel
+
+[bits 64]
+section .text
+
+%define xmask0f xmm15
+%define xgft4_lo xmm14
+%define xgft4_hi xmm13
+%define xgft5_lo xmm12
+%define xgft5_hi xmm11
+%define xgft6_lo xmm10
+%define xgft6_hi xmm9
+
+%define x0 xmm0
+%define xtmpa xmm1
+%define xtmph1 xmm2
+%define xtmpl1 xmm3
+%define xtmph2 xmm4
+%define xtmpl2 xmm5
+%define xtmph3 xmm6
+%define xtmpl3 xmm7
+%define xd1 xmm8
+%define xd2 xtmpl1
+%define xd3 xtmph1
+
+
+align 16
+global gf_6vect_mad_sse:function
+func(gf_6vect_mad_sse)
+ FUNC_SAVE
+ sub len, 16
+ jl .return_fail
+
+ xor pos, pos
+ movdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
+
+ mov tmp, vec
+ sal vec_i, 5 ;Multiply by 32
+ lea tmp3, [mul_array + vec_i]
+ sal tmp, 6 ;Multiply by 64
+
+ sal vec, 5 ;Multiply by 32
+ lea vec_i, [tmp + vec] ;vec_i = 96
+ lea mul_array, [tmp + vec_i] ;mul_array = 160
+
+ movdqu xgft5_lo, [tmp3+2*tmp] ;Load array Ex{00}, Ex{01}, ..., Ex{0f}
+ movdqu xgft5_hi, [tmp3+2*tmp+16] ; " Ex{00}, Ex{10}, ..., Ex{f0}
+ movdqu xgft4_lo, [tmp3+vec_i] ;Load array Dx{00}, Dx{01}, Dx{02}, ...
+ movdqu xgft4_hi, [tmp3+vec_i+16] ; " Dx{00}, Dx{10}, Dx{20}, ... , Dx{f0}
+ movdqu xgft6_lo, [tmp3+mul_array] ;Load array Fx{00}, Fx{01}, ..., Fx{0f}
+ movdqu xgft6_hi, [tmp3+mul_array+16] ; " Fx{00}, Fx{10}, ..., Fx{f0}
+
+ mov dest2, [dest1+PS]
+ mov dest3, [dest1+2*PS]
+ mov dest4, [dest1+3*PS] ; reuse mul_array
+ mov dest5, [dest1+4*PS]
+ mov dest6, [dest1+5*PS] ; reuse vec_i
+ mov dest1, [dest1]
+
+.loop16:
+ XLDR x0, [src+pos] ;Get next source vector
+
+ movdqu xtmpl1, [tmp3] ;Load array Ax{00}, Ax{01}, Ax{02}, ...
+ movdqu xtmph1, [tmp3+16] ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
+ movdqu xtmpl2, [tmp3+vec] ;Load array Bx{00}, Bx{01}, Bx{02}, ...
+ movdqu xtmph2, [tmp3+vec+16] ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
+ movdqu xtmpl3, [tmp3+2*vec] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
+ movdqu xtmph3, [tmp3+2*vec+16] ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
+ XLDR xd1, [dest1+pos] ;Get next dest vector
+
+ movdqa xtmpa, x0 ;Keep unshifted copy of src
+ psraw x0, 4 ;Shift to put high nibble into bits 4-0
+ pand x0, xmask0f ;Mask high src nibble in bits 4-0
+ pand xtmpa, xmask0f ;Mask low src nibble in bits 4-0
+
+ ;dest1
+ pshufb xtmph1, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl1, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph1, xtmpl1 ;GF add high and low partials
+ pxor xd1, xtmph1
+
+ XLDR xd2, [dest2+pos] ;reuse xtmpl1. Get next dest vector
+ XLDR xd3, [dest3+pos] ;reuse xtmph1. Get next dest3 vector
+
+ ;dest2
+ pshufb xtmph2, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl2, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph2, xtmpl2 ;GF add high and low partials
+ pxor xd2, xtmph2
+
+ ;dest3
+ pshufb xtmph3, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl3, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph3, xtmpl3 ;GF add high and low partials
+ pxor xd3, xtmph3
+
+ XSTR [dest1+pos], xd1 ;Store result into dest1
+ XSTR [dest2+pos], xd2 ;Store result into dest2
+ XSTR [dest3+pos], xd3 ;Store result into dest3
+
+ movdqa xtmph1, xgft4_hi ;Reload const array registers
+ movdqa xtmpl1, xgft4_lo ;Reload const array registers
+ movdqa xtmph2, xgft5_hi ;Reload const array registers
+ movdqa xtmpl2, xgft5_lo ;Reload const array registers
+ movdqa xtmph3, xgft6_hi ;Reload const array registers
+ movdqa xtmpl3, xgft6_lo ;Reload const array registers
+
+ ;dest4
+ XLDR xd1, [dest4+pos] ;Get next dest vector
+ pshufb xtmph1, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl1, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph1, xtmpl1 ;GF add high and low partials
+ pxor xd1, xtmph1
+
+ XLDR xd2, [dest5+pos] ;reuse xtmpl1. Get next dest vector
+ XLDR xd3, [dest6+pos] ;reuse xtmph1. Get next dest vector
+
+ ;dest5
+ pshufb xtmph2, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl2, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph2, xtmpl2 ;GF add high and low partials
+ pxor xd2, xtmph2
+
+ ;dest6
+ pshufb xtmph3, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl3, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph3, xtmpl3 ;GF add high and low partials
+ pxor xd3, xtmph3
+
+ XSTR [dest4+pos], xd1 ;Store result into dest4
+ XSTR [dest5+pos], xd2 ;Store result into dest5
+ XSTR [dest6+pos], xd3 ;Store result into dest6
+
+ add pos, 16 ;Loop on 16 bytes at a time
+ cmp pos, len
+ jle .loop16
+
+ lea tmp, [len + 16]
+ cmp pos, tmp
+ je .return_pass
+
+.lessthan16:
+ ;; Tail len
+ ;; Do one more overlap pass
+ ;; Overlapped offset length-16
+ mov tmp, len ;Backup len as len=rdi
+
+ XLDR x0, [src+tmp] ;Get next source vector
+ XLDR xd1, [dest4+tmp] ;Get next dest vector
+ XLDR xd2, [dest5+tmp] ;reuse xtmpl1. Get next dest vector
+ XLDR xd3, [dest6+tmp] ;reuse xtmph1. Get next dest vector
+
+ sub len, pos
+
+ movdqa xtmph3, [constip16] ;Load const of i + 16
+ pinsrb xtmpl3, len.w, 15
+ pshufb xtmpl3, xmask0f ;Broadcast len to all bytes
+ pcmpgtb xtmpl3, xtmph3
+
+ movdqa xtmpa, x0 ;Keep unshifted copy of src
+ psraw x0, 4 ;Shift to put high nibble into bits 4-0
+ pand x0, xmask0f ;Mask high src nibble in bits 4-0
+ pand xtmpa, xmask0f ;Mask low src nibble in bits 4-0
+
+ ;dest4
+ pshufb xgft4_hi, x0 ;Lookup mul table of high nibble
+ pshufb xgft4_lo, xtmpa ;Lookup mul table of low nibble
+ pxor xgft4_hi, xgft4_lo ;GF add high and low partials
+ pand xgft4_hi, xtmpl3
+ pxor xd1, xgft4_hi
+
+ ;dest5
+ pshufb xgft5_hi, x0 ;Lookup mul table of high nibble
+ pshufb xgft5_lo, xtmpa ;Lookup mul table of low nibble
+ pxor xgft5_hi, xgft5_lo ;GF add high and low partials
+ pand xgft5_hi, xtmpl3
+ pxor xd2, xgft5_hi
+
+ ;dest6
+ pshufb xgft6_hi, x0 ;Lookup mul table of high nibble
+ pshufb xgft6_lo, xtmpa ;Lookup mul table of low nibble
+ pxor xgft6_hi, xgft6_lo ;GF add high and low partials
+ pand xgft6_hi, xtmpl3
+ pxor xd3, xgft6_hi
+
+ XSTR [dest4+tmp], xd1 ;Store result into dest4
+ XSTR [dest5+tmp], xd2 ;Store result into dest5
+ XSTR [dest6+tmp], xd3 ;Store result into dest6
+
+ movdqu xgft4_lo, [tmp3] ;Load array Ax{00}, Ax{01}, Ax{02}, ...
+ movdqu xgft4_hi, [tmp3+16] ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
+ movdqu xgft5_lo, [tmp3+vec] ;Load array Bx{00}, Bx{01}, Bx{02}, ...
+ movdqu xgft5_hi, [tmp3+vec+16] ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
+ movdqu xgft6_lo, [tmp3+2*vec] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
+ movdqu xgft6_hi, [tmp3+2*vec+16] ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
+ XLDR xd1, [dest1+tmp] ;Get next dest vector
+ XLDR xd2, [dest2+tmp] ;reuse xtmpl1. Get next dest vector
+ XLDR xd3, [dest3+tmp] ;reuse xtmph1. Get next dest3 vector
+
+ ;dest1
+ pshufb xgft4_hi, x0 ;Lookup mul table of high nibble
+ pshufb xgft4_lo, xtmpa ;Lookup mul table of low nibble
+ pxor xgft4_hi, xgft4_lo ;GF add high and low partials
+ pand xgft4_hi, xtmpl3
+ pxor xd1, xgft4_hi
+
+ ;dest2
+ pshufb xgft5_hi, x0 ;Lookup mul table of high nibble
+ pshufb xgft5_lo, xtmpa ;Lookup mul table of low nibble
+ pxor xgft5_hi, xgft5_lo ;GF add high and low partials
+ pand xgft5_hi, xtmpl3
+ pxor xd2, xgft5_hi
+
+ ;dest3
+ pshufb xgft6_hi, x0 ;Lookup mul table of high nibble
+ pshufb xgft6_lo, xtmpa ;Lookup mul table of low nibble
+ pxor xgft6_hi, xgft6_lo ;GF add high and low partials
+ pand xgft6_hi, xtmpl3
+ pxor xd3, xgft6_hi
+
+ XSTR [dest1+tmp], xd1 ;Store result into dest1
+ XSTR [dest2+tmp], xd2 ;Store result into dest2
+ XSTR [dest3+tmp], xd3 ;Store result into dest3
+
+.return_pass:
+ FUNC_RESTORE
+ mov return, 0
+ ret
+
+.return_fail:
+ FUNC_RESTORE
+ mov return, 1
+ ret
+
+endproc_frame
+
+section .data
+
+align 16
+
+mask0f: ddq 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
+constip16:
+ ddq 0xf0f1f2f3f4f5f6f7f8f9fafbfcfdfeff
+
+%macro slversion 4
+global %1_slver_%2%3%4
+global %1_slver
+%1_slver:
+%1_slver_%2%3%4:
+ dw 0x%4
+ db 0x%3, 0x%2
+%endmacro
+;;; func core, ver, snum
+slversion gf_6vect_mad_sse, 00, 00, 020f
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_vect_dot_prod_avx(len, vec, *g_tbls, **buffs, *dest);
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
%define arg2 rdx
%define arg3 rcx
%define arg4 r8
- %define arg5 r9
%define tmp r11
%define tmp2 r10
%define tmp3 r9
%define return rax
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
%define PS 8
%define func(x) x:
%define FUNC_SAVE
%define tmp2 r10
%define tmp3 rdi ; must be saved and loaded
%define return rax
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
%define PS 8
%define frame_size 2*8
%define arg(x) [rsp + frame_size + PS + PS*x]
%endmacro
%endif
+%ifidn __OUTPUT_FORMAT__, elf32
+
+;;;================== High Address;
+;;; arg4
+;;; arg3
+;;; arg2
+;;; arg1
+;;; arg0
+;;; return
+;;;<================= esp of caller
+;;; ebp
+;;;<================= ebp = esp
+;;; esi
+;;; edi
+;;; ebx
+;;;<================= esp of callee
+;;;
+;;;================== Low Address;
+
+ %define PS 4
+ %define LOG_PS 2
+ %define func(x) x:
+ %define arg(x) [ebp + PS*2 + PS*x]
+
+ %define trans ecx ;trans is for the variables in stack
+ %define arg0 trans
+ %define arg0_m arg(0)
+ %define arg1 trans
+ %define arg1_m arg(1)
+ %define arg2 arg2_m
+ %define arg2_m arg(2)
+ %define arg3 ebx
+ %define arg4 trans
+ %define arg4_m arg(4)
+ %define tmp edx
+ %define tmp2 edi
+ %define tmp3 esi
+ %define return eax
+ %macro SLDR 2 ;; stack load/restore
+ mov %1, %2
+ %endmacro
+ %define SSTR SLDR
+
+ %macro FUNC_SAVE 0
+ push ebp
+ mov ebp, esp
+ push esi
+ push edi
+ push ebx
+ mov arg3, arg(3)
+ %endmacro
+
+ %macro FUNC_RESTORE 0
+ pop ebx
+ pop edi
+ pop esi
+ mov esp, ebp
+ pop ebp
+ %endmacro
+
+%endif ; output formats
%define len arg0
%define vec arg1
%define ptr tmp3
%define pos return
+ %ifidn PS,4 ;32-bit code
+ %define vec_m arg1_m
+ %define len_m arg0_m
+ %define dest_m arg4_m
+ %endif
+
%ifndef EC_ALIGNED_ADDR
;;; Use Un-aligned load/store
%define XLDR vmovdqu
%endif
%endif
+%ifidn PS,8 ; 64-bit code
+ default rel
+ [bits 64]
+%endif
-default rel
-
-[bits 64]
section .text
%define xmask0f xmm5
global gf_vect_dot_prod_avx:function
func(gf_vect_dot_prod_avx)
FUNC_SAVE
+ SLDR len, len_m
sub len, 16
+ SSTR len_m, len
jl .return_fail
xor pos, pos
vmovdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
xor vec_i, vec_i
.next_vect:
+
mov ptr, [src+vec_i*PS]
vmovdqu xgft_lo, [tmp] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
vmovdqu xgft_hi, [tmp+16] ; " Cx{00}, Cx{10}, ..., Cx{f0}
XLDR x0, [ptr+pos] ;Get next source vector
+
add tmp, 32
add vec_i, 1
vpshufb xgft_lo, xgft_lo, xtmpa ;Lookup mul table of low nibble
vpxor xgft_hi, xgft_hi, xgft_lo ;GF add high and low partials
vpxor xp, xp, xgft_hi ;xp += partial
+
+ SLDR vec, vec_m
cmp vec_i, vec
jl .next_vect
+ SLDR dest, dest_m
XSTR [dest+pos], xp
+
add pos, 16 ;Loop on 16 bytes at a time
+ SLDR len, len_m
cmp pos, len
jle .loop16
align 16
-poly:
mask0f:
ddq 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
db 0x%3, 0x%2
%endmacro
;;; func core, ver, snum
-slversion gf_vect_dot_prod_avx, 02, 03, 0061
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
+slversion gf_vect_dot_prod_avx, 02, 04, 0061
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_vect_dot_prod_avx2(len, vec, *g_tbls, **buffs, *dest);
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
%define tmp2 r10
%define tmp3 r9
%define return rax
- %define PS 8
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
+ %define PS 8
%define func(x) x:
%define FUNC_SAVE
%define FUNC_RESTORE
%define tmp2 r10
%define tmp3 rdi ; must be saved and loaded
%define return rax
- %define PS 8
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
+ %define PS 8
%define frame_size 2*8
%define arg(x) [rsp + frame_size + PS + PS*x]
%endmacro
%endif
+%ifidn __OUTPUT_FORMAT__, elf32
+
+;;;================== High Address;
+;;; arg4
+;;; arg3
+;;; arg2
+;;; arg1
+;;; arg0
+;;; return
+;;;<================= esp of caller
+;;; ebp
+;;;<================= ebp = esp
+;;; esi
+;;; edi
+;;; ebx
+;;;<================= esp of callee
+;;;
+;;;================== Low Address;
+
+ %define PS 4
+ %define LOG_PS 2
+ %define func(x) x:
+ %define arg(x) [ebp + PS*2 + PS*x]
+
+ %define trans ecx ;trans is for the variables in stack
+ %define arg0 trans
+ %define arg0_m arg(0)
+ %define arg1 trans
+ %define arg1_m arg(1)
+ %define arg2 arg2_m
+ %define arg2_m arg(2)
+ %define arg3 ebx
+ %define arg4 trans
+ %define arg4_m arg(4)
+ %define tmp edx
+ %define tmp.w edx
+ %define tmp.b dl
+ %define tmp2 edi
+ %define tmp3 esi
+ %define return eax
+ %macro SLDR 2 ;stack load/restore
+ mov %1, %2
+ %endmacro
+ %define SSTR SLDR
+
+ %macro FUNC_SAVE 0
+ push ebp
+ mov ebp, esp
+ push esi
+ push edi
+ push ebx
+ mov arg3, arg(3)
+ %endmacro
+
+ %macro FUNC_RESTORE 0
+ pop ebx
+ pop edi
+ pop esi
+ mov esp, ebp
+ pop ebp
+ %endmacro
+
+%endif ; output formats
%define len arg0
%define vec arg1
%define ptr tmp3
%define pos return
+%ifidn PS,4 ;32-bit code
+ %define vec_m arg1_m
+ %define len_m arg0_m
+ %define dest_m arg4_m
+%endif
+
%ifndef EC_ALIGNED_ADDR
;;; Use Un-aligned load/store
%define XLDR vmovdqu
%endif
%endif
+%ifidn PS,8 ;64-bit code
+ default rel
+ [bits 64]
+%endif
-default rel
-
-[bits 64]
section .text
%define xmask0f ymm3
global gf_vect_dot_prod_avx2:function
func(gf_vect_dot_prod_avx2)
FUNC_SAVE
+ SLDR len, len_m
sub len, 32
+ SSTR len_m, len
jl .return_fail
xor pos, pos
mov tmp.b, 0x0f
xor vec_i, vec_i
.next_vect:
+
mov ptr, [src+vec_i*PS]
vmovdqu xgft_lo, [tmp] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
vperm2i128 xgft_lo, xgft_lo, xgft_lo, 0x00 ; swapped to lo | lo
XLDR x0, [ptr+pos] ;Get next source vector
+
add tmp, 32
add vec_i, 1
vpshufb xgft_lo, xgft_lo, xtmpa ;Lookup mul table of low nibble
vpxor xgft_hi, xgft_hi, xgft_lo ;GF add high and low partials
vpxor xp, xp, xgft_hi ;xp += partial
+
+ SLDR vec, vec_m
cmp vec_i, vec
jl .next_vect
+ SLDR dest, dest_m
XSTR [dest+pos], xp
+
add pos, 32 ;Loop on 32 bytes at a time
+ SLDR len, len_m
cmp pos, len
jle .loop32
db 0x%3, 0x%2
%endmacro
;;; func core, ver, snum
-slversion gf_vect_dot_prod_avx2, 04, 03, 0190
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
+slversion gf_vect_dot_prod_avx2, 04, 04, 0190
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_vect_dot_prod_sse(len, vec, *g_tbls, **buffs, *dest);
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
%define tmp2 r10
%define tmp3 r9
%define return rax
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
%define PS 8
%define func(x) x:
%define FUNC_SAVE
%define tmp2 r10
%define tmp3 rdi ; must be saved and loaded
%define return rax
+ %macro SLDR 2
+ %endmacro
+ %define SSTR SLDR
%define PS 8
%define frame_size 2*8
%define arg(x) [rsp + frame_size + PS + PS*x]
%endmacro
%endif
+%ifidn __OUTPUT_FORMAT__, elf32
+
+;;;================== High Address;
+;;; arg4
+;;; arg3
+;;; arg2
+;;; arg1
+;;; arg0
+;;; return
+;;;<================= esp of caller
+;;; ebp
+;;;<================= ebp = esp
+;;; esi
+;;; edi
+;;; ebx
+;;;<================= esp of callee
+;;;
+;;;================== Low Address;
+
+ %define PS 4
+ %define LOG_PS 2
+ %define func(x) x:
+ %define arg(x) [ebp + PS*2 + PS*x]
+
+ %define trans ecx ;trans is for the variables in stack
+ %define arg0 trans
+ %define arg0_m arg(0)
+ %define arg1 trans
+ %define arg1_m arg(1)
+ %define arg2 arg2_m
+ %define arg2_m arg(2)
+ %define arg3 ebx
+ %define arg4 trans
+ %define arg4_m arg(4)
+ %define tmp edx
+ %define tmp2 edi
+ %define tmp3 esi
+ %define return eax
+ %macro SLDR 2 ;; stack load/restore
+ mov %1, %2
+ %endmacro
+ %define SSTR SLDR
+
+ %macro FUNC_SAVE 0
+ push ebp
+ mov ebp, esp
+ push esi
+ push edi
+ push ebx
+ mov arg3, arg(3)
+ %endmacro
+
+ %macro FUNC_RESTORE 0
+ pop ebx
+ pop edi
+ pop esi
+ mov esp, ebp
+ pop ebp
+ %endmacro
+
+%endif ; output formats
%define len arg0
%define vec arg1
%define ptr tmp3
%define pos return
+ %ifidn PS,4 ;32-bit code
+ %define vec_m arg1_m
+ %define len_m arg0_m
+ %define dest_m arg4_m
+ %endif
%ifndef EC_ALIGNED_ADDR
;;; Use Un-aligned load/store
%endif
%endif
+%ifidn PS,8 ;64-bit code
+ default rel
+ [bits 64]
+%endif
-default rel
-
-[bits 64]
section .text
%define xmask0f xmm5
global gf_vect_dot_prod_sse:function
func(gf_vect_dot_prod_sse)
FUNC_SAVE
+ SLDR len, len_m
sub len, 16
+ SSTR len_m, len
jl .return_fail
xor pos, pos
movdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
xor vec_i, vec_i
.next_vect:
+
mov ptr, [src+vec_i*PS]
movdqu xgft_lo, [tmp] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
movdqu xgft_hi, [tmp+16] ; " Cx{00}, Cx{10}, ..., Cx{f0}
XLDR x0, [ptr+pos] ;Get next source vector
+
add tmp, 32
add vec_i, 1
+
movdqa xtmpa, x0 ;Keep unshifted copy of src
psraw x0, 4 ;Shift to put high nibble into bits 4-0
pand x0, xmask0f ;Mask high src nibble in bits 4-0
pand xtmpa, xmask0f ;Mask low src nibble in bits 4-0
+
pshufb xgft_hi, x0 ;Lookup mul table of high nibble
pshufb xgft_lo, xtmpa ;Lookup mul table of low nibble
pxor xgft_hi, xgft_lo ;GF add high and low partials
pxor xp, xgft_hi ;xp += partial
+
+ SLDR vec, vec_m
cmp vec_i, vec
jl .next_vect
+ SLDR dest, dest_m
XSTR [dest+pos], xp
add pos, 16 ;Loop on 16 bytes at a time
+ SLDR len, len_m
cmp pos, len
jle .loop16
section .data
align 16
+
mask0f: ddq 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
%macro slversion 4
db 0x%3, 0x%2
%endmacro
;;; func core, ver, snum
-slversion gf_vect_dot_prod_sse, 00, 03, 0060
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
+slversion gf_vect_dot_prod_sse, 00, 04, 0060
--- /dev/null
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+; * Redistributions of source code must retain the above copyright
+; notice, this list of conditions and the following disclaimer.
+; * Redistributions in binary form must reproduce the above copyright
+; notice, this list of conditions and the following disclaimer in
+; the documentation and/or other materials provided with the
+; distribution.
+; * Neither the name of Intel Corporation nor the names of its
+; contributors may be used to endorse or promote products derived
+; from this software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;
+;;; gf_vect_mad_avx(len, vec, vec_i, mul_array, src, dest);
+;;;
+
+%ifidn __OUTPUT_FORMAT__, win64
+ %define arg0 rcx
+ %define arg0.w ecx
+ %define arg1 rdx
+ %define arg2 r8
+ %define arg3 r9
+ %define arg4 r12
+ %define arg5 r15
+ %define tmp r11
+ %define return rax
+ %define return.w eax
+ %define PS 8
+ %define stack_size 16*3 + 3*8
+ %define arg(x) [rsp + stack_size + PS + PS*x]
+ %define func(x) proc_frame x
+
+%macro FUNC_SAVE 0
+ sub rsp, stack_size
+ vmovdqa [rsp+16*0],xmm6
+ vmovdqa [rsp+16*1],xmm7
+ vmovdqa [rsp+16*2],xmm8
+ save_reg r12, 3*16 + 0*8
+ save_reg r15, 3*16 + 1*8
+ end_prolog
+ mov arg4, arg(4)
+ mov arg5, arg(5)
+%endmacro
+
+%macro FUNC_RESTORE 0
+ vmovdqa xmm6, [rsp+16*0]
+ vmovdqa xmm7, [rsp+16*1]
+ vmovdqa xmm8, [rsp+16*2]
+ mov r12, [rsp + 3*16 + 0*8]
+ mov r15, [rsp + 3*16 + 1*8]
+ add rsp, stack_size
+%endmacro
+
+%elifidn __OUTPUT_FORMAT__, elf64
+ %define arg0 rdi
+ %define arg0.w edi
+ %define arg1 rsi
+ %define arg2 rdx
+ %define arg3 rcx
+ %define arg4 r8
+ %define arg5 r9
+ %define tmp r11
+ %define return rax
+ %define return.w eax
+
+ %define func(x) x:
+ %define FUNC_SAVE
+ %define FUNC_RESTORE
+%endif
+
+;;; gf_vect_mad_avx(len, vec, vec_i, mul_array, src, dest)
+%define len arg0
+%define len.w arg0.w
+%define vec arg1
+%define vec_i arg2
+%define mul_array arg3
+%define src arg4
+%define dest arg5
+%define pos return
+%define pos.w return.w
+
+%ifndef EC_ALIGNED_ADDR
+;;; Use Un-aligned load/store
+ %define XLDR vmovdqu
+ %define XSTR vmovdqu
+%else
+;;; Use Non-temporal load/stor
+ %ifdef NO_NT_LDST
+ %define XLDR vmovdqa
+ %define XSTR vmovdqa
+ %else
+ %define XLDR vmovntdqa
+ %define XSTR vmovntdq
+ %endif
+%endif
+
+
+default rel
+
+[bits 64]
+section .text
+
+%define xmask0f xmm8
+%define xgft_lo xmm7
+%define xgft_hi xmm6
+
+%define x0 xmm0
+%define xtmpa xmm1
+%define xtmph xmm2
+%define xtmpl xmm3
+%define xd xmm4
+%define xtmpd xmm5
+
+align 16
+global gf_vect_mad_avx:function
+func(gf_vect_mad_avx)
+ FUNC_SAVE
+ sub len, 16
+ jl .return_fail
+
+ xor pos, pos
+ vmovdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
+
+ sal vec_i, 5 ;Multiply by 32
+ vmovdqu xgft_lo, [vec_i+mul_array] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
+ vmovdqu xgft_hi, [vec_i+mul_array+16] ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
+
+ XLDR xtmpd, [dest+len] ;backup the last 16 bytes in dest
+
+.loop16:
+ XLDR xd, [dest+pos] ;Get next dest vector
+.loop16_overlap:
+ XLDR x0, [src+pos] ;Get next source vector
+
+ vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+
+ vpshufb xtmph, xgft_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl, xgft_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph, xtmph, xtmpl ;GF add high and low partials
+ vpxor xd, xd, xtmph ;xd += partial
+
+ XSTR [dest+pos], xd
+ add pos, 16 ;Loop on 16 bytes at a time
+ cmp pos, len
+ jle .loop16
+
+ lea tmp, [len + 16]
+ cmp pos, tmp
+ je .return_pass
+
+ ;; Tail len
+ mov pos, len ;Overlapped offset length-16
+ vmovdqa xd, xtmpd ;Restore xd
+ jmp .loop16_overlap ;Do one more overlap pass
+
+.return_pass:
+ mov return, 0
+ FUNC_RESTORE
+ ret
+
+.return_fail:
+ mov return, 1
+ FUNC_RESTORE
+ ret
+
+endproc_frame
+
+section .data
+
+align 16
+
+mask0f: ddq 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
+
+%macro slversion 4
+global %1_slver_%2%3%4
+global %1_slver
+%1_slver:
+%1_slver_%2%3%4:
+ dw 0x%4
+ db 0x%3, 0x%2
+%endmacro
+;;; func core, ver, snum
+slversion gf_vect_mad_avx, 02, 00, 0201
--- /dev/null
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+; * Redistributions of source code must retain the above copyright
+; notice, this list of conditions and the following disclaimer.
+; * Redistributions in binary form must reproduce the above copyright
+; notice, this list of conditions and the following disclaimer in
+; the documentation and/or other materials provided with the
+; distribution.
+; * Neither the name of Intel Corporation nor the names of its
+; contributors may be used to endorse or promote products derived
+; from this software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;
+;;; gf_vect_mad_avx2(len, vec, vec_i, mul_array, src, dest);
+;;;
+
+%ifidn __OUTPUT_FORMAT__, win64
+ %define arg0 rcx
+ %define arg0.w ecx
+ %define arg1 rdx
+ %define arg2 r8
+ %define arg3 r9
+ %define arg4 r12 ; must be saved and loaded
+ %define arg5 r15
+
+ %define tmp r11
+ %define tmp.w r11d
+ %define tmp.b r11b
+ %define return rax
+ %define return.w eax
+ %define PS 8
+ %define stack_size 16*3 + 3*8
+ %define arg(x) [rsp + stack_size + PS + PS*x]
+ %define func(x) proc_frame x
+
+ %macro FUNC_SAVE 0
+ sub rsp, stack_size
+ vmovdqa [rsp+16*0],xmm6
+ vmovdqa [rsp+16*1],xmm7
+ vmovdqa [rsp+16*2],xmm8
+ save_reg r12, 3*16 + 0*8
+ save_reg r15, 3*16 + 1*8
+ end_prolog
+ mov arg4, arg(4)
+ mov arg5, arg(5)
+ %endmacro
+
+ %macro FUNC_RESTORE 0
+ vmovdqa xmm6, [rsp+16*0]
+ vmovdqa xmm7, [rsp+16*1]
+ vmovdqa xmm8, [rsp+16*2]
+ mov r12, [rsp + 3*16 + 0*8]
+ mov r15, [rsp + 3*16 + 1*8]
+ add rsp, stack_size
+ %endmacro
+
+%elifidn __OUTPUT_FORMAT__, elf64
+ %define arg0 rdi
+ %define arg0.w edi
+ %define arg1 rsi
+ %define arg2 rdx
+ %define arg3 rcx
+ %define arg4 r8
+ %define arg5 r9
+
+ %define tmp r11
+ %define tmp.w r11d
+ %define tmp.b r11b
+ %define return rax
+ %define return.w eax
+
+ %define func(x) x:
+ %define FUNC_SAVE
+ %define FUNC_RESTORE
+%endif
+
+
+;;; gf_vect_mad_avx2(len, vec, vec_i, mul_array, src, dest)
+%define len arg0
+%define len.w arg0.w
+%define vec arg1
+%define vec_i arg2
+%define mul_array arg3
+%define src arg4
+%define dest arg5
+%define pos return
+%define pos.w return.w
+
+%ifndef EC_ALIGNED_ADDR
+;;; Use Un-aligned load/store
+ %define XLDR vmovdqu
+ %define XSTR vmovdqu
+%else
+;;; Use Non-temporal load/stor
+ %ifdef NO_NT_LDST
+ %define XLDR vmovdqa
+ %define XSTR vmovdqa
+ %else
+ %define XLDR vmovntdqa
+ %define XSTR vmovntdq
+ %endif
+%endif
+
+
+default rel
+
+[bits 64]
+section .text
+
+%define xmask0f ymm8
+%define xmask0fx xmm8
+%define xgft_lo ymm7
+%define xgft_hi ymm6
+
+%define x0 ymm0
+%define xtmpa ymm1
+%define xtmph ymm2
+%define xtmpl ymm3
+%define xd ymm4
+%define xtmpd ymm5
+
+align 16
+global gf_vect_mad_avx2:function
+func(gf_vect_mad_avx2)
+ FUNC_SAVE
+ sub len, 32
+ jl .return_fail
+ xor pos, pos
+ mov tmp.b, 0x0f
+ vpinsrb xmask0fx, xmask0fx, tmp.w, 0
+ vpbroadcastb xmask0f, xmask0fx ;Construct mask 0x0f0f0f...
+
+ sal vec_i, 5 ;Multiply by 32
+ vmovdqu xgft_lo, [vec_i+mul_array] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
+ ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
+ vperm2i128 xgft_hi, xgft_lo, xgft_lo, 0x11 ; swapped to hi | hi
+ vperm2i128 xgft_lo, xgft_lo, xgft_lo, 0x00 ; swapped to lo | lo
+
+ XLDR xtmpd, [dest+len] ;backup the last 32 bytes in dest
+
+.loop32:
+ XLDR xd, [dest+pos] ;Get next dest vector
+.loop32_overlap:
+ XLDR x0, [src+pos] ;Get next source vector
+
+ vpand xtmpa, x0, xmask0f ;Mask low src nibble in bits 4-0
+ vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
+ vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
+
+ vpshufb xtmph, xgft_hi, x0 ;Lookup mul table of high nibble
+ vpshufb xtmpl, xgft_lo, xtmpa ;Lookup mul table of low nibble
+ vpxor xtmph, xtmph, xtmpl ;GF add high and low partials
+ vpxor xd, xd, xtmph ;xd += partial
+
+ XSTR [dest+pos], xd
+ add pos, 32 ;Loop on 32 bytes at a time
+ cmp pos, len
+ jle .loop32
+
+ lea tmp, [len + 32]
+ cmp pos, tmp
+ je .return_pass
+
+ ;; Tail len
+ mov pos, len ;Overlapped offset length-32
+ vmovdqa xd, xtmpd ;Restore xd
+ jmp .loop32_overlap ;Do one more overlap pass
+
+.return_pass:
+ mov return, 0
+ FUNC_RESTORE
+ ret
+
+.return_fail:
+ mov return, 1
+ FUNC_RESTORE
+ ret
+
+endproc_frame
+
+section .data
+
+%macro slversion 4
+global %1_slver_%2%3%4
+global %1_slver
+%1_slver:
+%1_slver_%2%3%4:
+ dw 0x%4
+ db 0x%3, 0x%2
+%endmacro
+;;; func core, ver, snum
+slversion gf_vect_mad_avx2, 04, 00, 0202
--- /dev/null
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions
+; are met:
+; * Redistributions of source code must retain the above copyright
+; notice, this list of conditions and the following disclaimer.
+; * Redistributions in binary form must reproduce the above copyright
+; notice, this list of conditions and the following disclaimer in
+; the documentation and/or other materials provided with the
+; distribution.
+; * Neither the name of Intel Corporation nor the names of its
+; contributors may be used to endorse or promote products derived
+; from this software without specific prior written permission.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;
+;;; gf_vect_mad_sse(len, vec, vec_i, mul_array, src, dest);
+;;;
+
+%ifidn __OUTPUT_FORMAT__, win64
+ %define arg0 rcx
+ %define arg0.w ecx
+ %define arg1 rdx
+ %define arg2 r8
+ %define arg3 r9
+ %define arg4 r12
+ %define arg5 r15
+ %define tmp r11
+ %define return rax
+ %define return.w eax
+ %define PS 8
+ %define stack_size 16*3 + 3*8
+ %define arg(x) [rsp + stack_size + PS + PS*x]
+ %define func(x) proc_frame x
+
+%macro FUNC_SAVE 0
+ sub rsp, stack_size
+ movdqa [rsp+16*0],xmm6
+ movdqa [rsp+16*1],xmm7
+ movdqa [rsp+16*2],xmm8
+ save_reg r12, 3*16 + 0*8
+ save_reg r15, 3*16 + 1*8
+ end_prolog
+ mov arg4, arg(4)
+ mov arg5, arg(5)
+%endmacro
+
+%macro FUNC_RESTORE 0
+ movdqa xmm6, [rsp+16*0]
+ movdqa xmm7, [rsp+16*1]
+ movdqa xmm8, [rsp+16*2]
+ mov r12, [rsp + 3*16 + 0*8]
+ mov r15, [rsp + 3*16 + 1*8]
+ add rsp, stack_size
+%endmacro
+
+%elifidn __OUTPUT_FORMAT__, elf64
+ %define arg0 rdi
+ %define arg0.w edi
+ %define arg1 rsi
+ %define arg2 rdx
+ %define arg3 rcx
+ %define arg4 r8
+ %define arg5 r9
+ %define tmp r11
+ %define return rax
+ %define return.w eax
+
+ %define func(x) x:
+ %define FUNC_SAVE
+ %define FUNC_RESTORE
+%endif
+
+;;; gf_vect_mad_sse(len, vec, vec_i, mul_array, src, dest)
+%define len arg0
+%define len.w arg0.w
+%define vec arg1
+%define vec_i arg2
+%define mul_array arg3
+%define src arg4
+%define dest arg5
+%define pos return
+%define pos.w return.w
+
+%ifndef EC_ALIGNED_ADDR
+;;; Use Un-aligned load/store
+ %define XLDR movdqu
+ %define XSTR movdqu
+%else
+;;; Use Non-temporal load/stor
+ %ifdef NO_NT_LDST
+ %define XLDR movdqa
+ %define XSTR movdqa
+ %else
+ %define XLDR movntdqa
+ %define XSTR movntdq
+ %endif
+%endif
+
+default rel
+
+[bits 64]
+section .text
+
+%define xmask0f xmm8
+%define xgft_lo xmm7
+%define xgft_hi xmm6
+
+%define x0 xmm0
+%define xtmpa xmm1
+%define xtmph xmm2
+%define xtmpl xmm3
+%define xd xmm4
+%define xtmpd xmm5
+
+
+align 16
+global gf_vect_mad_sse:function
+func(gf_vect_mad_sse)
+ FUNC_SAVE
+ sub len, 16
+ jl .return_fail
+
+ xor pos, pos
+ movdqa xmask0f, [mask0f] ;Load mask of lower nibble in each byte
+ sal vec_i, 5 ;Multiply by 32
+ movdqu xgft_lo, [vec_i+mul_array] ;Load array Cx{00}, Cx{01}, Cx{02}, ...
+ movdqu xgft_hi, [vec_i+mul_array+16] ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
+
+ XLDR xtmpd, [dest+len] ;backup the last 16 bytes in dest
+
+.loop16:
+ XLDR xd, [dest+pos] ;Get next dest vector
+.loop16_overlap:
+ XLDR x0, [src+pos] ;Get next source vector
+ movdqa xtmph, xgft_hi ;Reload const array registers
+ movdqa xtmpl, xgft_lo
+ movdqa xtmpa, x0 ;Keep unshifted copy of src
+ psraw x0, 4 ;Shift to put high nibble into bits 4-0
+ pand x0, xmask0f ;Mask high src nibble in bits 4-0
+ pand xtmpa, xmask0f ;Mask low src nibble in bits 4-0
+ pshufb xtmph, x0 ;Lookup mul table of high nibble
+ pshufb xtmpl, xtmpa ;Lookup mul table of low nibble
+ pxor xtmph, xtmpl ;GF add high and low partials
+
+ pxor xd, xtmph
+ XSTR [dest+pos], xd ;Store result
+
+ add pos, 16 ;Loop on 16 bytes at a time
+ cmp pos, len
+ jle .loop16
+
+ lea tmp, [len + 16]
+ cmp pos, tmp
+ je .return_pass
+
+ ;; Tail len
+ mov pos, len ;Overlapped offset length-16
+ movdqa xd, xtmpd ;Restore xd
+ jmp .loop16_overlap ;Do one more overlap pass
+
+.return_pass:
+ mov return, 0
+ FUNC_RESTORE
+ ret
+
+.return_fail:
+ mov return, 1
+ FUNC_RESTORE
+ ret
+
+endproc_frame
+
+section .data
+
+align 16
+
+mask0f: ddq 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
+
+%macro slversion 4
+global %1_slver_%2%3%4
+global %1_slver
+%1_slver:
+%1_slver_%2%3%4:
+ dw 0x%4
+ db 0x%3, 0x%2
+%endmacro
+;;; func core, ver, snum
+slversion gf_vect_mad_sse, 00, 00, 0200
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_vect_mul_avx(len, mul_array, src, dest)
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
%endmacro
;;; func core, ver, snum
slversion gf_vect_mul_avx, 01, 02, 0036
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
;;;
;;; gf_vect_mul_sse(len, mul_array, src, dest)
;;;
-;;; Author: Gregory Tucker
-
%ifidn __OUTPUT_FORMAT__, elf64
%define arg0 rdi
%endmacro
;;; func core, ver, snum
slversion gf_vect_mul_sse, 00, 02, 0034
-; inform linker that this doesn't require executable stack
-section .note.GNU-stack noalloc noexec nowrite progbits
/**********************************************************************
- Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+ Copyright(c) 2011-2015 Intel Corporation All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
void ec_init_tables(int k, int rows, unsigned char* a, unsigned char* gftbls);
/**
- * @brief Generate or decode erasure codes on blocks of data.
+ * @brief Generate or decode erasure codes on blocks of data, runs appropriate version.
*
* Given a list of source data blocks, generate one or multiple blocks of
* encoded data as specified by a matrix of GF(2^8) coefficients. When given a
* suitable set of coefficients, this function will perform the fast generation
* or decoding of Reed-Solomon type erasure codes.
*
- * @requires SSE4.1
+ * This function determines what instruction sets are enabled and
+ * selects the appropriate version at runtime.
+ *
* @param len Length of each block of data (vector) of source or dest data.
* @param k The number of vector sources or rows in the generator matrix
* for coding.
* @param rows The number of output vectors to concurrently encode/decode.
* @param gftbls Pointer to array of input tables generated from coding
- * coefficients in ec_init_tables(). Must be of size 32*k*rows
+ * coefficients in ec_init_tables(). Must be of size 32*k*rows
* @param data Array of pointers to source input buffers.
* @param coding Array of pointers to coded output buffers.
* @returns none
*/
-void ec_encode_data_sse(int len, int k, int rows, unsigned char *gftbls, unsigned char **data, unsigned char **coding);
+void ec_encode_data(int len, int k, int rows, unsigned char *gftbls, unsigned char **data,
+ unsigned char **coding);
+/**
+ * @brief Generate or decode erasure codes on blocks of data.
+ *
+ * Arch specific version of ec_encode_data() with same parameters.
+ * @requires SSE4.1
+ */
+void ec_encode_data_sse(int len, int k, int rows, unsigned char *gftbls, unsigned char **data,
+ unsigned char **coding);
/**
- * @brief Generate or decode erasure codes on blocks of data, runs appropriate version.
+ * @brief Generate or decode erasure codes on blocks of data.
*
- * Given a list of source data blocks, generate one or multiple blocks of
- * encoded data as specified by a matrix of GF(2^8) coefficients. When given a
- * suitable set of coefficients, this function will perform the fast generation
- * or decoding of Reed-Solomon type erasure codes.
+ * Arch specific version of ec_encode_data() with same parameters.
+ * @requires AVX
+ */
+void ec_encode_data_avx(int len, int k, int rows, unsigned char *gftbls, unsigned char **data,
+ unsigned char **coding);
+
+/**
+ * @brief Generate or decode erasure codes on blocks of data.
*
- * This function determines what instruction sets are enabled and
- * selects the appropriate version at runtime.
+ * Arch specific version of ec_encode_data() with same parameters.
+ * @requires AVX2
+ */
+void ec_encode_data_avx2(int len, int k, int rows, unsigned char *gftbls, unsigned char **data,
+ unsigned char **coding);
+
+/**
+ * @brief Generate or decode erasure codes on blocks of data, runs baseline version.
+ *
+ * Baseline version of ec_encode_data() with same parameters.
+ */
+void ec_encode_data_base(int len, int srcs, int dests, unsigned char *v, unsigned char **src,
+ unsigned char **dest);
+
+/**
+ * @brief Generate update for encode or decode of erasure codes from single source, runs appropriate version.
+ *
+ * Given one source data block, update one or multiple blocks of encoded data as
+ * specified by a matrix of GF(2^8) coefficients. When given a suitable set of
+ * coefficients, this function will perform the fast generation or decoding of
+ * Reed-Solomon type erasure codes from one input source at a time.
+ *
+ * This function determines what instruction sets are enabled and selects the
+ * appropriate version at runtime.
*
* @param len Length of each block of data (vector) of source or dest data.
* @param k The number of vector sources or rows in the generator matrix
* for coding.
* @param rows The number of output vectors to concurrently encode/decode.
- * @param gftbls Pointer to array of input tables generated from coding
+ * @param vec_i The vector index corresponding to the single input source.
+ * @param g_tbls Pointer to array of input tables generated from coding
* coefficients in ec_init_tables(). Must be of size 32*k*rows
- * @param data Array of pointers to source input buffers.
+ * @param data Pointer to single input source used to update output parity.
* @param coding Array of pointers to coded output buffers.
* @returns none
*/
+void ec_encode_data_update(int len, int k, int rows, int vec_i, unsigned char *g_tbls,
+ unsigned char *data, unsigned char **coding);
-void ec_encode_data(int len, int k, int rows, unsigned char *gftbls, unsigned char **data, unsigned char **coding);
+/**
+ * @brief Generate update for encode or decode of erasure codes from single source.
+ *
+ * Arch specific version of ec_encode_data_update() with same parameters.
+ * @requires SSE4.1
+ */
+void ec_encode_data_update_sse(int len, int k, int rows, int vec_i, unsigned char *g_tbls,
+ unsigned char *data, unsigned char **coding);
/**
- * @brief Generate or decode erasure codes on blocks of data, runs baseline version.
+ * @brief Generate update for encode or decode of erasure codes from single source.
*
- * Given a list of source data blocks, generate one or multiple blocks of
- * encoded data as specified by a matrix of GF(2^8) coefficients. When given a
- * suitable set of coefficients, this function will perform the fast generation
- * or decoding of Reed-Solomon type erasure codes.
+ * Arch specific version of ec_encode_data_update() with same parameters.
+ * @requires AVX
+ */
+
+void ec_encode_data_update_avx(int len, int k, int rows, int vec_i, unsigned char *g_tbls,
+ unsigned char *data, unsigned char **coding);
+
+/**
+ * @brief Generate update for encode or decode of erasure codes from single source.
*
- * @param len Length of each block of data (vector) of source or dest data.
- * @param srcs The number of vector sources or rows in the generator matrix
- * for coding.
- * @param dests The number of output vectors to concurrently encode/decode.
- * @param v Pointer to array of input tables generated from coding
- * coefficients in ec_init_tables(). Must be of size 32*k*rows
- * @param src Array of pointers to source input buffers.
- * @param dest Array of pointers to coded output buffers.
- * @returns none
+ * Arch specific version of ec_encode_data_update() with same parameters.
+ * @requires AVX2
+ */
+
+void ec_encode_data_update_avx2(int len, int k, int rows, int vec_i, unsigned char *g_tbls,
+ unsigned char *data, unsigned char **coding);
+
+/**
+ * @brief Generate update for encode or decode of erasure codes from single source.
+ *
+ * Baseline version of ec_encode_data_update().
*/
-void ec_encode_data_base(int len, int srcs, int dests, unsigned char *v, unsigned char **src, unsigned char **dest);
+void ec_encode_data_update_base(int len, int k, int rows, int vec_i, unsigned char *v,
+ unsigned char *data, unsigned char **dest);
/**
* set of coefficients to produce each byte of the output. Can be used for
* erasure coding encode and decode. Function requires pre-calculation of a
* 32*vlen byte constant array based on the input coefficients.
- *
* @requires SSE4.1
+ *
* @param len Length of each vector in bytes. Must be >= 16.
* @param vlen Number of vector sources.
* @param gftbls Pointer to 32*vlen byte array of pre-calculated constants based
* set of coefficients to produce each byte of the output. Can be used for
* erasure coding encode and decode. Function requires pre-calculation of a
* 32*vlen byte constant array based on the input coefficients.
- *
* @requires AVX
+ *
* @param len Length of each vector in bytes. Must be >= 16.
* @param vlen Number of vector sources.
* @param gftbls Pointer to 32*vlen byte array of pre-calculated constants based
* set of coefficients to produce each byte of the output. Can be used for
* erasure coding encode and decode. Function requires pre-calculation of a
* 32*vlen byte constant array based on the input coefficients.
- *
* @requires AVX2
+ *
* @param len Length of each vector in bytes. Must be >= 32.
* @param vlen Number of vector sources.
* @param gftbls Pointer to 32*vlen byte array of pre-calculated constants based
* sets of coefficients to produce each byte of the outputs. Can be used for
* erasure coding encode and decode. Function requires pre-calculation of a
* 2*32*vlen byte constant array based on the two sets of input coefficients.
- *
* @requires SSE4.1
+ *
* @param len Length of each vector in bytes. Must be >= 16.
* @param vlen Number of vector sources.
* @param gftbls Pointer to 2*32*vlen byte array of pre-calculated constants
* sets of coefficients to produce each byte of the outputs. Can be used for
* erasure coding encode and decode. Function requires pre-calculation of a
* 2*32*vlen byte constant array based on the two sets of input coefficients.
- *
* @requires AVX
+ *
* @param len Length of each vector in bytes. Must be >= 16.
* @param vlen Number of vector sources.
* @param gftbls Pointer to 2*32*vlen byte array of pre-calculated constants
* sets of coefficients to produce each byte of the outputs. Can be used for
* erasure coding encode and decode. Function requires pre-calculation of a
* 2*32*vlen byte constant array based on the two sets of input coefficients.
- *
* @requires AVX2
+ *
* @param len Length of each vector in bytes. Must be >= 32.
* @param vlen Number of vector sources.
* @param gftbls Pointer to 2*32*vlen byte array of pre-calculated constants
* sets of coefficients to produce each byte of the outputs. Can be used for
* erasure coding encode and decode. Function requires pre-calculation of a
* 3*32*vlen byte constant array based on the three sets of input coefficients.
- *
* @requires SSE4.1
+ *
* @param len Length of each vector in bytes. Must be >= 16.
* @param vlen Number of vector sources.
* @param gftbls Pointer to 3*32*vlen byte array of pre-calculated constants
* sets of coefficients to produce each byte of the outputs. Can be used for
* erasure coding encode and decode. Function requires pre-calculation of a
* 3*32*vlen byte constant array based on the three sets of input coefficients.
- *
* @requires AVX
+ *
* @param len Length of each vector in bytes. Must be >= 16.
* @param vlen Number of vector sources.
* @param gftbls Pointer to 3*32*vlen byte array of pre-calculated constants
* sets of coefficients to produce each byte of the outputs. Can be used for
* erasure coding encode and decode. Function requires pre-calculation of a
* 3*32*vlen byte constant array based on the three sets of input coefficients.
- *
* @requires AVX2
+ *
* @param len Length of each vector in bytes. Must be >= 32.
* @param vlen Number of vector sources.
* @param gftbls Pointer to 3*32*vlen byte array of pre-calculated constants
* sets of coefficients to produce each byte of the outputs. Can be used for
* erasure coding encode and decode. Function requires pre-calculation of a
* 4*32*vlen byte constant array based on the four sets of input coefficients.
- *
* @requires SSE4.1
+ *
* @param len Length of each vector in bytes. Must be >= 16.
* @param vlen Number of vector sources.
* @param gftbls Pointer to 4*32*vlen byte array of pre-calculated constants
* sets of coefficients to produce each byte of the outputs. Can be used for
* erasure coding encode and decode. Function requires pre-calculation of a
* 4*32*vlen byte constant array based on the four sets of input coefficients.
- *
* @requires AVX
+ *
* @param len Length of each vector in bytes. Must be >= 16.
* @param vlen Number of vector sources.
* @param gftbls Pointer to 4*32*vlen byte array of pre-calculated constants
* sets of coefficients to produce each byte of the outputs. Can be used for
* erasure coding encode and decode. Function requires pre-calculation of a
* 4*32*vlen byte constant array based on the four sets of input coefficients.
- *
* @requires AVX2
+ *
* @param len Length of each vector in bytes. Must be >= 32.
* @param vlen Number of vector sources.
* @param gftbls Pointer to 4*32*vlen byte array of pre-calculated constants
* sets of coefficients to produce each byte of the outputs. Can be used for
* erasure coding encode and decode. Function requires pre-calculation of a
* 5*32*vlen byte constant array based on the five sets of input coefficients.
- *
* @requires SSE4.1
+ *
* @param len Length of each vector in bytes. Must >= 16.
* @param vlen Number of vector sources.
* @param gftbls Pointer to 5*32*vlen byte array of pre-calculated constants
* sets of coefficients to produce each byte of the outputs. Can be used for
* erasure coding encode and decode. Function requires pre-calculation of a
* 5*32*vlen byte constant array based on the five sets of input coefficients.
- *
* @requires AVX
+ *
* @param len Length of each vector in bytes. Must >= 16.
* @param vlen Number of vector sources.
* @param gftbls Pointer to 5*32*vlen byte array of pre-calculated constants
* sets of coefficients to produce each byte of the outputs. Can be used for
* erasure coding encode and decode. Function requires pre-calculation of a
* 5*32*vlen byte constant array based on the five sets of input coefficients.
- *
* @requires AVX2
+ *
* @param len Length of each vector in bytes. Must >= 32.
* @param vlen Number of vector sources.
* @param gftbls Pointer to 5*32*vlen byte array of pre-calculated constants
* sets of coefficients to produce each byte of the outputs. Can be used for
* erasure coding encode and decode. Function requires pre-calculation of a
* 6*32*vlen byte constant array based on the six sets of input coefficients.
- *
* @requires SSE4.1
+ *
* @param len Length of each vector in bytes. Must be >= 16.
* @param vlen Number of vector sources.
* @param gftbls Pointer to 6*32*vlen byte array of pre-calculated constants
* sets of coefficients to produce each byte of the outputs. Can be used for
* erasure coding encode and decode. Function requires pre-calculation of a
* 6*32*vlen byte constant array based on the six sets of input coefficients.
- *
* @requires AVX
+ *
* @param len Length of each vector in bytes. Must be >= 16.
* @param vlen Number of vector sources.
* @param gftbls Pointer to 6*32*vlen byte array of pre-calculated constants
* sets of coefficients to produce each byte of the outputs. Can be used for
* erasure coding encode and decode. Function requires pre-calculation of a
* 6*32*vlen byte constant array based on the six sets of input coefficients.
- *
* @requires AVX2
+ *
* @param len Length of each vector in bytes. Must be >= 32.
* @param vlen Number of vector sources.
* @param gftbls Pointer to 6*32*vlen byte array of pre-calculated constants
void gf_vect_dot_prod(int len, int vlen, unsigned char *gftbls,
unsigned char **src, unsigned char *dest);
+
+/**
+ * @brief GF(2^8) vector multiply accumulate, runs appropriate version.
+ *
+ * Does a GF(2^8) multiply across each byte of input source with expanded
+ * constant and add to destination array. Can be used for erasure coding encode
+ * and decode update when only one source is available at a time. Function
+ * requires pre-calculation of a 32*vec byte constant array based on the input
+ * coefficients.
+ *
+ * This function determines what instruction sets are enabled and selects the
+ * appropriate version at runtime.
+ *
+ * @param len Length of each vector in bytes. Must be >= 32.
+ * @param vec The number of vector sources or rows in the generator matrix
+ * for coding.
+ * @param vec_i The vector index corresponding to the single input source.
+ * @param gftbls Pointer to array of input tables generated from coding
+ * coefficients in ec_init_tables(). Must be of size 32*vec.
+ * @param src Array of pointers to source inputs.
+ * @param dest Pointer to destination data array.
+ * @returns none
+ */
+
+void gf_vect_mad(int len, int vec, int vec_i, unsigned char *gftbls, unsigned char *src,
+ unsigned char *dest);
+
+/**
+ * @brief GF(2^8) vector multiply accumulate, arch specific version.
+ *
+ * Arch specific version of gf_vect_mad() with same parameters.
+ * @requires SSE4.1
+ */
+
+void gf_vect_mad_sse(int len, int vec, int vec_i, unsigned char *gftbls, unsigned char *src,
+ unsigned char *dest);
+/**
+ * @brief GF(2^8) vector multiply accumulate, arch specific version.
+ *
+ * Arch specific version of gf_vect_mad() with same parameters.
+ * @requires AVX
+ */
+
+void gf_vect_mad_avx(int len, int vec, int vec_i, unsigned char *gftbls, unsigned char *src,
+ unsigned char *dest);
+
+/**
+ * @brief GF(2^8) vector multiply accumulate, arch specific version.
+ *
+ * Arch specific version of gf_vect_mad() with same parameters.
+ * @requires AVX2
+ */
+
+void gf_vect_mad_avx2(int len, int vec, int vec_i, unsigned char *gftbls, unsigned char *src,
+ unsigned char *dest);
+
+/**
+ * @brief GF(2^8) vector multiply accumulate, baseline version.
+ *
+ * Baseline version of gf_vect_mad() with same parameters.
+ */
+
+void gf_vect_mad_base(int len, int vec, int vec_i, unsigned char *v, unsigned char *src,
+ unsigned char *dest);
+
+/**
+ * @brief GF(2^8) vector multiply with 2 accumulate. SSE version.
+ *
+ * Does a GF(2^8) multiply across each byte of input source with expanded
+ * constants and add to destination arrays. Can be used for erasure coding
+ * encode and decode update when only one source is available at a
+ * time. Function requires pre-calculation of a 32*vec byte constant array based
+ * on the input coefficients.
+ * @requires SSE4.1
+ *
+ * @param len Length of each vector in bytes. Must be >= 32.
+ * @param vec The number of vector sources or rows in the generator matrix
+ * for coding.
+ * @param vec_i The vector index corresponding to the single input source.
+ * @param gftbls Pointer to array of input tables generated from coding
+ * coefficients in ec_init_tables(). Must be of size 32*vec.
+ * @param src Pointer to source input array.
+ * @param dest Array of pointers to destination input/outputs.
+ * @returns none
+ */
+
+void gf_2vect_mad_sse(int len, int vec, int vec_i, unsigned char *gftbls, unsigned char *src,
+ unsigned char **dest);
+
+/**
+ * @brief GF(2^8) vector multiply with 2 accumulate. AVX version of gf_2vect_mad_sse().
+ * @requires AVX
+ */
+void gf_2vect_mad_avx(int len, int vec, int vec_i, unsigned char *gftbls, unsigned char *src,
+ unsigned char **dest);
+/**
+ * @brief GF(2^8) vector multiply with 2 accumulate. AVX2 version of gf_2vect_mad_sse().
+ * @requires AVX2
+ */
+void gf_2vect_mad_avx2(int len, int vec, int vec_i, unsigned char *gftbls, unsigned char *src,
+ unsigned char **dest);
+
+/**
+ * @brief GF(2^8) vector multiply with 3 accumulate. SSE version.
+ *
+ * Does a GF(2^8) multiply across each byte of input source with expanded
+ * constants and add to destination arrays. Can be used for erasure coding
+ * encode and decode update when only one source is available at a
+ * time. Function requires pre-calculation of a 32*vec byte constant array based
+ * on the input coefficients.
+ * @requires SSE4.1
+ *
+ * @param len Length of each vector in bytes. Must be >= 32.
+ * @param vec The number of vector sources or rows in the generator matrix
+ * for coding.
+ * @param vec_i The vector index corresponding to the single input source.
+ * @param gftbls Pointer to array of input tables generated from coding
+ * coefficients in ec_init_tables(). Must be of size 32*vec.
+ * @param src Pointer to source input array.
+ * @param dest Array of pointers to destination input/outputs.
+ * @returns none
+ */
+
+void gf_3vect_mad_sse(int len, int vec, int vec_i, unsigned char *gftbls, unsigned char *src,
+ unsigned char **dest);
+
+/**
+ * @brief GF(2^8) vector multiply with 3 accumulate. AVX version of gf_3vect_mad_sse().
+ * @requires AVX
+ */
+void gf_3vect_mad_avx(int len, int vec, int vec_i, unsigned char *gftbls, unsigned char *src,
+ unsigned char **dest);
+
+/**
+ * @brief GF(2^8) vector multiply with 3 accumulate. AVX2 version of gf_3vect_mad_sse().
+ * @requires AVX2
+ */
+void gf_3vect_mad_avx2(int len, int vec, int vec_i, unsigned char *gftbls, unsigned char *src,
+ unsigned char **dest);
+
+/**
+ * @brief GF(2^8) vector multiply with 4 accumulate. SSE version.
+ *
+ * Does a GF(2^8) multiply across each byte of input source with expanded
+ * constants and add to destination arrays. Can be used for erasure coding
+ * encode and decode update when only one source is available at a
+ * time. Function requires pre-calculation of a 32*vec byte constant array based
+ * on the input coefficients.
+ * @requires SSE4.1
+ *
+ * @param len Length of each vector in bytes. Must be >= 32.
+ * @param vec The number of vector sources or rows in the generator matrix
+ * for coding.
+ * @param vec_i The vector index corresponding to the single input source.
+ * @param gftbls Pointer to array of input tables generated from coding
+ * coefficients in ec_init_tables(). Must be of size 32*vec.
+ * @param src Pointer to source input array.
+ * @param dest Array of pointers to destination input/outputs.
+ * @returns none
+ */
+
+void gf_4vect_mad_sse(int len, int vec, int vec_i, unsigned char *gftbls, unsigned char *src,
+ unsigned char **dest);
+
+/**
+ * @brief GF(2^8) vector multiply with 4 accumulate. AVX version of gf_4vect_mad_sse().
+ * @requires AVX
+ */
+void gf_4vect_mad_avx(int len, int vec, int vec_i, unsigned char *gftbls, unsigned char *src,
+ unsigned char **dest);
+/**
+ * @brief GF(2^8) vector multiply with 4 accumulate. AVX2 version of gf_4vect_mad_sse().
+ * @requires AVX2
+ */
+void gf_4vect_mad_avx2(int len, int vec, int vec_i, unsigned char *gftbls, unsigned char *src,
+ unsigned char **dest);
+
+/**
+ * @brief GF(2^8) vector multiply with 5 accumulate. SSE version.
+ * @requires SSE4.1
+ */
+void gf_5vect_mad_sse(int len, int vec, int vec_i, unsigned char *gftbls, unsigned char *src,
+ unsigned char **dest);
+
+/**
+ * @brief GF(2^8) vector multiply with 5 accumulate. AVX version.
+ * @requires AVX
+ */
+void gf_5vect_mad_avx(int len, int vec, int vec_i, unsigned char *gftbls, unsigned char *src,
+ unsigned char **dest);
+/**
+ * @brief GF(2^8) vector multiply with 5 accumulate. AVX2 version.
+ * @requires AVX2
+ */
+void gf_5vect_mad_avx2(int len, int vec, int vec_i, unsigned char *gftbls, unsigned char *src,
+ unsigned char **dest);
+
+/**
+ * @brief GF(2^8) vector multiply with 6 accumulate. SSE version.
+ * @requires SSE4.1
+ */
+void gf_6vect_mad_sse(int len, int vec, int vec_i, unsigned char *gftbls, unsigned char *src,
+ unsigned char **dest);
+/**
+ * @brief GF(2^8) vector multiply with 6 accumulate. AVX version.
+ * @requires AVX
+ */
+void gf_6vect_mad_avx(int len, int vec, int vec_i, unsigned char *gftbls, unsigned char *src,
+ unsigned char **dest);
+
+/**
+ * @brief GF(2^8) vector multiply with 6 accumulate. AVX2 version.
+ * @requires AVX2
+ */
+void gf_6vect_mad_avx2(int len, int vec, int vec_i, unsigned char *gftbls, unsigned char *src,
+ unsigned char **dest);
+
+
/**********************************************************************
* The remaining are lib support functions used in GF(2^8) operations.
*/
int gf_invert_matrix(unsigned char *in, unsigned char *out, const int n);
+
/*************************************************************/
#ifdef __cplusplus
/**********************************************************************
- Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+ Copyright(c) 2011-2015 Intel Corporation All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
* 32-element constant array based on constant C. gftbl(C) = {C{00},
* C{01}, C{02}, ... , C{0f} }, {C{00}, C{10}, C{20}, ... , C{f0} }. Len
* and src must be aligned to 32B.
-
* @requires SSE4.1
+ *
* @param len Length of vector in bytes. Must be aligned to 32B.
* @param gftbl Pointer to 32-byte array of pre-calculated constants based on C.
* @param src Pointer to src data array. Must be aligned to 32B.
* 32-element constant array based on constant C. gftbl(C) = {C{00},
* C{01}, C{02}, ... , C{0f} }, {C{00}, C{10}, C{20}, ... , C{f0} }. Len
* and src must be aligned to 32B.
-
* @requires AVX
+ *
* @param len Length of vector in bytes. Must be aligned to 32B.
* @param gftbl Pointer to 32-byte array of pre-calculated constants based on C.
* @param src Pointer to src data array. Must be aligned to 32B.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-; Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions
/**********************************************************************
- Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+ Copyright(c) 2011-2015 Intel Corporation All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
projects / ceph.git / commitdiff