switch to scalar loads

Author: mkannwischer

dev/aarch64_clean/src/ntt.S

@@ -26,6 +26,24 @@
 #include "../../../common.h"
 #if defined(MLD_ARITH_BACKEND_AARCH64)
 
+.macro vins vec_out, gpr_in, lane
+        ins \vec_out\().d[\lane], \gpr_in
+.endm
+
+.macro ldr_vo vec, base, offset
+        ldr xtmp0, [\base, #\offset]
+        ldr xtmp1, [\base, #(\offset+8)]
+        vins \vec, xtmp0, 0
+        vins \vec, xtmp1, 1
+.endm
+
+.macro ldr_vi vec, base, inc
+        ldr xtmp0, [\base], #\inc
+        ldr xtmp1, [\base, #(-\inc+8)]
+        vins \vec, xtmp0, 0
+        vins \vec, xtmp1, 1
+.endm
+
 .macro mulmodq dst, src, const, idx0, idx1
         sqrdmulh t2.4s,      \src\().4s, \const\().s[\idx1\()]
         mul      \dst\().4s, \src\().4s, \const\().s[\idx0\()]
@@ -51,35 +69,35 @@
 .endm
 
 .macro load_roots_123
-        ldr q_root0, [r012345_ptr], #64
-        ldr q_root1, [r012345_ptr, #(-64 + 16)]
-        ldr q_root2, [r012345_ptr, #(-64 + 32)]
-        ldr q_root3, [r012345_ptr, #(-64 + 48)]
+        ldr_vi root0, r012345_ptr, 64
+        ldr_vo root1, r012345_ptr, (-64 + 16)
+        ldr_vo root2, r012345_ptr, (-64 + 32)
+        ldr_vo root3, r012345_ptr, (-64 + 48)
 .endm
 
 .macro load_roots_456
-        ldr q_root0, [r012345_ptr], #64
-        ldr q_root1, [r012345_ptr, #(-64 + 16)]
-        ldr q_root2, [r012345_ptr, #(-64 + 32)]
-        ldr q_root3, [r012345_ptr, #(-64 + 48)]
+        ldr_vi root0, r012345_ptr, 64
+        ldr_vo root1, r012345_ptr, (-64 + 16)
+        ldr_vo root2, r012345_ptr, (-64 + 32)
+        ldr_vo root3, r012345_ptr, (-64 + 48)
 .endm
 
 .macro load_roots_78_part1
-        ldr q_root0,    [r67_ptr], #(12*16)
-        ldr q_root0_tw, [r67_ptr, #(-12*16 + 1*16)]
-        ldr q_root1,    [r67_ptr, #(-12*16 + 2*16)]
-        ldr q_root1_tw, [r67_ptr, #(-12*16 + 3*16)]
-        ldr q_root2,    [r67_ptr, #(-12*16 + 4*16)]
-        ldr q_root2_tw, [r67_ptr, #(-12*16 + 5*16)]
+        ldr_vi root0,    r67_ptr, (12*16)
+        ldr_vo root0_tw, r67_ptr, (-12*16 + 1*16)
+        ldr_vo root1,    r67_ptr, (-12*16 + 2*16)
+        ldr_vo root1_tw, r67_ptr, (-12*16 + 3*16)
+        ldr_vo root2,    r67_ptr, (-12*16 + 4*16)
+        ldr_vo root2_tw, r67_ptr, (-12*16 + 5*16)
 .endm
 
 .macro load_roots_78_part2
-        ldr q_root0,    [r67_ptr, #(-12*16 +  6*16)]
-        ldr q_root0_tw, [r67_ptr, #(-12*16 +  7*16)]
-        ldr q_root1,    [r67_ptr, #(-12*16 +  8*16)]
-        ldr q_root1_tw, [r67_ptr, #(-12*16 +  9*16)]
-        ldr q_root2,    [r67_ptr, #(-12*16 + 10*16)]
-        ldr q_root2_tw, [r67_ptr, #(-12*16 + 11*16)]
+        ldr_vo root0,    r67_ptr, (-12*16 +  6*16)
+        ldr_vo root0_tw, r67_ptr, (-12*16 +  7*16)
+        ldr_vo root1,    r67_ptr, (-12*16 +  8*16)
+        ldr_vo root1_tw, r67_ptr, (-12*16 +  9*16)
+        ldr_vo root2,    r67_ptr, (-12*16 + 10*16)
+        ldr_vo root2_tw, r67_ptr, (-12*16 + 11*16)
 .endm
 
 .macro transpose4 data0, data1, data2, data3
@@ -129,6 +147,9 @@
         xtmp    .req x6
         wtmp    .req w6
 
+        xtmp0   .req x6
+        xtmp1   .req x7
+
         data0  .req v9
         data1  .req v10
         data2  .req v11
@@ -193,14 +214,14 @@ MLD_ASM_FN_SYMBOL(ntt_asm)
 
         .p2align 2
 layer123_start:
-        ldr q_data0, [in, #(0*(1024/8))]
-        ldr q_data1, [in, #(1*(1024/8))]
-        ldr q_data2, [in, #(2*(1024/8))]
-        ldr q_data3, [in, #(3*(1024/8))]
-        ldr q_data4, [in, #(4*(1024/8))]
-        ldr q_data5, [in, #(5*(1024/8))]
-        ldr q_data6, [in, #(6*(1024/8))]
-        ldr q_data7, [in, #(7*(1024/8))]
+        ldr_vo data0, in, (0*(1024/8))
+        ldr_vo data1, in, (1*(1024/8))
+        ldr_vo data2, in, (2*(1024/8))
+        ldr_vo data3, in, (3*(1024/8))
+        ldr_vo data4, in, (4*(1024/8))
+        ldr_vo data5, in, (5*(1024/8))
+        ldr_vo data6, in, (6*(1024/8))
+        ldr_vo data7, in, (7*(1024/8))
 
         ct_butterfly data0, data4, root0, 0, 1
         ct_butterfly data1, data5, root0, 0, 1
@@ -245,14 +266,14 @@ layer123_start:
 
         .p2align 2
 layer45678_start:
-        ldr q_data0, [in, #(64 + 16*0)]
-        ldr q_data1, [in, #(64 + 16*1)]
-        ldr q_data2, [in, #(64 + 16*2)]
-        ldr q_data3, [in, #(64 + 16*3)]
-        ldr q_data4, [inpp, #(64 + 16*0)]
-        ldr q_data5, [inpp, #(64 + 16*1)]
-        ldr q_data6, [inpp, #(64 + 16*2)]
-        ldr q_data7, [inpp, #(64 + 16*3)]
+        ldr_vo data0, in, (64 + 16*0)
+        ldr_vo data1, in, (64 + 16*1)
+        ldr_vo data2, in, (64 + 16*2)
+        ldr_vo data3, in, (64 + 16*3)
+        ldr_vo data4, inpp, (64 + 16*0)
+        ldr_vo data5, inpp, (64 + 16*1)
+        ldr_vo data6, inpp, (64 + 16*2)
+        ldr_vo data7, inpp, (64 + 16*3)
 
         add in, in, #64
         add inpp, inpp, #64

mldsa/native/aarch64/src/Makefile

@@ -20,7 +20,7 @@ SLOTHY_FLAGS=-c sw_pipelining.enabled=true \
              -c sw_pipelining.allow_post \
              -c variable_size \
              -c constraints.stalls_first_attempt=64 \
-             -c timeout=120 \
+             -c timeout=300 \
              $(SLOTHY_EXTRA_FLAGS)
 
 # For kernels which stash callee-saved v8-v15 but don't stash callee-saved GPRs x19-x30.