diff --git a/go.mod b/go.mod index ac8b53d..bbfaaa9 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.27.2 require ( github.com/go-fft/fft v0.28.11 - github.com/go-simd/floats v0.9.0 + github.com/go-simd/floats v0.10.0 ) require golang.org/x/sys v0.49.0 // indirect diff --git a/go.sum b/go.sum index 6671d94..512f12b 100644 --- a/go.sum +++ b/go.sum @@ -2,5 +2,7 @@ github.com/go-fft/fft v0.28.11 h1:ZfT78fSmPjZU2E9PgkSI/0f0E3sesYpl13GHmIFzRGc= github.com/go-fft/fft v0.28.11/go.mod h1:GN5V4TWCjV05ECqH1ONfjFM6Z1m2shqCdssxKGqQ41s= github.com/go-simd/floats v0.9.0 h1:P/sMlwYWwaqCah/cMcBRfffK1JcUxXZVj76f4AwQRN0= github.com/go-simd/floats v0.9.0/go.mod h1:Aq5a6oY5pUqm9Hpi+iV2nrNmaaUb0rwHvUHIerXWSvA= +github.com/go-simd/floats v0.10.0 h1:NFMzLl2wmc1+jNE9J0W9li0O+OmtW9TE1m9XS27068k= +github.com/go-simd/floats v0.10.0/go.mod h1:Aq5a6oY5pUqm9Hpi+iV2nrNmaaUb0rwHvUHIerXWSvA= golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= diff --git a/internal/kernels/asmgen/loong64/gen.go b/internal/kernels/asmgen/loong64/gen.go index 6d39ca4..e93d594 100644 --- a/internal/kernels/asmgen/loong64/gen.go +++ b/internal/kernels/asmgen/loong64/gen.go @@ -29,15 +29,14 @@ const ( func main() { f := emit.NewFile("loong64") - f.Add(sumKernel()) f.Add(dotKernel()) - f.Add(sqrtKernel()) + // The float64 sum, sqrt and Add/Sub/Mul/Div kernels live in go-simd/floats + // (lanes_loong64_gen.go there) since floats v0.10.0. for _, k := range []struct{ name, vec, sca string }{ {"addLASX", "XVADDD", "ADDD"}, {"subLASX", "XVSUBD", "SUBD"}, {"mulLASX", "XVMULD", "MULD"}, {"divLASX", "XVDIVD", "DIVD"}, } { - f.Add(binKernel(k.name, k.vec, k.sca)) - // "XVADDD" -> "VADDF", "ADDD" -> "ADDF": the LSX single-precision op. + //"XVADDD" -> "VADDF", "ADDD" -> "ADDF": the LSX single-precision op. f.Add(bin32Kernel(k.name[:3]+"32LSX", k.vec[1:len(k.vec)-1]+"F", k.sca[:len(k.sca)-1]+"F")) } f.Add(gemmKernel()) @@ -74,32 +73,6 @@ func zero(b *loong64.Builder, n int) { } } -// sumKernel builds sumLASX(a *float64, n int) float64: four LASX accumulators -// (16 doubles per iteration), folded, plus a scalar tail. A regrouping of the -// sum, held to a tolerance like the other targets' SIMD sums. -func sumKernel() *emit.Function { - sig := loong64.Layout([]string{"a", "n"}, []loong64.Type{loong64.Ptr, loong64.Int64}, - []string{"ret"}, []loong64.Type{loong64.Float64}) - b := loong64.NewFunc("sumLASX", sig, 0) - b.LoadArg("a", "R4").LoadArg("n", "R5") - zero(b, 4) - b.Label("sloop").Raw("MOVV $16, R12").Raw("BLT R5, R12, sfold") - for k := 0; k < 4; k++ { - b.Raw("XVMOVQ %d(R4), X%d", 32*k, 4+k) - } - for k := 0; k < 4; k++ { - b.Raw("XVADDD X%d, X%d, X%d", 4+k, k, k) - } - b.Raw("ADDV $128, R4").Raw("ADDV $-16, R5").Raw("JMP sloop") - b.Label("sfold") - fold(b) - b.Label("stail").Raw("BEQ R5, sdone"). - Raw("MOVD 0(R4), F1").Raw("ADDD F1, F0"). - Raw("ADDV $8, R4").Raw("ADDV $-1, R5").Raw("JMP stail") - b.Label("sdone").StoreRet("F0", "ret").Ret() - return b.Func() -} - // dotKernel builds dotLASX(a, b *float64, n int) float64: four fused // multiply-add accumulators (xvfmadd.d), folded, plus a fused scalar tail. func dotKernel() *emit.Function { @@ -126,59 +99,6 @@ func dotKernel() *emit.Function { return b.Func() } -// sqrtKernel builds sqrtLASX(dst, src *float64, n int): XVFSQRTD over 16 -// doubles per iteration, SQRTD for the tail. Both are the correctly rounded -// IEEE square root, so the result is bit-identical to math.Sqrt. -func sqrtKernel() *emit.Function { - sig := loong64.Layout([]string{"dst", "src", "n"}, []loong64.Type{loong64.Ptr, loong64.Ptr, loong64.Int64}, nil, nil) - b := loong64.NewFunc("sqrtLASX", sig, 0) - b.LoadArg("dst", "R4").LoadArg("src", "R6").LoadArg("n", "R5") - b.Label("qloop").Raw("MOVV $16, R12").Raw("BLT R5, R12, qtail") - for k := 0; k < 4; k++ { - b.Raw("XVMOVQ %d(R6), X%d", 32*k, k) - } - for k := 0; k < 4; k++ { - b.Raw("XVFSQRTD X%d, X%d", k, 4+k) - } - for k := 0; k < 4; k++ { - b.Raw("XVMOVQ X%d, %d(R4)", 4+k, 32*k) - } - b.Raw("ADDV $128, R4").Raw("ADDV $128, R6").Raw("ADDV $-16, R5").Raw("JMP qloop") - b.Label("qtail").Raw("BEQ R5, qdone"). - Raw("MOVD 0(R6), F1").Raw("SQRTD F1, F1").Raw("MOVD F1, 0(R4)"). - Raw("ADDV $8, R4").Raw("ADDV $8, R6").Raw("ADDV $-1, R5").Raw("JMP qtail") - b.Label("qdone").Ret() - return b.Func() -} - -// binKernel builds name(dst, a, b *float64, n int): dst[i] = a[i] OP b[i], 16 -// doubles per iteration with the LASX op, a scalar tail with sca. Each lane is -// one correctly rounded IEEE operation, bit-identical to the scalar oracle. -func binKernel(name, vec, sca string) *emit.Function { - sig := loong64.Layout([]string{"dst", "a", "b", "n"}, - []loong64.Type{loong64.Ptr, loong64.Ptr, loong64.Ptr, loong64.Int64}, nil, nil) - b := loong64.NewFunc(name, sig, 0) - b.LoadArg("dst", "R4").LoadArg("a", "R6").LoadArg("b", "R7").LoadArg("n", "R5") - l := name[:3] - b.Label(l+"loop").Raw("MOVV $16, R12").Raw("BLT R5, R12, %stail", l) - for k := 0; k < 4; k++ { - b.Raw("XVMOVQ %d(R6), X%d", 32*k, k) - b.Raw("XVMOVQ %d(R7), X%d", 32*k, 4+k) - } - for k := 0; k < 4; k++ { - b.Raw("%s X%d, X%d, X%d", vec, 4+k, k, 8+k) // X(8+k) = X(k) OP X(4+k) - } - for k := 0; k < 4; k++ { - b.Raw("XVMOVQ X%d, %d(R4)", 8+k, 32*k) - } - b.Raw("ADDV $128, R4").Raw("ADDV $128, R6").Raw("ADDV $128, R7").Raw("ADDV $-16, R5").Raw("JMP %sloop", l) - b.Label(l+"tail").Raw("BEQ R5, %sdone", l). - Raw("MOVD 0(R6), F1").Raw("MOVD 0(R7), F2").Raw("%s F2, F1", sca).Raw("MOVD F1, 0(R4)"). - Raw("ADDV $8, R4").Raw("ADDV $8, R6").Raw("ADDV $8, R7").Raw("ADDV $-1, R5").Raw("JMP %stail", l) - b.Label(l + "done").Ret() - return b.Func() -} - // lanes is a 16-byte vector constant: v in every lane of size bytes, each lane // little-endian. func lanes(size int, v uint64) []byte { diff --git a/internal/kernels/asmgen/ppc64le/gen.go b/internal/kernels/asmgen/ppc64le/gen.go index 1bb47a7..fac4cec 100644 --- a/internal/kernels/asmgen/ppc64le/gen.go +++ b/internal/kernels/asmgen/ppc64le/gen.go @@ -31,13 +31,12 @@ const ( func main() { f := emit.NewFile("ppc64le") - f.Add(sumKernel()) f.Add(dotKernel()) - f.Add(sqrtKernel()) + // The float64 sum, sqrt and Add/Sub/Mul/Div kernels live in go-simd/floats + // (lanes_ppc64le_gen.go there) since floats v0.10.0. for _, k := range []struct { name, vec, sca string }{{"addVSX", "add", "FADD"}, {"subVSX", "sub", "FSUB"}, {"mulVSX", "mul", "FMUL"}, {"divVSX", "div", "FDIV"}} { - f.Add(binKernel(k.name, k.vec, k.sca)) f.Add(bin32Kernel(k.name[:3]+"32VSX", k.vec, k.sca+"S")) } f.Add(gemmKernel()) @@ -85,33 +84,6 @@ func reduceSig() ppc64.Signature { []string{"ret"}, []ppc64.Type{ppc64.Float64}) } -// sumKernel builds sumVSX(a *float64, n int) float64: four VSX accumulators -// (8 doubles per iteration), folded, plus a scalar tail. A regrouping of the -// sum, held to a tolerance like the other targets' SIMD sums. -func sumKernel() *emit.Function { - b := ppc64.NewFunc("sumVSX", reduceSig(), 0) - b.LoadArg("a", "R3").LoadArg("n", "R4") - offsets(b, false) - for r := 0; r < 4; r++ { - b.Raw("XXLXOR VS%d, VS%d, VS%d", r, r, r) - } - b.Label("sloop").Raw("CMP R4, $8").Raw("BLT sfold") - for k := 0; k < 4; k++ { - b.Raw("LXVD2X (R3)(%s), VS%d", vidx(k), 4+k) - } - for k := 0; k < 4; k++ { - b.XVADDDP(k, k, 4+k) - } - b.Raw("ADD $64, R3").Raw("ADD $-8, R4").Raw("BR sloop") - b.Label("sfold") - fold(b) - b.Label("stail").Raw("CMP R4, $0").Raw("BEQ sdone"). - Raw("FMOVD 0(R3), F1").Raw("FADD F1, F0"). - Raw("ADD $8, R3").Raw("ADD $-1, R4").Raw("BR stail") - b.Label("sdone").StoreRet("F0", "ret").Ret() - return b.Func() -} - // dotKernel builds dotVSX(a, b *float64, n int) float64: four fused // multiply-add accumulators (XVMADDADP), folded, plus a scalar tail. func dotKernel() *emit.Function { @@ -141,70 +113,6 @@ func dotKernel() *emit.Function { return b.Func() } -// sqrtKernel builds sqrtVSX(dst, src *float64, n int): XVSQRTDP over 8 doubles -// per iteration, FSQRT for the tail. Both are the correctly rounded IEEE -// square root, so the result is bit-identical to math.Sqrt. -func sqrtKernel() *emit.Function { - sig := ppc64.Layout([]string{"dst", "src", "n"}, []ppc64.Type{ppc64.Ptr, ppc64.Ptr, ppc64.Int64}, nil, nil) - b := ppc64.NewFunc("sqrtVSX", sig, 0) - b.LoadArg("dst", "R3").LoadArg("src", "R5").LoadArg("n", "R4") - offsets(b, false) - b.Label("qloop").Raw("CMP R4, $8").Raw("BLT qtail") - for k := 0; k < 4; k++ { - b.Raw("LXVD2X (R5)(%s), VS%d", vidx(k), 32+k) - } - for k := 0; k < 4; k++ { - b.XVSQRTDP(36+k, 32+k) - } - for k := 0; k < 4; k++ { - b.Raw("STXVD2X VS%d, (R3)(%s)", 36+k, vidx(k)) - } - b.Raw("ADD $64, R3").Raw("ADD $64, R5").Raw("ADD $-8, R4").Raw("BR qloop") - b.Label("qtail").Raw("CMP R4, $0").Raw("BEQ qdone"). - Raw("FMOVD 0(R5), F1").Raw("FSQRT F1, F1").Raw("FMOVD F1, 0(R3)"). - Raw("ADD $8, R3").Raw("ADD $8, R5").Raw("ADD $-1, R4").Raw("BR qtail") - b.Label("qdone").Ret() - return b.Func() -} - -// binKernel builds name(dst, a, b *float64, n int): dst[i] = a[i] OP b[i], 8 -// doubles per iteration with the VSX op and a scalar tail with sca. Each lane -// is one correctly rounded IEEE operation, bit-identical to the scalar oracle. -func binKernel(name, op, sca string) *emit.Function { - sig := ppc64.Layout([]string{"dst", "a", "b", "n"}, - []ppc64.Type{ppc64.Ptr, ppc64.Ptr, ppc64.Ptr, ppc64.Int64}, nil, nil) - b := ppc64.NewFunc(name, sig, 0) - b.LoadArg("dst", "R3").LoadArg("a", "R5").LoadArg("b", "R6").LoadArg("n", "R4") - offsets(b, false) - l := name[:3] - b.Label(l+"loop").Raw("CMP R4, $8").Raw("BLT %stail", l) - for k := 0; k < 4; k++ { - b.Raw("LXVD2X (R5)(%s), VS%d", vidx(k), 32+k) - b.Raw("LXVD2X (R6)(%s), VS%d", vidx(k), 36+k) - } - for k := 0; k < 4; k++ { - switch op { - case "add": - b.XVADDDP(40+k, 32+k, 36+k) - case "sub": - b.XVSUBDP(40+k, 32+k, 36+k) - case "mul": - b.XVMULDP(40+k, 32+k, 36+k) - case "div": - b.XVDIVDP(40+k, 32+k, 36+k) - } - } - for k := 0; k < 4; k++ { - b.Raw("STXVD2X VS%d, (R3)(%s)", 40+k, vidx(k)) - } - b.Raw("ADD $64, R3").Raw("ADD $64, R5").Raw("ADD $64, R6").Raw("ADD $-8, R4").Raw("BR %sloop", l) - b.Label(l+"tail").Raw("CMP R4, $0").Raw("BEQ %sdone", l). - Raw("FMOVD 0(R5), F1").Raw("FMOVD 0(R6), F2").Raw("%s F2, F1", sca).Raw("FMOVD F1, 0(R3)"). - Raw("ADD $8, R3").Raw("ADD $8, R5").Raw("ADD $8, R6").Raw("ADD $-1, R4").Raw("BR %stail", l) - b.Label(l + "done").Ret() - return b.Func() -} - // lanes is a 16-byte vector constant: v in every lane of size bytes, each lane // little-endian, as LXVD2X/LXVW4X load it on ppc64le (every lane equal, so // their lane order does not matter). diff --git a/internal/kernels/asmgen/riscv64/gen.go b/internal/kernels/asmgen/riscv64/gen.go index f8ae17b..a85232e 100644 --- a/internal/kernels/asmgen/riscv64/gen.go +++ b/internal/kernels/asmgen/riscv64/gen.go @@ -26,14 +26,13 @@ import ( func main() { f := emit.NewFile("riscv64") - f.Add(reduceKernel("sumRVV", false)) f.Add(reduceKernel("dotRVV", true)) - f.Add(unaryKernel("sqrtRVV", "VFSQRTV V0, V16")) + // The float64 sum, sqrt and Add/Sub/Mul/Div kernels live in go-simd/floats + // (lanes_riscv64_gen.go there) since floats v0.10.0. for _, k := range []struct{ name, op string }{ {"addRVV", "VFADDVV"}, {"subRVV", "VFSUBVV"}, {"mulRVV", "VFMULVV"}, {"divRVV", "VFDIVVV"}, } { - f.Add(binKernel(k.name, k.op)) f.Add(bin32Kernel(k.name[:3]+"32RVV", k.op)) } f.Add(gemmKernel()) @@ -106,40 +105,6 @@ func reduceKernel(name string, dot bool) *emit.Function { return b.Func() } -// unaryKernel builds NAME(dst, src *float64, n int) for n >= 1: op reads V0 and -// writes V16. -func unaryKernel(name, op string) *emit.Function { - sig := riscv64.Layout([]string{"dst", "src", "n"}, - []riscv64.Type{riscv64.Ptr, riscv64.Ptr, riscv64.Int64}, nil, nil) - b := riscv64.NewFunc(name, sig, 0) - b.LoadArg("dst", "X10").LoadArg("src", "X11").LoadArg("n", "X13") - b.Raw("loop:"). - Raw("VSETVLI X13, E64, M8, TA, MA, X14"). - Raw("VLE64V (X11), V0"). - Raw(op). - Raw("VSE64V V16, (X10)") - advance(b, "X14", "X10", "X11") - b.Raw("SUB X14, X13, X13").Raw("BNEZ X13, loop").Ret() - return b.Func() -} - -// binKernel builds NAME(dst, a, b *float64, n int) for n >= 1: dst = a OP b. -func binKernel(name, op string) *emit.Function { - sig := riscv64.Layout([]string{"dst", "a", "b", "n"}, - []riscv64.Type{riscv64.Ptr, riscv64.Ptr, riscv64.Ptr, riscv64.Int64}, nil, nil) - b := riscv64.NewFunc(name, sig, 0) - b.LoadArg("dst", "X10").LoadArg("a", "X11").LoadArg("b", "X12").LoadArg("n", "X13") - b.Raw("loop:"). - Raw("VSETVLI X13, E64, M8, TA, MA, X14"). - Raw("VLE64V (X11), V0"). - Raw("VLE64V (X12), V8"). - Raw("%s V8, V0, V16", op). // V16 = a OP b - Raw("VSE64V V16, (X10)") - advance(b, "X14", "X10", "X11", "X12") - b.Raw("SUB X14, X13, X13").Raw("BNEZ X13, loop").Ret() - return b.Func() -} - // extremumKernel builds NAME(a *floatN, n int) floatN for n >= 1, the // NaN-propagating maximum (or minimum), N = bits: // diff --git a/internal/kernels/asmgen/s390x/gen.go b/internal/kernels/asmgen/s390x/gen.go index b3d4d46..fac63a7 100644 --- a/internal/kernels/asmgen/s390x/gen.go +++ b/internal/kernels/asmgen/s390x/gen.go @@ -27,14 +27,13 @@ import ( func main() { f := emit.NewFile("s390x") - f.Add(reduceKernel("sumVX", false)) f.Add(reduceKernel("dotVX", true)) - f.Add(sqrtKernel()) + // The float64 sum, sqrt and Add/Sub/Mul/Div kernels live in go-simd/floats + // (lanes_s390x_gen.go there) since floats v0.10.0. for _, k := range []struct{ name, vec, sca string }{ {"addVX", "VFADB", "FADD"}, {"subVX", "VFSDB", "FSUB"}, {"mulVX", "VFMDB", "FMUL"}, {"divVX", "VFDDB", "FDIV"}, } { - f.Add(binKernel(k.name, k.vec, k.sca)) f.Add(bin32Kernel(k.name[:3]+"32VXE", k.name[:3], k.sca+"S")) } f.Add(gemmKernel()) @@ -103,51 +102,6 @@ func reduceKernel(name string, dot bool) *emit.Function { return b.Func() } -// sqrtKernel builds sqrtVX(dst, src *float64, n int). -func sqrtKernel() *emit.Function { - sig := s390x.Layout([]string{"dst", "src", "n"}, - []s390x.Type{s390x.Ptr, s390x.Ptr, s390x.Int64}, nil, nil) - b := s390x.NewFunc("sqrtVX", sig, 0) - b.LoadArg("dst", "R1").LoadArg("src", "R2").LoadArg("n", "R4") - b.Raw("loop:").Raw("CMP R4, $8").Raw("BLT tail"). - Raw("VLM (R2), V0, V3") - for r := 0; r < 4; r++ { - b.Raw("VFSQDB V%d, V%d", r, r) - } - b.Raw("VSTM V0, V3, (R1)"). - Raw("ADD $64, R1").Raw("ADD $64, R2").Raw("SUB $8, R4").Raw("BR loop"). - Raw("tail:").Raw("CMP R4, $0").Raw("BEQ done"). - Raw("FMOVD (R2), F0").Raw("FSQRT F0, F0").Raw("FMOVD F0, (R1)"). - Raw("ADD $8, R1").Raw("ADD $8, R2").Raw("SUB $1, R4").Raw("BR tail"). - Raw("done:").Ret() - return b.Func() -} - -// binKernel builds NAME(dst, a, b *float64, n int): dst = a OP b. vec computes -// Vd = Vn OP Vm from "vec Vm, Vn, Vd"; sca computes F0 = F0 OP F1 from -// "sca F1, F0". -func binKernel(name, vec, sca string) *emit.Function { - sig := s390x.Layout([]string{"dst", "a", "b", "n"}, - []s390x.Type{s390x.Ptr, s390x.Ptr, s390x.Ptr, s390x.Int64}, nil, nil) - b := s390x.NewFunc(name, sig, 0) - b.LoadArg("dst", "R1").LoadArg("a", "R2").LoadArg("b", "R3").LoadArg("n", "R4") - b.Raw("loop:").Raw("CMP R4, $8").Raw("BLT tail"). - Raw("VLM (R2), V0, V3"). // a - Raw("VLM (R3), V4, V7") // b - for r := 0; r < 4; r++ { - b.Raw("%s V%d, V%d, V%d", vec, 4+r, r, r) // V_r = a OP b - } - b.Raw("VSTM V0, V3, (R1)"). - Raw("ADD $64, R1").Raw("ADD $64, R2").Raw("ADD $64, R3").Raw("SUB $8, R4").Raw("BR loop"). - Raw("tail:").Raw("CMP R4, $0").Raw("BEQ done"). - Raw("FMOVD (R2), F0").Raw("FMOVD (R3), F1"). - Raw(sca + " F1, F0"). - Raw("FMOVD F0, (R1)"). - Raw("ADD $8, R1").Raw("ADD $8, R2").Raw("ADD $8, R3").Raw("SUB $1, R4").Raw("BR tail"). - Raw("done:").Ret() - return b.Func() -} - // extremum32Kernel builds NAME(a *float32, n int) float32, the maximum (or // minimum) of a, for n >= 16. VFMAXSB/VFMINSB and their scalar W forms come // with the vector-enhancements facility 1 (z14), so the Go side gates them on diff --git a/internal/kernels/reduce_floats.go b/internal/kernels/reduce_floats.go index b265220..d3baf9a 100644 --- a/internal/kernels/reduce_floats.go +++ b/internal/kernels/reduce_floats.go @@ -11,9 +11,9 @@ import "github.com/go-simd/floats" // sqrtScalar, Max, Min of kernels.go) on every input, signed zeros and NaNs // included; the tests of this package hold them to it. // -// amd64 needs AVX2+FMA for them (floats' gate); on a CPU without it floats runs -// its portable loops, which the compiler does not vectorise (the SSE2 kernels -// this package had before ran there). +// amd64 needs AVX2+FMA for the AVX2 kernels (floats' gate); on a CPU without it +// floats runs its SSE2 kernels (floats v0.10.0, floats#31), as this package's +// own SSE2 kernels did before v0.37.0. // HaveReduceSIMD reports that this build ships SIMD reduction/elementwise // kernels (so the dispatch wrappers route through them rather than the scalar @@ -24,25 +24,13 @@ const HaveReduceSIMD = true func sqrtSIMD(dst, src []float64) { floats.Sqrt(dst, src[:len(dst)]) } // maxSIMD returns the NaN-propagating maximum of a (non-empty), the serial Max -// oracle's value (max(-0, +0) = +0, as NumPy). floats.Max leaves the sign of a -// zero result unspecified (its kernels compare zeros as equal), so a zero result, -// the one case where it matters, is resolved by the oracle, which costs nothing -// on any other result. -func maxSIMD(a []float64) float64 { - if r := floats.Max(a); r != 0 { - return r - } - return Max(a) -} +// oracle's value (max(-0, +0) = +0, as NumPy). floats.Max has that contract since +// v0.10.0 (floats#32), so no zero result needs resolving here any more. +func maxSIMD(a []float64) float64 { return floats.Max(a) } // minSIMD returns the NaN-propagating minimum of a (non-empty) (min(-0, +0) = -// -0), resolved like maxSIMD. -func minSIMD(a []float64) float64 { - if r := floats.Min(a); r != 0 { - return r - } - return Min(a) -} +// -0), floats.Min's contract since v0.10.0. +func minSIMD(a []float64) float64 { return floats.Min(a) } // addBin/subBin/mulBin/divBin write a[i] OP b[i] into dst[i]; len(dst) elements. func addBin(dst, a, b []float64) { floats.Add(dst, a[:len(dst)], b[:len(dst)]) } diff --git a/internal/kernels/reduce_floats_elem.go b/internal/kernels/reduce_floats_elem.go new file mode 100644 index 0000000..1d8a13a --- /dev/null +++ b/internal/kernels/reduce_floats_elem.go @@ -0,0 +1,20 @@ +//go:build ppc64le || loong64 || riscv64 || s390x + +package kernels + +import "github.com/go-simd/floats" + +// On ppc64le (VSX), loong64 (LASX), riscv64 (RVV) and s390x (VX) the float64 +// elementwise and square-root kernels are go-simd/floats' since v0.10.0 +// (floats#30): they moved there from this package's sum_.s, with the same +// instruction sequences, and floats gates them on the CPU feature itself. Each is +// bit-identical to the scalar oracle (Add, Sub, Mul, Div, sqrtScalar). + +// sqrtSIMD writes sqrt(src[i]) into dst[i] (correctly rounded, like math.Sqrt). +func sqrtSIMD(dst, src []float64) { floats.Sqrt(dst, src[:len(dst)]) } + +// addBin/subBin/mulBin/divBin write a[i] OP b[i] into dst[i]; len(dst) elements. +func addBin(dst, a, b []float64) { floats.Add(dst, a[:len(dst)], b[:len(dst)]) } +func subBin(dst, a, b []float64) { floats.Sub(dst, a[:len(dst)], b[:len(dst)]) } +func mulBin(dst, a, b []float64) { floats.Mul(dst, a[:len(dst)], b[:len(dst)]) } +func divBin(dst, a, b []float64) { floats.Div(dst, a[:len(dst)], b[:len(dst)]) } diff --git a/internal/kernels/reduce_loong64.go b/internal/kernels/reduce_loong64.go index f82cf6f..84a1148 100644 --- a/internal/kernels/reduce_loong64.go +++ b/internal/kernels/reduce_loong64.go @@ -29,16 +29,7 @@ func lasxFromAuxv() bool { } // The sum is floats.SumPairwise on every target (reduce_sum.go): NumPy's order. -// sumLASX stays in sum_loong64.s until the loong64 kernels move to go-simd/floats. - -func sqrtSIMD(dst, src []float64) { - if !useLASX || len(dst) == 0 { - sqrtScalar(dst, src) - return - } - _ = src[len(dst)-1] // the kernel reads len(dst) of src - sqrtLASX(&dst[0], &src[0], len(dst)) -} +// The elementwise kernels and sqrt are go-simd/floats' (reduce_floats_elem.go). // max/min stay on the four-accumulator scalar reducers. // maxSIMD / minSIMD reduce whole blocks of 8 with the LSX kernels @@ -89,57 +80,3 @@ func maxLSX(a *float64, n int, out *[2]int64) int64 //go:noescape func minLSX(a *float64, n int, out *[2]int64) int64 - -func addBin(dst, a, b []float64) { - if !useLASX || len(dst) == 0 { - Add(dst, a, b) - return - } - _, _ = a[len(dst)-1], b[len(dst)-1] // the kernel reads len(dst) of each - addLASX(&dst[0], &a[0], &b[0], len(dst)) -} - -func subBin(dst, a, b []float64) { - if !useLASX || len(dst) == 0 { - Sub(dst, a, b) - return - } - _, _ = a[len(dst)-1], b[len(dst)-1] // the kernel reads len(dst) of each - subLASX(&dst[0], &a[0], &b[0], len(dst)) -} - -func mulBin(dst, a, b []float64) { - if !useLASX || len(dst) == 0 { - Mul(dst, a, b) - return - } - _, _ = a[len(dst)-1], b[len(dst)-1] // the kernel reads len(dst) of each - mulLASX(&dst[0], &a[0], &b[0], len(dst)) -} - -func divBin(dst, a, b []float64) { - if !useLASX || len(dst) == 0 { - Div(dst, a, b) - return - } - _, _ = a[len(dst)-1], b[len(dst)-1] // the kernel reads len(dst) of each - divLASX(&dst[0], &a[0], &b[0], len(dst)) -} - -//go:noescape -func sumLASX(a *float64, n int) float64 - -//go:noescape -func sqrtLASX(dst, src *float64, n int) - -//go:noescape -func addLASX(dst, a, b *float64, n int) - -//go:noescape -func subLASX(dst, a, b *float64, n int) - -//go:noescape -func mulLASX(dst, a, b *float64, n int) - -//go:noescape -func divLASX(dst, a, b *float64, n int) diff --git a/internal/kernels/reduce_ppc64le.go b/internal/kernels/reduce_ppc64le.go index 95b1c3b..dc4b4d1 100644 --- a/internal/kernels/reduce_ppc64le.go +++ b/internal/kernels/reduce_ppc64le.go @@ -14,17 +14,7 @@ import "math" const HaveReduceSIMD = true // The sum is floats.SumPairwise on every target (reduce_sum.go): NumPy's order. -// sumVSX stays in sum_ppc64le.s until the ppc64le kernels move to go-simd/floats. - -// sqrtSIMD writes sqrt(src[i]) into dst[i] with XVSQRTDP, bit-identical to -// math.Sqrt (one correctly rounded operation per lane). -func sqrtSIMD(dst, src []float64) { - if len(dst) == 0 { - return - } - _ = src[len(dst)-1] // the kernel reads len(dst) of src - sqrtVSX(&dst[0], &src[0], len(dst)) -} +// The elementwise kernels and sqrt are go-simd/floats' (reduce_floats_elem.go). // maxSIMD / minSIMD reduce whole blocks of 8 with xvmaxdp/xvmindp // (sum_ppc64le.s) and fold the two lanes and the tail with Go's max/min. @@ -75,51 +65,3 @@ func maxVSX(a *float64, n int, out *[2]float64) int64 //go:noescape func minVSX(a *float64, n int, out *[2]float64) int64 - -// addBin, subBin, mulBin, divBin write dst[i] = a[i] OP b[i] with the VSX -// kernels, each lane one IEEE operation, bit-identical to the scalar oracle. -func addBin(dst, a, b []float64) { - if len(dst) > 0 { - _, _ = a[len(dst)-1], b[len(dst)-1] // the kernel reads len(dst) of each - addVSX(&dst[0], &a[0], &b[0], len(dst)) - } -} - -func subBin(dst, a, b []float64) { - if len(dst) > 0 { - _, _ = a[len(dst)-1], b[len(dst)-1] // the kernel reads len(dst) of each - subVSX(&dst[0], &a[0], &b[0], len(dst)) - } -} - -func mulBin(dst, a, b []float64) { - if len(dst) > 0 { - _, _ = a[len(dst)-1], b[len(dst)-1] // the kernel reads len(dst) of each - mulVSX(&dst[0], &a[0], &b[0], len(dst)) - } -} - -func divBin(dst, a, b []float64) { - if len(dst) > 0 { - _, _ = a[len(dst)-1], b[len(dst)-1] // the kernel reads len(dst) of each - divVSX(&dst[0], &a[0], &b[0], len(dst)) - } -} - -//go:noescape -func sumVSX(a *float64, n int) float64 - -//go:noescape -func sqrtVSX(dst, src *float64, n int) - -//go:noescape -func addVSX(dst, a, b *float64, n int) - -//go:noescape -func subVSX(dst, a, b *float64, n int) - -//go:noescape -func mulVSX(dst, a, b *float64, n int) - -//go:noescape -func divVSX(dst, a, b *float64, n int) diff --git a/internal/kernels/reduce_riscv64.go b/internal/kernels/reduce_riscv64.go index d4a0dd3..83cb3a8 100644 --- a/internal/kernels/reduce_riscv64.go +++ b/internal/kernels/reduce_riscv64.go @@ -32,19 +32,11 @@ func rvvFromAuxv() bool { return err == nil && hwcapFrom(auxv)&hwcapV != 0 } -// The sum is floats.SumPairwise on every target (reduce_sum.go): NumPy's order. -// sumRVV stays in sum_riscv64.s until the riscv64 kernels move to go-simd/floats -// (it regrouped [MaxInt64, MinInt64, -3] as float64 on a real SpacemiT X60: the -// mean was 0, NumPy's -1; SumPairwise adds n < 8 left to right as NumPy does). - -func sqrtSIMD(dst, src []float64) { - if !useRVV || len(dst) == 0 { - sqrtScalar(dst, src) - return - } - _ = src[len(dst)-1] // the kernel reads len(dst) of src - sqrtRVV(&dst[0], &src[0], len(dst)) -} +// The sum is floats.SumPairwise on every target (reduce_sum.go): NumPy's order +// (the former sumRVV regrouped [MaxInt64, MinInt64, -3] as float64 on a real +// SpacemiT X60: the mean was 0, NumPy's -1; SumPairwise adds n < 8 left to right +// as NumPy does). The elementwise kernels and sqrt are go-simd/floats' +// (reduce_floats_elem.go). // maxSIMD / minSIMD are the NaN-propagating extrema of a (non-empty): the RVV // kernels (vfmax/vfmin with a vmfne NaN count) when the CPU reports V, the @@ -69,42 +61,6 @@ func maxRVV(a *float64, n int) float64 //go:noescape func minRVV(a *float64, n int) float64 -func addBin(dst, a, b []float64) { - if !useRVV || len(dst) == 0 { - Add(dst, a, b) - return - } - _, _ = a[len(dst)-1], b[len(dst)-1] // the kernel reads len(dst) of each - addRVV(&dst[0], &a[0], &b[0], len(dst)) -} - -func subBin(dst, a, b []float64) { - if !useRVV || len(dst) == 0 { - Sub(dst, a, b) - return - } - _, _ = a[len(dst)-1], b[len(dst)-1] // the kernel reads len(dst) of each - subRVV(&dst[0], &a[0], &b[0], len(dst)) -} - -func mulBin(dst, a, b []float64) { - if !useRVV || len(dst) == 0 { - Mul(dst, a, b) - return - } - _, _ = a[len(dst)-1], b[len(dst)-1] // the kernel reads len(dst) of each - mulRVV(&dst[0], &a[0], &b[0], len(dst)) -} - -func divBin(dst, a, b []float64) { - if !useRVV || len(dst) == 0 { - Div(dst, a, b) - return - } - _, _ = a[len(dst)-1], b[len(dst)-1] // the kernel reads len(dst) of each - divRVV(&dst[0], &a[0], &b[0], len(dst)) -} - // dotSIMD is the inner product (RVV kernel, one fused accumulator per lane). func dotSIMD(a, b []float64) float64 { if !useRVV || len(a) == 0 { @@ -114,23 +70,5 @@ func dotSIMD(a, b []float64) float64 { return dotRVV(&a[0], &b[0], len(a)) } -//go:noescape -func sumRVV(a *float64, n int) float64 - //go:noescape func dotRVV(a, b *float64, n int) float64 - -//go:noescape -func sqrtRVV(dst, src *float64, n int) - -//go:noescape -func addRVV(dst, a, b *float64, n int) - -//go:noescape -func subRVV(dst, a, b *float64, n int) - -//go:noescape -func mulRVV(dst, a, b *float64, n int) - -//go:noescape -func divRVV(dst, a, b *float64, n int) diff --git a/internal/kernels/reduce_s390x.go b/internal/kernels/reduce_s390x.go index 73f4299..f81893c 100644 --- a/internal/kernels/reduce_s390x.go +++ b/internal/kernels/reduce_s390x.go @@ -31,16 +31,7 @@ func vxFromAuxv() bool { } // The sum is floats.SumPairwise on every target (reduce_sum.go): NumPy's order. -// sumVX stays in sum_s390x.s until the s390x kernels move to go-simd/floats. - -func sqrtSIMD(dst, src []float64) { - if !useVX || len(dst) == 0 { - sqrtScalar(dst, src) - return - } - _ = src[len(dst)-1] // the kernel reads len(dst) of src - sqrtVX(&dst[0], &src[0], len(dst)) -} +// The elementwise kernels and sqrt are go-simd/floats' (reduce_floats_elem.go). // hwcapVXE is HWCAP_S390_VXRS_EXT (bit 13), the vector-enhancements facility 1 // of the z14, which brings VFMAXDB/VFMINDB. @@ -71,42 +62,6 @@ func minSIMD(a []float64) float64 { return minVXE(&a[0], len(a)) } -func addBin(dst, a, b []float64) { - if !useVX || len(dst) == 0 { - Add(dst, a, b) - return - } - _, _ = a[len(dst)-1], b[len(dst)-1] // the kernel reads len(dst) of each - addVX(&dst[0], &a[0], &b[0], len(dst)) -} - -func subBin(dst, a, b []float64) { - if !useVX || len(dst) == 0 { - Sub(dst, a, b) - return - } - _, _ = a[len(dst)-1], b[len(dst)-1] // the kernel reads len(dst) of each - subVX(&dst[0], &a[0], &b[0], len(dst)) -} - -func mulBin(dst, a, b []float64) { - if !useVX || len(dst) == 0 { - Mul(dst, a, b) - return - } - _, _ = a[len(dst)-1], b[len(dst)-1] // the kernel reads len(dst) of each - mulVX(&dst[0], &a[0], &b[0], len(dst)) -} - -func divBin(dst, a, b []float64) { - if !useVX || len(dst) == 0 { - Div(dst, a, b) - return - } - _, _ = a[len(dst)-1], b[len(dst)-1] // the kernel reads len(dst) of each - divVX(&dst[0], &a[0], &b[0], len(dst)) -} - // dotSIMD is the inner product (vector kernel, four fused two-lane accumulators). func dotSIMD(a, b []float64) float64 { if !useVX || len(a) == 0 { @@ -116,27 +71,9 @@ func dotSIMD(a, b []float64) float64 { return dotVX(&a[0], &b[0], len(a)) } -//go:noescape -func sumVX(a *float64, n int) float64 - //go:noescape func dotVX(a, b *float64, n int) float64 -//go:noescape -func sqrtVX(dst, src *float64, n int) - -//go:noescape -func addVX(dst, a, b *float64, n int) - -//go:noescape -func subVX(dst, a, b *float64, n int) - -//go:noescape -func mulVX(dst, a, b *float64, n int) - -//go:noescape -func divVX(dst, a, b *float64, n int) - //go:noescape func maxVXE(a *float64, n int) float64 diff --git a/internal/kernels/sum_loong64.s b/internal/kernels/sum_loong64.s index 1c7257a..764b8a4 100644 --- a/internal/kernels/sum_loong64.s +++ b/internal/kernels/sum_loong64.s @@ -4,53 +4,6 @@ #include "textflag.h" -TEXT ·sumLASX(SB), NOSPLIT, $0-24 - MOVV a+0(FP), R4 - MOVV n+8(FP), R5 - XVXORV X0, X0, X0 - XVXORV X1, X1, X1 - XVXORV X2, X2, X2 - XVXORV X3, X3, X3 -sloop: - MOVV $16, R12 - BLT R5, R12, sfold - XVMOVQ 0(R4), X4 - XVMOVQ 32(R4), X5 - XVMOVQ 64(R4), X6 - XVMOVQ 96(R4), X7 - XVADDD X4, X0, X0 - XVADDD X5, X1, X1 - XVADDD X6, X2, X2 - XVADDD X7, X3, X3 - ADDV $128, R4 - ADDV $-16, R5 - JMP sloop -sfold: - XVADDD X1, X0, X0 - XVADDD X3, X2, X2 - XVADDD X2, X0, X0 - XVMOVQ X0.V[0], R12 - XVMOVQ X0.V[1], R13 - XVMOVQ X0.V[2], R14 - XVMOVQ X0.V[3], R15 - MOVV R12, F0 - MOVV R13, F1 - MOVV R14, F2 - MOVV R15, F3 - ADDD F1, F0 - ADDD F3, F2 - ADDD F2, F0 -stail: - BEQ R5, sdone - MOVD 0(R4), F1 - ADDD F1, F0 - ADDV $8, R4 - ADDV $-1, R5 - JMP stail -sdone: - MOVD F0, ret+16(FP) - RET - TEXT ·dotLASX(SB), NOSPLIT, $0-32 MOVV a+0(FP), R4 MOVV b+8(FP), R6 @@ -106,84 +59,6 @@ ddone: MOVD F0, ret+24(FP) RET -TEXT ·sqrtLASX(SB), NOSPLIT, $0-24 - MOVV dst+0(FP), R4 - MOVV src+8(FP), R6 - MOVV n+16(FP), R5 -qloop: - MOVV $16, R12 - BLT R5, R12, qtail - XVMOVQ 0(R6), X0 - XVMOVQ 32(R6), X1 - XVMOVQ 64(R6), X2 - XVMOVQ 96(R6), X3 - XVFSQRTD X0, X4 - XVFSQRTD X1, X5 - XVFSQRTD X2, X6 - XVFSQRTD X3, X7 - XVMOVQ X4, 0(R4) - XVMOVQ X5, 32(R4) - XVMOVQ X6, 64(R4) - XVMOVQ X7, 96(R4) - ADDV $128, R4 - ADDV $128, R6 - ADDV $-16, R5 - JMP qloop -qtail: - BEQ R5, qdone - MOVD 0(R6), F1 - SQRTD F1, F1 - MOVD F1, 0(R4) - ADDV $8, R4 - ADDV $8, R6 - ADDV $-1, R5 - JMP qtail -qdone: - RET - -TEXT ·addLASX(SB), NOSPLIT, $0-32 - MOVV dst+0(FP), R4 - MOVV a+8(FP), R6 - MOVV b+16(FP), R7 - MOVV n+24(FP), R5 -addloop: - MOVV $16, R12 - BLT R5, R12, addtail - XVMOVQ 0(R6), X0 - XVMOVQ 0(R7), X4 - XVMOVQ 32(R6), X1 - XVMOVQ 32(R7), X5 - XVMOVQ 64(R6), X2 - XVMOVQ 64(R7), X6 - XVMOVQ 96(R6), X3 - XVMOVQ 96(R7), X7 - XVADDD X4, X0, X8 - XVADDD X5, X1, X9 - XVADDD X6, X2, X10 - XVADDD X7, X3, X11 - XVMOVQ X8, 0(R4) - XVMOVQ X9, 32(R4) - XVMOVQ X10, 64(R4) - XVMOVQ X11, 96(R4) - ADDV $128, R4 - ADDV $128, R6 - ADDV $128, R7 - ADDV $-16, R5 - JMP addloop -addtail: - BEQ R5, adddone - MOVD 0(R6), F1 - MOVD 0(R7), F2 - ADDD F2, F1 - MOVD F1, 0(R4) - ADDV $8, R4 - ADDV $8, R6 - ADDV $8, R7 - ADDV $-1, R5 - JMP addtail -adddone: - RET - TEXT ·add32LSX(SB), NOSPLIT, $0-32 MOVV dst+0(FP), R4 MOVV a+8(FP), R6 @@ -227,49 +102,6 @@ ftail: fdone: RET -TEXT ·subLASX(SB), NOSPLIT, $0-32 - MOVV dst+0(FP), R4 - MOVV a+8(FP), R6 - MOVV b+16(FP), R7 - MOVV n+24(FP), R5 -subloop: - MOVV $16, R12 - BLT R5, R12, subtail - XVMOVQ 0(R6), X0 - XVMOVQ 0(R7), X4 - XVMOVQ 32(R6), X1 - XVMOVQ 32(R7), X5 - XVMOVQ 64(R6), X2 - XVMOVQ 64(R7), X6 - XVMOVQ 96(R6), X3 - XVMOVQ 96(R7), X7 - XVSUBD X4, X0, X8 - XVSUBD X5, X1, X9 - XVSUBD X6, X2, X10 - XVSUBD X7, X3, X11 - XVMOVQ X8, 0(R4) - XVMOVQ X9, 32(R4) - XVMOVQ X10, 64(R4) - XVMOVQ X11, 96(R4) - ADDV $128, R4 - ADDV $128, R6 - ADDV $128, R7 - ADDV $-16, R5 - JMP subloop -subtail: - BEQ R5, subdone - MOVD 0(R6), F1 - MOVD 0(R7), F2 - SUBD F2, F1 - MOVD F1, 0(R4) - ADDV $8, R4 - ADDV $8, R6 - ADDV $8, R7 - ADDV $-1, R5 - JMP subtail -subdone: - RET - TEXT ·sub32LSX(SB), NOSPLIT, $0-32 MOVV dst+0(FP), R4 MOVV a+8(FP), R6 @@ -313,49 +145,6 @@ ftail: fdone: RET -TEXT ·mulLASX(SB), NOSPLIT, $0-32 - MOVV dst+0(FP), R4 - MOVV a+8(FP), R6 - MOVV b+16(FP), R7 - MOVV n+24(FP), R5 -mulloop: - MOVV $16, R12 - BLT R5, R12, multail - XVMOVQ 0(R6), X0 - XVMOVQ 0(R7), X4 - XVMOVQ 32(R6), X1 - XVMOVQ 32(R7), X5 - XVMOVQ 64(R6), X2 - XVMOVQ 64(R7), X6 - XVMOVQ 96(R6), X3 - XVMOVQ 96(R7), X7 - XVMULD X4, X0, X8 - XVMULD X5, X1, X9 - XVMULD X6, X2, X10 - XVMULD X7, X3, X11 - XVMOVQ X8, 0(R4) - XVMOVQ X9, 32(R4) - XVMOVQ X10, 64(R4) - XVMOVQ X11, 96(R4) - ADDV $128, R4 - ADDV $128, R6 - ADDV $128, R7 - ADDV $-16, R5 - JMP mulloop -multail: - BEQ R5, muldone - MOVD 0(R6), F1 - MOVD 0(R7), F2 - MULD F2, F1 - MOVD F1, 0(R4) - ADDV $8, R4 - ADDV $8, R6 - ADDV $8, R7 - ADDV $-1, R5 - JMP multail -muldone: - RET - TEXT ·mul32LSX(SB), NOSPLIT, $0-32 MOVV dst+0(FP), R4 MOVV a+8(FP), R6 @@ -399,49 +188,6 @@ ftail: fdone: RET -TEXT ·divLASX(SB), NOSPLIT, $0-32 - MOVV dst+0(FP), R4 - MOVV a+8(FP), R6 - MOVV b+16(FP), R7 - MOVV n+24(FP), R5 -divloop: - MOVV $16, R12 - BLT R5, R12, divtail - XVMOVQ 0(R6), X0 - XVMOVQ 0(R7), X4 - XVMOVQ 32(R6), X1 - XVMOVQ 32(R7), X5 - XVMOVQ 64(R6), X2 - XVMOVQ 64(R7), X6 - XVMOVQ 96(R6), X3 - XVMOVQ 96(R7), X7 - XVDIVD X4, X0, X8 - XVDIVD X5, X1, X9 - XVDIVD X6, X2, X10 - XVDIVD X7, X3, X11 - XVMOVQ X8, 0(R4) - XVMOVQ X9, 32(R4) - XVMOVQ X10, 64(R4) - XVMOVQ X11, 96(R4) - ADDV $128, R4 - ADDV $128, R6 - ADDV $128, R7 - ADDV $-16, R5 - JMP divloop -divtail: - BEQ R5, divdone - MOVD 0(R6), F1 - MOVD 0(R7), F2 - DIVD F2, F1 - MOVD F1, 0(R4) - ADDV $8, R4 - ADDV $8, R6 - ADDV $8, R7 - ADDV $-1, R5 - JMP divtail -divdone: - RET - TEXT ·div32LSX(SB), NOSPLIT, $0-32 MOVV dst+0(FP), R4 MOVV a+8(FP), R6 diff --git a/internal/kernels/sum_ppc64le.s b/internal/kernels/sum_ppc64le.s index b7c60b6..76345f4 100644 --- a/internal/kernels/sum_ppc64le.s +++ b/internal/kernels/sum_ppc64le.s @@ -4,48 +4,6 @@ #include "textflag.h" -TEXT ·sumVSX(SB), NOSPLIT, $0-24 - MOVD a+0(FP), R3 - MOVD n+8(FP), R4 - MOVD $16, R14 - MOVD $32, R15 - MOVD $48, R16 - XXLXOR VS0, VS0, VS0 - XXLXOR VS1, VS1, VS1 - XXLXOR VS2, VS2, VS2 - XXLXOR VS3, VS3, VS3 -sloop: - CMP R4, $8 - BLT sfold - LXVD2X (R3)(R0), VS4 - LXVD2X (R3)(R14), VS5 - LXVD2X (R3)(R15), VS6 - LXVD2X (R3)(R16), VS7 - WORD $0xf0002300 // XVADDDP VS0, VS4, VS0 - WORD $0xf0212b00 // XVADDDP VS1, VS5, VS1 - WORD $0xf0423300 // XVADDDP VS2, VS6, VS2 - WORD $0xf0633b00 // XVADDDP VS3, VS7, VS3 - ADD $64, R3 - ADD $-8, R4 - BR sloop -sfold: - WORD $0xf0000b00 // XVADDDP VS0, VS1, VS0 - WORD $0xf0421b00 // XVADDDP VS2, VS3, VS2 - WORD $0xf0001300 // XVADDDP VS0, VS2, VS0 - XXPERMDI VS0, VS0, $2, VS1 - WORD $0xf0000b00 // XVADDDP VS0, VS1, VS0 -stail: - CMP R4, $0 - BEQ sdone - FMOVD 0(R3), F1 - FADD F1, F0 - ADD $8, R3 - ADD $-1, R4 - BR stail -sdone: - FMOVD F0, ret+16(FP) - RET - TEXT ·dotVSX(SB), NOSPLIT, $0-32 MOVD a+0(FP), R3 MOVD b+8(FP), R5 @@ -97,92 +55,6 @@ ddone: FMOVD F0, ret+24(FP) RET -TEXT ·sqrtVSX(SB), NOSPLIT, $0-24 - MOVD dst+0(FP), R3 - MOVD src+8(FP), R5 - MOVD n+16(FP), R4 - MOVD $16, R14 - MOVD $32, R15 - MOVD $48, R16 -qloop: - CMP R4, $8 - BLT qtail - LXVD2X (R5)(R0), VS32 - LXVD2X (R5)(R14), VS33 - LXVD2X (R5)(R15), VS34 - LXVD2X (R5)(R16), VS35 - WORD $0xf080032f // XVSQRTDP VS32, VS36 - WORD $0xf0a00b2f // XVSQRTDP VS33, VS37 - WORD $0xf0c0132f // XVSQRTDP VS34, VS38 - WORD $0xf0e01b2f // XVSQRTDP VS35, VS39 - STXVD2X VS36, (R3)(R0) - STXVD2X VS37, (R3)(R14) - STXVD2X VS38, (R3)(R15) - STXVD2X VS39, (R3)(R16) - ADD $64, R3 - ADD $64, R5 - ADD $-8, R4 - BR qloop -qtail: - CMP R4, $0 - BEQ qdone - FMOVD 0(R5), F1 - FSQRT F1, F1 - FMOVD F1, 0(R3) - ADD $8, R3 - ADD $8, R5 - ADD $-1, R4 - BR qtail -qdone: - RET - -TEXT ·addVSX(SB), NOSPLIT, $0-32 - MOVD dst+0(FP), R3 - MOVD a+8(FP), R5 - MOVD b+16(FP), R6 - MOVD n+24(FP), R4 - MOVD $16, R14 - MOVD $32, R15 - MOVD $48, R16 -addloop: - CMP R4, $8 - BLT addtail - LXVD2X (R5)(R0), VS32 - LXVD2X (R6)(R0), VS36 - LXVD2X (R5)(R14), VS33 - LXVD2X (R6)(R14), VS37 - LXVD2X (R5)(R15), VS34 - LXVD2X (R6)(R15), VS38 - LXVD2X (R5)(R16), VS35 - LXVD2X (R6)(R16), VS39 - WORD $0xf1002307 // XVADDDP VS32, VS36, VS40 - WORD $0xf1212b07 // XVADDDP VS33, VS37, VS41 - WORD $0xf1423307 // XVADDDP VS34, VS38, VS42 - WORD $0xf1633b07 // XVADDDP VS35, VS39, VS43 - STXVD2X VS40, (R3)(R0) - STXVD2X VS41, (R3)(R14) - STXVD2X VS42, (R3)(R15) - STXVD2X VS43, (R3)(R16) - ADD $64, R3 - ADD $64, R5 - ADD $64, R6 - ADD $-8, R4 - BR addloop -addtail: - CMP R4, $0 - BEQ adddone - FMOVD 0(R5), F1 - FMOVD 0(R6), F2 - FADD F2, F1 - FMOVD F1, 0(R3) - ADD $8, R3 - ADD $8, R5 - ADD $8, R6 - ADD $-1, R4 - BR addtail -adddone: - RET - TEXT ·add32VSX(SB), NOSPLIT, $0-32 MOVD dst+0(FP), R3 MOVD a+8(FP), R5 @@ -230,53 +102,6 @@ ftail: fdone: RET -TEXT ·subVSX(SB), NOSPLIT, $0-32 - MOVD dst+0(FP), R3 - MOVD a+8(FP), R5 - MOVD b+16(FP), R6 - MOVD n+24(FP), R4 - MOVD $16, R14 - MOVD $32, R15 - MOVD $48, R16 -subloop: - CMP R4, $8 - BLT subtail - LXVD2X (R5)(R0), VS32 - LXVD2X (R6)(R0), VS36 - LXVD2X (R5)(R14), VS33 - LXVD2X (R6)(R14), VS37 - LXVD2X (R5)(R15), VS34 - LXVD2X (R6)(R15), VS38 - LXVD2X (R5)(R16), VS35 - LXVD2X (R6)(R16), VS39 - WORD $0xf1002347 // XVSUBDP VS32, VS36, VS40 - WORD $0xf1212b47 // XVSUBDP VS33, VS37, VS41 - WORD $0xf1423347 // XVSUBDP VS34, VS38, VS42 - WORD $0xf1633b47 // XVSUBDP VS35, VS39, VS43 - STXVD2X VS40, (R3)(R0) - STXVD2X VS41, (R3)(R14) - STXVD2X VS42, (R3)(R15) - STXVD2X VS43, (R3)(R16) - ADD $64, R3 - ADD $64, R5 - ADD $64, R6 - ADD $-8, R4 - BR subloop -subtail: - CMP R4, $0 - BEQ subdone - FMOVD 0(R5), F1 - FMOVD 0(R6), F2 - FSUB F2, F1 - FMOVD F1, 0(R3) - ADD $8, R3 - ADD $8, R5 - ADD $8, R6 - ADD $-1, R4 - BR subtail -subdone: - RET - TEXT ·sub32VSX(SB), NOSPLIT, $0-32 MOVD dst+0(FP), R3 MOVD a+8(FP), R5 @@ -324,53 +149,6 @@ ftail: fdone: RET -TEXT ·mulVSX(SB), NOSPLIT, $0-32 - MOVD dst+0(FP), R3 - MOVD a+8(FP), R5 - MOVD b+16(FP), R6 - MOVD n+24(FP), R4 - MOVD $16, R14 - MOVD $32, R15 - MOVD $48, R16 -mulloop: - CMP R4, $8 - BLT multail - LXVD2X (R5)(R0), VS32 - LXVD2X (R6)(R0), VS36 - LXVD2X (R5)(R14), VS33 - LXVD2X (R6)(R14), VS37 - LXVD2X (R5)(R15), VS34 - LXVD2X (R6)(R15), VS38 - LXVD2X (R5)(R16), VS35 - LXVD2X (R6)(R16), VS39 - WORD $0xf1002387 // XVMULDP VS32, VS36, VS40 - WORD $0xf1212b87 // XVMULDP VS33, VS37, VS41 - WORD $0xf1423387 // XVMULDP VS34, VS38, VS42 - WORD $0xf1633b87 // XVMULDP VS35, VS39, VS43 - STXVD2X VS40, (R3)(R0) - STXVD2X VS41, (R3)(R14) - STXVD2X VS42, (R3)(R15) - STXVD2X VS43, (R3)(R16) - ADD $64, R3 - ADD $64, R5 - ADD $64, R6 - ADD $-8, R4 - BR mulloop -multail: - CMP R4, $0 - BEQ muldone - FMOVD 0(R5), F1 - FMOVD 0(R6), F2 - FMUL F2, F1 - FMOVD F1, 0(R3) - ADD $8, R3 - ADD $8, R5 - ADD $8, R6 - ADD $-1, R4 - BR multail -muldone: - RET - TEXT ·mul32VSX(SB), NOSPLIT, $0-32 MOVD dst+0(FP), R3 MOVD a+8(FP), R5 @@ -418,53 +196,6 @@ ftail: fdone: RET -TEXT ·divVSX(SB), NOSPLIT, $0-32 - MOVD dst+0(FP), R3 - MOVD a+8(FP), R5 - MOVD b+16(FP), R6 - MOVD n+24(FP), R4 - MOVD $16, R14 - MOVD $32, R15 - MOVD $48, R16 -divloop: - CMP R4, $8 - BLT divtail - LXVD2X (R5)(R0), VS32 - LXVD2X (R6)(R0), VS36 - LXVD2X (R5)(R14), VS33 - LXVD2X (R6)(R14), VS37 - LXVD2X (R5)(R15), VS34 - LXVD2X (R6)(R15), VS38 - LXVD2X (R5)(R16), VS35 - LXVD2X (R6)(R16), VS39 - WORD $0xf10023c7 // XVDIVDP VS32, VS36, VS40 - WORD $0xf1212bc7 // XVDIVDP VS33, VS37, VS41 - WORD $0xf14233c7 // XVDIVDP VS34, VS38, VS42 - WORD $0xf1633bc7 // XVDIVDP VS35, VS39, VS43 - STXVD2X VS40, (R3)(R0) - STXVD2X VS41, (R3)(R14) - STXVD2X VS42, (R3)(R15) - STXVD2X VS43, (R3)(R16) - ADD $64, R3 - ADD $64, R5 - ADD $64, R6 - ADD $-8, R4 - BR divloop -divtail: - CMP R4, $0 - BEQ divdone - FMOVD 0(R5), F1 - FMOVD 0(R6), F2 - FDIV F2, F1 - FMOVD F1, 0(R3) - ADD $8, R3 - ADD $8, R5 - ADD $8, R6 - ADD $-1, R4 - BR divtail -divdone: - RET - TEXT ·div32VSX(SB), NOSPLIT, $0-32 MOVD dst+0(FP), R3 MOVD a+8(FP), R5 diff --git a/internal/kernels/sum_riscv64.s b/internal/kernels/sum_riscv64.s index 03c43bc..a5f89ca 100644 --- a/internal/kernels/sum_riscv64.s +++ b/internal/kernels/sum_riscv64.s @@ -4,27 +4,6 @@ #include "textflag.h" -TEXT ·sumRVV(SB), NOSPLIT, $0-24 - MOV a+0(FP), X10 - MOV n+8(FP), X11 - VSETVLI X0, E64, M8, TA, MA, X13 - VMVVI $0, V16 - loop: - VSETVLI X11, E64, M8, TU, MA, X13 - VLE64V (X10), V0 - VFADDVV V0, V16, V16 - SLLI $3, X13, X17 - ADD X17, X10, X10 - SUB X13, X11, X11 - BNEZ X11, loop - VSETVLI X0, E64, M8, TA, MA, X13 - FMVDX X0, F0 - VFMVSF F0, V24 - VFREDUSUMVS V24, V16, V24 - VFMVFS V24, F0 - MOVD F0, ret+16(FP) - RET - TEXT ·dotRVV(SB), NOSPLIT, $0-32 MOV a+0(FP), X10 MOV b+8(FP), X11 @@ -49,41 +28,6 @@ TEXT ·dotRVV(SB), NOSPLIT, $0-32 MOVD F0, ret+24(FP) RET -TEXT ·sqrtRVV(SB), NOSPLIT, $0-24 - MOV dst+0(FP), X10 - MOV src+8(FP), X11 - MOV n+16(FP), X13 - loop: - VSETVLI X13, E64, M8, TA, MA, X14 - VLE64V (X11), V0 - VFSQRTV V0, V16 - VSE64V V16, (X10) - SLLI $3, X14, X17 - ADD X17, X10, X10 - ADD X17, X11, X11 - SUB X14, X13, X13 - BNEZ X13, loop - RET - -TEXT ·addRVV(SB), NOSPLIT, $0-32 - MOV dst+0(FP), X10 - MOV a+8(FP), X11 - MOV b+16(FP), X12 - MOV n+24(FP), X13 - loop: - VSETVLI X13, E64, M8, TA, MA, X14 - VLE64V (X11), V0 - VLE64V (X12), V8 - VFADDVV V8, V0, V16 - VSE64V V16, (X10) - SLLI $3, X14, X17 - ADD X17, X10, X10 - ADD X17, X11, X11 - ADD X17, X12, X12 - SUB X14, X13, X13 - BNEZ X13, loop - RET - TEXT ·add32RVV(SB), NOSPLIT, $0-32 MOV dst+0(FP), X10 MOV a+8(FP), X11 @@ -103,25 +47,6 @@ TEXT ·add32RVV(SB), NOSPLIT, $0-32 BNEZ X13, loop RET -TEXT ·subRVV(SB), NOSPLIT, $0-32 - MOV dst+0(FP), X10 - MOV a+8(FP), X11 - MOV b+16(FP), X12 - MOV n+24(FP), X13 - loop: - VSETVLI X13, E64, M8, TA, MA, X14 - VLE64V (X11), V0 - VLE64V (X12), V8 - VFSUBVV V8, V0, V16 - VSE64V V16, (X10) - SLLI $3, X14, X17 - ADD X17, X10, X10 - ADD X17, X11, X11 - ADD X17, X12, X12 - SUB X14, X13, X13 - BNEZ X13, loop - RET - TEXT ·sub32RVV(SB), NOSPLIT, $0-32 MOV dst+0(FP), X10 MOV a+8(FP), X11 @@ -141,25 +66,6 @@ TEXT ·sub32RVV(SB), NOSPLIT, $0-32 BNEZ X13, loop RET -TEXT ·mulRVV(SB), NOSPLIT, $0-32 - MOV dst+0(FP), X10 - MOV a+8(FP), X11 - MOV b+16(FP), X12 - MOV n+24(FP), X13 - loop: - VSETVLI X13, E64, M8, TA, MA, X14 - VLE64V (X11), V0 - VLE64V (X12), V8 - VFMULVV V8, V0, V16 - VSE64V V16, (X10) - SLLI $3, X14, X17 - ADD X17, X10, X10 - ADD X17, X11, X11 - ADD X17, X12, X12 - SUB X14, X13, X13 - BNEZ X13, loop - RET - TEXT ·mul32RVV(SB), NOSPLIT, $0-32 MOV dst+0(FP), X10 MOV a+8(FP), X11 @@ -179,25 +85,6 @@ TEXT ·mul32RVV(SB), NOSPLIT, $0-32 BNEZ X13, loop RET -TEXT ·divRVV(SB), NOSPLIT, $0-32 - MOV dst+0(FP), X10 - MOV a+8(FP), X11 - MOV b+16(FP), X12 - MOV n+24(FP), X13 - loop: - VSETVLI X13, E64, M8, TA, MA, X14 - VLE64V (X11), V0 - VLE64V (X12), V8 - VFDIVVV V8, V0, V16 - VSE64V V16, (X10) - SLLI $3, X14, X17 - ADD X17, X10, X10 - ADD X17, X11, X11 - ADD X17, X12, X12 - SUB X14, X13, X13 - BNEZ X13, loop - RET - TEXT ·div32RVV(SB), NOSPLIT, $0-32 MOV dst+0(FP), X10 MOV a+8(FP), X11 diff --git a/internal/kernels/sum_s390x.s b/internal/kernels/sum_s390x.s index 46619e0..ad3f6a2 100644 --- a/internal/kernels/sum_s390x.s +++ b/internal/kernels/sum_s390x.s @@ -4,42 +4,6 @@ #include "textflag.h" -TEXT ·sumVX(SB), NOSPLIT, $0-24 - MOVD a+0(FP), R1 - MOVD n+8(FP), R2 - VZERO V16 - VZERO V17 - VZERO V18 - VZERO V19 - loop: - CMP R2, $8 - BLT fold - VLM (R1), V0, V3 - VFADB V0, V16, V16 - VFADB V1, V17, V17 - VFADB V2, V18, V18 - VFADB V3, V19, V19 - ADD $64, R1 - SUB $8, R2 - BR loop - fold: - VFADB V17, V16, V16 - VFADB V19, V18, V18 - VFADB V18, V16, V0 - VREPG $1, V0, V1 - WFADB V0, V1, V0 - tail: - CMP R2, $0 - BEQ done - FMOVD (R1), F1 - FADD F1, F0 - ADD $8, R1 - SUB $1, R2 - BR tail - done: - FMOVD F0, ret+16(FP) - RET - TEXT ·dotVX(SB), NOSPLIT, $0-32 MOVD a+0(FP), R1 MOVD b+8(FP), R3 @@ -82,71 +46,6 @@ TEXT ·dotVX(SB), NOSPLIT, $0-32 FMOVD F0, ret+24(FP) RET -TEXT ·sqrtVX(SB), NOSPLIT, $0-24 - MOVD dst+0(FP), R1 - MOVD src+8(FP), R2 - MOVD n+16(FP), R4 - loop: - CMP R4, $8 - BLT tail - VLM (R2), V0, V3 - VFSQDB V0, V0 - VFSQDB V1, V1 - VFSQDB V2, V2 - VFSQDB V3, V3 - VSTM V0, V3, (R1) - ADD $64, R1 - ADD $64, R2 - SUB $8, R4 - BR loop - tail: - CMP R4, $0 - BEQ done - FMOVD (R2), F0 - FSQRT F0, F0 - FMOVD F0, (R1) - ADD $8, R1 - ADD $8, R2 - SUB $1, R4 - BR tail - done: - RET - -TEXT ·addVX(SB), NOSPLIT, $0-32 - MOVD dst+0(FP), R1 - MOVD a+8(FP), R2 - MOVD b+16(FP), R3 - MOVD n+24(FP), R4 - loop: - CMP R4, $8 - BLT tail - VLM (R2), V0, V3 - VLM (R3), V4, V7 - VFADB V4, V0, V0 - VFADB V5, V1, V1 - VFADB V6, V2, V2 - VFADB V7, V3, V3 - VSTM V0, V3, (R1) - ADD $64, R1 - ADD $64, R2 - ADD $64, R3 - SUB $8, R4 - BR loop - tail: - CMP R4, $0 - BEQ done - FMOVD (R2), F0 - FMOVD (R3), F1 - FADD F1, F0 - FMOVD F0, (R1) - ADD $8, R1 - ADD $8, R2 - ADD $8, R3 - SUB $1, R4 - BR tail - done: - RET - TEXT ·add32VXE(SB), NOSPLIT, $0-32 MOVD dst+0(FP), R1 MOVD a+8(FP), R2 @@ -182,41 +81,6 @@ TEXT ·add32VXE(SB), NOSPLIT, $0-32 done: RET -TEXT ·subVX(SB), NOSPLIT, $0-32 - MOVD dst+0(FP), R1 - MOVD a+8(FP), R2 - MOVD b+16(FP), R3 - MOVD n+24(FP), R4 - loop: - CMP R4, $8 - BLT tail - VLM (R2), V0, V3 - VLM (R3), V4, V7 - VFSDB V4, V0, V0 - VFSDB V5, V1, V1 - VFSDB V6, V2, V2 - VFSDB V7, V3, V3 - VSTM V0, V3, (R1) - ADD $64, R1 - ADD $64, R2 - ADD $64, R3 - SUB $8, R4 - BR loop - tail: - CMP R4, $0 - BEQ done - FMOVD (R2), F0 - FMOVD (R3), F1 - FSUB F1, F0 - FMOVD F0, (R1) - ADD $8, R1 - ADD $8, R2 - ADD $8, R3 - SUB $1, R4 - BR tail - done: - RET - TEXT ·sub32VXE(SB), NOSPLIT, $0-32 MOVD dst+0(FP), R1 MOVD a+8(FP), R2 @@ -252,41 +116,6 @@ TEXT ·sub32VXE(SB), NOSPLIT, $0-32 done: RET -TEXT ·mulVX(SB), NOSPLIT, $0-32 - MOVD dst+0(FP), R1 - MOVD a+8(FP), R2 - MOVD b+16(FP), R3 - MOVD n+24(FP), R4 - loop: - CMP R4, $8 - BLT tail - VLM (R2), V0, V3 - VLM (R3), V4, V7 - VFMDB V4, V0, V0 - VFMDB V5, V1, V1 - VFMDB V6, V2, V2 - VFMDB V7, V3, V3 - VSTM V0, V3, (R1) - ADD $64, R1 - ADD $64, R2 - ADD $64, R3 - SUB $8, R4 - BR loop - tail: - CMP R4, $0 - BEQ done - FMOVD (R2), F0 - FMOVD (R3), F1 - FMUL F1, F0 - FMOVD F0, (R1) - ADD $8, R1 - ADD $8, R2 - ADD $8, R3 - SUB $1, R4 - BR tail - done: - RET - TEXT ·mul32VXE(SB), NOSPLIT, $0-32 MOVD dst+0(FP), R1 MOVD a+8(FP), R2 @@ -322,41 +151,6 @@ TEXT ·mul32VXE(SB), NOSPLIT, $0-32 done: RET -TEXT ·divVX(SB), NOSPLIT, $0-32 - MOVD dst+0(FP), R1 - MOVD a+8(FP), R2 - MOVD b+16(FP), R3 - MOVD n+24(FP), R4 - loop: - CMP R4, $8 - BLT tail - VLM (R2), V0, V3 - VLM (R3), V4, V7 - VFDDB V4, V0, V0 - VFDDB V5, V1, V1 - VFDDB V6, V2, V2 - VFDDB V7, V3, V3 - VSTM V0, V3, (R1) - ADD $64, R1 - ADD $64, R2 - ADD $64, R3 - SUB $8, R4 - BR loop - tail: - CMP R4, $0 - BEQ done - FMOVD (R2), F0 - FMOVD (R3), F1 - FDIV F1, F0 - FMOVD F0, (R1) - ADD $8, R1 - ADD $8, R2 - ADD $8, R3 - SUB $1, R4 - BR tail - done: - RET - TEXT ·div32VXE(SB), NOSPLIT, $0-32 MOVD dst+0(FP), R1 MOVD a+8(FP), R2