From bcc46a7187e718d69a5c5ef03f79fa14743c09d0 Mon Sep 17 00:00:00 2001 From: superme1one <2459380227@qq.com> Date: Thu, 5 Nov 2026 16:28:12 +0800 Subject: [PATCH 1/5] perf: specialize Iluvatar BF16 Gate-Up and Down GEMV --- docs/performance/iluvatar-mlp-gemv.md | 62 +++++++++++++++++++ src/native/cuda/iluvatar/ops/gemm/cublas.h | 39 +++++++++++- .../cuda/iluvatar/ops/gemm/gemv_policy.h | 30 +++++++++ src/native/cuda/iluvatar/ops/gemm/kernel.cuh | 54 ++++++++++++++++ src/native/cuda/iluvatar/ops/gemm/kernel.h | 27 ++++++++ tests/host/iluvatar_gemv_policy_test.cpp | 40 ++++++++++++ tests/test_iluvatar_mlp_gemv.py | 52 ++++++++++++++++ 7 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 docs/performance/iluvatar-mlp-gemv.md create mode 100644 src/native/cuda/iluvatar/ops/gemm/gemv_policy.h create mode 100644 src/native/cuda/iluvatar/ops/gemm/kernel.cuh create mode 100644 src/native/cuda/iluvatar/ops/gemm/kernel.h create mode 100644 tests/host/iluvatar_gemv_policy_test.cpp create mode 100644 tests/test_iluvatar_mlp_gemv.py diff --git a/docs/performance/iluvatar-mlp-gemv.md b/docs/performance/iluvatar-mlp-gemv.md new file mode 100644 index 000000000..9357c342a --- /dev/null +++ b/docs/performance/iluvatar-mlp-gemv.md @@ -0,0 +1,62 @@ +# Iluvatar small-batch MLP GEMV + +The default Iluvatar GEMM implementation specializes two BF16 decode shapes: + +| Projection | M | K | N | +| --- | --- | --- | --- | +| Combined Gate/Up | 1 or 2 | 4096 | 32768 | +| Down | 1 or 2 | 16384 | 4096 | + +A and Y are row-major; B has strides `(1, K)`. The fast path requires +alpha=1, beta=0, no C/bias, no transpose, matching devices and aligned BF16-pair +loads. All other calls use the existing BLAS implementation. The kernel shares +each weight load between the two rows and keeps four FP32 accumulators per lane. +This changes the GEMM accumulation order relative to BLAS; token equality is not +an acceptance requirement. + +## Validation and measurements + +The measured private implementation on TG-V200/CoreX used these same kernel +shapes and arithmetic. The submission factors the two identical algorithms into +one template, removes experimental counters, and adds an alignment fallback. +Its final integration on current upstream still requires an Iluvatar smoke build +and tests; the PR is a draft until that check is available. + +Historical same-card measurements, using two fixed GSM8K tasks with a 512-token +greedy budget, found Gate/Up reduced application completion time by +9.22–9.28%. Adding Down to that version reduced time by a further 6.58–6.75%. +These are sequential experiments with different baselines, not additive gains. + +The subsequent **four-optimization combination** (these two GEMVs, fused +Prefill causal Softmax, and Decode QK reduction) was compared against InfiniLM +`6e7f9537173cd33bb30539ebf8d92ab6d869660d` on 100 new fixed GSM8K targets. +Both versions for each target ran on the same card. The first 20 targets ran +serially on one card; the remaining 80 were sharded over two cards. The sum of +paired formal batch makespans was 749.203818 → 505.326616 seconds (-32.5515%); +request throughput increased 48.2613%. Output lengths changed, so token +throughput (+51.6653%) is reported separately. + +Accuracy was 82/100 → 84/100; all requests completed without errors or +truncation. There were 34 token-different targets, including one correct→wrong +(1097: reference 480, output 480→448) and three wrong→correct +(672, 1259, 1203). These combined results do not isolate either GEMV's quality +effect. Eight demonstrations came from a disjoint subset of the same test +corpus; this is an internal paired evaluation, not the standard benchmark split. +Generation budget, EOS and strict last-`####` numeric scoring were unchanged. + +InfiniLM main advanced to `cc6d52ae04f4c9ff17229bc7fc13a9ad105aca6d` +(auto-tuning initialization) after that experiment. This PR is based on InfiniOps +`1a68b261d19f0ed6f76bbabeab7a25f479f05e07`; the historical timings do not +measure this rebased revision. + +## Reproduce focused checks + +```sh +c++ -std=c++17 -Isrc tests/host/iluvatar_gemv_policy_test.cpp -o /tmp/gemv-policy +/tmp/gemv-policy +python -m pytest tests/test_iluvatar_mlp_gemv.py -q +``` + +The portable test checks eligibility/fallback boundaries. The device test covers +both shapes and batch sizes against a deterministic FP32 reference, plus the +bias/scalar BLAS fallback. Existing `tests/test_gemm.py` covers general GEMM. diff --git a/src/native/cuda/iluvatar/ops/gemm/cublas.h b/src/native/cuda/iluvatar/ops/gemm/cublas.h index d086cf4f4..18d68c55d 100644 --- a/src/native/cuda/iluvatar/ops/gemm/cublas.h +++ b/src/native/cuda/iluvatar/ops/gemm/cublas.h @@ -2,6 +2,8 @@ #define INFINI_OPS_ILUVATAR_GEMM_CUBLAS_H_ #include "native/cuda/iluvatar/blas.h" +#include "native/cuda/iluvatar/ops/gemm/gemv_policy.h" +#include "native/cuda/iluvatar/ops/gemm/kernel.h" #include "native/cuda/ops/gemm/blas.h" namespace infini::ops { @@ -9,8 +11,43 @@ namespace infini::ops { template <> class Operator : public BlasGemm> { + using Parent = BlasGemm>; + public: - using BlasGemm>::BlasGemm; + using Parent::Parent; + + using Parent::operator(); + + void operator()(const Tensor a, const Tensor b, const std::optional c, + std::optional alpha, std::optional beta, + std::optional trans_a, std::optional trans_b, + Tensor y) const override { + if (a.ndim() == 2 && b.ndim() == 2 && y.ndim() == 2 && + a.size(1) == b.size(0) && a.size(0) == y.size(0) && + b.size(1) == y.size(1) && + iluvatar::CanUseSmallBatchGemv( + a.dtype() == DataType::kBFloat16 && + b.dtype() == DataType::kBFloat16 && + y.dtype() == DataType::kBFloat16, + a.device() == b.device() && a.device() == y.device(), c.has_value(), + alpha.value_or(alpha_), beta.value_or(beta_), + trans_a.value_or(trans_a_), trans_b.value_or(trans_b_), a.size(0), + a.size(1), y.size(1), a.stride(0), a.stride(1), b.stride(0), + b.stride(1), y.stride(0), y.stride(1), + reinterpret_cast(a.data()), + reinterpret_cast(b.data()))) { + auto stream = static_cast(stream_ ? stream_ : 0); + if (a.size(1) == 4096) { + iluvatar::LaunchSmallBatchGemv<4096, 32768>( + a.data(), b.data(), y.data(), a.size(0), stream); + } else { + iluvatar::LaunchSmallBatchGemv<16384, 4096>( + a.data(), b.data(), y.data(), a.size(0), stream); + } + return; + } + Parent::operator()(a, b, c, alpha, beta, trans_a, trans_b, y); + } }; } // namespace infini::ops diff --git a/src/native/cuda/iluvatar/ops/gemm/gemv_policy.h b/src/native/cuda/iluvatar/ops/gemm/gemv_policy.h new file mode 100644 index 000000000..fed4e5179 --- /dev/null +++ b/src/native/cuda/iluvatar/ops/gemm/gemv_policy.h @@ -0,0 +1,30 @@ +#ifndef INFINI_OPS_ILUVATAR_GEMM_GEMV_POLICY_H_ +#define INFINI_OPS_ILUVATAR_GEMM_GEMV_POLICY_H_ + +#include +#include + +namespace infini::ops::iluvatar { + +inline bool CanUseSmallBatchGemv(bool bf16, bool same_device, bool has_bias, + float alpha, float beta, int trans_a, + int trans_b, std::size_t m, std::size_t k, + std::size_t n, std::ptrdiff_t a0, + std::ptrdiff_t a1, std::ptrdiff_t b0, + std::ptrdiff_t b1, std::ptrdiff_t y0, + std::ptrdiff_t y1, std::uintptr_t a_address, + std::uintptr_t b_address) { + // Each lane loads two adjacent BF16 values. Reject offset views that are + // not aligned for the packed load even when their strides match. + return bf16 && same_device && !has_bias && alpha == 1.0f && beta == 0.0f && + trans_a == 0 && trans_b == 0 && (m == 1 || m == 2) && + ((k == 4096 && n == 32768) || (k == 16384 && n == 4096)) && + a0 == static_cast(k) && a1 == 1 && b0 == 1 && + b1 == static_cast(k) && + y0 == static_cast(n) && y1 == 1 && + a_address % 4 == 0 && b_address % 4 == 0; +} + +} // namespace infini::ops::iluvatar + +#endif diff --git a/src/native/cuda/iluvatar/ops/gemm/kernel.cuh b/src/native/cuda/iluvatar/ops/gemm/kernel.cuh new file mode 100644 index 000000000..f7aadbc11 --- /dev/null +++ b/src/native/cuda/iluvatar/ops/gemm/kernel.cuh @@ -0,0 +1,54 @@ +#ifndef INFINI_OPS_ILUVATAR_GEMM_KERNEL_CUH_ +#define INFINI_OPS_ILUVATAR_GEMM_KERNEL_CUH_ + +#include +#include + +namespace infini::ops::iluvatar { + +template +__global__ void SmallBatchGemvKernel(const __nv_bfloat16* x, + const __nv_bfloat16* w, __nv_bfloat16* y) { + static_assert(kBatch == 1 || kBatch == 2); + static_assert((kInput == 4096 && kOutput == 32768) || + (kInput == 16384 && kOutput == 4096)); + const int lane = threadIdx.x & 31; + const int output = blockIdx.x * 4 + threadIdx.x / 32; + const auto* w2 = reinterpret_cast(w + output * kInput); + const auto* x2 = reinterpret_cast(x); + float s0[4] = {0.f, 0.f, 0.f, 0.f}; + float s1[4] = {0.f, 0.f, 0.f, 0.f}; + for (int k = lane; k < kInput / 2; k += 128) { +#pragma unroll + for (int j = 0; j < 4; ++j) { + float2 a = __bfloat1622float2(w2[k + j * 32]); + float2 b = __bfloat1622float2(x2[k + j * 32]); + s0[j] = fmaf(a.x, b.x, s0[j]); + s0[j] = fmaf(a.y, b.y, s0[j]); + if constexpr (kBatch == 2) { + float2 c = __bfloat1622float2(x2[kInput / 2 + k + j * 32]); + s1[j] = fmaf(a.x, c.x, s1[j]); + s1[j] = fmaf(a.y, c.y, s1[j]); + } + } + } + // Keep the four-accumulator FP32 tree used in the measured specialization. + float a = (s0[0] + s0[1]) + (s0[2] + s0[3]); + float b = (s1[0] + s1[1]) + (s1[2] + s1[3]); + for (int delta = 16; delta > 0; delta /= 2) { + a += __shfl_down_sync(0xffffffffu, a, delta); + if constexpr (kBatch == 2) { + b += __shfl_down_sync(0xffffffffu, b, delta); + } + } + if (lane == 0) { + y[output] = __float2bfloat16(a); + if constexpr (kBatch == 2) { + y[kOutput + output] = __float2bfloat16(b); + } + } +} + +} // namespace infini::ops::iluvatar + +#endif diff --git a/src/native/cuda/iluvatar/ops/gemm/kernel.h b/src/native/cuda/iluvatar/ops/gemm/kernel.h new file mode 100644 index 000000000..18b8c60ce --- /dev/null +++ b/src/native/cuda/iluvatar/ops/gemm/kernel.h @@ -0,0 +1,27 @@ +#ifndef INFINI_OPS_ILUVATAR_GEMM_KERNEL_H_ +#define INFINI_OPS_ILUVATAR_GEMM_KERNEL_H_ + +#include + +#include "native/cuda/iluvatar/ops/gemm/kernel.cuh" + +namespace infini::ops::iluvatar { + +template +inline void LaunchSmallBatchGemv(const void* a, const void* b, void* y, + std::size_t batch, cudaStream_t stream) { + const auto* x = static_cast(a); + const auto* w = static_cast(b); + auto* out = static_cast<__nv_bfloat16*>(y); + if (batch == 1) { + SmallBatchGemvKernel<1, kInput, kOutput> + <<>>(x, w, out); + } else { + SmallBatchGemvKernel<2, kInput, kOutput> + <<>>(x, w, out); + } +} + +} // namespace infini::ops::iluvatar + +#endif diff --git a/tests/host/iluvatar_gemv_policy_test.cpp b/tests/host/iluvatar_gemv_policy_test.cpp new file mode 100644 index 000000000..92e0d67be --- /dev/null +++ b/tests/host/iluvatar_gemv_policy_test.cpp @@ -0,0 +1,40 @@ +#include +#include +#include + +#include "native/cuda/iluvatar/ops/gemm/gemv_policy.h" + +int main() { + using infini::ops::iluvatar::CanUseSmallBatchGemv; + for (int down : {0, 1}) { + const int k = down ? 16384 : 4096; + const int n = down ? 4096 : 32768; + for (int m : {1, 2}) { + const auto valid = [&](bool bf16 = true, bool same = true, + bool bias = false, float alpha = 1, float beta = 0, + int ta = 0, int tb = 0, int rows = 0, + int a_stride = 0, std::uintptr_t address = 4096) { + return CanUseSmallBatchGemv( + bf16, same, bias, alpha, beta, ta, tb, rows ? rows : m, k, n, + a_stride ? a_stride : k, 1, 1, k, n, 1, address, 8192); + }; + assert(valid()); + assert(!valid(false)); + assert(!valid(true, false)); + assert(!valid(true, true, true)); + assert(!valid(true, true, false, 0.5f)); + assert(!valid(true, true, false, 1, 1)); + assert(!valid(true, true, false, 1, 0, 1)); + assert(!valid(true, true, false, 1, 0, 0, 1)); + assert(!valid(true, true, false, 1, 0, 0, 0, 3)); + assert(!valid(true, true, false, 1, 0, 0, 0, m, k + 1)); + assert(!valid(true, true, false, 1, 0, 0, 0, m, k, 4098)); + assert( + !valid(true, true, false, std::numeric_limits::quiet_NaN())); + assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, m, k, n + 1, + k, 1, 1, k, n + 1, 1, 4096, 8192)); + assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, m, k, n, k, 1, + n, 1, n, 1, 4096, 8192)); + } + } +} diff --git a/tests/test_iluvatar_mlp_gemv.py b/tests/test_iluvatar_mlp_gemv.py new file mode 100644 index 000000000..ee511318e --- /dev/null +++ b/tests/test_iluvatar_mlp_gemv.py @@ -0,0 +1,52 @@ +import infini.ops +import pytest +import torch + +from tests.utils import get_stream + + +@pytest.mark.parametrize("m", (1, 2)) +@pytest.mark.parametrize("k,n", ((4096, 32768), (16384, 4096))) +def test_iluvatar_mlp_gemv(m, k, n, device): + if device != "cuda" or not infini.ops.Gemm.active_implementation_indices( + "iluvatar" + ): + pytest.skip("requires the Iluvatar backend") + + # Bounded dyadic values exercise both signs without tolerance masking errors. + x = ((torch.arange(m * k, device=device) % 7 - 3).float() / 16).reshape(m, k) + w = ((torch.arange(n * k, device=device) % 5 - 2).float() / 16).reshape(n, k) + a = x.to(torch.bfloat16) + b = w.to(torch.bfloat16).t() + out = torch.empty((m, n), dtype=torch.bfloat16, device=device) + expected = (a.float() @ b.float()).to(torch.bfloat16) + infini.ops.gemm( + a, + b, + None, + 1.0, + 0.0, + False, + False, + out, + stream=get_stream(a.device), + implementation_index=0, + ) + torch.testing.assert_close(out, expected, rtol=0, atol=0) + + # Bias/scalar calls must retain BLAS semantics outside the specialization. + bias = torch.ones_like(out) + infini.ops.gemm( + a, + b, + bias, + 0.5, + 1.0, + False, + False, + out, + stream=get_stream(a.device), + implementation_index=0, + ) + expected = (0.5 * (a.float() @ b.float()) + bias.float()).to(torch.bfloat16) + torch.testing.assert_close(out, expected, rtol=1e-2, atol=1e-2) From 029ebc0816951d93362c6e063e09effc51e44241 Mon Sep 17 00:00:00 2001 From: superme1one <2459380227@qq.com> Date: Fri, 6 Nov 2026 10:27:56 +0800 Subject: [PATCH 2/5] test: bind Iluvatar regression to implementation zero --- tests/test_iluvatar_mlp_gemv.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_iluvatar_mlp_gemv.py b/tests/test_iluvatar_mlp_gemv.py index ee511318e..e7be41782 100644 --- a/tests/test_iluvatar_mlp_gemv.py +++ b/tests/test_iluvatar_mlp_gemv.py @@ -7,7 +7,8 @@ @pytest.mark.parametrize("m", (1, 2)) @pytest.mark.parametrize("k,n", ((4096, 32768), (16384, 4096))) -def test_iluvatar_mlp_gemv(m, k, n, device): +@pytest.mark.parametrize("implementation_index", (0,)) +def test_iluvatar_mlp_gemv(m, k, n, device, implementation_index): if device != "cuda" or not infini.ops.Gemm.active_implementation_indices( "iluvatar" ): @@ -30,7 +31,7 @@ def test_iluvatar_mlp_gemv(m, k, n, device): False, out, stream=get_stream(a.device), - implementation_index=0, + implementation_index=implementation_index, ) torch.testing.assert_close(out, expected, rtol=0, atol=0) @@ -46,7 +47,7 @@ def test_iluvatar_mlp_gemv(m, k, n, device): False, out, stream=get_stream(a.device), - implementation_index=0, + implementation_index=implementation_index, ) expected = (0.5 * (a.float() @ b.float()) + bias.float()).to(torch.bfloat16) torch.testing.assert_close(out, expected, rtol=1e-2, atol=1e-2) From 9cc5e9adcf38411f6597a41b4d759d48e9eb03eb Mon Sep 17 00:00:00 2001 From: superme1one <2459380227@qq.com> Date: Fri, 6 Nov 2026 10:34:27 +0800 Subject: [PATCH 3/5] test: respect current GEMM no-C input contract --- tests/test_iluvatar_mlp_gemv.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_iluvatar_mlp_gemv.py b/tests/test_iluvatar_mlp_gemv.py index e7be41782..35d3e239a 100644 --- a/tests/test_iluvatar_mlp_gemv.py +++ b/tests/test_iluvatar_mlp_gemv.py @@ -35,19 +35,19 @@ def test_iluvatar_mlp_gemv(m, k, n, device, implementation_index): ) torch.testing.assert_close(out, expected, rtol=0, atol=0) - # Bias/scalar calls must retain BLAS semantics outside the specialization. - bias = torch.ones_like(out) + # Non-default alpha must retain the supported BLAS fallback. + # The current base GEMM contract requires C=None and forces beta=0. infini.ops.gemm( a, b, - bias, + None, 0.5, - 1.0, + 0.0, False, False, out, stream=get_stream(a.device), implementation_index=implementation_index, ) - expected = (0.5 * (a.float() @ b.float()) + bias.float()).to(torch.bfloat16) + expected = (0.5 * (a.float() @ b.float())).to(torch.bfloat16) torch.testing.assert_close(out, expected, rtol=1e-2, atol=1e-2) From 15a1a69b6d522b1bac631bca3171dbfb934eff3c Mon Sep 17 00:00:00 2001 From: superme1one <2459380227@qq.com> Date: Fri, 6 Nov 2026 11:12:39 +0800 Subject: [PATCH 4/5] docs(perf): record current CoreX build and GPU validation --- docs/performance/iluvatar-mlp-gemv.md | 48 +++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/docs/performance/iluvatar-mlp-gemv.md b/docs/performance/iluvatar-mlp-gemv.md index 9357c342a..9ead41e3c 100644 --- a/docs/performance/iluvatar-mlp-gemv.md +++ b/docs/performance/iluvatar-mlp-gemv.md @@ -19,8 +19,7 @@ an acceptance requirement. The measured private implementation on TG-V200/CoreX used these same kernel shapes and arithmetic. The submission factors the two identical algorithms into one template, removes experimental counters, and adds an alignment fallback. -Its final integration on current upstream still requires an Iluvatar smoke build -and tests; the PR is a draft until that check is available. +Its current submitted-code CoreX build and GPU checks are recorded below. Historical same-card measurements, using two fixed GSM8K tasks with a 512-token greedy budget, found Gate/Up reduced application completion time by @@ -59,4 +58,47 @@ python -m pytest tests/test_iluvatar_mlp_gemv.py -q The portable test checks eligibility/fallback boundaries. The device test covers both shapes and batch sizes against a deterministic FP32 reference, plus the -bias/scalar BLAS fallback. Existing `tests/test_gemm.py` covers general GEMM. +supported no-C scalar BLAS fallback. Existing `tests/test_gemm.py` covers general GEMM. + +## Current submitted-code CoreX validation + +CoreX current-code validation completed on 2026-11-06 against InfiniLM +`12a4492f70dd33235e3d142e333ad4a9629d3b7b` and InfiniOps +`1a68b261d19f0ed6f76bbabeab7a25f479f05e07`. Both complete SDKs compiled +and imported. Both roles used the identical documented CoreX validation adapter +and ABI0, with automatic tuning disabled. The adapter includes existing CUDA +Argmax/Triu/Tril registrations, unchanged rejecting unavailable factories, a +copy-on-write read-only-view loader, and the existing paged-only FlashVarlen +provider's metadata gate. See the companion InfiniLM +`docs/performance/corex-validation/README.md` for exact scope. + +The affected native GPU tests passed without skips: GEMV 4, causal Softmax 8, +Decode Attention 8, plus 7 common Argmax/mask checks. The submitted device tests +now explicitly bind implementation index 0. GEMV fallback uses the supported +no-C, alpha=0.5 contract; this backend rejects C/bias at its base operator. + +Original tasks 554/172 ran on one TG-V200 in forward and reverse role order. +Each of the four model processes used one simultaneous two-request warmup wave +and one simultaneous two-request measured wave: 16 requests total, 8 measured. +All 16 answers were correct with natural EOS, zero errors/truncations, normal +scope return, final library maps, atexit and actual OS exit0. The original +512-token greedy budget, EOS=[2,73440], strict last-#### numeric scorer, inputs +and weights were unchanged. Two physical rows, full Prefill and graph off were +held fixed. No new independent quality questions were added. + +Mean two-request measured-wave time was **14.976775177 -> +9.782192262 seconds (-34.6843%)**. +Completed request throughput was **0.133540096 -> 0.204453148 +requests/s (+53.1024%)**. Published token throughput +was 20.631944884 -> 31.588011330 tokens/s and is reported separately. +The time denominator includes queue and text processing and excludes loading, +warmup and final shutdown. Both role orders reduced application time; this is +the combined four-optimization gain, not an individual-PR or component sum. + +| Order | Official wave, s | Combined wave, s | Time reduction | +| --- | ---: | ---: | ---: | +| forward | 14.973374084 | 9.782256734 | 34.6690% | +| reverse | 14.980176271 | 9.782127790 | 34.6995% | + +Token-different task IDs across the two measured orders: none. Answers and token differences are recorded separately. +Raw clocks, output IDs, strict scores, native receipts and ordinary exits were independently recomputed. This two-question integration check is kept separate from the historical 100-question experiment above. From de7b6cb8335ef26efb794cdf58e04e6befd1cf7a Mon Sep 17 00:00:00 2001 From: superme1one <2459380227@qq.com> Date: Mon, 9 Nov 2026 11:46:43 +0800 Subject: [PATCH 5/5] perf(iluvatar): generalize small-batch GEMV after review --- docs/performance/iluvatar-mlp-gemv.md | 104 ------------------ src/native/cuda/iluvatar/ops/gemm/cublas.h | 17 +-- .../cuda/iluvatar/ops/gemm/gemv_policy.h | 13 ++- src/native/cuda/iluvatar/ops/gemm/kernel.cuh | 24 ++-- src/native/cuda/iluvatar/ops/gemm/kernel.h | 23 +++- tests/host/iluvatar_gemv_policy_test.cpp | 49 ++++++++- tests/test_iluvatar_mlp_gemv.py | 93 +++++++++++++++- 7 files changed, 183 insertions(+), 140 deletions(-) delete mode 100644 docs/performance/iluvatar-mlp-gemv.md diff --git a/docs/performance/iluvatar-mlp-gemv.md b/docs/performance/iluvatar-mlp-gemv.md deleted file mode 100644 index 9ead41e3c..000000000 --- a/docs/performance/iluvatar-mlp-gemv.md +++ /dev/null @@ -1,104 +0,0 @@ -# Iluvatar small-batch MLP GEMV - -The default Iluvatar GEMM implementation specializes two BF16 decode shapes: - -| Projection | M | K | N | -| --- | --- | --- | --- | -| Combined Gate/Up | 1 or 2 | 4096 | 32768 | -| Down | 1 or 2 | 16384 | 4096 | - -A and Y are row-major; B has strides `(1, K)`. The fast path requires -alpha=1, beta=0, no C/bias, no transpose, matching devices and aligned BF16-pair -loads. All other calls use the existing BLAS implementation. The kernel shares -each weight load between the two rows and keeps four FP32 accumulators per lane. -This changes the GEMM accumulation order relative to BLAS; token equality is not -an acceptance requirement. - -## Validation and measurements - -The measured private implementation on TG-V200/CoreX used these same kernel -shapes and arithmetic. The submission factors the two identical algorithms into -one template, removes experimental counters, and adds an alignment fallback. -Its current submitted-code CoreX build and GPU checks are recorded below. - -Historical same-card measurements, using two fixed GSM8K tasks with a 512-token -greedy budget, found Gate/Up reduced application completion time by -9.22–9.28%. Adding Down to that version reduced time by a further 6.58–6.75%. -These are sequential experiments with different baselines, not additive gains. - -The subsequent **four-optimization combination** (these two GEMVs, fused -Prefill causal Softmax, and Decode QK reduction) was compared against InfiniLM -`6e7f9537173cd33bb30539ebf8d92ab6d869660d` on 100 new fixed GSM8K targets. -Both versions for each target ran on the same card. The first 20 targets ran -serially on one card; the remaining 80 were sharded over two cards. The sum of -paired formal batch makespans was 749.203818 → 505.326616 seconds (-32.5515%); -request throughput increased 48.2613%. Output lengths changed, so token -throughput (+51.6653%) is reported separately. - -Accuracy was 82/100 → 84/100; all requests completed without errors or -truncation. There were 34 token-different targets, including one correct→wrong -(1097: reference 480, output 480→448) and three wrong→correct -(672, 1259, 1203). These combined results do not isolate either GEMV's quality -effect. Eight demonstrations came from a disjoint subset of the same test -corpus; this is an internal paired evaluation, not the standard benchmark split. -Generation budget, EOS and strict last-`####` numeric scoring were unchanged. - -InfiniLM main advanced to `cc6d52ae04f4c9ff17229bc7fc13a9ad105aca6d` -(auto-tuning initialization) after that experiment. This PR is based on InfiniOps -`1a68b261d19f0ed6f76bbabeab7a25f479f05e07`; the historical timings do not -measure this rebased revision. - -## Reproduce focused checks - -```sh -c++ -std=c++17 -Isrc tests/host/iluvatar_gemv_policy_test.cpp -o /tmp/gemv-policy -/tmp/gemv-policy -python -m pytest tests/test_iluvatar_mlp_gemv.py -q -``` - -The portable test checks eligibility/fallback boundaries. The device test covers -both shapes and batch sizes against a deterministic FP32 reference, plus the -supported no-C scalar BLAS fallback. Existing `tests/test_gemm.py` covers general GEMM. - -## Current submitted-code CoreX validation - -CoreX current-code validation completed on 2026-11-06 against InfiniLM -`12a4492f70dd33235e3d142e333ad4a9629d3b7b` and InfiniOps -`1a68b261d19f0ed6f76bbabeab7a25f479f05e07`. Both complete SDKs compiled -and imported. Both roles used the identical documented CoreX validation adapter -and ABI0, with automatic tuning disabled. The adapter includes existing CUDA -Argmax/Triu/Tril registrations, unchanged rejecting unavailable factories, a -copy-on-write read-only-view loader, and the existing paged-only FlashVarlen -provider's metadata gate. See the companion InfiniLM -`docs/performance/corex-validation/README.md` for exact scope. - -The affected native GPU tests passed without skips: GEMV 4, causal Softmax 8, -Decode Attention 8, plus 7 common Argmax/mask checks. The submitted device tests -now explicitly bind implementation index 0. GEMV fallback uses the supported -no-C, alpha=0.5 contract; this backend rejects C/bias at its base operator. - -Original tasks 554/172 ran on one TG-V200 in forward and reverse role order. -Each of the four model processes used one simultaneous two-request warmup wave -and one simultaneous two-request measured wave: 16 requests total, 8 measured. -All 16 answers were correct with natural EOS, zero errors/truncations, normal -scope return, final library maps, atexit and actual OS exit0. The original -512-token greedy budget, EOS=[2,73440], strict last-#### numeric scorer, inputs -and weights were unchanged. Two physical rows, full Prefill and graph off were -held fixed. No new independent quality questions were added. - -Mean two-request measured-wave time was **14.976775177 -> -9.782192262 seconds (-34.6843%)**. -Completed request throughput was **0.133540096 -> 0.204453148 -requests/s (+53.1024%)**. Published token throughput -was 20.631944884 -> 31.588011330 tokens/s and is reported separately. -The time denominator includes queue and text processing and excludes loading, -warmup and final shutdown. Both role orders reduced application time; this is -the combined four-optimization gain, not an individual-PR or component sum. - -| Order | Official wave, s | Combined wave, s | Time reduction | -| --- | ---: | ---: | ---: | -| forward | 14.973374084 | 9.782256734 | 34.6690% | -| reverse | 14.980176271 | 9.782127790 | 34.6995% | - -Token-different task IDs across the two measured orders: none. Answers and token differences are recorded separately. -Raw clocks, output IDs, strict scores, native receipts and ordinary exits were independently recomputed. This two-question integration check is kept separate from the historical 100-question experiment above. diff --git a/src/native/cuda/iluvatar/ops/gemm/cublas.h b/src/native/cuda/iluvatar/ops/gemm/cublas.h index 18d68c55d..4680a5cb9 100644 --- a/src/native/cuda/iluvatar/ops/gemm/cublas.h +++ b/src/native/cuda/iluvatar/ops/gemm/cublas.h @@ -11,12 +11,12 @@ namespace infini::ops { template <> class Operator : public BlasGemm> { - using Parent = BlasGemm>; + using Base = BlasGemm>; public: - using Parent::Parent; + using Base::Base; - using Parent::operator(); + using Base::operator(); void operator()(const Tensor a, const Tensor b, const std::optional c, std::optional alpha, std::optional beta, @@ -37,16 +37,11 @@ class Operator reinterpret_cast(a.data()), reinterpret_cast(b.data()))) { auto stream = static_cast(stream_ ? stream_ : 0); - if (a.size(1) == 4096) { - iluvatar::LaunchSmallBatchGemv<4096, 32768>( - a.data(), b.data(), y.data(), a.size(0), stream); - } else { - iluvatar::LaunchSmallBatchGemv<16384, 4096>( - a.data(), b.data(), y.data(), a.size(0), stream); - } + iluvatar::LaunchSmallBatchGemv(a.data(), b.data(), y.data(), a.size(0), + a.size(1), y.size(1), stream); return; } - Parent::operator()(a, b, c, alpha, beta, trans_a, trans_b, y); + Base::operator()(a, b, c, alpha, beta, trans_a, trans_b, y); } }; diff --git a/src/native/cuda/iluvatar/ops/gemm/gemv_policy.h b/src/native/cuda/iluvatar/ops/gemm/gemv_policy.h index fed4e5179..b94e724bd 100644 --- a/src/native/cuda/iluvatar/ops/gemm/gemv_policy.h +++ b/src/native/cuda/iluvatar/ops/gemm/gemv_policy.h @@ -3,10 +3,11 @@ #include #include +#include namespace infini::ops::iluvatar { -inline bool CanUseSmallBatchGemv(bool bf16, bool same_device, bool has_bias, +inline bool CanUseSmallBatchGemv(bool all_bf16, bool same_device, bool has_bias, float alpha, float beta, int trans_a, int trans_b, std::size_t m, std::size_t k, std::size_t n, std::ptrdiff_t a0, @@ -16,9 +17,13 @@ inline bool CanUseSmallBatchGemv(bool bf16, bool same_device, bool has_bias, std::uintptr_t b_address) { // Each lane loads two adjacent BF16 values. Reject offset views that are // not aligned for the packed load even when their strides match. - return bf16 && same_device && !has_bias && alpha == 1.0f && beta == 0.0f && - trans_a == 0 && trans_b == 0 && (m == 1 || m == 2) && - ((k == 4096 && n == 32768) || (k == 16384 && n == 4096)) && + constexpr std::size_t kMinWeightElements = 8 * 1024 * 1024; + return all_bf16 && same_device && !has_bias && alpha == 1.0f && + beta == 0.0f && trans_a == 0 && trans_b == 0 && (m == 1 || m == 2) && + k >= 1024 && k % 64 == 0 && n >= 1024 && n % 4 == 0 && + k <= static_cast(std::numeric_limits::max()) && + n <= static_cast(std::numeric_limits::max()) && + k >= (kMinWeightElements + n - 1) / n && a0 == static_cast(k) && a1 == 1 && b0 == 1 && b1 == static_cast(k) && y0 == static_cast(n) && y1 == 1 && diff --git a/src/native/cuda/iluvatar/ops/gemm/kernel.cuh b/src/native/cuda/iluvatar/ops/gemm/kernel.cuh index f7aadbc11..e966d28a2 100644 --- a/src/native/cuda/iluvatar/ops/gemm/kernel.cuh +++ b/src/native/cuda/iluvatar/ops/gemm/kernel.cuh @@ -4,29 +4,35 @@ #include #include +#include + namespace infini::ops::iluvatar { -template +template __global__ void SmallBatchGemvKernel(const __nv_bfloat16* x, - const __nv_bfloat16* w, __nv_bfloat16* y) { + const __nv_bfloat16* w, __nv_bfloat16* y, + std::size_t input, std::size_t outputs) { static_assert(kBatch == 1 || kBatch == 2); - static_assert((kInput == 4096 && kOutput == 32768) || - (kInput == 16384 && kOutput == 4096)); const int lane = threadIdx.x & 31; - const int output = blockIdx.x * 4 + threadIdx.x / 32; - const auto* w2 = reinterpret_cast(w + output * kInput); + const std::size_t output = blockIdx.x * 4 + threadIdx.x / 32; + const auto* w2 = reinterpret_cast(w + output * input); const auto* x2 = reinterpret_cast(x); float s0[4] = {0.f, 0.f, 0.f, 0.f}; float s1[4] = {0.f, 0.f, 0.f, 0.f}; - for (int k = lane; k < kInput / 2; k += 128) { + for (std::size_t k = lane; k < input / 2; k += 128) { #pragma unroll for (int j = 0; j < 4; ++j) { + if constexpr (kHasTail) { + if (k + j * 32 >= input / 2) { + continue; + } + } float2 a = __bfloat1622float2(w2[k + j * 32]); float2 b = __bfloat1622float2(x2[k + j * 32]); s0[j] = fmaf(a.x, b.x, s0[j]); s0[j] = fmaf(a.y, b.y, s0[j]); if constexpr (kBatch == 2) { - float2 c = __bfloat1622float2(x2[kInput / 2 + k + j * 32]); + float2 c = __bfloat1622float2(x2[input / 2 + k + j * 32]); s1[j] = fmaf(a.x, c.x, s1[j]); s1[j] = fmaf(a.y, c.y, s1[j]); } @@ -44,7 +50,7 @@ __global__ void SmallBatchGemvKernel(const __nv_bfloat16* x, if (lane == 0) { y[output] = __float2bfloat16(a); if constexpr (kBatch == 2) { - y[kOutput + output] = __float2bfloat16(b); + y[outputs + output] = __float2bfloat16(b); } } } diff --git a/src/native/cuda/iluvatar/ops/gemm/kernel.h b/src/native/cuda/iluvatar/ops/gemm/kernel.h index 18b8c60ce..65ac9640d 100644 --- a/src/native/cuda/iluvatar/ops/gemm/kernel.h +++ b/src/native/cuda/iluvatar/ops/gemm/kernel.h @@ -7,18 +7,29 @@ namespace infini::ops::iluvatar { -template +template +inline void LaunchSmallBatchGemv(const __nv_bfloat16* x, const __nv_bfloat16* w, + __nv_bfloat16* out, std::size_t input, + std::size_t output, cudaStream_t stream) { + if (input % 256 == 0) { + SmallBatchGemvKernel + <<>>(x, w, out, input, output); + } else { + SmallBatchGemvKernel + <<>>(x, w, out, input, output); + } +} + inline void LaunchSmallBatchGemv(const void* a, const void* b, void* y, - std::size_t batch, cudaStream_t stream) { + std::size_t batch, std::size_t input, + std::size_t output, cudaStream_t stream) { const auto* x = static_cast(a); const auto* w = static_cast(b); auto* out = static_cast<__nv_bfloat16*>(y); if (batch == 1) { - SmallBatchGemvKernel<1, kInput, kOutput> - <<>>(x, w, out); + LaunchSmallBatchGemv<1>(x, w, out, input, output, stream); } else { - SmallBatchGemvKernel<2, kInput, kOutput> - <<>>(x, w, out); + LaunchSmallBatchGemv<2>(x, w, out, input, output, stream); } } diff --git a/tests/host/iluvatar_gemv_policy_test.cpp b/tests/host/iluvatar_gemv_policy_test.cpp index 92e0d67be..7c702a08e 100644 --- a/tests/host/iluvatar_gemv_policy_test.cpp +++ b/tests/host/iluvatar_gemv_policy_test.cpp @@ -1,21 +1,37 @@ #include #include #include +#include #include "native/cuda/iluvatar/ops/gemm/gemv_policy.h" int main() { using infini::ops::iluvatar::CanUseSmallBatchGemv; - for (int down : {0, 1}) { - const int k = down ? 16384 : 4096; - const int n = down ? 4096 : 32768; + for (const auto [k, n] : {std::pair{2048, 4096}, + {4160, 2052}, + {4224, 2056}, + {4288, 2060}, + {2048, 11008}, + {5504, 2048}, + {3072, 16384}, + {8192, 3072}, + {4096, 22016}, + {11008, 4096}, + {4096, 28672}, + {14336, 4096}, + {5120, 27648}, + {13824, 5120}, + {8192, 57344}, + {28672, 8192}, + {4096, 32768}, + {16384, 4096}}) { for (int m : {1, 2}) { - const auto valid = [&](bool bf16 = true, bool same = true, + const auto valid = [&](bool all_bf16 = true, bool same = true, bool bias = false, float alpha = 1, float beta = 0, int ta = 0, int tb = 0, int rows = 0, int a_stride = 0, std::uintptr_t address = 4096) { return CanUseSmallBatchGemv( - bf16, same, bias, alpha, beta, ta, tb, rows ? rows : m, k, n, + all_bf16, same, bias, alpha, beta, ta, tb, rows ? rows : m, k, n, a_stride ? a_stride : k, 1, 1, k, n, 1, address, 8192); }; assert(valid()); @@ -35,6 +51,29 @@ int main() { k, 1, 1, k, n + 1, 1, 4096, 8192)); assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, m, k, n, k, 1, n, 1, n, 1, 4096, 8192)); + assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, m, k, n, k, 1, + 1, k, n, 1, 4096, 8194)); + assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, m, k, n, k, 1, + 1, k, n + 1, 1, 4096, 8192)); } } + for (std::size_t m : {0, 3, 4}) { + assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, m, 2048, 4096, + 2048, 1, 1, 2048, 4096, 1, 4096, 8192)); + } + const std::size_t overflow = + static_cast(std::numeric_limits::max()) + 1; + for (std::size_t k : + {std::size_t{0}, std::size_t{768}, std::size_t{1025}, overflow}) { + assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, 1, k, 16384, k, + 1, 1, k, 16384, 1, 4096, 8192)); + } + for (std::size_t n : + {std::size_t{0}, std::size_t{1020}, std::size_t{1025}, overflow}) { + assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, 1, 16384, n, + 16384, 1, 1, 16384, n, 1, 4096, 8192)); + } + // Weights below the measured size threshold retain the BLAS implementation. + assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, 1, 2048, 4092, + 2048, 1, 1, 2048, 4092, 1, 4096, 8192)); } diff --git a/tests/test_iluvatar_mlp_gemv.py b/tests/test_iluvatar_mlp_gemv.py index 35d3e239a..d5f1a5bab 100644 --- a/tests/test_iluvatar_mlp_gemv.py +++ b/tests/test_iluvatar_mlp_gemv.py @@ -5,8 +5,31 @@ from tests.utils import get_stream +@pytest.mark.smoke @pytest.mark.parametrize("m", (1, 2)) -@pytest.mark.parametrize("k,n", ((4096, 32768), (16384, 4096))) +@pytest.mark.parametrize( + "k,n", + ( + (2048, 4096), + (4160, 2052), + (4224, 2056), + (4288, 2060), + (2048, 11008), + (5504, 2048), + (3072, 16384), + (8192, 3072), + (4096, 22016), + (11008, 4096), + (4096, 28672), + (14336, 4096), + (5120, 27648), + (13824, 5120), + (8192, 57344), + (28672, 8192), + (4096, 32768), + (16384, 4096), + ), +) @pytest.mark.parametrize("implementation_index", (0,)) def test_iluvatar_mlp_gemv(m, k, n, device, implementation_index): if device != "cuda" or not infini.ops.Gemm.active_implementation_indices( @@ -51,3 +74,71 @@ def test_iluvatar_mlp_gemv(m, k, n, device, implementation_index): ) expected = (0.5 * (a.float() @ b.float())).to(torch.bfloat16) torch.testing.assert_close(out, expected, rtol=1e-2, atol=1e-2) + + a = (torch.randn((m, k), device=device) / k**0.5).to(torch.bfloat16) + b = torch.randn((n, k), device=device, dtype=torch.bfloat16).t() + expected = (a.float() @ b.float()).to(torch.bfloat16) + infini.ops.gemm( + a, + b, + None, + 1.0, + 0.0, + False, + False, + out, + stream=get_stream(a.device), + implementation_index=implementation_index, + ) + torch.testing.assert_close(out, expected, rtol=1e-2, atol=1e-2) + + +@pytest.mark.smoke +@pytest.mark.parametrize( + "m,k,n,layout", + ( + (1, 1024, 1024, "column"), + (2, 1088, 1028, "column"), + (3, 2048, 4096, "column"), + (1, 768, 16384, "column"), + (1, 2049, 4096, "column"), + (1, 2048, 4097, "column"), + (1, 2048, 4096, "row"), + (2, 2048, 4096, "padded_a"), + (1, 2048, 4096, "padded_b"), + ), +) +@pytest.mark.parametrize("implementation_index", (0,)) +def test_iluvatar_mlp_gemv_fallback(m, k, n, layout, device, implementation_index): + if device != "cuda" or not infini.ops.Gemm.active_implementation_indices( + "iluvatar" + ): + pytest.skip("requires the Iluvatar backend") + + a = torch.randn((m, k), device=device, dtype=torch.bfloat16) + b = torch.randn((n, k), device=device, dtype=torch.bfloat16).t() + + if layout == "row": + b = b.contiguous() + elif layout == "padded_a": + storage = torch.empty((m, k + 8), device=device, dtype=torch.bfloat16) + a = storage[:, :k].copy_(a) + elif layout == "padded_b": + storage = torch.empty((n, k + 8), device=device, dtype=torch.bfloat16) + b = storage[:, :k].t().copy_(b) + + out = torch.empty((m, n), device=device, dtype=torch.bfloat16) + expected = (a.float() @ b.float()).to(torch.bfloat16) + infini.ops.gemm( + a, + b, + None, + 1.0, + 0.0, + False, + False, + out, + stream=get_stream(a.device), + implementation_index=implementation_index, + ) + torch.testing.assert_close(out, expected, rtol=1e-2, atol=1e-2)