diff --git a/src/native/cuda/iluvatar/ops/gemm/cublas.h b/src/native/cuda/iluvatar/ops/gemm/cublas.h index d086cf4f4..4680a5cb9 100644 --- a/src/native/cuda/iluvatar/ops/gemm/cublas.h +++ b/src/native/cuda/iluvatar/ops/gemm/cublas.h @@ -2,6 +2,8 @@ #define INFINI_OPS_ILUVATAR_GEMM_CUBLAS_H_ #include "native/cuda/iluvatar/blas.h" +#include "native/cuda/iluvatar/ops/gemm/gemv_policy.h" +#include "native/cuda/iluvatar/ops/gemm/kernel.h" #include "native/cuda/ops/gemm/blas.h" namespace infini::ops { @@ -9,8 +11,38 @@ namespace infini::ops { template <> class Operator : public BlasGemm> { + using Base = BlasGemm>; + public: - using BlasGemm>::BlasGemm; + using Base::Base; + + using Base::operator(); + + void operator()(const Tensor a, const Tensor b, const std::optional c, + std::optional alpha, std::optional beta, + std::optional trans_a, std::optional trans_b, + Tensor y) const override { + if (a.ndim() == 2 && b.ndim() == 2 && y.ndim() == 2 && + a.size(1) == b.size(0) && a.size(0) == y.size(0) && + b.size(1) == y.size(1) && + iluvatar::CanUseSmallBatchGemv( + a.dtype() == DataType::kBFloat16 && + b.dtype() == DataType::kBFloat16 && + y.dtype() == DataType::kBFloat16, + a.device() == b.device() && a.device() == y.device(), c.has_value(), + alpha.value_or(alpha_), beta.value_or(beta_), + trans_a.value_or(trans_a_), trans_b.value_or(trans_b_), a.size(0), + a.size(1), y.size(1), a.stride(0), a.stride(1), b.stride(0), + b.stride(1), y.stride(0), y.stride(1), + reinterpret_cast(a.data()), + reinterpret_cast(b.data()))) { + auto stream = static_cast(stream_ ? stream_ : 0); + iluvatar::LaunchSmallBatchGemv(a.data(), b.data(), y.data(), a.size(0), + a.size(1), y.size(1), stream); + return; + } + Base::operator()(a, b, c, alpha, beta, trans_a, trans_b, y); + } }; } // namespace infini::ops diff --git a/src/native/cuda/iluvatar/ops/gemm/gemv_policy.h b/src/native/cuda/iluvatar/ops/gemm/gemv_policy.h new file mode 100644 index 000000000..b94e724bd --- /dev/null +++ b/src/native/cuda/iluvatar/ops/gemm/gemv_policy.h @@ -0,0 +1,35 @@ +#ifndef INFINI_OPS_ILUVATAR_GEMM_GEMV_POLICY_H_ +#define INFINI_OPS_ILUVATAR_GEMM_GEMV_POLICY_H_ + +#include +#include +#include + +namespace infini::ops::iluvatar { + +inline bool CanUseSmallBatchGemv(bool all_bf16, bool same_device, bool has_bias, + float alpha, float beta, int trans_a, + int trans_b, std::size_t m, std::size_t k, + std::size_t n, std::ptrdiff_t a0, + std::ptrdiff_t a1, std::ptrdiff_t b0, + std::ptrdiff_t b1, std::ptrdiff_t y0, + std::ptrdiff_t y1, std::uintptr_t a_address, + std::uintptr_t b_address) { + // Each lane loads two adjacent BF16 values. Reject offset views that are + // not aligned for the packed load even when their strides match. + constexpr std::size_t kMinWeightElements = 8 * 1024 * 1024; + return all_bf16 && same_device && !has_bias && alpha == 1.0f && + beta == 0.0f && trans_a == 0 && trans_b == 0 && (m == 1 || m == 2) && + k >= 1024 && k % 64 == 0 && n >= 1024 && n % 4 == 0 && + k <= static_cast(std::numeric_limits::max()) && + n <= static_cast(std::numeric_limits::max()) && + k >= (kMinWeightElements + n - 1) / n && + a0 == static_cast(k) && a1 == 1 && b0 == 1 && + b1 == static_cast(k) && + y0 == static_cast(n) && y1 == 1 && + a_address % 4 == 0 && b_address % 4 == 0; +} + +} // namespace infini::ops::iluvatar + +#endif diff --git a/src/native/cuda/iluvatar/ops/gemm/kernel.cuh b/src/native/cuda/iluvatar/ops/gemm/kernel.cuh new file mode 100644 index 000000000..e966d28a2 --- /dev/null +++ b/src/native/cuda/iluvatar/ops/gemm/kernel.cuh @@ -0,0 +1,60 @@ +#ifndef INFINI_OPS_ILUVATAR_GEMM_KERNEL_CUH_ +#define INFINI_OPS_ILUVATAR_GEMM_KERNEL_CUH_ + +#include +#include + +#include + +namespace infini::ops::iluvatar { + +template +__global__ void SmallBatchGemvKernel(const __nv_bfloat16* x, + const __nv_bfloat16* w, __nv_bfloat16* y, + std::size_t input, std::size_t outputs) { + static_assert(kBatch == 1 || kBatch == 2); + const int lane = threadIdx.x & 31; + const std::size_t output = blockIdx.x * 4 + threadIdx.x / 32; + const auto* w2 = reinterpret_cast(w + output * input); + const auto* x2 = reinterpret_cast(x); + float s0[4] = {0.f, 0.f, 0.f, 0.f}; + float s1[4] = {0.f, 0.f, 0.f, 0.f}; + for (std::size_t k = lane; k < input / 2; k += 128) { +#pragma unroll + for (int j = 0; j < 4; ++j) { + if constexpr (kHasTail) { + if (k + j * 32 >= input / 2) { + continue; + } + } + float2 a = __bfloat1622float2(w2[k + j * 32]); + float2 b = __bfloat1622float2(x2[k + j * 32]); + s0[j] = fmaf(a.x, b.x, s0[j]); + s0[j] = fmaf(a.y, b.y, s0[j]); + if constexpr (kBatch == 2) { + float2 c = __bfloat1622float2(x2[input / 2 + k + j * 32]); + s1[j] = fmaf(a.x, c.x, s1[j]); + s1[j] = fmaf(a.y, c.y, s1[j]); + } + } + } + // Keep the four-accumulator FP32 tree used in the measured specialization. + float a = (s0[0] + s0[1]) + (s0[2] + s0[3]); + float b = (s1[0] + s1[1]) + (s1[2] + s1[3]); + for (int delta = 16; delta > 0; delta /= 2) { + a += __shfl_down_sync(0xffffffffu, a, delta); + if constexpr (kBatch == 2) { + b += __shfl_down_sync(0xffffffffu, b, delta); + } + } + if (lane == 0) { + y[output] = __float2bfloat16(a); + if constexpr (kBatch == 2) { + y[outputs + output] = __float2bfloat16(b); + } + } +} + +} // namespace infini::ops::iluvatar + +#endif diff --git a/src/native/cuda/iluvatar/ops/gemm/kernel.h b/src/native/cuda/iluvatar/ops/gemm/kernel.h new file mode 100644 index 000000000..65ac9640d --- /dev/null +++ b/src/native/cuda/iluvatar/ops/gemm/kernel.h @@ -0,0 +1,38 @@ +#ifndef INFINI_OPS_ILUVATAR_GEMM_KERNEL_H_ +#define INFINI_OPS_ILUVATAR_GEMM_KERNEL_H_ + +#include + +#include "native/cuda/iluvatar/ops/gemm/kernel.cuh" + +namespace infini::ops::iluvatar { + +template +inline void LaunchSmallBatchGemv(const __nv_bfloat16* x, const __nv_bfloat16* w, + __nv_bfloat16* out, std::size_t input, + std::size_t output, cudaStream_t stream) { + if (input % 256 == 0) { + SmallBatchGemvKernel + <<>>(x, w, out, input, output); + } else { + SmallBatchGemvKernel + <<>>(x, w, out, input, output); + } +} + +inline void LaunchSmallBatchGemv(const void* a, const void* b, void* y, + std::size_t batch, std::size_t input, + std::size_t output, cudaStream_t stream) { + const auto* x = static_cast(a); + const auto* w = static_cast(b); + auto* out = static_cast<__nv_bfloat16*>(y); + if (batch == 1) { + LaunchSmallBatchGemv<1>(x, w, out, input, output, stream); + } else { + LaunchSmallBatchGemv<2>(x, w, out, input, output, stream); + } +} + +} // namespace infini::ops::iluvatar + +#endif diff --git a/tests/host/iluvatar_gemv_policy_test.cpp b/tests/host/iluvatar_gemv_policy_test.cpp new file mode 100644 index 000000000..7c702a08e --- /dev/null +++ b/tests/host/iluvatar_gemv_policy_test.cpp @@ -0,0 +1,79 @@ +#include +#include +#include +#include + +#include "native/cuda/iluvatar/ops/gemm/gemv_policy.h" + +int main() { + using infini::ops::iluvatar::CanUseSmallBatchGemv; + for (const auto [k, n] : {std::pair{2048, 4096}, + {4160, 2052}, + {4224, 2056}, + {4288, 2060}, + {2048, 11008}, + {5504, 2048}, + {3072, 16384}, + {8192, 3072}, + {4096, 22016}, + {11008, 4096}, + {4096, 28672}, + {14336, 4096}, + {5120, 27648}, + {13824, 5120}, + {8192, 57344}, + {28672, 8192}, + {4096, 32768}, + {16384, 4096}}) { + for (int m : {1, 2}) { + const auto valid = [&](bool all_bf16 = true, bool same = true, + bool bias = false, float alpha = 1, float beta = 0, + int ta = 0, int tb = 0, int rows = 0, + int a_stride = 0, std::uintptr_t address = 4096) { + return CanUseSmallBatchGemv( + all_bf16, same, bias, alpha, beta, ta, tb, rows ? rows : m, k, n, + a_stride ? a_stride : k, 1, 1, k, n, 1, address, 8192); + }; + assert(valid()); + assert(!valid(false)); + assert(!valid(true, false)); + assert(!valid(true, true, true)); + assert(!valid(true, true, false, 0.5f)); + assert(!valid(true, true, false, 1, 1)); + assert(!valid(true, true, false, 1, 0, 1)); + assert(!valid(true, true, false, 1, 0, 0, 1)); + assert(!valid(true, true, false, 1, 0, 0, 0, 3)); + assert(!valid(true, true, false, 1, 0, 0, 0, m, k + 1)); + assert(!valid(true, true, false, 1, 0, 0, 0, m, k, 4098)); + assert( + !valid(true, true, false, std::numeric_limits::quiet_NaN())); + assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, m, k, n + 1, + k, 1, 1, k, n + 1, 1, 4096, 8192)); + assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, m, k, n, k, 1, + n, 1, n, 1, 4096, 8192)); + assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, m, k, n, k, 1, + 1, k, n, 1, 4096, 8194)); + assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, m, k, n, k, 1, + 1, k, n + 1, 1, 4096, 8192)); + } + } + for (std::size_t m : {0, 3, 4}) { + assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, m, 2048, 4096, + 2048, 1, 1, 2048, 4096, 1, 4096, 8192)); + } + const std::size_t overflow = + static_cast(std::numeric_limits::max()) + 1; + for (std::size_t k : + {std::size_t{0}, std::size_t{768}, std::size_t{1025}, overflow}) { + assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, 1, k, 16384, k, + 1, 1, k, 16384, 1, 4096, 8192)); + } + for (std::size_t n : + {std::size_t{0}, std::size_t{1020}, std::size_t{1025}, overflow}) { + assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, 1, 16384, n, + 16384, 1, 1, 16384, n, 1, 4096, 8192)); + } + // Weights below the measured size threshold retain the BLAS implementation. + assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, 1, 2048, 4092, + 2048, 1, 1, 2048, 4092, 1, 4096, 8192)); +} diff --git a/tests/test_iluvatar_mlp_gemv.py b/tests/test_iluvatar_mlp_gemv.py new file mode 100644 index 000000000..d5f1a5bab --- /dev/null +++ b/tests/test_iluvatar_mlp_gemv.py @@ -0,0 +1,144 @@ +import infini.ops +import pytest +import torch + +from tests.utils import get_stream + + +@pytest.mark.smoke +@pytest.mark.parametrize("m", (1, 2)) +@pytest.mark.parametrize( + "k,n", + ( + (2048, 4096), + (4160, 2052), + (4224, 2056), + (4288, 2060), + (2048, 11008), + (5504, 2048), + (3072, 16384), + (8192, 3072), + (4096, 22016), + (11008, 4096), + (4096, 28672), + (14336, 4096), + (5120, 27648), + (13824, 5120), + (8192, 57344), + (28672, 8192), + (4096, 32768), + (16384, 4096), + ), +) +@pytest.mark.parametrize("implementation_index", (0,)) +def test_iluvatar_mlp_gemv(m, k, n, device, implementation_index): + if device != "cuda" or not infini.ops.Gemm.active_implementation_indices( + "iluvatar" + ): + pytest.skip("requires the Iluvatar backend") + + # Bounded dyadic values exercise both signs without tolerance masking errors. + x = ((torch.arange(m * k, device=device) % 7 - 3).float() / 16).reshape(m, k) + w = ((torch.arange(n * k, device=device) % 5 - 2).float() / 16).reshape(n, k) + a = x.to(torch.bfloat16) + b = w.to(torch.bfloat16).t() + out = torch.empty((m, n), dtype=torch.bfloat16, device=device) + expected = (a.float() @ b.float()).to(torch.bfloat16) + infini.ops.gemm( + a, + b, + None, + 1.0, + 0.0, + False, + False, + out, + stream=get_stream(a.device), + implementation_index=implementation_index, + ) + torch.testing.assert_close(out, expected, rtol=0, atol=0) + + # Non-default alpha must retain the supported BLAS fallback. + # The current base GEMM contract requires C=None and forces beta=0. + infini.ops.gemm( + a, + b, + None, + 0.5, + 0.0, + False, + False, + out, + stream=get_stream(a.device), + implementation_index=implementation_index, + ) + expected = (0.5 * (a.float() @ b.float())).to(torch.bfloat16) + torch.testing.assert_close(out, expected, rtol=1e-2, atol=1e-2) + + a = (torch.randn((m, k), device=device) / k**0.5).to(torch.bfloat16) + b = torch.randn((n, k), device=device, dtype=torch.bfloat16).t() + expected = (a.float() @ b.float()).to(torch.bfloat16) + infini.ops.gemm( + a, + b, + None, + 1.0, + 0.0, + False, + False, + out, + stream=get_stream(a.device), + implementation_index=implementation_index, + ) + torch.testing.assert_close(out, expected, rtol=1e-2, atol=1e-2) + + +@pytest.mark.smoke +@pytest.mark.parametrize( + "m,k,n,layout", + ( + (1, 1024, 1024, "column"), + (2, 1088, 1028, "column"), + (3, 2048, 4096, "column"), + (1, 768, 16384, "column"), + (1, 2049, 4096, "column"), + (1, 2048, 4097, "column"), + (1, 2048, 4096, "row"), + (2, 2048, 4096, "padded_a"), + (1, 2048, 4096, "padded_b"), + ), +) +@pytest.mark.parametrize("implementation_index", (0,)) +def test_iluvatar_mlp_gemv_fallback(m, k, n, layout, device, implementation_index): + if device != "cuda" or not infini.ops.Gemm.active_implementation_indices( + "iluvatar" + ): + pytest.skip("requires the Iluvatar backend") + + a = torch.randn((m, k), device=device, dtype=torch.bfloat16) + b = torch.randn((n, k), device=device, dtype=torch.bfloat16).t() + + if layout == "row": + b = b.contiguous() + elif layout == "padded_a": + storage = torch.empty((m, k + 8), device=device, dtype=torch.bfloat16) + a = storage[:, :k].copy_(a) + elif layout == "padded_b": + storage = torch.empty((n, k + 8), device=device, dtype=torch.bfloat16) + b = storage[:, :k].t().copy_(b) + + out = torch.empty((m, n), device=device, dtype=torch.bfloat16) + expected = (a.float() @ b.float()).to(torch.bfloat16) + infini.ops.gemm( + a, + b, + None, + 1.0, + 0.0, + False, + False, + out, + stream=get_stream(a.device), + implementation_index=implementation_index, + ) + torch.testing.assert_close(out, expected, rtol=1e-2, atol=1e-2)