Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 33 additions & 1 deletion src/native/cuda/iluvatar/ops/gemm/cublas.h
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,47 @@
#define INFINI_OPS_ILUVATAR_GEMM_CUBLAS_H_

#include "native/cuda/iluvatar/blas.h"
#include "native/cuda/iluvatar/ops/gemm/gemv_policy.h"
#include "native/cuda/iluvatar/ops/gemm/kernel.h"
#include "native/cuda/ops/gemm/blas.h"

namespace infini::ops {

template <>
class Operator<Gemm, Device::Type::kIluvatar>
: public BlasGemm<Blas<Device::Type::kIluvatar>> {
using Base = BlasGemm<Blas<Device::Type::kIluvatar>>;

public:
using BlasGemm<Blas<Device::Type::kIluvatar>>::BlasGemm;
using Base::Base;

using Base::operator();

void operator()(const Tensor a, const Tensor b, const std::optional<Tensor> c,
std::optional<float> alpha, std::optional<float> beta,
std::optional<int> trans_a, std::optional<int> trans_b,
Tensor y) const override {
if (a.ndim() == 2 && b.ndim() == 2 && y.ndim() == 2 &&
a.size(1) == b.size(0) && a.size(0) == y.size(0) &&
b.size(1) == y.size(1) &&
iluvatar::CanUseSmallBatchGemv(
a.dtype() == DataType::kBFloat16 &&
b.dtype() == DataType::kBFloat16 &&
y.dtype() == DataType::kBFloat16,
a.device() == b.device() && a.device() == y.device(), c.has_value(),
alpha.value_or(alpha_), beta.value_or(beta_),
trans_a.value_or(trans_a_), trans_b.value_or(trans_b_), a.size(0),
a.size(1), y.size(1), a.stride(0), a.stride(1), b.stride(0),
b.stride(1), y.stride(0), y.stride(1),
reinterpret_cast<std::uintptr_t>(a.data()),
reinterpret_cast<std::uintptr_t>(b.data()))) {
auto stream = static_cast<cudaStream_t>(stream_ ? stream_ : 0);
iluvatar::LaunchSmallBatchGemv(a.data(), b.data(), y.data(), a.size(0),
a.size(1), y.size(1), stream);
return;
}
Base::operator()(a, b, c, alpha, beta, trans_a, trans_b, y);
}
};

} // namespace infini::ops
Expand Down
35 changes: 35 additions & 0 deletions src/native/cuda/iluvatar/ops/gemm/gemv_policy.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
#ifndef INFINI_OPS_ILUVATAR_GEMM_GEMV_POLICY_H_
#define INFINI_OPS_ILUVATAR_GEMM_GEMV_POLICY_H_

#include <cstddef>
#include <cstdint>
#include <limits>

namespace infini::ops::iluvatar {

inline bool CanUseSmallBatchGemv(bool all_bf16, bool same_device, bool has_bias,
float alpha, float beta, int trans_a,
int trans_b, std::size_t m, std::size_t k,
std::size_t n, std::ptrdiff_t a0,
std::ptrdiff_t a1, std::ptrdiff_t b0,
std::ptrdiff_t b1, std::ptrdiff_t y0,
std::ptrdiff_t y1, std::uintptr_t a_address,
std::uintptr_t b_address) {
// Each lane loads two adjacent BF16 values. Reject offset views that are
// not aligned for the packed load even when their strides match.
constexpr std::size_t kMinWeightElements = 8 * 1024 * 1024;
return all_bf16 && same_device && !has_bias && alpha == 1.0f &&
beta == 0.0f && trans_a == 0 && trans_b == 0 && (m == 1 || m == 2) &&
k >= 1024 && k % 64 == 0 && n >= 1024 && n % 4 == 0 &&
k <= static_cast<std::size_t>(std::numeric_limits<int>::max()) &&
n <= static_cast<std::size_t>(std::numeric_limits<int>::max()) &&
k >= (kMinWeightElements + n - 1) / n &&
a0 == static_cast<std::ptrdiff_t>(k) && a1 == 1 && b0 == 1 &&
b1 == static_cast<std::ptrdiff_t>(k) &&
y0 == static_cast<std::ptrdiff_t>(n) && y1 == 1 &&
a_address % 4 == 0 && b_address % 4 == 0;
}

} // namespace infini::ops::iluvatar

#endif
60 changes: 60 additions & 0 deletions src/native/cuda/iluvatar/ops/gemm/kernel.cuh
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#ifndef INFINI_OPS_ILUVATAR_GEMM_KERNEL_CUH_
#define INFINI_OPS_ILUVATAR_GEMM_KERNEL_CUH_

#include <cuda_bf16.h>
#include <cuda_runtime.h>

#include <cstddef>

namespace infini::ops::iluvatar {

template <int kBatch, bool kHasTail>
__global__ void SmallBatchGemvKernel(const __nv_bfloat16* x,
const __nv_bfloat16* w, __nv_bfloat16* y,
std::size_t input, std::size_t outputs) {
static_assert(kBatch == 1 || kBatch == 2);
const int lane = threadIdx.x & 31;
const std::size_t output = blockIdx.x * 4 + threadIdx.x / 32;
const auto* w2 = reinterpret_cast<const __nv_bfloat162*>(w + output * input);
const auto* x2 = reinterpret_cast<const __nv_bfloat162*>(x);
float s0[4] = {0.f, 0.f, 0.f, 0.f};
float s1[4] = {0.f, 0.f, 0.f, 0.f};
for (std::size_t k = lane; k < input / 2; k += 128) {
#pragma unroll
for (int j = 0; j < 4; ++j) {
if constexpr (kHasTail) {
if (k + j * 32 >= input / 2) {
continue;
}
}
float2 a = __bfloat1622float2(w2[k + j * 32]);
float2 b = __bfloat1622float2(x2[k + j * 32]);
s0[j] = fmaf(a.x, b.x, s0[j]);
s0[j] = fmaf(a.y, b.y, s0[j]);
if constexpr (kBatch == 2) {
float2 c = __bfloat1622float2(x2[input / 2 + k + j * 32]);
s1[j] = fmaf(a.x, c.x, s1[j]);
s1[j] = fmaf(a.y, c.y, s1[j]);
}
}
}
// Keep the four-accumulator FP32 tree used in the measured specialization.
float a = (s0[0] + s0[1]) + (s0[2] + s0[3]);
float b = (s1[0] + s1[1]) + (s1[2] + s1[3]);
for (int delta = 16; delta > 0; delta /= 2) {
a += __shfl_down_sync(0xffffffffu, a, delta);
if constexpr (kBatch == 2) {
b += __shfl_down_sync(0xffffffffu, b, delta);
}
}
if (lane == 0) {
y[output] = __float2bfloat16(a);
if constexpr (kBatch == 2) {
y[outputs + output] = __float2bfloat16(b);
}
}
}

} // namespace infini::ops::iluvatar

#endif
38 changes: 38 additions & 0 deletions src/native/cuda/iluvatar/ops/gemm/kernel.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#ifndef INFINI_OPS_ILUVATAR_GEMM_KERNEL_H_
#define INFINI_OPS_ILUVATAR_GEMM_KERNEL_H_

#include <cstddef>

#include "native/cuda/iluvatar/ops/gemm/kernel.cuh"

namespace infini::ops::iluvatar {

template <int kBatch>
inline void LaunchSmallBatchGemv(const __nv_bfloat16* x, const __nv_bfloat16* w,
__nv_bfloat16* out, std::size_t input,
std::size_t output, cudaStream_t stream) {
if (input % 256 == 0) {
SmallBatchGemvKernel<kBatch, false>
<<<output / 4, 128, 0, stream>>>(x, w, out, input, output);
} else {
SmallBatchGemvKernel<kBatch, true>
<<<output / 4, 128, 0, stream>>>(x, w, out, input, output);
}
}

inline void LaunchSmallBatchGemv(const void* a, const void* b, void* y,
std::size_t batch, std::size_t input,
std::size_t output, cudaStream_t stream) {
const auto* x = static_cast<const __nv_bfloat16*>(a);
const auto* w = static_cast<const __nv_bfloat16*>(b);
auto* out = static_cast<__nv_bfloat16*>(y);
if (batch == 1) {
LaunchSmallBatchGemv<1>(x, w, out, input, output, stream);
} else {
LaunchSmallBatchGemv<2>(x, w, out, input, output, stream);
}
}

} // namespace infini::ops::iluvatar

#endif
79 changes: 79 additions & 0 deletions tests/host/iluvatar_gemv_policy_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
#include <cassert>
#include <initializer_list>
#include <limits>
#include <utility>

#include "native/cuda/iluvatar/ops/gemm/gemv_policy.h"

int main() {
using infini::ops::iluvatar::CanUseSmallBatchGemv;
for (const auto [k, n] : {std::pair{2048, 4096},
{4160, 2052},
{4224, 2056},
{4288, 2060},
{2048, 11008},
{5504, 2048},
{3072, 16384},
{8192, 3072},
{4096, 22016},
{11008, 4096},
{4096, 28672},
{14336, 4096},
{5120, 27648},
{13824, 5120},
{8192, 57344},
{28672, 8192},
{4096, 32768},
{16384, 4096}}) {
for (int m : {1, 2}) {
const auto valid = [&](bool all_bf16 = true, bool same = true,
bool bias = false, float alpha = 1, float beta = 0,
int ta = 0, int tb = 0, int rows = 0,
int a_stride = 0, std::uintptr_t address = 4096) {
return CanUseSmallBatchGemv(
all_bf16, same, bias, alpha, beta, ta, tb, rows ? rows : m, k, n,
a_stride ? a_stride : k, 1, 1, k, n, 1, address, 8192);
};
assert(valid());
assert(!valid(false));
assert(!valid(true, false));
assert(!valid(true, true, true));
assert(!valid(true, true, false, 0.5f));
assert(!valid(true, true, false, 1, 1));
assert(!valid(true, true, false, 1, 0, 1));
assert(!valid(true, true, false, 1, 0, 0, 1));
assert(!valid(true, true, false, 1, 0, 0, 0, 3));
assert(!valid(true, true, false, 1, 0, 0, 0, m, k + 1));
assert(!valid(true, true, false, 1, 0, 0, 0, m, k, 4098));
assert(
!valid(true, true, false, std::numeric_limits<float>::quiet_NaN()));
assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, m, k, n + 1,
k, 1, 1, k, n + 1, 1, 4096, 8192));
assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, m, k, n, k, 1,
n, 1, n, 1, 4096, 8192));
assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, m, k, n, k, 1,
1, k, n, 1, 4096, 8194));
assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, m, k, n, k, 1,
1, k, n + 1, 1, 4096, 8192));
}
}
for (std::size_t m : {0, 3, 4}) {
assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, m, 2048, 4096,
2048, 1, 1, 2048, 4096, 1, 4096, 8192));
}
const std::size_t overflow =
static_cast<std::size_t>(std::numeric_limits<int>::max()) + 1;
for (std::size_t k :
{std::size_t{0}, std::size_t{768}, std::size_t{1025}, overflow}) {
assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, 1, k, 16384, k,
1, 1, k, 16384, 1, 4096, 8192));
}
for (std::size_t n :
{std::size_t{0}, std::size_t{1020}, std::size_t{1025}, overflow}) {
assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, 1, 16384, n,
16384, 1, 1, 16384, n, 1, 4096, 8192));
}
// Weights below the measured size threshold retain the BLAS implementation.
assert(!CanUseSmallBatchGemv(true, true, false, 1, 0, 0, 0, 1, 2048, 4092,
2048, 1, 1, 2048, 4092, 1, 4096, 8192));
}
Loading