Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions gemma/configs.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1350,6 +1350,8 @@ std::string KVEncodingToString(KVEncoding encoding) {
return "BF16MatrixAccumulation";
case KVEncoding::kInt8MatrixAccumulation:
return "Int8MatrixAccumulation";
case KVEncoding::kInt4:
return "Int4";
default:
return "Unknown";
}
Expand Down
2 changes: 2 additions & 0 deletions gemma/configs.h
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,8 @@ enum class KVEncoding {
kBF16MatrixAccumulation = 7,
kInt8MatrixAccumulation = 8,
kInt8VNNITwoTranspositions = 9,
// Same layout as kInt8, but two 4-bit values per byte (see kv_transcoding.h).
kInt4 = 10,
};

// Returns a string representation of the KVEncoding.
Expand Down
107 changes: 107 additions & 0 deletions gemma/kv_transcoding.cc
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
#include <algorithm>
#include <cmath>
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <optional>

Expand Down Expand Up @@ -39,6 +40,9 @@ std::optional<size_t> GetTileSizeBytes(gcpp::KVEncoding encoding,
case gcpp::KVEncoding::kF32:
case gcpp::KVEncoding::kF32TwoTranspositions:
return tileElements * sizeof(float);
case gcpp::KVEncoding::kInt4:
// Two 4-bit values per byte, plus the same scales as kInt8.
return tileElements / 2 + kTileSize * 2 * sizeof(gcpp::KV_microscale_t);
default:
return std::nullopt;
}
Expand Down Expand Up @@ -67,6 +71,31 @@ int8_t Quantize(float v, float inv_scale) {
return hwy::ConvertScalarTo<int8_t>(scaled);
}

// Symmetric 4-bit quantization: returns q in [-7, 7].
int8_t QuantizeInt4(float v, float inv_scale) {
float scaled = std::nearbyint(v * inv_scale);
if (scaled > 7.0f) return 7;
if (scaled < -7.0f) return -7;
return hwy::ConvertScalarTo<int8_t>(scaled);
}

// kInt4 stores q + 8 in a nibble. Element `offset` (in the kInt8 layout) is in
// byte `offset / 2`: the low nibble if `offset` is even, else the high nibble.
// K offsets pair adjacent tokens and V offsets pair adjacent dims, because
// kTileSize and qkv_dim are even.
inline void StoreNibble(uint8_t* data, size_t offset, int8_t q) {
const uint8_t nibble = static_cast<uint8_t>(q + 8) & 0xF;
uint8_t& byte = data[offset / 2];
byte = (offset % 2 == 0) ? static_cast<uint8_t>((byte & 0xF0) | nibble)
: static_cast<uint8_t>((byte & 0x0F) | (nibble << 4));
}

inline int LoadNibble(const uint8_t* data, size_t offset) {
const uint8_t byte = data[offset / 2];
const int nibble = (offset % 2 == 0) ? (byte & 0xF) : (byte >> 4);
return nibble - 8;
}

template <typename DecodeKFn, typename DecodeVFn>
inline void DecodeTileWithFn(size_t qkv_dim, DecodedTile* out,
const DecodeKFn& decode_k,
Expand Down Expand Up @@ -282,6 +311,76 @@ void EncodeTileInt8(bool transposed, size_t qkv_dim, const DecodedTile& decoded,
});
}

void EncodeTileInt4(size_t qkv_dim, const DecodedTile& decoded,
hwy::Span<char> out_encoded_tile_data) {
HWY_DASSERT(qkv_dim % 2 == 0);
uint8_t* k_data = HWY_RCAST_ALIGNED(uint8_t*, out_encoded_tile_data.data());
uint8_t* v_data = k_data + qkv_dim * kTileSize / 2;
gcpp::KV_microscale_t* scales = HWY_RCAST_ALIGNED(
gcpp::KV_microscale_t*, v_data + kTileSize * qkv_dim / 2);
gcpp::KV_microscale_t* k_scales = scales;
gcpp::KV_microscale_t* v_scales = scales + kTileSize;

AlignedFloatVector inv_scales_k(kTileSize);
AlignedFloatVector inv_scales_v(kTileSize);
for (size_t token = 0; token < kTileSize; ++token) {
float k_max_abs = 0.0f;
float v_max_abs = 0.0f;
for (size_t dim = 0; dim < qkv_dim; ++dim) {
k_max_abs = std::max(k_max_abs, std::abs(decoded.k_elem(token, dim)));
v_max_abs = std::max(v_max_abs, std::abs(decoded.v_elem(token, dim)));
}
const float scale_k = k_max_abs == 0.0f ? 1.0f : k_max_abs / 7.0f;
k_scales[token] = hwy::ConvertScalarTo<gcpp::KV_microscale_t>(scale_k);
// Quantize with the stored (bf16) scale, as for kInt8.
const float stored_k = hwy::ConvertScalarTo<float>(k_scales[token]);
inv_scales_k[token] = stored_k == 0.0f ? 0.0f : 1.0f / stored_k;

const float scale_v = v_max_abs == 0.0f ? 1.0f : v_max_abs / 7.0f;
v_scales[token] = hwy::ConvertScalarTo<gcpp::KV_microscale_t>(scale_v);
const float stored_v = hwy::ConvertScalarTo<float>(v_scales[token]);
inv_scales_v[token] = stored_v == 0.0f ? 0.0f : 1.0f / stored_v;
}

EncodeTileWithFn(
qkv_dim, decoded,
[&](size_t dim, size_t token, float val) HWY_ATTR {
StoreNibble(k_data, KOffset(/*transposed=*/false, qkv_dim, dim, token),
QuantizeInt4(val, inv_scales_k[token]));
},
[&](size_t dim, size_t token, float val) HWY_ATTR {
StoreNibble(v_data, VOffset(/*transposed=*/false, qkv_dim, dim, token),
QuantizeInt4(val, inv_scales_v[token]));
});
}

void DecodeTileInt4(size_t qkv_dim, hwy::Span<const char> encoded_tile_data,
DecodedTile* out) {
HWY_DASSERT(qkv_dim % 2 == 0);
const uint8_t* k_data =
HWY_RCAST_ALIGNED(const uint8_t*, encoded_tile_data.data());
const uint8_t* v_data = k_data + qkv_dim * kTileSize / 2;
const gcpp::KV_microscale_t* scales = HWY_RCAST_ALIGNED(
const gcpp::KV_microscale_t*, v_data + kTileSize * qkv_dim / 2);
const gcpp::KV_microscale_t* k_scales = scales;
const gcpp::KV_microscale_t* v_scales = scales + kTileSize;

DecodeTileWithFn(
qkv_dim, out,
[&](size_t dim, size_t token) HWY_ATTR {
const float scale = hwy::ConvertScalarTo<float>(k_scales[token]);
return LoadNibble(k_data,
KOffset(/*transposed=*/false, qkv_dim, dim, token)) *
scale;
},
[&](size_t dim, size_t token) HWY_ATTR {
const float scale = hwy::ConvertScalarTo<float>(v_scales[token]);
return LoadNibble(v_data,
VOffset(/*transposed=*/false, qkv_dim, dim, token)) *
scale;
});
}

void EncodeTileInt8VNNI(size_t qkv_dim, const DecodedTile& decoded,
hwy::Span<char> out_encoded_tile_data) {
int8_t* k_data = HWY_RCAST_ALIGNED(int8_t*, out_encoded_tile_data.data());
Expand Down Expand Up @@ -555,6 +654,10 @@ bool DecodeTile(KVEncoding encoding, hwy::Span<const char> encoded_tile_data,
DecodeTileInt8MatrixAccumulation(qkv_dim, encoded_tile_data, out);
return true;
}
case gcpp::KVEncoding::kInt4: {
DecodeTileInt4(qkv_dim, encoded_tile_data, out);
return true;
}
default:
return false;
}
Expand Down Expand Up @@ -597,6 +700,10 @@ bool EncodeTile(gcpp::KVEncoding encoding, const DecodedTile& decoded,
EncodeTileInt8MatrixAccumulation(qkv_dim, decoded, out_encoded_tile_data);
return true;
}
case gcpp::KVEncoding::kInt4: {
EncodeTileInt4(qkv_dim, decoded, out_encoded_tile_data);
return true;
}
default:
return false;
}
Expand Down
6 changes: 6 additions & 0 deletions gemma/kv_transcoding.h
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,12 @@ namespace gcpp {

// Returns the size in bytes of a single KV cache tile for a given encoding.
// Returns std::nullopt if the encoding is unsupported.
//
// KVEncoding::kInt4 uses the kInt8 layout (K: [qkv_dim, tile_size],
// V: [tile_size, qkv_dim], then K and V per-token bf16 scales) with two 4-bit
// values per byte: element i of the kInt8 layout is in byte i / 2, in the low
// nibble if i is even. A nibble stores q + 8 for q in [-7, 7]; the value is
// q * scale, with scale = bf16(absmax / 7) per token.
std::optional<size_t> GetTileSizeBytes(gcpp::KVEncoding encoding,
size_t qkv_dim);

Expand Down
205 changes: 204 additions & 1 deletion gemma/kv_transcoding_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,9 @@ INSTANTIATE_TEST_SUITE_P(
EncodingTestCase{gcpp::KVEncoding::kInt8, 0.1f},
EncodingTestCase{gcpp::KVEncoding::kInt8TwoTranspositions, 0.1f},
EncodingTestCase{gcpp::KVEncoding::kInt8VNNITwoTranspositions, 0.1f},
EncodingTestCase{gcpp::KVEncoding::kInt8MatrixAccumulation, 0.02f}));
EncodingTestCase{gcpp::KVEncoding::kInt8MatrixAccumulation, 0.02f},
// Half a step is 1.5 / 7 / 2 ~= 0.107 for this pattern's absmax.
EncodingTestCase{gcpp::KVEncoding::kInt4, 0.11f}));

class Int8EncodingTest : public TestWithParam<gcpp::KVEncoding> {};

Expand Down Expand Up @@ -160,6 +162,207 @@ INSTANTIATE_TEST_SUITE_P(AllInt8Encodings, Int8EncodingTest,
gcpp::KVEncoding::kInt8VNNITwoTranspositions,
gcpp::KVEncoding::kInt8MatrixAccumulation));

// Fills a tile with per-token magnitudes that vary, so that absmax / qmax is
// rarely exact in bf16.
void FillVaryingTile(DecodedTile& tile) {
for (size_t token = 0; token < tile.tile_size; ++token) {
for (size_t dim = 0; dim < tile.qkv_dim; ++dim) {
const float i = static_cast<float>(dim * tile.tile_size + token);
tile.k_elem(token, dim) = std::sin(i * 0.37f) * (1.0f + token);
tile.v_elem(token, dim) = std::cos(i * 0.53f) * (3.0f + 0.7f * token);
}
}
}

// Reference per-token symmetric 4-bit quant-dequant of one row: bf16 scale
// absmax / 7, quantized with the stored scale, values in [-7, 7].
void RefQuantDequantInt4Row(float* row, size_t d) {
float max_abs = 0.0f;
for (size_t j = 0; j < d; ++j) max_abs = std::max(max_abs, std::abs(row[j]));
const float scale = hwy::ConvertScalarTo<float>(
hwy::ConvertScalarTo<hwy::bfloat16_t>(max_abs == 0.0f ? 1.0f
: max_abs / 7.0f));
const float inv = scale == 0.0f ? 0.0f : 1.0f / scale;
for (size_t j = 0; j < d; ++j) {
const float q = std::clamp(std::nearbyint(row[j] * inv), -7.0f, 7.0f);
row[j] = q * scale;
}
}

TEST(Int4EncodingTest, RoundTripIsWithinHalfAStep) {
constexpr size_t kTileSize = 32;
constexpr size_t qkv_dim = 256;
const KVEncoding encoding = KVEncoding::kInt4;

DecodedTile original(qkv_dim, kTileSize);
FillVaryingTile(original);

hwy::AlignedVector<char> encoded(GetTileSizeBytes(encoding, qkv_dim).value());
ASSERT_TRUE(EncodeTile(encoding, original, qkv_dim,
hwy::Span<char>(encoded.data(), encoded.size())));
DecodedTile decoded(qkv_dim, kTileSize);
ASSERT_TRUE(DecodeTile(encoding,
hwy::Span<const char>(encoded.data(), encoded.size()),
qkv_dim, &decoded));

for (size_t token = 0; token < kTileSize; ++token) {
float k_max = 0.0f, v_max = 0.0f;
for (size_t dim = 0; dim < qkv_dim; ++dim) {
k_max = std::max(k_max, std::abs(original.k_elem(token, dim)));
v_max = std::max(v_max, std::abs(original.v_elem(token, dim)));
}
// Half a step, plus slack for rounding the scale to bf16.
const float k_bound = 0.51f * k_max / 7.0f;
const float v_bound = 0.51f * v_max / 7.0f;
for (size_t dim = 0; dim < qkv_dim; ++dim) {
ASSERT_LE(
std::abs(decoded.k_elem(token, dim) - original.k_elem(token, dim)),
k_bound)
<< "k token=" << token << " dim=" << dim;
ASSERT_LE(
std::abs(decoded.v_elem(token, dim) - original.v_elem(token, dim)),
v_bound)
<< "v token=" << token << " dim=" << dim;
}
}
}

// The packed encoding must reproduce the simulated (quant-dequant) INT4 that
// the accuracy evaluation used, bit for bit.
TEST(Int4EncodingTest, MatchesReferenceQuantDequantExactly) {
constexpr size_t kTileSize = 32;
constexpr size_t qkv_dim = 512;
const KVEncoding encoding = KVEncoding::kInt4;

DecodedTile original(qkv_dim, kTileSize);
FillVaryingTile(original);
DecodedTile expected = original;
for (size_t token = 0; token < kTileSize; ++token) {
RefQuantDequantInt4Row(&expected.k_elem(token, 0), qkv_dim);
RefQuantDequantInt4Row(&expected.v_elem(token, 0), qkv_dim);
}

hwy::AlignedVector<char> encoded(GetTileSizeBytes(encoding, qkv_dim).value());
ASSERT_TRUE(EncodeTile(encoding, original, qkv_dim,
hwy::Span<char>(encoded.data(), encoded.size())));
DecodedTile decoded(qkv_dim, kTileSize);
ASSERT_TRUE(DecodeTile(encoding,
hwy::Span<const char>(encoded.data(), encoded.size()),
qkv_dim, &decoded));

EXPECT_THAT(decoded.k, Pointwise(::testing::FloatEq(), expected.k));
EXPECT_THAT(decoded.v, Pointwise(::testing::FloatEq(), expected.v));
}

TEST(Int4EncodingTest, TileIsHalfOfInt8PlusScales) {
for (size_t qkv_dim : {256, 512}) {
const size_t scales = 32 * 2 * sizeof(hwy::bfloat16_t);
const size_t int8 = GetTileSizeBytes(KVEncoding::kInt8, qkv_dim).value();
const size_t int4 = GetTileSizeBytes(KVEncoding::kInt4, qkv_dim).value();
EXPECT_EQ(int4, (int8 - scales) / 2 + scales) << "qkv_dim=" << qkv_dim;
}
EXPECT_EQ(KVEncodingToString(KVEncoding::kInt4), "Int4");
}

TEST(Int4EncodingTest, LayoutValidation) {
constexpr size_t kTileSize = 32;
constexpr size_t qkv_dim = 4;
const KVEncoding encoding = KVEncoding::kInt4;

// Token 0 has K = {7, -7, 0, 3.5} and V = {-14, 14, 2, 0}; other tokens are
// zero. Scales: K 7/7 = 1, V 14/7 = 2.
DecodedTile original(qkv_dim, kTileSize);
for (size_t token = 0; token < kTileSize; ++token) {
for (size_t dim = 0; dim < qkv_dim; ++dim) {
original.k_elem(token, dim) = 0.0f;
original.v_elem(token, dim) = 0.0f;
}
}
const float k0[] = {7.0f, -7.0f, 0.0f, 3.25f};
const float v0[] = {-14.0f, 14.0f, 2.0f, 0.0f};
for (size_t dim = 0; dim < qkv_dim; ++dim) {
original.k_elem(0, dim) = k0[dim];
original.v_elem(0, dim) = v0[dim];
}
// Token 1, dim 0: K = 5 (alone in its row, so q = 7, scale 5/7).
original.k_elem(1, 0) = 5.0f;

hwy::AlignedVector<char> encoded(GetTileSizeBytes(encoding, qkv_dim).value());
ASSERT_TRUE(EncodeTile(encoding, original, qkv_dim,
hwy::Span<char>(encoded.data(), encoded.size())));
const uint8_t* data = HWY_RCAST_ALIGNED(const uint8_t*, encoded.data());

// K is [qkv_dim, tile_size] nibbles: byte (dim * 32 + token) / 2, low nibble
// for even tokens. Stored value is q + 8.
EXPECT_EQ(data[0], (8 + 7) | ((8 + 7) << 4)); // dim 0: t0 q=7, t1 q=7
EXPECT_EQ(data[16], (8 - 7) | (8 << 4)); // dim 1: t0 q=-7, t1 q=0
EXPECT_EQ(data[32], 8 | (8 << 4)); // dim 2: t0 q=0
EXPECT_EQ(data[48], (8 + 3) | (8 << 4)); // dim 3: t0 3.25 -> q=3
// V is [tile_size, qkv_dim] nibbles starting at qkv_dim * 32 / 2 = 64; low
// nibble for even dims.
EXPECT_EQ(data[64], (8 - 7) | ((8 + 7) << 4)); // t0: d0 q=-7, d1 q=7
EXPECT_EQ(data[65], (8 + 1) | (8 << 4)); // t0: d2 q=1, d3 q=0
// Scales follow at 128: K scales then V scales, bf16.
const hwy::bfloat16_t* scales =
HWY_RCAST_ALIGNED(const hwy::bfloat16_t*, data + 128);
EXPECT_EQ(hwy::ConvertScalarTo<float>(scales[0]), 1.0f);
EXPECT_EQ(hwy::ConvertScalarTo<float>(scales[32]), 2.0f);
}

TEST(Int4EncodingTest, TranscodesToAndFromInt8) {
constexpr size_t kTileSize = 32;
constexpr size_t qkv_dim = 256;
DecodedTile original(qkv_dim, kTileSize);
FillVaryingTile(original);

hwy::AlignedVector<char> int4(
GetTileSizeBytes(KVEncoding::kInt4, qkv_dim).value());
hwy::AlignedVector<char> int8(
GetTileSizeBytes(KVEncoding::kInt8, qkv_dim).value());
hwy::AlignedVector<char> int4_again(int4.size());
ASSERT_TRUE(EncodeTile(KVEncoding::kInt4, original, qkv_dim,
hwy::Span<char>(int4.data(), int4.size())));
ASSERT_TRUE(TranscodeTile(KVEncoding::kInt4,
hwy::Span<const char>(int4.data(), int4.size()),
KVEncoding::kInt8,
hwy::Span<char>(int8.data(), int8.size()),
qkv_dim));
ASSERT_TRUE(TranscodeTile(
KVEncoding::kInt8, hwy::Span<const char>(int8.data(), int8.size()),
KVEncoding::kInt4, hwy::Span<char>(int4_again.data(), int4_again.size()),
qkv_dim));

// The INT8 hop moves each value by at most half an INT8 step (~absmax / 254),
// far below half an INT4 step (~absmax / 14), so re-encoding to INT4 must
// give the same values up to rounding of the scale.
DecodedTile a(qkv_dim, kTileSize), b(qkv_dim, kTileSize);
ASSERT_TRUE(DecodeTile(KVEncoding::kInt4,
hwy::Span<const char>(int4.data(), int4.size()),
qkv_dim, &a));
ASSERT_TRUE(DecodeTile(
KVEncoding::kInt4,
hwy::Span<const char>(int4_again.data(), int4_again.size()), qkv_dim,
&b));
for (size_t token = 0; token < kTileSize; ++token) {
float k_max = 0.0f;
for (size_t dim = 0; dim < qkv_dim; ++dim) {
k_max = std::max(k_max, std::abs(a.k_elem(token, dim)));
}
for (size_t dim = 0; dim < qkv_dim; ++dim) {
ASSERT_NEAR(a.k_elem(token, dim), b.k_elem(token, dim), 0.02f * k_max)
<< "k token=" << token << " dim=" << dim;
}
float v_max = 0.0f;
for (size_t dim = 0; dim < qkv_dim; ++dim) {
v_max = std::max(v_max, std::abs(a.v_elem(token, dim)));
}
for (size_t dim = 0; dim < qkv_dim; ++dim) {
ASSERT_NEAR(a.v_elem(token, dim), b.v_elem(token, dim), 0.02f * v_max)
<< "v token=" << token << " dim=" << dim;
}
}
}

TEST(KVEncodingTest, ConvertTileFloat32ToBfloat16) {
constexpr size_t kTileSize = 32;
constexpr size_t qkv_dim = 256;
Expand Down
Loading