diff --git a/gemma/configs.cc b/gemma/configs.cc index 9110eb58..d2cdc379 100644 --- a/gemma/configs.cc +++ b/gemma/configs.cc @@ -1350,6 +1350,8 @@ std::string KVEncodingToString(KVEncoding encoding) { return "BF16MatrixAccumulation"; case KVEncoding::kInt8MatrixAccumulation: return "Int8MatrixAccumulation"; + case KVEncoding::kInt4: + return "Int4"; default: return "Unknown"; } diff --git a/gemma/configs.h b/gemma/configs.h index f1f63510..a7a18432 100644 --- a/gemma/configs.h +++ b/gemma/configs.h @@ -133,6 +133,8 @@ enum class KVEncoding { kBF16MatrixAccumulation = 7, kInt8MatrixAccumulation = 8, kInt8VNNITwoTranspositions = 9, + // Same layout as kInt8, but two 4-bit values per byte (see kv_transcoding.h). + kInt4 = 10, }; // Returns a string representation of the KVEncoding. diff --git a/gemma/kv_transcoding.cc b/gemma/kv_transcoding.cc index dcc299f3..10849275 100644 --- a/gemma/kv_transcoding.cc +++ b/gemma/kv_transcoding.cc @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -39,6 +40,9 @@ std::optional GetTileSizeBytes(gcpp::KVEncoding encoding, case gcpp::KVEncoding::kF32: case gcpp::KVEncoding::kF32TwoTranspositions: return tileElements * sizeof(float); + case gcpp::KVEncoding::kInt4: + // Two 4-bit values per byte, plus the same scales as kInt8. + return tileElements / 2 + kTileSize * 2 * sizeof(gcpp::KV_microscale_t); default: return std::nullopt; } @@ -67,6 +71,31 @@ int8_t Quantize(float v, float inv_scale) { return hwy::ConvertScalarTo(scaled); } +// Symmetric 4-bit quantization: returns q in [-7, 7]. +int8_t QuantizeInt4(float v, float inv_scale) { + float scaled = std::nearbyint(v * inv_scale); + if (scaled > 7.0f) return 7; + if (scaled < -7.0f) return -7; + return hwy::ConvertScalarTo(scaled); +} + +// kInt4 stores q + 8 in a nibble. Element `offset` (in the kInt8 layout) is in +// byte `offset / 2`: the low nibble if `offset` is even, else the high nibble. +// K offsets pair adjacent tokens and V offsets pair adjacent dims, because +// kTileSize and qkv_dim are even. +inline void StoreNibble(uint8_t* data, size_t offset, int8_t q) { + const uint8_t nibble = static_cast(q + 8) & 0xF; + uint8_t& byte = data[offset / 2]; + byte = (offset % 2 == 0) ? static_cast((byte & 0xF0) | nibble) + : static_cast((byte & 0x0F) | (nibble << 4)); +} + +inline int LoadNibble(const uint8_t* data, size_t offset) { + const uint8_t byte = data[offset / 2]; + const int nibble = (offset % 2 == 0) ? (byte & 0xF) : (byte >> 4); + return nibble - 8; +} + template inline void DecodeTileWithFn(size_t qkv_dim, DecodedTile* out, const DecodeKFn& decode_k, @@ -282,6 +311,76 @@ void EncodeTileInt8(bool transposed, size_t qkv_dim, const DecodedTile& decoded, }); } +void EncodeTileInt4(size_t qkv_dim, const DecodedTile& decoded, + hwy::Span out_encoded_tile_data) { + HWY_DASSERT(qkv_dim % 2 == 0); + uint8_t* k_data = HWY_RCAST_ALIGNED(uint8_t*, out_encoded_tile_data.data()); + uint8_t* v_data = k_data + qkv_dim * kTileSize / 2; + gcpp::KV_microscale_t* scales = HWY_RCAST_ALIGNED( + gcpp::KV_microscale_t*, v_data + kTileSize * qkv_dim / 2); + gcpp::KV_microscale_t* k_scales = scales; + gcpp::KV_microscale_t* v_scales = scales + kTileSize; + + AlignedFloatVector inv_scales_k(kTileSize); + AlignedFloatVector inv_scales_v(kTileSize); + for (size_t token = 0; token < kTileSize; ++token) { + float k_max_abs = 0.0f; + float v_max_abs = 0.0f; + for (size_t dim = 0; dim < qkv_dim; ++dim) { + k_max_abs = std::max(k_max_abs, std::abs(decoded.k_elem(token, dim))); + v_max_abs = std::max(v_max_abs, std::abs(decoded.v_elem(token, dim))); + } + const float scale_k = k_max_abs == 0.0f ? 1.0f : k_max_abs / 7.0f; + k_scales[token] = hwy::ConvertScalarTo(scale_k); + // Quantize with the stored (bf16) scale, as for kInt8. + const float stored_k = hwy::ConvertScalarTo(k_scales[token]); + inv_scales_k[token] = stored_k == 0.0f ? 0.0f : 1.0f / stored_k; + + const float scale_v = v_max_abs == 0.0f ? 1.0f : v_max_abs / 7.0f; + v_scales[token] = hwy::ConvertScalarTo(scale_v); + const float stored_v = hwy::ConvertScalarTo(v_scales[token]); + inv_scales_v[token] = stored_v == 0.0f ? 0.0f : 1.0f / stored_v; + } + + EncodeTileWithFn( + qkv_dim, decoded, + [&](size_t dim, size_t token, float val) HWY_ATTR { + StoreNibble(k_data, KOffset(/*transposed=*/false, qkv_dim, dim, token), + QuantizeInt4(val, inv_scales_k[token])); + }, + [&](size_t dim, size_t token, float val) HWY_ATTR { + StoreNibble(v_data, VOffset(/*transposed=*/false, qkv_dim, dim, token), + QuantizeInt4(val, inv_scales_v[token])); + }); +} + +void DecodeTileInt4(size_t qkv_dim, hwy::Span encoded_tile_data, + DecodedTile* out) { + HWY_DASSERT(qkv_dim % 2 == 0); + const uint8_t* k_data = + HWY_RCAST_ALIGNED(const uint8_t*, encoded_tile_data.data()); + const uint8_t* v_data = k_data + qkv_dim * kTileSize / 2; + const gcpp::KV_microscale_t* scales = HWY_RCAST_ALIGNED( + const gcpp::KV_microscale_t*, v_data + kTileSize * qkv_dim / 2); + const gcpp::KV_microscale_t* k_scales = scales; + const gcpp::KV_microscale_t* v_scales = scales + kTileSize; + + DecodeTileWithFn( + qkv_dim, out, + [&](size_t dim, size_t token) HWY_ATTR { + const float scale = hwy::ConvertScalarTo(k_scales[token]); + return LoadNibble(k_data, + KOffset(/*transposed=*/false, qkv_dim, dim, token)) * + scale; + }, + [&](size_t dim, size_t token) HWY_ATTR { + const float scale = hwy::ConvertScalarTo(v_scales[token]); + return LoadNibble(v_data, + VOffset(/*transposed=*/false, qkv_dim, dim, token)) * + scale; + }); +} + void EncodeTileInt8VNNI(size_t qkv_dim, const DecodedTile& decoded, hwy::Span out_encoded_tile_data) { int8_t* k_data = HWY_RCAST_ALIGNED(int8_t*, out_encoded_tile_data.data()); @@ -555,6 +654,10 @@ bool DecodeTile(KVEncoding encoding, hwy::Span encoded_tile_data, DecodeTileInt8MatrixAccumulation(qkv_dim, encoded_tile_data, out); return true; } + case gcpp::KVEncoding::kInt4: { + DecodeTileInt4(qkv_dim, encoded_tile_data, out); + return true; + } default: return false; } @@ -597,6 +700,10 @@ bool EncodeTile(gcpp::KVEncoding encoding, const DecodedTile& decoded, EncodeTileInt8MatrixAccumulation(qkv_dim, decoded, out_encoded_tile_data); return true; } + case gcpp::KVEncoding::kInt4: { + EncodeTileInt4(qkv_dim, decoded, out_encoded_tile_data); + return true; + } default: return false; } diff --git a/gemma/kv_transcoding.h b/gemma/kv_transcoding.h index 4fef0018..98b6f06c 100644 --- a/gemma/kv_transcoding.h +++ b/gemma/kv_transcoding.h @@ -12,6 +12,12 @@ namespace gcpp { // Returns the size in bytes of a single KV cache tile for a given encoding. // Returns std::nullopt if the encoding is unsupported. +// +// KVEncoding::kInt4 uses the kInt8 layout (K: [qkv_dim, tile_size], +// V: [tile_size, qkv_dim], then K and V per-token bf16 scales) with two 4-bit +// values per byte: element i of the kInt8 layout is in byte i / 2, in the low +// nibble if i is even. A nibble stores q + 8 for q in [-7, 7]; the value is +// q * scale, with scale = bf16(absmax / 7) per token. std::optional GetTileSizeBytes(gcpp::KVEncoding encoding, size_t qkv_dim); diff --git a/gemma/kv_transcoding_test.cc b/gemma/kv_transcoding_test.cc index eeaf8d9e..71571683 100644 --- a/gemma/kv_transcoding_test.cc +++ b/gemma/kv_transcoding_test.cc @@ -103,7 +103,9 @@ INSTANTIATE_TEST_SUITE_P( EncodingTestCase{gcpp::KVEncoding::kInt8, 0.1f}, EncodingTestCase{gcpp::KVEncoding::kInt8TwoTranspositions, 0.1f}, EncodingTestCase{gcpp::KVEncoding::kInt8VNNITwoTranspositions, 0.1f}, - EncodingTestCase{gcpp::KVEncoding::kInt8MatrixAccumulation, 0.02f})); + EncodingTestCase{gcpp::KVEncoding::kInt8MatrixAccumulation, 0.02f}, + // Half a step is 1.5 / 7 / 2 ~= 0.107 for this pattern's absmax. + EncodingTestCase{gcpp::KVEncoding::kInt4, 0.11f})); class Int8EncodingTest : public TestWithParam {}; @@ -160,6 +162,207 @@ INSTANTIATE_TEST_SUITE_P(AllInt8Encodings, Int8EncodingTest, gcpp::KVEncoding::kInt8VNNITwoTranspositions, gcpp::KVEncoding::kInt8MatrixAccumulation)); +// Fills a tile with per-token magnitudes that vary, so that absmax / qmax is +// rarely exact in bf16. +void FillVaryingTile(DecodedTile& tile) { + for (size_t token = 0; token < tile.tile_size; ++token) { + for (size_t dim = 0; dim < tile.qkv_dim; ++dim) { + const float i = static_cast(dim * tile.tile_size + token); + tile.k_elem(token, dim) = std::sin(i * 0.37f) * (1.0f + token); + tile.v_elem(token, dim) = std::cos(i * 0.53f) * (3.0f + 0.7f * token); + } + } +} + +// Reference per-token symmetric 4-bit quant-dequant of one row: bf16 scale +// absmax / 7, quantized with the stored scale, values in [-7, 7]. +void RefQuantDequantInt4Row(float* row, size_t d) { + float max_abs = 0.0f; + for (size_t j = 0; j < d; ++j) max_abs = std::max(max_abs, std::abs(row[j])); + const float scale = hwy::ConvertScalarTo( + hwy::ConvertScalarTo(max_abs == 0.0f ? 1.0f + : max_abs / 7.0f)); + const float inv = scale == 0.0f ? 0.0f : 1.0f / scale; + for (size_t j = 0; j < d; ++j) { + const float q = std::clamp(std::nearbyint(row[j] * inv), -7.0f, 7.0f); + row[j] = q * scale; + } +} + +TEST(Int4EncodingTest, RoundTripIsWithinHalfAStep) { + constexpr size_t kTileSize = 32; + constexpr size_t qkv_dim = 256; + const KVEncoding encoding = KVEncoding::kInt4; + + DecodedTile original(qkv_dim, kTileSize); + FillVaryingTile(original); + + hwy::AlignedVector encoded(GetTileSizeBytes(encoding, qkv_dim).value()); + ASSERT_TRUE(EncodeTile(encoding, original, qkv_dim, + hwy::Span(encoded.data(), encoded.size()))); + DecodedTile decoded(qkv_dim, kTileSize); + ASSERT_TRUE(DecodeTile(encoding, + hwy::Span(encoded.data(), encoded.size()), + qkv_dim, &decoded)); + + for (size_t token = 0; token < kTileSize; ++token) { + float k_max = 0.0f, v_max = 0.0f; + for (size_t dim = 0; dim < qkv_dim; ++dim) { + k_max = std::max(k_max, std::abs(original.k_elem(token, dim))); + v_max = std::max(v_max, std::abs(original.v_elem(token, dim))); + } + // Half a step, plus slack for rounding the scale to bf16. + const float k_bound = 0.51f * k_max / 7.0f; + const float v_bound = 0.51f * v_max / 7.0f; + for (size_t dim = 0; dim < qkv_dim; ++dim) { + ASSERT_LE( + std::abs(decoded.k_elem(token, dim) - original.k_elem(token, dim)), + k_bound) + << "k token=" << token << " dim=" << dim; + ASSERT_LE( + std::abs(decoded.v_elem(token, dim) - original.v_elem(token, dim)), + v_bound) + << "v token=" << token << " dim=" << dim; + } + } +} + +// The packed encoding must reproduce the simulated (quant-dequant) INT4 that +// the accuracy evaluation used, bit for bit. +TEST(Int4EncodingTest, MatchesReferenceQuantDequantExactly) { + constexpr size_t kTileSize = 32; + constexpr size_t qkv_dim = 512; + const KVEncoding encoding = KVEncoding::kInt4; + + DecodedTile original(qkv_dim, kTileSize); + FillVaryingTile(original); + DecodedTile expected = original; + for (size_t token = 0; token < kTileSize; ++token) { + RefQuantDequantInt4Row(&expected.k_elem(token, 0), qkv_dim); + RefQuantDequantInt4Row(&expected.v_elem(token, 0), qkv_dim); + } + + hwy::AlignedVector encoded(GetTileSizeBytes(encoding, qkv_dim).value()); + ASSERT_TRUE(EncodeTile(encoding, original, qkv_dim, + hwy::Span(encoded.data(), encoded.size()))); + DecodedTile decoded(qkv_dim, kTileSize); + ASSERT_TRUE(DecodeTile(encoding, + hwy::Span(encoded.data(), encoded.size()), + qkv_dim, &decoded)); + + EXPECT_THAT(decoded.k, Pointwise(::testing::FloatEq(), expected.k)); + EXPECT_THAT(decoded.v, Pointwise(::testing::FloatEq(), expected.v)); +} + +TEST(Int4EncodingTest, TileIsHalfOfInt8PlusScales) { + for (size_t qkv_dim : {256, 512}) { + const size_t scales = 32 * 2 * sizeof(hwy::bfloat16_t); + const size_t int8 = GetTileSizeBytes(KVEncoding::kInt8, qkv_dim).value(); + const size_t int4 = GetTileSizeBytes(KVEncoding::kInt4, qkv_dim).value(); + EXPECT_EQ(int4, (int8 - scales) / 2 + scales) << "qkv_dim=" << qkv_dim; + } + EXPECT_EQ(KVEncodingToString(KVEncoding::kInt4), "Int4"); +} + +TEST(Int4EncodingTest, LayoutValidation) { + constexpr size_t kTileSize = 32; + constexpr size_t qkv_dim = 4; + const KVEncoding encoding = KVEncoding::kInt4; + + // Token 0 has K = {7, -7, 0, 3.5} and V = {-14, 14, 2, 0}; other tokens are + // zero. Scales: K 7/7 = 1, V 14/7 = 2. + DecodedTile original(qkv_dim, kTileSize); + for (size_t token = 0; token < kTileSize; ++token) { + for (size_t dim = 0; dim < qkv_dim; ++dim) { + original.k_elem(token, dim) = 0.0f; + original.v_elem(token, dim) = 0.0f; + } + } + const float k0[] = {7.0f, -7.0f, 0.0f, 3.25f}; + const float v0[] = {-14.0f, 14.0f, 2.0f, 0.0f}; + for (size_t dim = 0; dim < qkv_dim; ++dim) { + original.k_elem(0, dim) = k0[dim]; + original.v_elem(0, dim) = v0[dim]; + } + // Token 1, dim 0: K = 5 (alone in its row, so q = 7, scale 5/7). + original.k_elem(1, 0) = 5.0f; + + hwy::AlignedVector encoded(GetTileSizeBytes(encoding, qkv_dim).value()); + ASSERT_TRUE(EncodeTile(encoding, original, qkv_dim, + hwy::Span(encoded.data(), encoded.size()))); + const uint8_t* data = HWY_RCAST_ALIGNED(const uint8_t*, encoded.data()); + + // K is [qkv_dim, tile_size] nibbles: byte (dim * 32 + token) / 2, low nibble + // for even tokens. Stored value is q + 8. + EXPECT_EQ(data[0], (8 + 7) | ((8 + 7) << 4)); // dim 0: t0 q=7, t1 q=7 + EXPECT_EQ(data[16], (8 - 7) | (8 << 4)); // dim 1: t0 q=-7, t1 q=0 + EXPECT_EQ(data[32], 8 | (8 << 4)); // dim 2: t0 q=0 + EXPECT_EQ(data[48], (8 + 3) | (8 << 4)); // dim 3: t0 3.25 -> q=3 + // V is [tile_size, qkv_dim] nibbles starting at qkv_dim * 32 / 2 = 64; low + // nibble for even dims. + EXPECT_EQ(data[64], (8 - 7) | ((8 + 7) << 4)); // t0: d0 q=-7, d1 q=7 + EXPECT_EQ(data[65], (8 + 1) | (8 << 4)); // t0: d2 q=1, d3 q=0 + // Scales follow at 128: K scales then V scales, bf16. + const hwy::bfloat16_t* scales = + HWY_RCAST_ALIGNED(const hwy::bfloat16_t*, data + 128); + EXPECT_EQ(hwy::ConvertScalarTo(scales[0]), 1.0f); + EXPECT_EQ(hwy::ConvertScalarTo(scales[32]), 2.0f); +} + +TEST(Int4EncodingTest, TranscodesToAndFromInt8) { + constexpr size_t kTileSize = 32; + constexpr size_t qkv_dim = 256; + DecodedTile original(qkv_dim, kTileSize); + FillVaryingTile(original); + + hwy::AlignedVector int4( + GetTileSizeBytes(KVEncoding::kInt4, qkv_dim).value()); + hwy::AlignedVector int8( + GetTileSizeBytes(KVEncoding::kInt8, qkv_dim).value()); + hwy::AlignedVector int4_again(int4.size()); + ASSERT_TRUE(EncodeTile(KVEncoding::kInt4, original, qkv_dim, + hwy::Span(int4.data(), int4.size()))); + ASSERT_TRUE(TranscodeTile(KVEncoding::kInt4, + hwy::Span(int4.data(), int4.size()), + KVEncoding::kInt8, + hwy::Span(int8.data(), int8.size()), + qkv_dim)); + ASSERT_TRUE(TranscodeTile( + KVEncoding::kInt8, hwy::Span(int8.data(), int8.size()), + KVEncoding::kInt4, hwy::Span(int4_again.data(), int4_again.size()), + qkv_dim)); + + // The INT8 hop moves each value by at most half an INT8 step (~absmax / 254), + // far below half an INT4 step (~absmax / 14), so re-encoding to INT4 must + // give the same values up to rounding of the scale. + DecodedTile a(qkv_dim, kTileSize), b(qkv_dim, kTileSize); + ASSERT_TRUE(DecodeTile(KVEncoding::kInt4, + hwy::Span(int4.data(), int4.size()), + qkv_dim, &a)); + ASSERT_TRUE(DecodeTile( + KVEncoding::kInt4, + hwy::Span(int4_again.data(), int4_again.size()), qkv_dim, + &b)); + for (size_t token = 0; token < kTileSize; ++token) { + float k_max = 0.0f; + for (size_t dim = 0; dim < qkv_dim; ++dim) { + k_max = std::max(k_max, std::abs(a.k_elem(token, dim))); + } + for (size_t dim = 0; dim < qkv_dim; ++dim) { + ASSERT_NEAR(a.k_elem(token, dim), b.k_elem(token, dim), 0.02f * k_max) + << "k token=" << token << " dim=" << dim; + } + float v_max = 0.0f; + for (size_t dim = 0; dim < qkv_dim; ++dim) { + v_max = std::max(v_max, std::abs(a.v_elem(token, dim))); + } + for (size_t dim = 0; dim < qkv_dim; ++dim) { + ASSERT_NEAR(a.v_elem(token, dim), b.v_elem(token, dim), 0.02f * v_max) + << "v token=" << token << " dim=" << dim; + } + } +} + TEST(KVEncodingTest, ConvertTileFloat32ToBfloat16) { constexpr size_t kTileSize = 32; constexpr size_t qkv_dim = 256;