Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions gemma/model_store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -401,6 +401,13 @@ bool ModelStore::ReadMatPtrs(BlobReader& reader) {

// Retrieve actual key index because a writer may have written other
// blobs before the tensor data.
HWY_ASSERT(result.missing_fields <= 1);
HWY_ASSERT(EnumValid(mat.GetType()) &&
mat.GetType() != Type::kUnknown);
const size_t stride =
(result.missing_fields == 1) ? mat.Cols() : mat.Stride();
mat.SetType(mat.GetType());
mat.SetPtr(nullptr, stride);
const BlobRange* range = reader.Find(mat.Name());
HWY_ASSERT(range);
const size_t key_idx = range->key_idx;
Expand Down
98 changes: 98 additions & 0 deletions gemma/weights_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -285,5 +285,103 @@ TEST(WeightsTest, MappingRequiresBlobAlignment) {
WeightsPtrs::Mode::kReadBF16);
}

static void WriteModelBlobWithToc(const Path& path, ThreadingContext& ctx,
const std::vector<uint32_t>& toc) {
ModelConfig config(Model::GEMMA2_2B, Type::kBF16, PromptWrapping::GEMMA_PT);
GemmaTokenizer tokenizer(kMockTokenizer);
const std::array<BF16, 16> tensor_data = {};
BlobWriter writer(path, ctx);
writer.Add("w_25", tensor_data.data(), sizeof(tensor_data));
WriteSingleFile(config, tokenizer, toc, writer);
}

TEST(WeightsTest, SetTypeRejectsNumElementsOverflow) {
// 524296 * 8192 = 4,295,032,832 > UINT32_MAX (b/583871800).
MatPtr mat("emb", Type::kUnknown, Extents2D(524296, 8192));
EXPECT_DEATH(mat.SetType(Type::kBF16), "extents\\.rows <=");

EXPECT_DEATH(
MatPtr("big", Type::kBF16, Extents2D(uint64_t{UINT32_MAX} + 1, 1)),
"extents\\.rows <=");

MatPtr non_empty("ne", Type::kBF16, Extents2D(4, 8));
EXPECT_DEATH(non_empty.OverrideRows(0), "rows > 0");
EXPECT_DEATH(non_empty.OverrideCols(0), "cols > 0");

MatPtr default_mat;
EXPECT_EQ(default_mat.GetType(), Type::kUnknown);
EXPECT_EQ(default_mat.Stride(), 0u);

MatPtr reshaped("rs", Type::kBF16, Extents2D(6, 4));
reshaped.ReshapePackedRowsToCols(2);
EXPECT_EQ(reshaped.Rows(), 3u);
EXPECT_EQ(reshaped.Cols(), 8u);
EXPECT_EQ(reshaped.Stride(), 8u);
EXPECT_EQ(reshaped.PackedBytes(), 48u);

MatPtr odd("odd", Type::kBF16, Extents2D(5, 8));
EXPECT_DEATH(odd.ReshapePackedRowsToCols(2), "private_rows_ % factor");

MatPtr overridden("ov", Type::kBF16, Extents2D(4, 8));
overridden.OverrideRows(2);
EXPECT_DEATH(overridden.ReshapePackedRowsToCols(2), "override_rows_ == 0");
}

TEST(WeightsTest, ModelStoreValidatesDeserializedTocEntries) {
ThreadingContext ctx = MakeContext();
MatPtr valid("w_25", Type::kBF16, Extents2D(4, 4));
valid.SetScale(1.5f);
const std::vector<uint32_t> base_toc = valid.Write();
ASSERT_EQ(base_toc.size(), 10u);

// Full TOC roundtrip and legacy TOC omitting `stride_` (missing_fields == 1).
std::vector<uint32_t> legacy_toc = base_toc;
legacy_toc.pop_back();
legacy_toc[0] -= 1;
for (const std::vector<uint32_t>& toc : {base_toc, legacy_toc}) {
TemporaryBlob blob;
WriteModelBlobWithToc(blob.path(), ctx, toc);
BlobReader reader(blob.path());
ModelStore store(reader);
const MatPtr* loaded = store.FindMat("w_25");
ASSERT_NE(loaded, nullptr);
EXPECT_EQ(loaded->GetType(), Type::kBF16);
EXPECT_EQ(loaded->Rows(), 4u);
EXPECT_EQ(loaded->Cols(), 4u);
EXPECT_EQ(loaded->Stride(), 4u);
EXPECT_EQ(loaded->PackedBytes(), 32u);
EXPECT_FLOAT_EQ(loaded->Scale(), 1.5f);
}

const auto expect_toc_death = [&](const std::vector<uint32_t>& bad_toc,
const char* regex) {
TemporaryBlob blob;
WriteModelBlobWithToc(blob.path(), ctx, bad_toc);
BlobReader reader(blob.path());
EXPECT_DEATH({ ModelStore store(reader); }, regex);
};

// rows * cols = 524296 * 8192 > UINT32_MAX (b/583871800).
std::vector<uint32_t> overflow_toc = base_toc;
overflow_toc[6] = 524296;
overflow_toc[7] = 8192;
overflow_toc[9] = 8192;
expect_toc_death(overflow_toc, "extents\\.rows <=");

// stride < cols.
std::vector<uint32_t> bad_stride_toc = base_toc;
bad_stride_toc[9] = 2;
expect_toc_death(bad_stride_toc, "stride 2 < cols 4");

// Type::kUnknown in TOC.
std::vector<uint32_t> unknown_type_toc = base_toc;
unknown_type_toc[3] = static_cast<uint32_t>(Type::kUnknown);
expect_toc_death(unknown_type_toc, "mat\\.GetType\\(\\) != Type::kUnknown");

// Truncated TOC entry stopping right after `name_` (missing_fields == 7).
std::vector<uint32_t> truncated_toc = {2, 1, base_toc[2]};
expect_toc_death(truncated_toc, "result\\.missing_fields <= 1");
}

} // namespace
} // namespace gcpp
51 changes: 37 additions & 14 deletions util/mat.h
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,10 @@ class MatPtr : public IFields {
MatPtr() = default;
// `name`: see `SetName`. Note that `stride` is initially `cols` and only
// differs after deserializing, or calling `SetPtr`.
MatPtr(const char* name, Type type, Extents2D extents)
: private_rows_(static_cast<uint32_t>(extents.rows)),
cols_(static_cast<uint32_t>(extents.cols)) {
MatPtr(const char* name, Type type, Extents2D extents) {
HWY_ASSERT(extents.rows <= UINT32_MAX && extents.cols <= UINT32_MAX);
private_rows_ = static_cast<uint32_t>(extents.rows);
cols_ = static_cast<uint32_t>(extents.cols);
SetName(name);
SetType(type);
SetPtr(nullptr, cols_);
Expand All @@ -98,6 +99,7 @@ class MatPtr : public IFields {
if (stride < Cols()) {
HWY_ABORT("%s: stride %zu < cols %zu\n", Name(), stride, Cols());
}
HWY_ASSERT(stride <= UINT32_MAX);
ptr_ = ptr;
stride_ = static_cast<uint32_t>(stride);

Expand Down Expand Up @@ -163,15 +165,18 @@ class MatPtr : public IFields {
// Works for any kind of padding and element type.
uint8_t* RowBytes(size_t row) {
HWY_DASSERT(row < Rows());
return static_cast<uint8_t*>(ptr_) + row * (stride_ * element_bytes_);
return static_cast<uint8_t*>(ptr_) +
row * static_cast<size_t>(stride_) * element_bytes_;
}
const uint8_t* RowBytes(size_t row) const {
HWY_DASSERT(row < Rows());
return static_cast<const uint8_t*>(ptr_) + row * (stride_ * element_bytes_);
return static_cast<const uint8_t*>(ptr_) +
row * static_cast<size_t>(stride_) * element_bytes_;
}

Type GetType() const { return type_; }
void SetType(Type type) {
HWY_ASSERT(EnumValid(type));
type_ = type;
if (type == Type::kUnknown) {
// Temporary invalid state. Happens during weights.h construction, before
Expand All @@ -181,7 +186,9 @@ class MatPtr : public IFields {
return;
}
element_bytes_ = static_cast<uint32_t>(hwy::DivCeil(TypeBits(type), 8));
num_elements_ = static_cast<uint32_t>(ComputeNumElements(type, Extents()));
const size_t num_elements = ComputeNumElements(type, Extents());
HWY_ASSERT(num_elements <= UINT32_MAX);
num_elements_ = static_cast<uint32_t>(num_elements);
HWY_DASSERT(0 != element_bytes_ && element_bytes_ <= 16);
}

Expand Down Expand Up @@ -212,27 +219,41 @@ class MatPtr : public IFields {
HWY_ABORT("%s: rows %zu > private_rows_ %u\n", name_.c_str(), rows,
private_rows_);
}
HWY_ASSERT(rows > 0 || private_rows_ == 0);
override_rows_ = static_cast<uint32_t>(rows);
num_elements_ = static_cast<uint32_t>(ComputeNumElements(type_, Extents()));
const size_t num_elements = ComputeNumElements(type_, Extents());
HWY_ASSERT(num_elements <= UINT32_MAX);
num_elements_ = static_cast<uint32_t>(num_elements);
}

void OverrideCols(size_t cols) {
if (HWY_UNLIKELY(cols > stride_)) {
HWY_ABORT("%s: cols %zu > stride_ %u\n", name_.c_str(), cols,
stride_);
}
HWY_ASSERT(cols > 0 || cols_ == 0);
override_cols_ = static_cast<uint32_t>(cols);
num_elements_ = static_cast<uint32_t>(ComputeNumElements(type_, Extents()));
const size_t num_elements = ComputeNumElements(type_, Extents());
HWY_ASSERT(num_elements <= UINT32_MAX);
num_elements_ = static_cast<uint32_t>(num_elements);
}

// Changes the number of rows and columns without reallocating the memory.
// Increases cols by factor and reduces rows by factor.
// The rows must be divisible by factor and the matrix must be packed.
void ReshapePackedRowsToCols(size_t factor) {
HWY_ASSERT(factor > 0 && factor <= UINT32_MAX);
HWY_ASSERT(override_rows_ == 0 && override_cols_ == 0);
HWY_ASSERT(IsPacked());
private_rows_ = hwy::DivCeil(private_rows_, factor);
cols_ *= factor;
stride_ *= factor; // safe because we verified IsPacked
HWY_ASSERT((private_rows_ % factor) == 0);
HWY_ASSERT(static_cast<uint64_t>(cols_) * factor <= UINT32_MAX);
HWY_ASSERT(static_cast<uint64_t>(stride_) * factor <= UINT32_MAX);
private_rows_ = static_cast<uint32_t>(private_rows_ / factor);
cols_ = static_cast<uint32_t>(cols_ * factor);
stride_ = static_cast<uint32_t>(stride_ * factor); // safe: IsPacked
const size_t num_elements = ComputeNumElements(type_, Extents());
HWY_ASSERT(num_elements <= UINT32_MAX);
num_elements_ = static_cast<uint32_t>(num_elements);
}

// Offset by which to advance pointers to the next row.
Expand Down Expand Up @@ -276,11 +297,12 @@ class MatPtr : public IFields {
// padding, which is anyway not supported for NUQ because `compress-inl.h`
// assumes a contiguous stream for its group indexing.
static size_t ComputeNumElements(Type type, Extents2D extents) {
HWY_ASSERT(extents.cols == 0 || extents.rows <= UINT32_MAX / extents.cols);
return PackedEnd(type, extents.Area());
}

std::string name_; // See `SetName`.
Type type_;
Type type_ = Type::kUnknown;

// Most members are u32 because that is the preferred type of fields.h.

Expand All @@ -306,7 +328,7 @@ class MatPtr : public IFields {
uint8_t** row_ptrs_ = nullptr; // not serialized

// Offset by which to advance pointers to the next row, >= `cols_`.
uint32_t stride_;
uint32_t stride_ = 0;

float scale_ = 1.0f; // multiplier for each value, for MatMul.

Expand Down Expand Up @@ -399,7 +421,8 @@ RowPtrs<T> GetOrSetTempRowPtrs(
"%s.AttachRowPtrs() was not called.\n",
mat.Name());
}
HWY_DASSERT(mat.HasPtr());
HWY_ASSERT(mat.HasPtr());
HWY_ASSERT(mat.Rows() <= kMaxBatchSize);
for (size_t r = 0; r < mat.Rows(); ++r) {
storage[r] = reinterpret_cast<uint8_t*>(const_cast<T*>(mat.Row(r)));
}
Expand Down
Loading