Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .bazelversion
Original file line number Diff line number Diff line change
@@ -1 +1 @@
7.1.1
9.2.0
20 changes: 19 additions & 1 deletion BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,22 @@ cc_test(
],
)

cc_test(
name = "sentencepiece_tokenizer_test",
srcs = ["tokenizer/sentencepiece_tokenizer_test.cc"],
data = [
"tokenizer/testdata/tokenizer.model",
],
deps = [
":tokenizer",
"@googletest//:gtest_main", # buildcleaner: keep
"//io",
"@highway//:hwy",
"@com_google_sentencepiece//:sentencepiece_model_converters",
"@com_google_sentencepiece//src:sentencepiece_model_cc_proto",
],
)

cc_test(
name = "bpe_tokenizer_benchmark_test",
srcs = ["tokenizer/bpe_tokenizer_benchmark_test.cc"],
Expand Down Expand Up @@ -322,7 +338,9 @@ cc_library(
"//io:fields",
"@highway//:hwy",
"@highway//:profiler",
"@com_google_sentencepiece//:sentencepiece_processor",
"@com_google_sentencepiece//:sentencepiece_lite",
"@com_google_sentencepiece//:sentencepiece_model_converters",
"@com_google_sentencepiece//src:sentencepiece_model_cc_proto",
],
)

Expand Down
12 changes: 9 additions & 3 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ endif()

## Note: absl needs to be installed by sentencepiece. This will only happen if
## cmake is invoked with -DSPM_ENABLE_SHARED=OFF and -DSPM_ABSL_PROVIDER=module
gemma_fetch(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG e0cce7d37b065b5140349dbe12c6bcf6192fdd78 EXCLUDE_FROM_ALL)
gemma_fetch(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG eafd99cf3b8bfa5101c2c5f42aab2cc39947938d EXCLUDE_FROM_ALL) # v0.2.3

gemma_fetch(json GIT_REPOSITORY https://github.com/nlohmann/json.git GIT_TAG 9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03 EXCLUDE_FROM_ALL)

Expand Down Expand Up @@ -271,14 +271,19 @@ endif()

FetchContent_GetProperties(sentencepiece)

# tokenizer/sentencepiece_tokenizer.cc uses SentencePiece Lite: the runtime
# (sentencepiece_lite) and the ModelProto -> FlatBuffer converter
# (sentencepiece_lite_fb_converter). Both are separate from sentencepiece-static.
set(GEMMA_SENTENCEPIECE_LIBS sentencepiece_lite sentencepiece_lite_fb_converter)

## Library Target

add_library(libgemma ${SOURCES})
set_property(TARGET libgemma PROPERTY CXX_STANDARD ${GEMMA_CXX_STANDARD})
set_target_properties(libgemma PROPERTIES PREFIX "")
set_property(TARGET libgemma PROPERTY POSITION_INDEPENDENT_CODE ON)
target_include_directories(libgemma PUBLIC ./)
target_link_libraries(libgemma hwy hwy_contrib sentencepiece-static
target_link_libraries(libgemma hwy hwy_contrib ${GEMMA_SENTENCEPIECE_LIBS}
nlohmann_json::nlohmann_json)
target_include_directories(libgemma PUBLIC ${sentencepiece_SOURCE_DIR})
target_compile_definitions(libgemma PRIVATE $<$<PLATFORM_ID:Windows>:_CRT_SECURE_NO_WARNINGS NOMINMAX>)
Expand Down Expand Up @@ -306,7 +311,8 @@ target_include_directories(gemma_shared PUBLIC ./)
target_link_libraries(gemma_shared PRIVATE
$<LINK_LIBRARY:WHOLE_ARCHIVE,hwy>
$<LINK_LIBRARY:WHOLE_ARCHIVE,hwy_contrib>
$<LINK_LIBRARY:WHOLE_ARCHIVE,sentencepiece-static>
$<LINK_LIBRARY:WHOLE_ARCHIVE,sentencepiece_lite>
$<LINK_LIBRARY:WHOLE_ARCHIVE,sentencepiece_lite_fb_converter>
)
target_include_directories(gemma_shared PUBLIC ${sentencepiece_SOURCE_DIR})
target_compile_definitions(gemma_shared
Expand Down
40 changes: 9 additions & 31 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,15 @@ bazel_dep(name = "rules_cc", version = "0.2.0")
bazel_dep(name = "rules_license", version = "1.0.0")
bazel_dep(name = "rules_python", version = "1.6.3")
bazel_dep(name = "google_benchmark", version = "1.8.5")
bazel_dep(name = "sentencepiece", version = "0.2.3", repo_name = "com_google_sentencepiece")

# SentencePiece is not in the Bazel Central Registry yet.
archive_override(
module_name = "sentencepiece",
integrity = "sha256-rqv5sq4ad82m5yGrH7n25EuLpN241zQ9xbAq7HD6syQ=",
strip_prefix = "sentencepiece-0.2.3",
urls = ["https://github.com/google/sentencepiece/archive/refs/tags/v0.2.3.tar.gz"],
)

# Require a more recent version.
git_override(
Expand Down Expand Up @@ -63,37 +72,6 @@ http_archive(
],
)

http_archive(
name = "com_google_sentencepiece",
build_file = "@//bazel:sentencepiece.bazel",
patch_args = ["-p1"],
patches = ["@//bazel:sentencepiece.patch"],
sha256 = "8409b0126ebd62b256c685d5757150cf7fcb2b92a2f2b98efb3f38fc36719754",
strip_prefix = "sentencepiece-0.1.96",
urls = ["https://github.com/google/sentencepiece/archive/refs/tags/v0.1.96.zip"],
)

# For sentencepiece
http_archive(
name = "darts_clone",
build_file_content = """
licenses(["notice"])
exports_files(["LICENSE"])
package(default_visibility = ["//visibility:public"])
cc_library(
name = "darts_clone",
hdrs = [
"include/darts.h",
],
)
""",
sha256 = "c97f55d05c98da6fcaf7f9ecc6a6dc6bc5b18b8564465f77abff8879d446491c",
strip_prefix = "darts-clone-e40ce4627526985a7767444b6ed6893ab6ff8983",
urls = [
"https://github.com/s-yata/darts-clone/archive/e40ce4627526985a7767444b6ed6893ab6ff8983.zip",
],
)

pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
pip.parse(
hub_name = "compression_deps",
Expand Down
2 changes: 1 addition & 1 deletion bazel/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Required for referencing bazel:sentencepiece.patch
# Required for referencing the build files in this directory.
package(
default_applicable_licenses = ["//:license"],
default_visibility = ["//:__subpackages__"],
Expand Down
97 changes: 0 additions & 97 deletions bazel/sentencepiece.bazel

This file was deleted.

Loading
Loading