diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3709ac4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,12 @@ +* text=auto +*.png binary +*.jpg binary +*.npy binary +*.npz binary +*.safetensors binary +*.gguf binary +*.bin binary +third_party/** linguist-vendored +tests/data/** linguist-generated +tests/golden/** linguist-generated +tests/reference_images/** linguist-generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..d1bfe83 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Review requests for pull requests. The last matching line wins. +* @xycld diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..2515b81 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,11 @@ +## What changes + + + +## Tests + + + +## Speed and memory + + diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c30ae1a..dffd438 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -249,29 +249,29 @@ jobs: ctest --preset "$preset" -L '^cpu$' --parallel 4 - name: Install - run: cmake --install "build-$preset" --prefix "build-$preset/install" + run: cmake --install "build/$preset" --prefix "build/$preset/install" - name: Install the system-CUDA variant # Everything but the `cuda` component (cmake/modules/packaging.cmake): no cuBLASLt, no CUDA EULA. - run: cmake --install "build-$preset" --component Unspecified --prefix "build-$preset/install-system-cuda" + run: cmake --install "build/$preset" --component Unspecified --prefix "build/$preset/install-system-cuda" - name: AC1 / exports / AC17 / find_package consumer run: | cuda="${{ matrix.cuda }}" - python .github/scripts/check_package.py --prefix "build-$preset/install" --cuda-major "${cuda%%.*}" \ - --consumer examples/consumer-cmake --consumer-build "build-$preset/consumer" --run-consumer + python .github/scripts/check_package.py --prefix "build/$preset/install" --cuda-major "${cuda%%.*}" \ + --consumer examples/consumer-cmake --consumer-build "build/$preset/consumer" --run-consumer - name: AC1 / exports / AC17 / no CUDA libraries (system-CUDA variant) - run: python .github/scripts/check_package.py --prefix "build-$preset/install-system-cuda" --system-cuda + run: python .github/scripts/check_package.py --prefix "build/$preset/install-system-cuda" --system-cuda - name: Package - working-directory: build-${{ env.preset }} + working-directory: build/${{ env.preset }} run: cpack --config CPackConfig.cmake - name: Package the system-CUDA variant # The CPack archives' layout (one top-level directory named like the archive) and checksum files, from the # system-CUDA install: mossinfer---cu--system-cuda.{zip,tar.gz}. - working-directory: build-${{ env.preset }} + working-directory: build/${{ env.preset }} run: | full=$(ls mossinfer-*.zip | grep -v -- '-system-cuda\.zip$') [ "$(echo "$full" | wc -l)" -eq 1 ] || { echo "expected one CPack zip, got: $full"; exit 1; } @@ -290,7 +290,7 @@ jobs: - name: AC17 size of the archive run: | - for archive in build-"$preset"/mossinfer-*.zip build-"$preset"/mossinfer-*.tar.gz; do + for archive in build/"$preset"/mossinfer-*.zip build/"$preset"/mossinfer-*.tar.gz; do [ -f "$archive" ] || continue case "$archive" in *-system-cuda.*) python .github/scripts/check_package.py --archive "$archive" --system-cuda ;; @@ -304,9 +304,9 @@ jobs: with: name: mossinfer-${{ matrix.platform }}-${{ matrix.flavour }} path: | - build-${{ env.preset }}/mossinfer-*.zip - build-${{ env.preset }}/mossinfer-*.tar.gz - build-${{ env.preset }}/mossinfer-*.sha256 + build/${{ env.preset }}/mossinfer-*.zip + build/${{ env.preset }}/mossinfer-*.tar.gz + build/${{ env.preset }}/mossinfer-*.sha256 if-no-files-found: error retention-days: ${{ inputs.retention-days }} compression-level: 0 diff --git a/.mailmap b/.mailmap new file mode 100644 index 0000000..bef9a48 --- /dev/null +++ b/.mailmap @@ -0,0 +1 @@ +xycld xy_cloud <85053859+xycld@users.noreply.github.com> diff --git a/CMakePresets.json b/CMakePresets.json index 38898f7..b6cddc2 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -6,8 +6,8 @@ "name": "base", "hidden": true, "generator": "Ninja", - "binaryDir": "${sourceDir}/build-${presetName}", - "installDir": "${sourceDir}/build-${presetName}/install", + "binaryDir": "${sourceDir}/build/${presetName}", + "installDir": "${sourceDir}/build/${presetName}/install", "cacheVariables": { "CMAKE_EXPORT_COMPILE_COMMANDS": "ON" } }, { @@ -97,7 +97,7 @@ "name": "windows-msvc-cuda13-sm75", "displayName": "Windows release, CUDA 13.x with Turing (sm_75)", "inherits": "windows-msvc-cuda13", - "binaryDir": "${sourceDir}/build-${presetName}", + "binaryDir": "${sourceDir}/build/${presetName}", "cacheVariables": { "MOSSINFER_SM75": "ON" } }, { diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..bbfbcb4 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,51 @@ +# Contributing + +## Build and test + +CUDA 12.x or 13.x, CMake ≥ 3.28 and Ninja; on Linux GCC ≥ 12 and GoogleTest, on Windows a Visual Studio 2022 x64 +developer prompt. + +```sh +cmake --preset dev && cmake --build --preset dev +CUDA_VISIBLE_DEVICES="" ctest --preset dev -L '^cpu$' # what CI runs +ctest --preset dev # adds the GPU tests on your GPU +``` + +The `dev` preset builds for compute capability 8.9; pass `-DCMAKE_CUDA_ARCHITECTURES=` for another GPU. Build +trees go to `build/`. Tests that need model files skip when the files are absent. + +CI builds every release package (Linux and Windows, CUDA 12 and 13, the `-sm75` and `-sm70` builds, Vulkan and HIP) +and runs the CPU tests and the package checks. It has no GPU: a change to a kernel, the planner or a pipeline path +needs a GPU run, named in the pull request. + +## Branches + +Branch from `main` and name the branch for the kind of change: `feat/`, `fix/`, `perf/`, `refactor/`, `docs/`, `ci/`, +`test/`, and `release/X.Y.Z` for a version bump. Open a draft pull request to run CI on work in progress. + +## Commits + +Subject line: `area: what changes`, in the imperative, without a final period. Areas: `cuda`, `kernels`, `pipeline`, +`planner`, `weights`, `quantize`, `capi`, `server`, `cli`, `sm75`, `sm70`, `vulkan`, `hip`, `build`, `ci`, `release`, +`docs`, `tests`. + +``` +vulkan: count the DiT's weights in --vram-budget with every block streamed +capi: report the preload's plan before its first phase +``` + +The body says why, and what the change costs in speed or memory if anything. + +## Pull requests + +`main` takes changes through pull requests only. Small changes are squash-merged with the pull request's title as the +commit subject; a branch whose commits each build and tell a story (a new backend, a series of kernels) is merged with +a merge commit. A change on a hot path reports s/step and peak VRAM before and after, on the same GPU, model and +settings: no feature may cost speed. + +## Releases + +Maintainers release from `main`. A `release/X.Y.Z` pull request bumps `MI_VERSION_*` in +`include/mossinfer/mossinfer.h` and the version strings in the docs. After it merges and CI passes on `main`, a signed +`vX.Y.Z` tag on the merge commit starts `.github/workflows/release.yml`, which builds, checks and publishes the +packages. `vX.Y.Z-rcN` tags publish pre-releases. Release tags cannot be moved or deleted. diff --git a/docs/integration.md b/docs/integration.md index d0db201..2ba1cbc 100644 --- a/docs/integration.md +++ b/docs/integration.md @@ -203,14 +203,14 @@ On top of that, the weights are read through the operating system's page cache. cmake --preset linux-cuda13 cmake --build --preset linux-cuda13 -j8 CUDA_VISIBLE_DEVICES="" ctest --preset linux-cuda13 -L '^cpu$' -cmake --install build-linux-cuda13 # -> build-linux-cuda13/install -cmake --install build-linux-cuda13 --component Unspecified --prefix build-linux-cuda13/install-system-cuda -(cd build-linux-cuda13 && cpack --config CPackConfig.cmake) # -> mossinfer-2.2.0-linux-cu13-x86_64.{tar.gz,zip} +cmake --install build/linux-cuda13 # -> build/linux-cuda13/install +cmake --install build/linux-cuda13 --component Unspecified --prefix build/linux-cuda13/install-system-cuda +(cd build/linux-cuda13 && cpack --config CPackConfig.cmake) # -> mossinfer-2.2.0-linux-cu13-x86_64.{tar.gz,zip} ``` - **glibc:** the archive needs the glibc of the machine that built it, or a newer one. Build on the oldest distribution you want to support. CI uses Ubuntu 22.04, which gives glibc 2.35. A build from a rolling distribution such as Arch will not start on Ubuntu LTS. - **Linux CUDA 12 with Volta:** use the `linux-cuda12-sm70` configure/build/test/package presets. This produces a `-sm70` package; CUDA 13 cannot build or run the Volta backend. See [sm70.md](sm70.md#build). -- **Windows CUDA 13 with Turing:** from a Visual Studio x64 developer prompt, run `cmake --preset windows-msvc-cuda13-sm75`, `cmake --build --preset windows-msvc-cuda13-sm75`, `ctest --preset windows-msvc-cuda13-sm75`, then `cpack --config build-windows-msvc-cuda13-sm75/CPackConfig.cmake -B build-windows-msvc-cuda13-sm75`. This creates the full `-sm75.zip` package; the ordinary presets remain Ampere-and-newer. Any preset takes `-DMOSSINFER_SM75=ON`; what the build adds is in [sm75.md](sm75.md#build). +- **Windows CUDA 13 with Turing:** from a Visual Studio x64 developer prompt, run `cmake --preset windows-msvc-cuda13-sm75`, `cmake --build --preset windows-msvc-cuda13-sm75`, `ctest --preset windows-msvc-cuda13-sm75`, then `cpack --config build/windows-msvc-cuda13-sm75/CPackConfig.cmake -B build/windows-msvc-cuda13-sm75`. This creates the full `-sm75.zip` package; the ordinary presets remain Ampere-and-newer. Any preset takes `-DMOSSINFER_SM75=ON`; what the build adds is in [sm75.md](sm75.md#build). - **Other presets:** `linux-cuda12`, `windows-msvc-cuda13` and `windows-msvc-cuda12`. The Windows presets run from a Visual Studio 2022 x64 developer prompt. ### Package contents diff --git a/docs/sm70.md b/docs/sm70.md index 84acfb6..091ef6d 100644 --- a/docs/sm70.md +++ b/docs/sm70.md @@ -15,10 +15,10 @@ machine whose supported devices are all Volta, the probe recommends `cu12` even cmake --preset linux-cuda12-sm70 -DCMAKE_CUDA_COMPILER=/usr/local/cuda-12.9/bin/nvcc cmake --build --preset linux-cuda12-sm70 CUDA_VISIBLE_DEVICES="" ctest --preset linux-cuda12-sm70 -L '^cpu$' -build-linux-cuda12-sm70/sm70_check all -build-linux-cuda12-sm70/sm70_check big -build-linux-cuda12-sm70/mossinfer_tests --gtest_filter='Sm70Build.*:GpuSm70.*:GpuSm75Convert.*' -cpack --config build-linux-cuda12-sm70/CPackConfig.cmake -B build-linux-cuda12-sm70 +build/linux-cuda12-sm70/sm70_check all +build/linux-cuda12-sm70/sm70_check big +build/linux-cuda12-sm70/mossinfer_tests --gtest_filter='Sm70Build.*:GpuSm70.*:GpuSm75Convert.*' +cpack --config build/linux-cuda12-sm70/CPackConfig.cmake -B build/linux-cuda12-sm70 ``` The package name ends in `-sm70`. The build/release matrix includes the Linux CUDA 12 SM70 preset. The CUDA architecture list of the ordinary core must stay sm_80 or newer; do not diff --git a/docs/sm75.md b/docs/sm75.md index c462457..d228f9a 100644 --- a/docs/sm75.md +++ b/docs/sm75.md @@ -16,8 +16,8 @@ The Windows CUDA 13 preset builds a complete package and names it with a `-sm75` cmake --preset windows-msvc-cuda13-sm75 cmake --build --preset windows-msvc-cuda13-sm75 ctest --preset windows-msvc-cuda13-sm75 -L '^cpu$' -cpack --config build-windows-msvc-cuda13-sm75/CPackConfig.cmake -B build-windows-msvc-cuda13-sm75 -# -> build-windows-msvc-cuda13-sm75/mossinfer-2.2.0-windows-cu13-x86_64-sm75.zip +cpack --config build/windows-msvc-cuda13-sm75/CPackConfig.cmake -B build/windows-msvc-cuda13-sm75 +# -> build/windows-msvc-cuda13-sm75/mossinfer-2.2.0-windows-cu13-x86_64-sm75.zip ``` Any other preset takes the option too (`-DMOSSINFER_SM75=ON`), e.g. a Linux build from a custom preset; the ordinary diff --git a/tests/server/test_install_tree.py b/tests/server/test_install_tree.py index 9f8d67f..81f5d5f 100644 --- a/tests/server/test_install_tree.py +++ b/tests/server/test_install_tree.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Install-tree test of the cuBLASLt search (src/backend/cuda_libs.h). Standard library only; needs no GPU. - python3 tests/server/test_install_tree.py --build build-dev --work build-dev/install_tree_test \\ + python3 tests/server/test_install_tree.py --build build/dev --work build/dev/install_tree_test \\ [--check-package .github/scripts/check_package.py] Installs the build twice: the full tree (every component, cuBLASLt bundled) and the system-CUDA variant