diff --git a/docker-compose/pasarguard-mariadb.yml b/docker-compose/pasarguard-mariadb.yml index 6710a68..9146b61 100644 --- a/docker-compose/pasarguard-mariadb.yml +++ b/docker-compose/pasarguard-mariadb.yml @@ -26,7 +26,7 @@ services: - --collation_server=utf8mb4_unicode_ci # Defines collation for Unicode - --host-cache-size=0 # Disables host cache to prevent DNS issues - --innodb-open-files=1024 # Sets the limit for InnoDB open files - - --innodb-buffer-pool-size=256M # Allocates buffer pool size for InnoDB + - --innodb-buffer-pool-size=${MYSQL_BUFFER_POOL_SIZE:-256M} # InnoDB cache; the installer sets about 25% of RAM - --binlog_expire_logs_seconds=1209600 # Sets binary log expiration to 14 days (2 weeks) - --innodb-doublewrite=0 # Disables doublewrite buffer (reduces disk I/O; may increase data loss risk) - --general_log=0 # Disables general query log to reduce disk usage diff --git a/docker-compose/pasarguard-mysql.yml b/docker-compose/pasarguard-mysql.yml index 0851123..e64103c 100644 --- a/docker-compose/pasarguard-mysql.yml +++ b/docker-compose/pasarguard-mysql.yml @@ -27,7 +27,8 @@ services: - --collation_server=utf8mb4_unicode_ci # Defines collation for Unicode - --host-cache-size=0 # Disables host cache to prevent DNS issues - --innodb-open-files=1024 # Sets the limit for InnoDB open files - - --innodb-buffer-pool-size=256M # Allocates buffer pool size for InnoDB + - --innodb-buffer-pool-size=${MYSQL_BUFFER_POOL_SIZE:-256M} # InnoDB cache; the installer sets about 25% of RAM + - --innodb-buffer-pool-instances=1 # Keeps the pool at the set size (several instances round it up) - --general_log=0 # Disables general query log for lower disk usage - --slow_query_log=1 # Enables slow query log for performance analysis - --slow_query_log_file=/var/lib/mysql/slow.log # Logs slow queries for troubleshooting diff --git a/docs/database-configurations.md b/docs/database-configurations.md index 1f39261..89db728 100644 --- a/docs/database-configurations.md +++ b/docs/database-configurations.md @@ -63,8 +63,10 @@ MySQL and MariaDB run in dedicated Docker containers with optional phpMyAdmin ac DB_USER="pasarguard" DB_PASSWORD="your-strong-password" MYSQL_ROOT_PASSWORD="generated-root-password" + MYSQL_BUFFER_POOL_SIZE="1024M" SQLALCHEMY_DATABASE_URL="mysql+pymysql://pasarguard:your-strong-password@127.0.0.1:3306/pasarguard" ``` +- **Buffer pool**: `MYSQL_BUFFER_POOL_SIZE` sets `--innodb-buffer-pool-size`, the InnoDB cache for tables and indexes. The installer sets it to about 25% of the host RAM (or of a lower cgroup memory limit), between `256M` and `4096M` (for example `1024M` on a 4 GB host). The MySQL template also sets `--innodb-buffer-pool-instances=1`, because with several instances MySQL rounds the pool up (4096M became 4480M in a test); MariaDB always uses one instance. Without the variable the container uses `256M`. A large panel needs more: in a test with 100k users and 33M usage rows on 2 CPUs and 4 GB, one usage write took about 11.6 s with `256M` (the job runs every 10 s) and 1.7 s with `2048M`. To change it, edit `/opt/pasarguard/.env` and run `pasarguard restart`. Installs created before this change keep the fixed `256M` in their compose file: replace `--innodb-buffer-pool-size=256M` with `--innodb-buffer-pool-size=${MYSQL_BUFFER_POOL_SIZE:-256M}` in `/opt/pasarguard/docker-compose.yml` (`pasarguard edit`) and add the variable to `.env`. --- diff --git a/docs/environment-variables.md b/docs/environment-variables.md index d9ffd21..963b066 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -34,6 +34,7 @@ Complete reference for configuration parameters found in `/opt/pasarguard/.env` | `DB_USER` | Primary database user. | `pasarguard` | | `DB_PASSWORD` | Primary database user password. | `Secr3tP@ssw0rd!` | | `MYSQL_ROOT_PASSWORD` | Root password for MySQL or MariaDB containers. | `RootP@ssw0rd!` | +| `MYSQL_BUFFER_POOL_SIZE` | InnoDB buffer pool size for MySQL or MariaDB. The installer sets about 25% of RAM (256M to 4096M); the compose default is `256M`. | `1024M` | --- diff --git a/pasarguard.sh b/pasarguard.sh index 385197a..a963649 100755 --- a/pasarguard.sh +++ b/pasarguard.sh @@ -1126,6 +1126,59 @@ set_pasarguard_panel_image() { fi } +# Print the lowest cgroup memory limit in bytes on the path of this process, from its +# own cgroup up to the mount root (cgroup v2 or v1). Prints nothing when there is no limit. +# Arguments: +# $1 - cgroup mount (default /sys/fs/cgroup). +# $2 - cgroup membership file (default /proc/self/cgroup). +cgroup_memory_limit_bytes() { + local root="${1:-/sys/fs/cgroup}" membership="${2:-/proc/self/cgroup}" + local base file path value lowest="" + if [ -f "$root/cgroup.controllers" ]; then + base="$root" file="memory.max" + path=$(awk -F: '$1 == "0" { print $3; exit }' "$membership" 2>/dev/null || true) + else + base="$root/memory" file="memory.limit_in_bytes" + path=$(awk -F: '$2 ~ /(^|,)memory(,|$)/ { print $3; exit }' "$membership" 2>/dev/null || true) + fi + path="${path%/}" + while :; do + # cgroup v2 writes "max" for no limit; v1 writes a huge number, which the caller's min ignores + value=$(cat "$base$path/$file" 2>/dev/null || true) + if [[ "$value" =~ ^[0-9]+$ ]] && { [ -z "$lowest" ] || [ "$value" -lt "$lowest" ]; }; then + lowest="$value" + fi + [ -z "$path" ] && break + path="${path%/*}" + done + [ -n "$lowest" ] && echo "$lowest" + return 0 +} + +# Print an InnoDB buffer pool size for this host: about 25% of RAM in 128M steps, +# kept between 256M and 4096M. A cgroup memory limit lower than the RAM is used +# instead of it. Falls back to 256M when RAM cannot be read. +# Arguments: +# $1 - meminfo file (default /proc/meminfo). +# $2 - cgroup mount (default /sys/fs/cgroup). +# $3 - cgroup membership file (default /proc/self/cgroup). +mysql_buffer_pool_size_for_ram() { + local meminfo="${1:-/proc/meminfo}" total_kb limit_bytes mb + total_kb=$(awk '$1 == "MemTotal:" { print $2; exit }' "$meminfo" 2>/dev/null || true) + if ! [[ "$total_kb" =~ ^[0-9]+$ ]]; then + echo "256M" + return + fi + limit_bytes=$(cgroup_memory_limit_bytes "${2:-}" "${3:-}") + if [[ "$limit_bytes" =~ ^[0-9]+$ ]] && [ $((limit_bytes / 1024)) -lt "$total_kb" ]; then + total_kb=$((limit_bytes / 1024)) + fi + mb=$((total_kb / 1024 / 4 / 128 * 128)) + [ "$mb" -lt 256 ] && mb=256 + [ "$mb" -gt 4096 ] && mb=4096 + echo "${mb}M" +} + # Download template files, configure database, and set image tag for PasarGuard installation. # Arguments: # $1 - Version string. @@ -1193,6 +1246,9 @@ install_pasarguard() { DB_PORT="3306" MYSQL_ROOT_PASSWORD=$(LC_ALL=C tr -dc 'A-Za-z0-9' >"$ENV_FILE" + MYSQL_BUFFER_POOL_SIZE=$(mysql_buffer_pool_size_for_ram) + echo "MYSQL_BUFFER_POOL_SIZE=\"$MYSQL_BUFFER_POOL_SIZE\"" >>"$ENV_FILE" + colorized_echo green "$db_name buffer pool: $MYSQL_BUFFER_POOL_SIZE (change MYSQL_BUFFER_POOL_SIZE in .env)" fi if [[ "$database_type" =~ ^(postgresql|timescaledb)$ ]]; then diff --git a/tests/unit_pasarguard.sh b/tests/unit_pasarguard.sh index df69bc5..67be95f 100644 --- a/tests/unit_pasarguard.sh +++ b/tests/unit_pasarguard.sh @@ -752,6 +752,45 @@ assert_true "run_all.sh: increments FAILED for missing suites" \ assert_true "run_all.sh: records missing suites in FAILED_LIST" \ contains "$missing_branch" 'FAILED_LIST+=("$suite (missing)")' +# ----------------------------------------------------------------------- +# mysql_buffer_pool_size_for_ram +# ----------------------------------------------------------------------- +meminfo_with() { printf 'MemTotal: %s kB\nMemFree: 1000 kB\n' "$1" >"$WORK_DIR/meminfo"; echo "$WORK_DIR/meminfo"; } +assert_eq "$(mysql_buffer_pool_size_for_ram "$(meminfo_with 1000000)" "$WORK_DIR/no-cgroup")" "256M" "buffer_pool: 1 GB host gets the 256M floor" +assert_eq "$(mysql_buffer_pool_size_for_ram "$(meminfo_with 2014000)" "$WORK_DIR/no-cgroup")" "384M" "buffer_pool: 2 GB host gets 384M" +assert_eq "$(mysql_buffer_pool_size_for_ram "$(meminfo_with 4194304)" "$WORK_DIR/no-cgroup")" "1024M" "buffer_pool: 4 GB host gets 1024M" +assert_eq "$(mysql_buffer_pool_size_for_ram "$(meminfo_with 8388608)" "$WORK_DIR/no-cgroup")" "2048M" "buffer_pool: 8 GB host gets 2048M" +assert_eq "$(mysql_buffer_pool_size_for_ram "$(meminfo_with 67108864)" "$WORK_DIR/no-cgroup")" "4096M" "buffer_pool: 64 GB host is capped at 4096M" +assert_eq "$(mysql_buffer_pool_size_for_ram "$WORK_DIR/no-such-meminfo")" "256M" "buffer_pool: unreadable meminfo falls back to 256M" +# Fake cgroup trees: v2 has cgroup.controllers at the root, v1 a memory/ hierarchy +CG="$WORK_DIR/cg" +mkdir -p "$CG/v2/outer/inner" "$CG/v2max/a" "$CG/v1/memory/outer/inner" +touch "$CG/v2/cgroup.controllers" "$CG/v2max/cgroup.controllers" +echo max >"$CG/v2/outer/inner/memory.max" +echo 2147483648 >"$CG/v2/outer/memory.max" +echo max >"$CG/v2max/a/memory.max" +echo 9223372036854771712 >"$CG/v1/memory/memory.limit_in_bytes" +echo 1073741824 >"$CG/v1/memory/outer/memory.limit_in_bytes" +echo 9223372036854771712 >"$CG/v1/memory/outer/inner/memory.limit_in_bytes" +printf '0::/outer/inner\n' >"$CG/self-v2" +printf '0::/a\n' >"$CG/self-v2max" +printf '5:cpu,cpuacct:/x\n4:memory:/outer/inner\n' >"$CG/self-v1" +printf '0::/\n' >"$CG/self-root" +mem32=$(meminfo_with 33554432) +assert_eq "$(mysql_buffer_pool_size_for_ram "$mem32" "$CG/v2" "$CG/self-v2")" "512M" "buffer_pool: cgroup v2 ancestor limit of 2 GB wins over 32 GB RAM" +assert_eq "$(mysql_buffer_pool_size_for_ram "$mem32" "$CG/v1" "$CG/self-v1")" "256M" "buffer_pool: cgroup v1 ancestor limit of 1 GB wins over 32 GB RAM" +assert_eq "$(mysql_buffer_pool_size_for_ram "$mem32" "$CG/v2max" "$CG/self-v2max")" "4096M" "buffer_pool: cgroup v2 'max' means no limit" +assert_eq "$(mysql_buffer_pool_size_for_ram "$mem32" "$CG/v2" "$CG/self-root")" "4096M" "buffer_pool: root cgroup v2 has no limit" +assert_eq "$(mysql_buffer_pool_size_for_ram "$(meminfo_with 8388608)" "$WORK_DIR/no-cgroup" "$CG/none")" "2048M" "buffer_pool: no cgroup files uses RAM" +for f in pasarguard-mysql.yml pasarguard-mariadb.yml; do + assert_true "$f: buffer pool reads MYSQL_BUFFER_POOL_SIZE" \ + grep -Fq -- '--innodb-buffer-pool-size=${MYSQL_BUFFER_POOL_SIZE:-256M}' "$ROOT_DIR/docker-compose/$f" +done +assert_true "pasarguard-mysql.yml: one buffer pool instance keeps the exact size" \ + grep -Fq -- '--innodb-buffer-pool-instances=1' "$ROOT_DIR/docker-compose/pasarguard-mysql.yml" +assert_false "pasarguard-mariadb.yml: no removed instances option" \ + grep -Fq -- 'innodb-buffer-pool-instances' "$ROOT_DIR/docker-compose/pasarguard-mariadb.yml" + echo "" echo "Results: $PASS passed, $FAIL failed" [ "$FAIL" -eq 0 ] || exit 1