feat(server): scope mutation locks and bound lock waits

.github/workflows/branch-checks.yml

@@ -237,6 +237,36 @@ jobs:
           cargo nextest run --locked --profile ci --workspace --features openshell-server/test-support
           cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path examples/supervisor-middleware-content-guard/Cargo.toml
 
+  rust-postgres-test:
+    name: Rust PostgreSQL tests
+    needs: pr_metadata
+    if: needs.pr_metadata.outputs.should_run == 'true'
+    # Bare runner: the test script starts a disposable PostgreSQL container
+    # with the host's Docker.
+    runs-on: linux-amd64-cpu8
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - uses: ./.github/actions/setup-nix
+        with:
+          prepare-shell: "true"
+          cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
+      - uses: ./.github/actions/setup-rust
+        with:
+          cache-key: rust-checks-v2
+          save-cache: "false"
+
+      - name: Test
+        shell: nix develop -c bash -euo pipefail {0}
+        env:
+          CONTAINER_ENGINE: docker
+          OPENSHELL_TELEMETRY_ENABLED: "false"
+        run: tasks/scripts/run-postgres-tests.sh --locked --profile ci
+
   rust-build-modes:
     name: Rust build modes (${{ matrix.system }})
     needs: pr_metadata

TESTING.md

@@ -48,6 +48,28 @@ mise run test:rust     # cargo test --workspace
 
 Rust validation checks tracked Cargo lockfiles; run `mise run rust:lockfiles:check` to check them directly. If one is stale, refresh it with Cargo using its adjacent manifest, review the diff, and commit the update.
 
+### PostgreSQL-backed tests
+
+Tests that need a real PostgreSQL server, such as advisory-lock concurrency
+across two stores, are `#[ignore]`d and named `postgres_*`. Run them with:
+
+```shell
+mise run test:rust:postgres
+```
+
+The task starts a disposable PostgreSQL container with Docker or Podman
+(set `CONTAINER_ENGINE` to choose), runs the tests one at a time, and removes
+the container. Each test works in its own temporary schema, except the replay
+and provider refresh lock tests, which use the default schema. To use your own
+disposable database, set `OPENSHELL_TEST_POSTGRES_URL`. The task overrides
+`OPENSHELL_REPLAY_TEST_DATABASE_URL` so legacy tests use that same database.
+Never point it at a database that a running gateway uses: the tests take
+fleet-wide advisory locks.
+Branch Checks runs them on x86_64 Linux.
+Load-sensitive capacity checks are named `bench_postgres_*` instead, so that
+task skips them. Run them on an idle machine with
+`mise run test:rust:postgres:bench`.
+
 ### Native Windows validation
 
 Use `mise run --skip-tools pre-commit` with the existing Rust/MSVC toolchain.

crates/openshell-core/src/config.rs

@@ -243,6 +243,24 @@ pub struct Config {
     /// Database URL for persistence.
     pub database_url: String,
 
+    /// Connection ceiling for the persistence pool.
+    ///
+    /// `None` leaves the backend's built-in default in place. The right
+    /// ceiling depends on the deployment — how many gateway replicas share
+    /// the database, and what `max_connections` a Postgres server itself
+    /// allows — so it cannot be one number baked into the binary. An
+    /// in-memory `SQLite` database ignores it: the database lives in its single
+    /// connection.
+    pub database_max_connections: Option<u32>,
+
+    /// Connection ceiling for the `PostgreSQL` mutation lock pool.
+    ///
+    /// `None` keeps the built-in default of 4. Each mutation guard holds one
+    /// lock connection, so this bounds how many guarded mutations a replica
+    /// runs or waits on in `PostgreSQL` at once. `SQLite` has no lock pool and
+    /// ignores it.
+    pub database_lock_max_connections: Option<u32>,
+
     /// Explicit compute driver configured for the gateway.
     /// `None` enables runtime auto-detection.
     pub compute_driver: Option<String>,
@@ -892,6 +910,8 @@ impl Config {
             mtls_auth: MtlsAuthConfig::default(),
             gateway_jwt: None,
             database_url: String::new(),
+            database_max_connections: None,
+            database_lock_max_connections: None,
             compute_driver: None,
             compute_driver_endpoints: BTreeMap::new(),
             credential_drivers: Vec::new(),
@@ -944,6 +964,27 @@ impl Config {
         self
     }
 
+    /// Create a new configuration with a database pool connection ceiling.
+    ///
+    /// `None` keeps the persistence backend's default.
+    #[must_use]
+    pub const fn with_database_max_connections(mut self, max_connections: Option<u32>) -> Self {
+        self.database_max_connections = max_connections;
+        self
+    }
+
+    /// Create a new configuration with a mutation lock pool connection ceiling.
+    ///
+    /// `None` keeps the built-in default.
+    #[must_use]
+    pub const fn with_database_lock_max_connections(
+        mut self,
+        max_connections: Option<u32>,
+    ) -> Self {
+        self.database_lock_max_connections = max_connections;
+        self
+    }
+
     /// Create a new configuration with an explicit compute driver.
     #[must_use]
     pub fn with_compute_driver(mut self, driver: impl ToString) -> Self {

crates/openshell-server/README.md

@@ -70,7 +70,8 @@ cargo test -p openshell-server --features test-support,prebuilt-z3
 cargo clippy -p openshell-server --all-targets --features test-support,prebuilt-z3 -- -D warnings
 ```
 
-The ignored `refresh_locks_coordinate_independent_replica_sessions` test requires
+The ignored `postgres_refresh_locks_coordinate_independent_replica_sessions` test
+runs under `mise run test:rust:postgres` (see `TESTING.md`) and still accepts
 `OPENSHELL_REFRESH_TEST_DATABASE_URL` pointing to a disposable PostgreSQL database.
 It exercises independent sessions without the local mutex. Do not point this
 fixture at an existing deployment database.

crates/openshell-server/src/cli.rs

@@ -128,6 +128,36 @@ struct RunArgs {
     #[arg(long, env = "OPENSHELL_DB_URL")]
     db_url: Option<String>,
 
+    /// Connection ceiling for the database pool, shared by every
+    /// database-backed request.
+    ///
+    /// When unset, the gateway uses its backend default: 10 with Postgres,
+    /// 5 with an on-disk `SQLite` database. An in-memory `SQLite` database is
+    /// always a single connection. Postgres needs at least 2, because the SSH
+    /// identity lock keeps one connection while it queries the pool, so the
+    /// gateway rejects 1 there at startup. Each replica opens its own pool,
+    /// plus a mutation lock pool on Postgres (`--db-lock-max-connections`), so
+    /// keep the total below what the database server itself admits.
+    #[arg(
+        long,
+        env = "OPENSHELL_DB_MAX_CONNECTIONS",
+        value_parser = clap::value_parser!(u32).range(1..)
+    )]
+    db_max_connections: Option<u32>,
+
+    /// Connection ceiling for the Postgres mutation lock pool.
+    ///
+    /// Each mutation guard holds one lock connection, so this bounds how many
+    /// guarded mutations a replica runs or waits on in Postgres at once.
+    /// When unset, the gateway uses 4. Each replica opens these on top of
+    /// its database pool. `SQLite` has no lock pool and ignores it.
+    #[arg(
+        long,
+        env = "OPENSHELL_DB_LOCK_MAX_CONNECTIONS",
+        value_parser = clap::value_parser!(u32).range(1..)
+    )]
+    db_lock_max_connections: Option<u32>,
+
     /// Compute driver configured for this gateway.
     ///
     /// Accepts one registered driver name. When unset, the gateway runs
@@ -516,7 +546,10 @@ fn prepare_server_config_with_drivers(
         config = config.with_metrics_bind_address(addr);
     }
 
-    config = config.with_database_url(db_url);
+    config = config
+        .with_database_url(db_url)
+        .with_database_max_connections(args.db_max_connections)
+        .with_database_lock_max_connections(args.db_lock_max_connections);
     if let Some(driver) = &args.compute_driver {
         config = config.with_compute_driver(driver);
     }
@@ -1267,6 +1300,18 @@ fn merge_file_into_args(args: &mut RunArgs, file: &GatewayFileSection, matches:
     {
         args.grpc_rate_limit_window_seconds = Some(window);
     }
+    if let Some(max_connections) = file.database_max_connections
+        && args.db_max_connections.is_none()
+        && arg_defaulted(matches, "db_max_connections")
+    {
+        args.db_max_connections = Some(max_connections);
+    }
+    if let Some(max_connections) = file.database_lock_max_connections
+        && args.db_lock_max_connections.is_none()
+        && arg_defaulted(matches, "db_lock_max_connections")
+    {
+        args.db_lock_max_connections = Some(max_connections);
+    }
 }
 
 fn validate_grpc_rate_limit_args(requests: Option<u64>, window_seconds: Option<u64>) -> Result<()> {
@@ -3133,6 +3178,137 @@ grpc_rate_limit_window_seconds = 30
         assert_eq!(args.grpc_rate_limit_window_seconds, Some(30));
     }
 
+    #[test]
+    fn file_db_max_connections_populates_args_when_cli_omits() {
+        Environment::new()
+            .remove("OPENSHELL_DB_MAX_CONNECTIONS")
+            .run(|| {
+                let (mut args, matches) =
+                    parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]);
+                assert_eq!(
+                    args.db_max_connections, None,
+                    "default leaves the pool alone"
+                );
+                let file = config_file_from_toml(
+                    r"
+[openshell.gateway]
+database_max_connections = 64
+",
+                );
+                merge_file_into_args(&mut args, &file.openshell.gateway, &matches);
+
+                assert_eq!(args.db_max_connections, Some(64));
+            });
+    }
+
+    #[test]
+    fn env_db_max_connections_overrides_file_value() {
+        Environment::new()
+            .set("OPENSHELL_DB_MAX_CONNECTIONS", "128")
+            .run(|| {
+                let (mut args, matches) =
+                    parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]);
+                let file = config_file_from_toml(
+                    r"
+[openshell.gateway]
+database_max_connections = 64
+",
+                );
+                merge_file_into_args(&mut args, &file.openshell.gateway, &matches);
+
+                assert_eq!(args.db_max_connections, Some(128));
+            });
+    }
+
+    #[test]
+    fn db_max_connections_rejects_a_zero_or_unparseable_ceiling() {
+        Environment::new()
+            .remove("OPENSHELL_DB_MAX_CONNECTIONS")
+            .run(|| {
+                // Zero would deadlock every `acquire`, and a silent fallback to the
+                // default would reproduce the ceiling the operator is trying to lift.
+                for bad in ["0", "-5", "many"] {
+                    assert!(
+                        command()
+                            .try_get_matches_from([
+                                "openshell-gateway",
+                                "--db-url",
+                                "sqlite::memory:",
+                                "--db-max-connections",
+                                bad,
+                            ])
+                            .is_err(),
+                        "input {bad:?} must be rejected"
+                    );
+                }
+            });
+    }
+
+    #[test]
+    fn file_db_lock_max_connections_populates_args_when_cli_omits() {
+        Environment::new()
+            .remove("OPENSHELL_DB_LOCK_MAX_CONNECTIONS")
+            .run(|| {
+                let (mut args, matches) =
+                    parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]);
+                assert_eq!(
+                    args.db_lock_max_connections, None,
+                    "default leaves the lock pool alone"
+                );
+                let file = config_file_from_toml(
+                    r"
+[openshell.gateway]
+database_lock_max_connections = 8
+",
+                );
+                merge_file_into_args(&mut args, &file.openshell.gateway, &matches);
+
+                assert_eq!(args.db_lock_max_connections, Some(8));
+            });
+    }
+
+    #[test]
+    fn env_db_lock_max_connections_overrides_file_value() {
+        Environment::new()
+            .set("OPENSHELL_DB_LOCK_MAX_CONNECTIONS", "16")
+            .run(|| {
+                let (mut args, matches) =
+                    parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]);
+                let file = config_file_from_toml(
+                    r"
+[openshell.gateway]
+database_lock_max_connections = 8
+",
+                );
+                merge_file_into_args(&mut args, &file.openshell.gateway, &matches);
+
+                assert_eq!(args.db_lock_max_connections, Some(16));
+            });
+    }
+
+    #[test]
+    fn db_lock_max_connections_rejects_a_zero_or_unparseable_ceiling() {
+        Environment::new()
+            .remove("OPENSHELL_DB_LOCK_MAX_CONNECTIONS")
+            .run(|| {
+                // A zero-sized lock pool would time out every guarded mutation.
+                for bad in ["0", "-5", "many"] {
+                    assert!(
+                        command()
+                            .try_get_matches_from([
+                                "openshell-gateway",
+                                "--db-url",
+                                "sqlite::memory:",
+                                "--db-lock-max-connections",
+                                bad,
+                            ])
+                            .is_err(),
+                        "input {bad:?} must be rejected"
+                    );
+                }
+            });
+    }
+
     #[test]
     fn aux_listener_preserves_file_ip_against_public_bind() {
         use std::net::SocketAddr;