From 5b47c7200a4a213a562b69a7283206e3f90a562b Mon Sep 17 00:00:00 2001 From: Juan Lee Date: Thu, 5 Nov 2026 06:27:57 -0700 Subject: [PATCH] test: add KMS Encryption plugin integration tests Add container tests for the KMS Encryption plugin on PostgreSQL and MySQL, including SQLAlchemy and Django, against LocalStack's KMS, a real KMS key and Aurora. The harness gets the RUN_KMS_ENCRYPTION_TESTS_ONLY and KMS_LOCALSTACK features, three Gradle tasks, a LocalStack container pinned by digest, PostgreSQL statement logging and a JUnit XML report. No product code changes. Add a compose lane, tests/integration/kms_local/run.sh, that runs the LocalStack modules without Gradle. It replaces tests/integration/test_kms_encryption_live.py. A new workflow runs the LocalStack job on pull requests, the real-key job on pushes to main and on dispatch, and the Aurora job on dispatch only. Each job fails when no KMS test ran. Two strict xfails track product defects: fetchmany() without a size returns no rows on mysql-connector's C extension, and on Aurora PostgreSQL the plugin's metadata lookup after a writer failover stops failover_v2 from running. The real-key jobs need one-time administrator setup: a symmetric key, an OIDC role trusted only for main and limited to that key, a Decrypt-only role, and the KMS_KEY_ARN, KMS_REGION, KMS_IT_ROLE_ARN and KMS_DECRYPT_ONLY_ROLE_ARN repository variables. --- .../kms_encryption_integration_tests.yml | 248 +++ docs/development-guide/IntegrationTests.md | 40 +- .../UsingTheKmsEncryptionPlugin.md | 2 + tests/integration/container/conftest.py | 15 + .../django/test_django_kms_encryption.py | 150 ++ .../test_sqlalchemy_kms_encryption.py | 294 +++ .../container/test_kms_encryption_aurora.py | 299 +++ .../test_kms_encryption_key_management.py | 524 +++++ .../container/test_kms_encryption_kms.py | 327 +++ .../test_kms_encryption_lifecycle.py | 438 ++++ .../container/test_kms_encryption_real_kms.py | 422 ++++ .../test_kms_encryption_round_trip.py | 465 ++++ .../test_kms_encryption_server_logs.py | 294 +++ .../test_kms_encryption_statements.py | 519 +++++ .../container/test_kms_encryption_triggers.py | 346 +++ .../container/utils/kms_encryption_helper.py | 690 ++++++ .../container/utils/kms_encryption_schema.py | 83 + .../container/utils/test_environment.py | 2 +- .../utils/test_environment_features.py | 2 + tests/integration/host/build.gradle.kts | 49 + .../integration/TestEnvironmentFeatures.java | 4 +- .../integration/host/TestEnvironment.java | 38 +- .../host/TestEnvironmentConfiguration.java | 9 + .../host/TestEnvironmentProvider.java | 22 +- .../integration/util/ContainerHelper.java | 36 +- tests/integration/kms_local/compose.yaml | 27 + tests/integration/kms_local/run.sh | 153 ++ tests/integration/test_kms_encryption_live.py | 1912 ----------------- ...est_kms_encryption_schema_matches_guide.py | 69 + 29 files changed, 5554 insertions(+), 1925 deletions(-) create mode 100644 .github/workflows/kms_encryption_integration_tests.yml create mode 100644 tests/integration/container/django/test_django_kms_encryption.py create mode 100644 tests/integration/container/sqlalchemy/test_sqlalchemy_kms_encryption.py create mode 100644 tests/integration/container/test_kms_encryption_aurora.py create mode 100644 tests/integration/container/test_kms_encryption_key_management.py create mode 100644 tests/integration/container/test_kms_encryption_kms.py create mode 100644 tests/integration/container/test_kms_encryption_lifecycle.py create mode 100644 tests/integration/container/test_kms_encryption_real_kms.py create mode 100644 tests/integration/container/test_kms_encryption_round_trip.py create mode 100644 tests/integration/container/test_kms_encryption_server_logs.py create mode 100644 tests/integration/container/test_kms_encryption_statements.py create mode 100644 tests/integration/container/test_kms_encryption_triggers.py create mode 100644 tests/integration/container/utils/kms_encryption_helper.py create mode 100644 tests/integration/container/utils/kms_encryption_schema.py create mode 100644 tests/integration/kms_local/compose.yaml create mode 100755 tests/integration/kms_local/run.sh delete mode 100644 tests/integration/test_kms_encryption_live.py create mode 100644 tests/unit/test_kms_encryption_schema_matches_guide.py diff --git a/.github/workflows/kms_encryption_integration_tests.yml b/.github/workflows/kms_encryption_integration_tests.yml new file mode 100644 index 00000000..11d4c64d --- /dev/null +++ b/.github/workflows/kms_encryption_integration_tests.yml @@ -0,0 +1,248 @@ +name: KMS Encryption Integration Tests + +# The LocalStack job needs no AWS account and no secrets, so it also runs on pull requests from forks. +# The real-key and Aurora jobs read the key ARN, its region and the roles from repository variables, +# which an administrator sets up once; CI never creates or changes KMS keys. + +on: + pull_request: + paths: + - 'aws_advanced_python_wrapper/**' + - 'tests/integration/**' + - 'pyproject.toml' + - 'poetry.lock' + - '.github/workflows/kms_encryption_integration_tests.yml' + push: + branches: [ main ] + workflow_dispatch: + inputs: + lane: + description: 'Which KMS environment to run' + type: choice + options: [ localstack, real-kms, aurora ] + default: localstack + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.lane }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + kms-localstack: + name: Run KMS Encryption Tests with LocalStack + if: github.event_name != 'workflow_dispatch' || inputs.lane == 'localstack' + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - { engine: pg, skip: mysql } + - { engine: mysql, skip: pg } + + steps: + - name: 'Clone repository' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: 'Set up JDK 8' + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: + distribution: 'corretto' + java-version: 8 + + - name: 'Set up Python 3.12' + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + + - name: Install poetry + shell: bash + run: | + pipx install poetry==1.8.2 + poetry config virtualenvs.prefer-active-python true + + - name: Install dependencies + run: poetry install + + - name: 'Run the KMS tests on Docker with LocalStack' + run: | + DB_PASSWORD="$(openssl rand -hex 16)" + echo "::add-mask::${DB_PASSWORD}" + export DB_PASSWORD DB_USERNAME=test_user DB_DATABASE_NAME=test_db + ./gradlew --no-parallel --no-daemon test-kms-encryption-docker \ + -Dexclude-${{ matrix.skip }}-engine=true -Dexclude-${{ matrix.skip }}-driver=true + + # pytest exits 0 when every test is skipped, so count the KMS test cases that ran. + - name: 'Fail when no KMS test ran' + run: | + python3 -c 'import glob,sys,xml.etree.ElementTree as E; c=[t for f in glob.glob("tests/integration/container/reports/*.xml") for t in E.parse(f).iter("testcase") if "kms_encryption" in t.get("classname","") and t.find("skipped") is None]; print("KMS tests run:", len(c)); sys.exit(0 if c else 1)' + + - name: 'Archive results' + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: kms-localstack-${{ matrix.engine }} + path: ./tests/integration/container/reports + retention-days: 5 + + kms-real-key: + name: Run KMS Encryption Tests with a real KMS key + if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || inputs.lane == 'real-kms' + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + strategy: + fail-fast: false + matrix: + include: + - { engine: pg, skip: mysql } + - { engine: mysql, skip: pg } + + steps: + - name: 'Clone repository' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: 'Set up JDK 8' + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: + distribution: 'corretto' + java-version: 8 + + - name: 'Set up Python 3.12' + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + + - name: Install poetry + shell: bash + run: | + pipx install poetry==1.8.2 + poetry config virtualenvs.prefer-active-python true + + - name: Install dependencies + run: poetry install + + - name: 'Configure AWS Credentials' + uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6 + with: + role-to-assume: ${{ vars.KMS_IT_ROLE_ARN }} + role-session-name: python_kms_integration_tests + role-duration-seconds: 3600 + aws-region: ${{ vars.KMS_REGION }} + + - name: 'Run the KMS tests on Docker with a real KMS key' + run: | + DB_PASSWORD="$(openssl rand -hex 16)" + echo "::add-mask::${DB_PASSWORD}" + export DB_PASSWORD DB_USERNAME=test_user DB_DATABASE_NAME=test_db + ./gradlew --no-parallel --no-daemon test-kms-encryption-docker-kms \ + -Dexclude-${{ matrix.skip }}-engine=true -Dexclude-${{ matrix.skip }}-driver=true + env: + KMS_KEY_ARN: ${{ vars.KMS_KEY_ARN }} + KMS_REGION: ${{ vars.KMS_REGION }} + KMS_DECRYPT_ONLY_ROLE_ARN: ${{ vars.KMS_DECRYPT_ONLY_ROLE_ARN }} + + - name: 'Fail when no KMS test ran' + run: | + python3 -c 'import glob,sys,xml.etree.ElementTree as E; c=[t for f in glob.glob("tests/integration/container/reports/*.xml") for t in E.parse(f).iter("testcase") if "kms_encryption" in t.get("classname","") and t.find("skipped") is None]; print("KMS tests run:", len(c)); sys.exit(0 if c else 1)' + + - name: 'Archive results' + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: kms-real-key-${{ matrix.engine }} + path: ./tests/integration/container/reports + retention-days: 5 + + kms-aurora: + name: Run KMS Encryption Tests on Aurora + if: inputs.lane == 'aurora' + runs-on: ubuntu-latest + timeout-minutes: 150 + permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + strategy: + fail-fast: false + matrix: + include: + - { engine: pg, skip: mysql } + - { engine: mysql, skip: pg } + concurrency: + group: kms-aurora-${{ matrix.engine }} + cancel-in-progress: false + env: + # The harness doesn't delete its cluster when a run is cancelled, so the last step deletes this one by name. + RDS_DB_NAME: kms-it-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.engine }} + + steps: + - name: 'Clone repository' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: 'Set up JDK 8' + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: + distribution: 'corretto' + java-version: 8 + + - name: 'Set up Python 3.12' + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + + - name: Install poetry + shell: bash + run: | + pipx install poetry==1.8.2 + poetry config virtualenvs.prefer-active-python true + + - name: Install dependencies + run: poetry install + + - name: 'Configure AWS Credentials' + uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} + role-session-name: python_integration_tests + role-duration-seconds: 21600 + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + + - name: 'Run the KMS tests on Aurora' + run: | + DB_PASSWORD="$(openssl rand -hex 16)" + echo "::add-mask::${DB_PASSWORD}" + export DB_PASSWORD DB_USERNAME=test_user DB_DATABASE_NAME=test_db + ./gradlew --no-parallel --no-daemon test-kms-encryption-aurora \ + -Dexclude-${{ matrix.skip }}-engine=true -Dexclude-${{ matrix.skip }}-driver=true + env: + NUM_INSTANCES: 2 + RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + KMS_KEY_ARN: ${{ vars.KMS_KEY_ARN }} + KMS_REGION: ${{ vars.KMS_REGION }} + + - name: 'Fail when no KMS test ran' + run: | + python3 -c 'import glob,sys,xml.etree.ElementTree as E; c=[t for f in glob.glob("tests/integration/container/reports/*.xml") for t in E.parse(f).iter("testcase") if "kms_encryption" in t.get("classname","") and t.find("skipped") is None]; print("KMS tests run:", len(c)); sys.exit(0 if c else 1)' + + - name: 'Archive results' + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: kms-aurora-${{ matrix.engine }} + path: ./tests/integration/container/reports + retention-days: 5 + + # Deletes the cluster named RDS_DB_NAME and its member instances, never clusters matched by a name prefix. + - name: 'Delete the cluster of this run' + if: always() + run: | + instances="$(aws rds describe-db-clusters --db-cluster-identifier "${RDS_DB_NAME}" \ + --query 'DBClusters[0].DBClusterMembers[].DBInstanceIdentifier' --output text 2>/dev/null)" || exit 0 + for instance in ${instances}; do + aws rds delete-db-instance --db-instance-identifier "${instance}" --skip-final-snapshot > /dev/null || true + done + aws rds delete-db-cluster --db-cluster-identifier "${RDS_DB_NAME}" --skip-final-snapshot > /dev/null || true diff --git a/docs/development-guide/IntegrationTests.md b/docs/development-guide/IntegrationTests.md index 3e018297..990ef297 100644 --- a/docs/development-guide/IntegrationTests.md +++ b/docs/development-guide/IntegrationTests.md @@ -66,7 +66,10 @@ unset FILTER # Done testing the IAM tests, unset FILTER ## Running async integration tests -The wrapper ships async counterparts to every sync integration test file. Async tests exercise `AsyncAwsWrapperConnection` (raw) and `create_async_engine` with the wrapper's dialects (`postgresql+aws_wrapper_psycopg` — shared with sync via `get_async_dialect_cls`; `mysql+aws_wrapper_aiomysql`). They are invoked via dedicated Gradle tasks, independent of the sync tasks: +The wrapper ships an async counterpart for each sync test file at the top level of `tests/integration/container`, except the KMS Encryption Plugin's files, because the plugin is sync only. +The `sqlalchemy/` and `django/` folders have no async counterparts. +Async tests exercise `AsyncAwsWrapperConnection` (raw) and `create_async_engine` with the wrapper's dialects: `postgresql+aws_wrapper_psycopg`, shared with sync through `get_async_dialect_cls`, and `mysql+aws_wrapper_aiomysql`. +They run through their own Gradle tasks, separate from the sync tasks: | Deployment | Engine | Sync task | Async task | |---|---|---|---| @@ -85,6 +88,29 @@ Async tests do not introduce any new pytest dependencies — they follow the rep 2. **Cold-start latency.** Serverless v2 can take 30–60 seconds to scale from 0 ACUs on the first connection. Warm the cluster with a trivial query (`SELECT 1`) before running timing-sensitive tests so that warmup cost isn't charged against a test's 600-second timeout. 3. **Topology-gated tests skip environmentally.** Multi-instance tests (failover, some read/write splitting assertions) need ≥2 instances to execute; they skip on single-writer topologies. Blue/Green and Global Database tests require specific engine versions / cluster configurations. Tests skipped under these conditions are environmental, not regressions — they are equivalent to the sync suite's existing behavior under the same cluster topology. +## Running the KMS encryption tests + +The KMS Encryption Plugin's tests are the `test_kms_encryption_*.py` files in `tests/integration/container`, plus `sqlalchemy/test_sqlalchemy_kms_encryption.py` and `django/test_django_kms_encryption.py`. +Each environment has its own Gradle task. + +| Task | Databases | KMS | AWS credentials | +|---|---|---|---| +| `test-kms-encryption-docker` | Docker | LocalStack | None | +| `test-kms-encryption-docker-kms` | Docker | A real key | Limited to the test key | +| `test-kms-encryption-aurora` | Aurora | A real key | As for the Aurora tasks | + +The LocalStack task runs every module except `test_kms_encryption_real_kms.py`, which runs only on the real-key Docker task, and `test_kms_encryption_aurora.py`, which runs only on the Aurora task. +The LocalStack task needs no AWS account. +The harness gives the test container dummy credentials and passes LocalStack's address to the tests as `KMS_ENDPOINT_URL`. +The KMS tests skip outside these tasks, and fail inside them when the endpoint or the key is missing. +The real-key tasks read the key ARN from `KMS_KEY_ARN`, its region from `KMS_REGION`, and the ARN of a role that may only decrypt from `KMS_DECRYPT_ONLY_ROLE_ARN`. +Use full key ARNs, not aliases. +The tests create keys only in LocalStack, so an administrator creates the real test key, once. +A key costs $1 a month, and a key scheduled for deletion waits 7 to 30 days before AWS deletes it. +Give the test credentials only `kms:GenerateDataKey`, `kms:Decrypt` and `kms:DescribeKey` on the test key, and `sts:AssumeRole` on the decrypt-only role. +Reports keep failure output, which can include the database password, so use throwaway database passwords and keep reports private. +To run the same tests against local servers and LocalStack without the harness, use `tests/integration/kms_local/run.sh`. + ## Debugging Aurora Integration Tests - Pycharm > [!WARNING]\ @@ -144,13 +170,15 @@ Async tests do not introduce any new pytest dependencies — they follow the rep | `RDS_DB_NAME` | Yes | The database identifier for your Aurora or RDS cluster. Must be a unique value to avoid conflicting with existing clusters. | `db-identifier` | | `RDS_DB_DOMAIN` | No | The existing database connection suffix. Use this variable to run against an existing database. | `XYZ.us-east-2.rds.amazonaws.com` | | `IAM_USER` | No | User within the database that is identified with AWSAuthenticationPlugin. This is used for AWS IAM Authentication and is optional | `example_user_name` | -| `AWS_ACCESS_KEY_ID` | Yes | An AWS access key associated with an IAM user or role with RDS permissions. | `ASIAIOSFODNN7EXAMPLE` | -| `AWS_SECRET_ACCESS_KEY` | Yes | The secret key associated with the provided AWS_ACCESS_KEY_ID. | `wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY` | +| `AWS_ACCESS_KEY_ID` | Aurora and RDS only | An AWS access key associated with an IAM user or role with RDS permissions. | `ASIAIOSFODNN7EXAMPLE` | +| `AWS_SECRET_ACCESS_KEY` | Aurora and RDS only | The secret key associated with the provided AWS_ACCESS_KEY_ID. | `wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY` | | `AWS_SESSION_TOKEN` | No | AWS Session Token for CLI, SDK, & API access. This value is for MFA credentials only. See: [temporary AWS credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html). | `AQoDYXdzEJr...` | | | `REUSE_RDS_DB` | Yes | Set to true if you would like to use an existing cluster for your tests. | `false` | | `RDS_DB_REGION` | Yes | The database region. | `us-east-2` | | `DEBUG_ENV` | No | The IDE you will be using to debug the tests, values are either `PYCHARM` or `VSCODE` | `PYCHARM` | +Docker tasks don't read the AWS credential variables, except the KMS task that uses a real key (see [Running the KMS encryption tests](#running-the-kms-encryption-tests)). + ## Managing Environment Variables After following steps 1-3 from the [Run Integration Tests](#running-aurora-integration-tests) section, your default @@ -236,6 +264,6 @@ can also be useful. ## Viewing test results -The test results can be viewed in the test report generated at -`tests/integration/container/reports/integration_tests.html`. The logs can be viewed from the console where you executed -the tests or by clicking `show details` on a given test in the test report. +Each run writes its test report to `tests/integration/container/reports/.html`, with a JUnit XML report, `.xml`, beside it. +`` names the deployment, the engine, the number of instances and the Python version. +The logs are in the console where you ran the tests, and under `show details` for each test in the report. diff --git a/docs/using-the-python-wrapper/using-plugins/UsingTheKmsEncryptionPlugin.md b/docs/using-the-python-wrapper/using-plugins/UsingTheKmsEncryptionPlugin.md index 2785a26b..f7f2efc5 100644 --- a/docs/using-the-python-wrapper/using-plugins/UsingTheKmsEncryptionPlugin.md +++ b/docs/using-the-python-wrapper/using-plugins/UsingTheKmsEncryptionPlugin.md @@ -344,6 +344,8 @@ The trigger checks a new value against the column's current HMAC key, and rotati Three controls decide how much the encryption protects: which KMS keys the application can use, who can write the metadata schema, and who can change the trigger. The first three steps cover them, and the fourth covers what none of them stops. - **KMS keys.** The plugin takes a column's master key from its `key_storage` row, not from its own settings, so the allow-list and the application's IAM policy keep a changed row from pointing it at another key. List every master key the columns use in `encryption_allowed_master_key_arns`, in the form `key_storage.master_key_arn` holds. The plugin refuses any other key before calling KMS (`UnauthorizedMasterKeyError`), on reads and writes alike, and `encryption_return_unverified_data` doesn't change that. Add a new master key before rotating a column onto it, and keep the old one listed, with `kms:Decrypt` allowed, while values written under it remain. Give the application's credentials only `kms:Decrypt`, on those key ARNs, never `"Resource": "*"`, and use a separate key per environment. Run `KeyManagementUtility`, which also calls `kms:GenerateDataKey`, `kms:CreateKey`, `kms:CreateAlias` and `kms:DescribeKey`, under a separate administrative credential. Don't add a `kms:ViaService` condition to these policies. It matches only requests an AWS service makes on your behalf, and the plugin calls KMS directly, so every `Decrypt` would be denied. + Store key ARNs, not alias ARNs, in `key_storage` and in `encryption_allowed_master_key_arns`. + Repointing an alias makes every value under the old key unreadable. ```json {"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Action": "kms:Decrypt", diff --git a/tests/integration/container/conftest.py b/tests/integration/container/conftest.py index 7fae7b4d..89f681d0 100644 --- a/tests/integration/container/conftest.py +++ b/tests/integration/container/conftest.py @@ -44,6 +44,7 @@ import boto3 import pytest +from .utils import kms_encryption_helper from .utils.connection_utils import ConnectionUtils from .utils.database_engine_deployment import DatabaseEngineDeployment from .utils.proxy_helper import ProxyHelper @@ -98,6 +99,7 @@ def pytest_runtest_setup(item): test_name = item.callspec.id else: TestEnvironment.get_current().set_current_driver(None) + test_name = item.name logger.info("Starting test preparation for: " + test_name) @@ -192,13 +194,26 @@ def pytest_generate_tests(metafunc): metafunc.parametrize("test_driver", allowed_drivers) +def pytest_configure(config): + config.addinivalue_line("markers", "kms_encryption: a KMS Encryption Plugin test") + # Makes the KMS fixtures available to every module. + config.pluginmanager.register(kms_encryption_helper, "kms_encryption_helper") + + def pytest_collection_modifyitems(config, items): features = TestEnvironment.get_current().get_features() skip_async_files = TestEnvironmentFeatures.SKIP_ASYNC_DRIVER_TESTS in features async_file_skip = pytest.mark.skip( reason="async test excluded by SKIP_ASYNC_DRIVER_TESTS env feature") + kms_only = TestEnvironmentFeatures.RUN_KMS_ENCRYPTION_TESTS_ONLY in features + not_kms_skip = pytest.mark.skip( + reason="not a kms_encryption test, excluded by RUN_KMS_ENCRYPTION_TESTS_ONLY env feature") for item in items: + if kms_only and item.get_closest_marker("kms_encryption") is None: + item.add_marker(not_kms_skip) + continue + is_async_file = "_async.py" in str(item.fspath) # (1) Legacy guard: when async drivers are disabled entirely, skip every diff --git a/tests/integration/container/django/test_django_kms_encryption.py b/tests/integration/container/django/test_django_kms_encryption.py new file mode 100644 index 00000000..4fdb65c6 --- /dev/null +++ b/tests/integration/container/django/test_django_kms_encryption.py @@ -0,0 +1,150 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The KMS Encryption Plugin through the wrapper's Django MySQL backend on a live server and KMS. + +Django is configured once per process, by the helper's ``kms_django_settings`` when nothing configured it yet, with +``default`` set up as ``test_django_basic.py`` does; the plugin runs only in a ``kms`` alias copied from ``default``, +which is never edited. +""" + +from __future__ import annotations + +from typing import Any, Iterator + +import django.db +import pytest +from django.conf import settings +from django.db import connections, models + +from aws_advanced_python_wrapper.errors import (IntegrityCheckError, + MetadataError) +from ..utils.conditions import enable_on_engines, enable_on_features +from ..utils.database_engine import DatabaseEngine +from ..utils.kms_encryption_helper import (DbUser, KmsSettings, + PayloadVerifier, encrypted_table, + kms_builds, plain_connect) +from ..utils.test_environment_features import TestEnvironmentFeatures + +pytestmark = [pytest.mark.kms_encryption, + enable_on_features([TestEnvironmentFeatures.RUN_KMS_ENCRYPTION_TESTS_ONLY, + TestEnvironmentFeatures.KMS_LOCALSTACK]), + enable_on_engines([DatabaseEngine.MYSQL])] + +_TABLE = "kms_dj_users" + + +@pytest.fixture(scope="module") +def user_model(kms_django_settings: None, kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str) -> Iterator[Any]: + """A model on ``kms_dj_users``, whose ``ssn`` and ``data`` columns are encrypted (without a trigger, so a test + can tamper with a stored value).""" + class KmsUser(models.Model): + name: Any = models.CharField(max_length=100) + ssn: Any = models.CharField(max_length=100, null=True) + data: Any = models.BinaryField(null=True) + + class Meta: + app_label = "kms_encryption_tests" + db_table = _TABLE + managed = False + + with encrypted_table(_TABLE, "name VARCHAR(100), ssn VARBINARY(1024), data VARBINARY(1024)", ("ssn", "data"), + kms_settings, kms_schema, kms_master_key_arn): + yield KmsUser + + +@pytest.fixture +def kms_alias(build: str, user_model: Any, kms_app_user: DbUser, kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str) -> Iterator[str]: + """The ``kms`` alias: ``default``'s settings with the application user and the plugin in OPTIONS.""" + alias = connections["default"].settings_dict.copy() + alias.update(USER=kms_app_user.name, PASSWORD=kms_app_user.password, + OPTIONS={**alias["OPTIONS"], "plugins": "kms_encryption", "encryption_metadata_schema": kms_schema, + "encryption_allowed_master_key_arns": kms_master_key_arn, **kms_settings.props(), + "use_pure": build == "mysql-pure"}) + connections.settings["kms"] = alias + try: + yield "kms" + finally: + connections["kms"].close() + del connections["kms"] + del connections.settings["kms"] + with plain_connect() as conn: + with conn.cursor() as cur: + cur.execute(f"DELETE FROM {_TABLE}") + conn.commit() + + +@pytest.fixture(params=[False, True], ids=["debug-off", "debug-on"]) +def debug(request: pytest.FixtureRequest) -> Iterator[bool]: + saved = settings.DEBUG + settings.DEBUG = request.param + try: + yield request.param + finally: + settings.DEBUG = saved + + +@kms_builds() +def test_django_orm_round_trip(build: str, debug: bool, user_model: Any, kms_alias: str, + kms_verifier: PayloadVerifier) -> None: + users = user_model.objects.using(kms_alias) + created = users.create(name="one", ssn="pt-dj-1") + users.bulk_create([user_model(name="two", ssn="pt-dj-2"), user_model(name="three", ssn="pt-dj-3")]) + created.ssn = "pt-dj-1-saved" + created.save(using=kms_alias) + users.filter(name="two").update(ssn="pt-dj-2-updated") + assert users.get(pk=created.pk).ssn == "pt-dj-1-saved", "save() and get() round-trip" + assert list(users.order_by("id").values_list("ssn", flat=True)) == [ + "pt-dj-1-saved", "pt-dj-2-updated", "pt-dj-3"], "bulk_create(), save() and update() write every value" + assert [u.ssn for u in users.order_by("id").iterator(chunk_size=1)] == [ + "pt-dj-1-saved", "pt-dj-2-updated", "pt-dj-3"], "iterator() decrypts each chunk" + assert [u.ssn for u in users.raw(f"SELECT id, name, ssn FROM {_TABLE} ORDER BY id")] == [ + "pt-dj-1-saved", "pt-dj-2-updated", "pt-dj-3"], "raw() decrypts" + with connections[kms_alias].cursor() as cur: + cur.execute(f"SELECT ssn FROM {_TABLE} ORDER BY id") + assert [row[0] for row in cur.fetchall()] == ["pt-dj-1-saved", "pt-dj-2-updated", "pt-dj-3"], \ + "connection.cursor() decrypts" + stored = kms_verifier.stored(f"SELECT ssn FROM {_TABLE} ORDER BY id") + assert [kms_verifier.decode(_TABLE, "ssn", payload)[1] for payload in stored] == [ + "pt-dj-1-saved", "pt-dj-2-updated", "pt-dj-3"], "every value Django wrote is stored encrypted" + + binary = users.create(name="binary", data=b"pt-dj-binary") + if debug: + recorded = connections[kms_alias].queries[-1]["sql"] + assert recorded.startswith("INSERT") and recorded.count("_binary") == 1, \ + "under DEBUG, the BinaryField's value is recorded with one _binary, Django's, and none added twice" + assert bytes(users.get(pk=binary.pk).data) == b"pt-dj-binary", "a BinaryField round-trips" + + +@kms_builds() +def test_django_bulk_update_is_refused_and_integrity_errors_are_not_translated( + build: str, user_model: Any, kms_alias: str, kms_verifier: PayloadVerifier) -> None: + users = user_model.objects.using(kms_alias) + created = users.create(name="bulk", ssn="pt-dj-bulk") + created.ssn = "pt-dj-bulk-new" + with pytest.raises(MetadataError) as refused: + users.bulk_update([created], ["ssn"]) + assert not isinstance(refused.value, django.db.DatabaseError), "Django doesn't translate the plugin's refusal" + assert users.get(pk=created.pk).ssn == "pt-dj-bulk", "bulk_update() on an encrypted column is refused" + + with plain_connect(admin=True) as admin, admin.cursor() as cur: + cur.execute(f"SELECT ssn FROM {_TABLE} WHERE id = %s", (created.pk,)) + payload = bytes(cur.fetchone()[0]) + cur.execute(f"UPDATE {_TABLE} SET ssn = %s WHERE id = %s", (payload[:-1] + bytes([payload[-1] ^ 1]), created.pk)) + admin.commit() + with pytest.raises(IntegrityCheckError) as tampered: + users.get(pk=created.pk) + assert not isinstance(tampered.value, django.db.DatabaseError), "Django doesn't translate IntegrityCheckError" diff --git a/tests/integration/container/sqlalchemy/test_sqlalchemy_kms_encryption.py b/tests/integration/container/sqlalchemy/test_sqlalchemy_kms_encryption.py new file mode 100644 index 00000000..ab7be739 --- /dev/null +++ b/tests/integration/container/sqlalchemy/test_sqlalchemy_kms_encryption.py @@ -0,0 +1,294 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The KMS Encryption Plugin through SQLAlchemy's wrapper dialects on a live server and KMS.""" + +from __future__ import annotations + +from typing import Any, Callable, Dict, Iterator, List, Optional, Type + +import pytest +from sqlalchemy import (Column, ForeignKey, Integer, MetaData, String, Table, + create_engine, exc, insert, select, text) +from sqlalchemy.engine import URL, Connection, Engine +from sqlalchemy.orm import (DeclarativeBase, Mapped, Session, joinedload, + mapped_column, relationship) +from sqlalchemy.types import UserDefinedType + +from aws_advanced_python_wrapper.encryption.key_management_utility import \ + KeyManagementUtility +from aws_advanced_python_wrapper.errors import (EncryptionFailedError, + IntegrityCheckError, + KeyManagementError, + MetadataError) +from ..utils.conditions import enable_on_features +from ..utils.kms_encryption_helper import (DbUser, KmsSettings, + PayloadVerifier, + create_column_trigger, + encrypt_column, encryption_config, + is_pg, kms_builds, kms_client, + plain_connect) +from ..utils.test_environment import TestEnvironment +from ..utils.test_environment_features import TestEnvironmentFeatures + +pytestmark = [pytest.mark.kms_encryption, + enable_on_features([TestEnvironmentFeatures.RUN_KMS_ENCRYPTION_TESTS_ONLY, + TestEnvironmentFeatures.KMS_LOCALSTACK])] + +_USERS, _CARDS, _GUARDED = "kms_sa_users", "kms_sa_cards", "kms_sa_guarded" + + +class EncryptedValue(UserDefinedType): + """The guide's pass-through type for an encrypted column, so PostgreSQL gets no cast on the parameter.""" + cache_ok = True + + def get_col_spec(self, **kw: Any) -> str: + return "BYTEA" if is_pg() else "VARBINARY(1024)" + + +class Base(DeclarativeBase): + pass + + +class User(Base): + __tablename__ = _USERS + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(String(100)) + ssn: Mapped[Optional[str]] = mapped_column(EncryptedValue()) + cards: Mapped[List[Card]] = relationship(back_populates="user", order_by="Card.id") + + +class Card(Base): + __tablename__ = _CARDS + id: Mapped[int] = mapped_column(primary_key=True) + user_id: Mapped[int] = mapped_column(ForeignKey(f"{_USERS}.id")) + number: Mapped[Optional[str]] = mapped_column(EncryptedValue()) + user: Mapped[User] = relationship(back_populates="cards") + + +_guarded = Table(_GUARDED, MetaData(), Column("id", Integer, primary_key=True), Column("name", String(100)), + Column("ssn", EncryptedValue())) + + +def _url(user: DbUser, **query: str) -> URL: + env = TestEnvironment.get_current() + info, writer = env.get_database_info(), env.get_writer() + return URL.create("postgresql+aws_wrapper_psycopg" if is_pg() else "mysql+aws_wrapper_mysqlconnector", + username=user.name, password=user.password, host=writer.get_host(), port=writer.get_port(), + database=info.get_default_db_name(), query=query) + + +@pytest.fixture +def tables(kms_trigger_installed: None, kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str) -> Iterator[Dict[str, int]]: + """New ``kms_sa_users.ssn`` and ``kms_sa_cards.number`` encrypted without a trigger, so a test can tamper with a + stored value, and ``kms_sa_guarded.ssn`` encrypted with the trigger. Returns each column's key row.""" + id_type = "SERIAL PRIMARY KEY" if is_pg() else "INT AUTO_INCREMENT PRIMARY KEY" + binary = EncryptedValue().get_col_spec() + with plain_connect() as conn: + with conn.cursor() as cur: + for name in (_CARDS, _USERS, _GUARDED): # tables a crashed run left behind + cur.execute(f"DROP TABLE IF EXISTS {name}") + cur.execute(f"CREATE TABLE {_USERS} (id {id_type}, name VARCHAR(100), ssn {binary})") + cur.execute(f"CREATE TABLE {_CARDS} (id {id_type}, user_id INT REFERENCES {_USERS} (id), number {binary})") + cur.execute(f"CREATE TABLE {_GUARDED} (id {id_type}, name VARCHAR(100), ssn {binary})") + conn.commit() + create_column_trigger(conn, _GUARDED, "ssn") + key_ids = {f"{table}.{column}": encrypt_column(conn, kms_settings, kms_schema, table, column, + kms_master_key_arn) + for table, column in ((_USERS, "ssn"), (_CARDS, "number"), (_GUARDED, "ssn"))} + try: + yield key_ids + finally: + with plain_connect() as conn: + with conn.cursor() as cur: + for name in (_CARDS, _USERS, _GUARDED): + cur.execute(f"DROP TABLE IF EXISTS {name}") + cur.execute(f"DELETE FROM {kms_schema}.encryption_metadata WHERE table_name IN (%s, %s, %s)", + (_CARDS, _USERS, _GUARDED)) + conn.commit() + + +@pytest.fixture +def plugin_props(kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str) -> Dict[str, str]: + """The plugin's properties other than ``plugins``.""" + return {"encryption_metadata_schema": kms_schema, "encryption_allowed_master_key_arns": kms_master_key_arn, + **kms_settings.props()} + + +@pytest.fixture +def make_engine(build: str, tables: Dict[str, int], kms_app_user: DbUser, + plugin_props: Dict[str, str]) -> Iterator[Callable[..., Engine]]: + """Creates engines on the wrapper dialect as the application user, with the plugin and its properties in + ``connect_args`` unless a test passes its own; every engine is disposed of afterwards.""" + engines: List[Engine] = [] + + def create(*, query: Optional[Dict[str, str]] = None, connect_args: Optional[Dict[str, Any]] = None) -> Engine: + args: Dict[str, Any] = {"plugins": "kms_encryption", **plugin_props} if connect_args is None \ + else dict(connect_args) + if not is_pg(): + args["use_pure"] = build == "mysql-pure" + engine = create_engine(_url(kms_app_user, **(query or {})), connect_args=args, hide_parameters=True) + engines.append(engine) + return engine + + yield create + for engine in engines: + engine.dispose() + + +def _stored_names(verifier: PayloadVerifier) -> List[str]: + return verifier.stored(f"SELECT name FROM {_GUARDED} ORDER BY id") + + +@kms_builds() +def test_wrapper_plugins_query_parameter_loads_the_plugin(build: str, make_engine: Callable[..., Engine], + tables: Dict[str, int], plugin_props: Dict[str, str], + kms_verifier: PayloadVerifier) -> None: + props = plugin_props + dropped = make_engine(query={"plugins": "kms_encryption", **props}, connect_args={}) + with pytest.raises(exc.DBAPIError, match="does not carry a valid HMAC tag"), dropped.begin() as conn: + conn.execute(insert(_guarded), {"name": "plugins-query", "ssn": "pt-sa-dropped"}) + assert _stored_names(kms_verifier) == [], \ + "with ?plugins= SQLAlchemy drops the parameter, the plugin isn't loaded and the trigger rejects the plaintext" + + for label, engine in (("wrapper-plugins", make_engine(query={"wrapper_plugins": "kms_encryption", **props}, + connect_args={})), + ("connect-args", make_engine())): + with engine.begin() as conn: + conn.execute(insert(_guarded), {"name": label, "ssn": f"pt-sa-{label}"}) + (payload,) = kms_verifier.stored(f"SELECT ssn FROM {_GUARDED} WHERE name = %s", (label,)) + assert kms_verifier.decode(_GUARDED, "ssn", payload) == (tables[f"{_GUARDED}.ssn"], f"pt-sa-{label}"), \ + f"with {label} the plugin is loaded and encrypts the write" + + +@kms_builds() +def test_sqlalchemy_core_and_orm_round_trip(build: str, make_engine: Callable[..., Engine], tables: Dict[str, int], + kms_verifier: PayloadVerifier) -> None: + engine = make_engine(query={"plugins": "kms_encryption"}) + with engine.begin() as conn: + conn.execute(insert(User), {"name": "core-1", "ssn": "pt-sa-core-1"}) + conn.execute(insert(User), [{"name": "core-2", "ssn": "pt-sa-core-2"}, + {"name": "core-3", "ssn": "pt-sa-core-3"}]) + rows = [(row.name, row.ssn) for row in conn.execute(select(User.name, User.ssn).order_by(User.id))] + assert rows == [("core-1", "pt-sa-core-1"), ("core-2", "pt-sa-core-2"), ("core-3", "pt-sa-core-3")], \ + "Core insert, executemany and result iteration round-trip with ?plugins= dropped and connect_args used" + + with Session(engine) as session: + session.add_all([User(name="orm-1", ssn="pt-sa-orm-1", + cards=[Card(number="pt-sa-card-1a"), Card(number="pt-sa-card-1b")]), + User(name="orm-2", ssn="pt-sa-orm-2", cards=[Card(number="pt-sa-card-2a")])]) + session.commit() + first = session.scalars(select(User).filter_by(name="orm-1")).one() + first_id = first.id + assert first.ssn == "pt-sa-orm-1", "an ORM filter_by reads the decrypted value" + first.ssn = "pt-sa-orm-1-updated" + session.commit() + with Session(engine) as session: + got = session.get(User, first_id) + assert got is not None and got.ssn == "pt-sa-orm-1-updated", "an ORM attribute update round-trips" + loaded = session.scalars(select(User).options(joinedload(User.cards)).where(User.name.like("orm-%")) + .order_by(User.id).limit(2)).unique().all() + assert [(u.ssn, [c.number for c in u.cards]) for u in loaded] == [ + ("pt-sa-orm-1-updated", ["pt-sa-card-1a", "pt-sa-card-1b"]), ("pt-sa-orm-2", ["pt-sa-card-2a"])], \ + "joinedload with LIMIT decrypts both tables' columns" + + stored = kms_verifier.stored(f"SELECT ssn FROM {_USERS} ORDER BY id") + assert [kms_verifier.decode(_USERS, "ssn", payload)[1] for payload in stored] == [ + "pt-sa-core-1", "pt-sa-core-2", "pt-sa-core-3", "pt-sa-orm-1-updated", "pt-sa-orm-2"], \ + "every value SQLAlchemy wrote is stored encrypted" + + +@kms_builds() +def test_plugin_errors_arrive_as_the_sqlalchemy_classes_the_guide_names( + build: str, make_engine: Callable[..., Engine], tables: Dict[str, int], kms_settings: KmsSettings, + kms_schema: str) -> None: + other_key = kms_client(kms_settings).create_key(Description="kms container test key outside the allow-list") + with plain_connect() as admin: + with admin.cursor() as cur: + cur.execute(f"ALTER TABLE {_USERS} ADD COLUMN pin {EncryptedValue().get_col_spec()}") + admin.commit() + encrypt_column(admin, kms_settings, kms_schema, _USERS, "pin", other_key["KeyMetadata"]["Arn"]) + engine = make_engine() + with engine.begin() as conn: + conn.execute(insert(User), {"name": "tamper", "ssn": "pt-sa-tamper"}) + with plain_connect() as admin, admin.cursor() as cur: + cur.execute(f"SELECT ssn FROM {_USERS} WHERE name = 'tamper'") + payload = bytes(cur.fetchone()[0]) + cur.execute(f"UPDATE {_USERS} SET ssn = %s WHERE name = 'tamper'", (payload[:-1] + bytes([payload[-1] ^ 1]),)) + admin.commit() + + def assert_wrapped(caught: pytest.ExceptionInfo, wrapper_class: Type[exc.DBAPIError], plugin_class: type, + conn: Connection) -> None: + conn.rollback() + assert type(caught.value) is wrapper_class and isinstance(caught.value.orig, plugin_class), \ + f"{plugin_class.__name__} arrives as sqlalchemy.exc.{wrapper_class.__name__} with the plugin's error as orig" + assert conn.execute(text("SELECT 1")).scalar() == 1, "the connection stays usable after the error" + + with engine.connect() as conn: + with pytest.raises(exc.IntegrityError) as tampered: + conn.execute(text(f"SELECT ssn FROM {_USERS} WHERE name = 'tamper'")).fetchall() + assert_wrapped(tampered, exc.IntegrityError, IntegrityCheckError, conn) + + with pytest.raises(exc.OperationalError) as literal: + conn.execute(text(f"INSERT INTO {_USERS} (name, ssn) VALUES ('literal', 'pt-sa-literal')")) + assert_wrapped(literal, exc.OperationalError, MetadataError, conn) + + with pytest.raises(exc.OperationalError) as no_exact_form: + conn.execute(text(f"INSERT INTO {_USERS} (name, ssn) VALUES (:name, :ssn)"), + {"name": "generic", "ssn": complex(1, 2)}) + assert_wrapped(no_exact_form, exc.OperationalError, EncryptionFailedError, conn) + + with pytest.raises(exc.OperationalError) as unauthorized: + conn.execute(text(f"INSERT INTO {_USERS} (name, pin) VALUES (:name, :pin)"), + {"name": "unauthorized", "pin": "pt-sa-pin"}) + assert_wrapped(unauthorized, exc.OperationalError, KeyManagementError, conn) + + +@kms_builds() +def test_utility_runs_on_an_engine_raw_connection(build: str, make_engine: Callable[..., Engine], + kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str, kms_verifier: PayloadVerifier) -> None: + with plain_connect() as admin: + with admin.cursor() as cur: + for column in ("raw_a", "raw_b"): + cur.execute(f"ALTER TABLE {_GUARDED} ADD COLUMN {column} {EncryptedValue().get_col_spec()}") + admin.commit() + client, config = kms_client(kms_settings), encryption_config(kms_settings, kms_schema) + # A setup script's engine on SQLAlchemy's own dialect: its raw connection is a pool proxy around the driver's. + setup_engine = create_engine("postgresql+psycopg://" if is_pg() else "mysql+mysqlconnector://", + creator=lambda: plain_connect(build=build)) + try: + for column, utility_for in ( + ("raw_a", lambda raw: KeyManagementUtility(raw.driver_connection, client, config)), + ("raw_b", lambda raw: KeyManagementUtility(raw, client, config, + engine="postgresql" if is_pg() else "mysql"))): + raw = setup_engine.raw_connection() + try: + utility_for(raw).initialize_encryption_for_column(_GUARDED, column, kms_master_key_arn) + raw.commit() + finally: + raw.close() + with setup_engine.connect() as conn: + assert conn.execute(text("SELECT 1")).scalar() == 1, \ + "closing the raw connection after commit gave the pool a working connection back" + finally: + setup_engine.dispose() + with make_engine().begin() as conn: + conn.execute(text(f"INSERT INTO {_GUARDED} (name, raw_a, raw_b) VALUES (:name, :a, :b)"), + {"name": "raw", "a": "pt-sa-raw-a", "b": "pt-sa-raw-b"}) + for column in ("raw_a", "raw_b"): + (payload,) = kms_verifier.stored(f"SELECT {column} FROM {_GUARDED} WHERE name = 'raw'") + assert kms_verifier.decode(_GUARDED, column, payload)[1] == f"pt-sa-raw-{column[-1]}", \ + f"the utility set up {column} on the setup engine's raw connection, so the plugin encrypts it" diff --git a/tests/integration/container/test_kms_encryption_aurora.py b/tests/integration/container/test_kms_encryption_aurora.py new file mode 100644 index 00000000..24edf102 --- /dev/null +++ b/tests/integration/container/test_kms_encryption_aurora.py @@ -0,0 +1,299 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The KMS Encryption Plugin on an Aurora cluster with a real KMS key. + +IAM and Secrets Manager authentication, required TLS, the default parameter group and real topology changes exist +only on AWS. +The cluster's admin user is the test user; it sets up the metadata schema, the tables and the trigger, and the +plugin runs as a DML-only application user unless a test logs in another way. +Each connection the plugin opens for itself is observed at ``PluginConnectionSource._open``, as the server sees it. +""" + +from __future__ import annotations + +import json +from contextlib import closing +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional +from uuid import uuid4 + +import boto3 +import pytest + +from aws_advanced_python_wrapper.encryption.connection_source import \ + PluginConnectionSource +from aws_advanced_python_wrapper.errors import MetadataError +from .utils.conditions import (disable_on_features, enable_on_deployments, + enable_on_features, enable_on_num_instances) +from .utils.database_engine_deployment import DatabaseEngineDeployment +from .utils.kms_encryption_helper import (TRIGGER_REJECTION, DbUser, + PayloadVerifier, binary_type, + create_column_trigger, create_table, + drop_table, encrypt_column, + install_trigger_sql, is_pg, + kms_builds, plain_connect, raised, + read_column, wrapper_connect, write) +from .utils.rds_test_utility import RdsTestUtility +from .utils.retry_helper import retry_until +from .utils.test_environment import TestEnvironment +from .utils.test_environment_features import TestEnvironmentFeatures + +if TYPE_CHECKING: + from .utils.kms_encryption_helper import KmsSettings + +pytestmark = [pytest.mark.kms_encryption, + enable_on_features([TestEnvironmentFeatures.RUN_KMS_ENCRYPTION_TESTS_ONLY]), + disable_on_features([TestEnvironmentFeatures.KMS_LOCALSTACK]), + enable_on_deployments([DatabaseEngineDeployment.AURORA])] + +_TABLE = "kms_aurora_users" +_GUARDED_TABLE = "kms_aurora_guarded" +_INSERT = f"INSERT INTO {_TABLE} (name, ssn) VALUES (%s, %s)" +_SELECT = f"SELECT ssn FROM {_TABLE} WHERE name = %s ORDER BY id" +# The secret holds the credentials under names other than the plugin's defaults, ``username`` and ``password``. +_SECRET_USER_KEY, _SECRET_PASSWORD_KEY = "aurora_user", "aurora_password" + + +@dataclass(frozen=True) +class Session: + """The server's view of one connection.""" + user: str + tls: bool + tls_version: str + instance: str + + +def _session(conn: Any) -> Session: + """How the server sees the driver connection ``conn``; it leaves no transaction open.""" + with conn.cursor() as cur: + if is_pg(): + cur.execute("SELECT current_user, s.ssl, COALESCE(s.version, '') FROM pg_catalog.pg_stat_ssl s " + "WHERE s.pid = pg_catalog.pg_backend_pid()") + user, tls, version = cur.fetchall()[0] + else: + cur.execute("SELECT CURRENT_USER()") + user = str(cur.fetchall()[0][0]).split("@")[0] + cur.execute("SHOW SESSION STATUS LIKE 'Ssl_version'") + version = cur.fetchall()[0][1] or "" + tls = bool(version) + cur.execute(RdsTestUtility.get_instance_id_query()) + instance = cur.fetchall()[0][0] + if not getattr(conn, "autocommit", True): + conn.commit() + return Session(str(user), bool(tls), str(version), str(instance)) + + +@pytest.fixture(scope="module") +def table(kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, kms_app_user: DbUser) -> Iterator[int]: + """The module's table, ``ssn`` encrypted under the real key; yields its ``key_storage`` id.""" + create_table(_TABLE, f"name VARCHAR(100), ssn {binary_type()}") + with plain_connect() as conn: + key_storage_id = encrypt_column(conn, kms_settings, kms_schema, _TABLE, "ssn", kms_master_key_arn) + try: + yield key_storage_id + finally: + drop_table(_TABLE) + + +@pytest.fixture(scope="module") +def iam_user(table: int, kms_schema: str) -> Iterator[str]: + """The harness's IAM database user, granted what the plugin and the test need for the module. + On MySQL the grants are revoked at the end, since a schema grant outlives ``DROP DATABASE``; PostgreSQL's go with + the schema and the table.""" + info = TestEnvironment.get_current().get_info() + name, database = info.get_iam_user_name(), info.get_database_info().get_default_db_name() + revokes: List[str] = [] + if is_pg(): + statements = [f"GRANT USAGE ON SCHEMA {kms_schema} TO {name}", + f"GRANT SELECT ON ALL TABLES IN SCHEMA {kms_schema} TO {name}", + f"GRANT SELECT, INSERT, UPDATE ON {_TABLE} TO {name}", + f"GRANT USAGE, SELECT ON SEQUENCE {_TABLE}_id_seq TO {name}"] + else: + grants = [("SELECT", f"{kms_schema}.*"), ("SELECT, INSERT, UPDATE", f"{database}.{_TABLE}")] + statements = [f"GRANT {privileges} ON {target} TO '{name}'@'%'" for privileges, target in grants] + revokes = [f"REVOKE {privileges} ON {target} FROM '{name}'@'%'" for privileges, target in grants] + with plain_connect() as conn, conn.cursor() as cur: + for statement in statements: + cur.execute(statement) + conn.commit() + try: + yield name + finally: + if revokes: + with plain_connect() as conn, conn.cursor() as cur: + for statement in revokes: + cur.execute(statement) + conn.commit() + + +@pytest.fixture(scope="module") +def secret_arn() -> Iterator[str]: + """A Secrets Manager secret of the module's own holding the admin user's credentials, deleted at the end, as + test_aws_secrets_manager.py does.""" + env = TestEnvironment.get_current() + info = env.get_info().get_database_info() + client = boto3.client("secretsmanager", region_name=env.get_info().get_region()) + secret = client.create_secret(Name=f"TestSecret-kms-{uuid4()}", SecretString=json.dumps({ + "engine": "postgres" if is_pg() else "mysql", "dbname": info.get_default_db_name(), + "host": info.get_cluster_endpoint(), _SECRET_USER_KEY: info.get_username(), + _SECRET_PASSWORD_KEY: info.get_password()})) + try: + yield secret["ARN"] + finally: + client.delete_secret(SecretId=secret["ARN"], ForceDeleteWithoutRecovery=True) + + +@pytest.fixture(scope="module") +def guarded_table(kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, kms_app_user: DbUser + ) -> Iterator[int]: + """A table whose ``ssn`` is encrypted and guarded by the package's trigger SQL, installed by the admin user; + yields the ``key_storage`` id.""" + create_table(_GUARDED_TABLE, f"name VARCHAR(100), ssn {binary_type()}") + with plain_connect() as conn: + key_storage_id = encrypt_column(conn, kms_settings, kms_schema, _GUARDED_TABLE, "ssn", kms_master_key_arn) + install_trigger_sql(conn, kms_schema) + create_column_trigger(conn, _GUARDED_TABLE, "ssn") + try: + yield key_storage_id + finally: + drop_table(_GUARDED_TABLE) + + +@pytest.fixture +def plugin_sessions(monkeypatch: pytest.MonkeyPatch) -> List[Session]: + """The server's view of each connection the plugin opens for itself during the test.""" + seen: List[Session] = [] + original = PluginConnectionSource._open + + def observed(source: PluginConnectionSource) -> Any: + conn = original(source) + seen.append(_session(conn)) + return conn + + monkeypatch.setattr(PluginConnectionSource, "_open", observed) + return seen + + +@enable_on_features([TestEnvironmentFeatures.IAM, TestEnvironmentFeatures.SECRETS_MANAGER]) +@kms_builds() +def test_plugin_connections_carry_the_applications_auth_and_tls( + build: str, table: int, iam_user: str, secret_arn: str, plugin_sessions: List[Session], + monkeypatch: pytest.MonkeyPatch, kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser) -> None: + # libpq reads PGSSLMODE only for a connection that doesn't set sslmode itself, so a plugin connection that lost + # the application's setting goes without TLS, or is refused where the cluster forces it. + monkeypatch.setenv("PGSSLMODE", "disable") + # mysql-connector negotiates the newest TLS version both sides offer, so only a pinned older one shows a plugin + # connection that lost the application's TLS settings. + tls: Dict[str, Any] = {"sslmode": "require"} if is_pg() else {"ssl_disabled": False, "tls_versions": ["TLSv1.2"]} + region = TestEnvironment.get_current().get_info().get_region() + # The Secrets Manager plugin reads the secret's key names at each connect, so a plugin connection without the + # secrets_manager_* settings looks for ``username`` in the secret and fails to log in. + logins = {"password": (kms_app_user, {}), + "secrets manager": (DbUser("IncorrectUser", "IncorrectPassword"), + {"plugins": "aws_secrets_manager,kms_encryption", + "secrets_manager_secret_id": secret_arn, "secrets_manager_region": region, + "secrets_manager_secret_username_key": _SECRET_USER_KEY, + "secrets_manager_secret_password_key": _SECRET_PASSWORD_KEY})} + if build != "mysql-pure": # the pure MySQL driver truncates IAM tokens (test_iam_authentication.py) + logins["iam"] = (DbUser(iam_user, ""), {"plugins": "iam,kms_encryption"}) + seen: Dict[str, Any] = {} + for login, (user, props) in logins.items(): + name, value = f"it65-{login.replace(' ', '-')}-{build}", f"pt-it65-{login}" + plugin_sessions.clear() + with wrapper_connect(build, user, kms_settings, kms_schema, kms_master_key_arn, + encryption_metadata_cache_enabled=False, **tls, **props) as conn: + app = _session(conn.target_connection) + write(conn, _INSERT, (name, value)) + read = read_column(conn, _SELECT, name) + differing = [(s.user, s.tls, s.tls_version) for s in plugin_sessions + if (s.user, s.tls, s.tls_version) != (app.user, app.tls, app.tls_version)] + seen[login] = (read, app.tls, len(plugin_sessions) > 0, differing) + assert seen == {login: ([f"pt-it65-{login}"], True, True, []) for login in logins}, \ + "each login's value round-trips over TLS, and every connection the plugin opens for itself logs in as the " \ + "application's does, with the same TLS" + + +@enable_on_num_instances(min_instances=2) +@enable_on_features([TestEnvironmentFeatures.FAILOVER_SUPPORTED]) +@pytest.mark.xfail(is_pg(), strict=True, raises=MetadataError, reason=( + "on psycopg the plugin's metadata lookup on the dead writer raises MetadataError before failover_v2 runs, so " + "failover never starts")) +@kms_builds() +def test_failover_and_read_write_splitting_keep_encrypting( + build: str, table: int, plugin_sessions: List[Session], kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str, kms_app_user: DbUser) -> None: + aurora = RdsTestUtility(TestEnvironment.get_current().get_info().get_region()) + before, during, after = (f"it66-{phase}-{build}" for phase in ("before", "during", "after")) + initial_writer = aurora.get_cluster_writer_instance_id() + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn, + plugins="read_write_splitting,failover_v2,kms_encryption", autocommit=True, + connect_timeout=10, socket_timeout=10, encryption_metadata_cache_enabled=False) as conn: + write(conn, _INSERT, (before, "pt-it66-before")) + aurora.failover_cluster_and_wait_until_writer_changed() + # conftest does this before each test; here it lets the verifier and the module's teardown reach the new writer. + TestEnvironment.get_current().get_database_info().move_instance_first(aurora.get_cluster_writer_instance_id()) + failover: Optional[BaseException] = None + try: + write(conn, _INSERT, (during, "pt-it66-during")) + except Exception as err: # noqa: BLE001 - the test asserts on the error + failover = err + plugin_sessions.clear() + write(conn, _INSERT, (after, "pt-it66-after")) + writer_lookups = sorted({s.instance for s in plugin_sessions}) + writer = aurora.query_instance_id(conn) + # On two instances the only reader is the old writer, which takes a while to come back. + aurora.make_sure_instances_up([initial_writer]) + + def switched_to_a_reader() -> bool: + conn.read_only = False + conn.read_only = True + return aurora.query_instance_id(conn) != writer + + # Past the 5 minutes for which the wrapper keeps the old writer marked unavailable after the failover. + on_a_reader = retry_until(switched_to_a_reader, timeout_sec=390, delay_sec=5) + expected = {before: ["pt-it66-before"], after: ["pt-it66-after"]} + replicated = retry_until(lambda: {name: read_column(conn, _SELECT, name) for name in expected} == expected, + timeout_sec=30, delay_sec=1) + conn.read_only = False + assert failover is not None and "pt-it66-during" not in f"{failover} {failover!r}", \ + "the statement that meets the failover raises an error that quotes no plaintext" + assert writer != initial_writer and retry_until(lambda: aurora.is_db_instance_writer(writer)), \ + "after the failover the connection writes on the new writer" + assert writer_lookups == [writer], "after the failover the plugin's lookups go to the new writer" + assert (on_a_reader, replicated) == (True, True), \ + "with read_only set, a reader decrypts the values written before and after the failover" + with closing(PayloadVerifier(kms_settings, kms_schema, kms_master_key_arn)) as verifier: + opened = {name: verifier.opened(_TABLE, "ssn", name) for name in (before, after)} + assert opened == {before: [(table, "pt-it66-before")], after: [(table, "pt-it66-after")]}, \ + "the writer stored both values encrypted under the column's key row" + + +@kms_builds("mysql-pure") +def test_round_trip_on_aurora(build: str, guarded_table: int, kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str, kms_app_user: DbUser, kms_verifier: PayloadVerifier) -> None: + """The packaged trigger installs as the admin user, and a value round-trips on Aurora.""" + name, plain_name, value = f"it67-{build}", f"it67-plain-{build}", f"pt-it67-{build}" + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + write(conn, f"INSERT INTO {_GUARDED_TABLE} (name, ssn) VALUES (%s, %s)", (name, value)) + read = read_column(conn, f"SELECT ssn FROM {_GUARDED_TABLE} WHERE name = %s", name) + assert (read, kms_verifier.opened(_GUARDED_TABLE, "ssn", name)) == ([value], [(guarded_table, value)]), \ + "a value written through the plugin is stored encrypted under the column's key row and reads back" + with plain_connect() as conn: + refused = raised(conn, lambda: write(conn, f"INSERT INTO {_GUARDED_TABLE} (name, ssn) VALUES (%s, %s)", + (plain_name, value.encode()))) + assert TRIGGER_REJECTION in str(refused) and kms_verifier.stored( + f"SELECT ssn FROM {_GUARDED_TABLE} WHERE name = %s", (plain_name,)) == [], \ + "the trigger refuses a plaintext write that doesn't come through the plugin" diff --git a/tests/integration/container/test_kms_encryption_key_management.py b/tests/integration/container/test_kms_encryption_key_management.py new file mode 100644 index 00000000..6935819d --- /dev/null +++ b/tests/integration/container/test_kms_encryption_key_management.py @@ -0,0 +1,524 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The KMS Encryption Plugin's metadata tables, KeyManagementUtility and key rows on a live server and KMS.""" + +from __future__ import annotations + +import base64 +import secrets +import uuid +from typing import Any, Callable, Iterator, List, Optional + +import pytest +import sqlalchemy + +from aws_advanced_python_wrapper.encryption.algorithm import \ + EncryptionAlgorithm +from aws_advanced_python_wrapper.encryption.encryption_service import \ + EncryptionService +from aws_advanced_python_wrapper.encryption.key_management_utility import \ + KeyManagementUtility +from aws_advanced_python_wrapper.errors import (IntegrityCheckError, + MetadataError) +from .utils.conditions import enable_on_features +from .utils.kms_encryption_helper import (DB_ERRORS, TRIGGER_REJECTION, DbUser, + KmsSettings, PayloadVerifier, + binary_type, create_column_trigger, + encrypt_column, encryption_config, + is_pg, key_management_utility, + kms_builds, kms_client, + plain_connect, run_script, + wrapper_connect, write, writes) +from .utils.kms_encryption_schema import ddl_for +from .utils.retry_helper import retry_until +from .utils.test_environment import TestEnvironment +from .utils.test_environment_features import TestEnvironmentFeatures + +pytestmark = [pytest.mark.kms_encryption, + enable_on_features([TestEnvironmentFeatures.RUN_KMS_ENCRYPTION_TESTS_ONLY, + TestEnvironmentFeatures.KMS_LOCALSTACK])] + + +def _read(conn: Any, sql: str, params: Any = None) -> List[Any]: + with conn.cursor() as cur: + cur.execute(sql, params) + return cur.fetchall() + + +def _insert(table: str, column: str = "ssn") -> str: + return f"INSERT INTO {table} (name, {column}) VALUES (%s, %s)" + + +def _metadata_key_ids(verifier: PayloadVerifier, schema: str, table: str) -> List[Any]: + return verifier.stored(f"SELECT key_id FROM {schema}.encryption_metadata WHERE table_name = %s " + "ORDER BY column_name", (table,)) + + +@pytest.fixture +def new_table(kms_trigger_installed: None) -> Iterator[Callable[..., str]]: + """Creates tables ``(id, name, ssn, card)``, ``ssn`` of ``ssn_type`` and ``card`` binary, with the trigger on + both when ``triggered``; nothing is encrypted. Every table is dropped afterwards.""" + created: List[str] = [] + + def create(ssn_type: Optional[str] = None, *, triggered: bool = False) -> str: + name = f"kms_km_{secrets.token_hex(4)}" + id_type = "SERIAL PRIMARY KEY" if is_pg() else "INT AUTO_INCREMENT PRIMARY KEY" + with plain_connect() as conn: + write(conn, f"CREATE TABLE {name} (id {id_type}, name VARCHAR(100), ssn {ssn_type or binary_type()}, " + f"card {binary_type()})") + created.append(name) + if triggered: + create_column_trigger(conn, name, "ssn") + create_column_trigger(conn, name, "card") + return name + + yield create + with plain_connect() as conn: + for name in created: + write(conn, f"DROP TABLE IF EXISTS {name}") + + +@kms_builds("mysql-pure") +def test_metadata_ddl_from_the_guide_creates_tables_the_plugin_reads( + build: str, new_table: Callable[..., str], kms_connect: Callable[..., Any], kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, kms_verifier: PayloadVerifier) -> None: + columns = kms_verifier.stored("SELECT concat(table_name, '.', column_name) FROM information_schema.columns " + "WHERE table_schema = %s ORDER BY 1", (kms_schema,)) + assert columns == sorted( + [f"encryption_metadata.{c}" for c in ("column_name", "created_at", "encryption_algorithm", "key_id", + "table_name", "updated_at")] + + [f"key_storage.{c}" for c in ("created_at", "encrypted_data_key", "hmac_key", "id", "key_id", "key_spec", + "last_used_at", "master_key_arn", "name")]), \ + "the guide's DDL creates the two metadata tables with the guide's columns" + table = new_table() + with plain_connect(build=build) as conn: + key_id = encrypt_column(conn, kms_settings, kms_schema, table, "ssn", kms_master_key_arn) + assert _metadata_key_ids(kms_verifier, kms_schema, table) == [key_id], \ + "KeyManagementUtility fills the tables, and encryption_metadata.key_id is the key row's integer id" + with kms_connect() as conn: + write(conn, _insert(table), ("ddl", "pt-ddl")) + assert _read(conn, f"SELECT ssn FROM {table}") == [("pt-ddl",)], "the plugin reads the metadata the DDL holds" + (payload,) = kms_verifier.stored(f"SELECT ssn FROM {table}") + assert kms_verifier.decode(table, "ssn", payload) == (key_id, "pt-ddl"), "the stored payload names the key row" + + +@kms_builds() +def test_least_privilege_grants_from_the_guide_are_enough_and_hold( + build: str, new_table: Callable[..., str], kms_connect: Callable[..., Any], kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, kms_app_user: DbUser, kms_verifier: PayloadVerifier) -> None: + table = new_table(triggered=True) + with plain_connect() as conn: + encrypt_column(conn, kms_settings, kms_schema, table, "ssn", kms_master_key_arn) + with kms_connect() as conn: + write(conn, _insert(table), ("app", "pt-least")) + assert _read(conn, f"SELECT ssn FROM {table}") == [("pt-least",)], \ + "the guide's run-time grants (SELECT on the metadata, DML on the table) are enough for the plugin" + + if is_pg(): + refused = [f"ALTER TABLE {table} DISABLE TRIGGER {table}_ssn_encrypted", + f"DROP TRIGGER {table}_ssn_encrypted ON {table}"] + else: + refused = [f"DROP TRIGGER {table}_ssn_hmac_insert"] + refused += [f"INSERT INTO {kms_schema}.key_storage (key_id, name, master_key_arn, encrypted_data_key, hmac_key) " + "VALUES ('k', 'k', 'arn', 'e', 'h')", + f"UPDATE {kms_schema}.key_storage SET name = 'changed'", + f"DELETE FROM {kms_schema}.encryption_metadata"] + with plain_connect(kms_app_user, build=build) as conn: + for statement in refused: + with pytest.raises(DB_ERRORS, match="permission denied|must be owner|command denied"): + write(conn, statement) + conn.rollback() + with pytest.raises(DB_ERRORS, match=TRIGGER_REJECTION): + write(conn, _insert(table), ("app-plain", b"pt-least-plain-" + b"x" * 64)) + conn.rollback() + assert kms_verifier.stored(f"SELECT name FROM {table} ORDER BY id") == ["app"], \ + "the application user can't change the key rows or the trigger, so its plaintext is still rejected" + + +# IT-26 in three legs. Each leg checks "not before" on a connection with the default refresh interval (300 s) and "at +# refresh" on one that refreshes every second, so no check depends on where the refresh timer is. +@kms_builds() +def test_open_connections_pick_up_metadata_changes_at_refresh_register( + build: str, new_table: Callable[..., str], kms_connect: Callable[..., Any], kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, kms_verifier: PayloadVerifier) -> None: + table = new_table(triggered=True) + with plain_connect() as admin: + encrypt_column(admin, kms_settings, kms_schema, table, "ssn", kms_master_key_arn) + with kms_connect() as stale, kms_connect(encryption_metadata_cache_refresh_interval_sec=1) as refreshing: + write(stale, _insert(table), ("stale-loaded", "pt-refresh-ssn")) + write(refreshing, _insert(table), ("refreshing-loaded", "pt-refresh-ssn")) + with plain_connect() as admin: + card_key_id = encrypt_column(admin, kms_settings, kms_schema, table, "card", kms_master_key_arn) + assert not writes(stale, _insert(table, "card"), ("registered-early", "pt-refresh-early")), \ + "before its refresh, an open connection still treats the new column as plain, so the trigger rejects it" + assert retry_until(lambda: writes(refreshing, _insert(table, "card"), ("registered", "pt-refresh-card")), + timeout_sec=15, delay_sec=0.25), \ + "after its refresh, an open connection encrypts the newly registered column" + (payload,) = kms_verifier.stored(f"SELECT card FROM {table} WHERE name = 'registered'") + assert kms_verifier.decode(table, "card", payload) == (card_key_id, "pt-refresh-card"), \ + "the refreshed connection's write names the new column's key row" + + +@kms_builds() +def test_open_connections_pick_up_metadata_changes_at_refresh_rotate( + build: str, new_table: Callable[..., str], kms_connect: Callable[..., Any], kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, kms_verifier: PayloadVerifier) -> None: + table = new_table(triggered=True) + with plain_connect() as admin: + encrypt_column(admin, kms_settings, kms_schema, table, "card", kms_master_key_arn) + with kms_connect() as stale, kms_connect(encryption_metadata_cache_refresh_interval_sec=1) as refreshing: + write(stale, _insert(table, "card"), ("stale-loaded", "pt-refresh-card")) + write(refreshing, _insert(table, "card"), ("refreshing-loaded", "pt-refresh-card")) + with plain_connect() as admin: + rotated_key_id = key_management_utility(admin, kms_settings, kms_schema).rotate_data_key( + table, "card", kms_master_key_arn) + admin.commit() + assert not writes(stale, _insert(table, "card"), ("rotated-early", "pt-refresh-early")), \ + "before its refresh, an open connection still writes with the old key, which the trigger rejects" + assert retry_until(lambda: writes(refreshing, _insert(table, "card"), ("rotated", "pt-refresh-rotated")), + timeout_sec=15, delay_sec=0.25), \ + "after its refresh, an open connection writes with the rotated key" + (payload,) = kms_verifier.stored(f"SELECT card FROM {table} WHERE name = 'rotated'") + assert kms_verifier.decode(table, "card", payload) == (rotated_key_id, "pt-refresh-rotated"), \ + "the refreshed connection's write names the rotated key row" + + +@kms_builds() +def test_open_connections_pick_up_metadata_changes_at_refresh_remove( + build: str, new_table: Callable[..., str], kms_connect: Callable[..., Any], kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, kms_verifier: PayloadVerifier) -> None: + table = new_table(triggered=True) + with plain_connect() as admin: + card_key_id = encrypt_column(admin, kms_settings, kms_schema, table, "card", kms_master_key_arn) + with kms_connect() as stale, kms_connect(encryption_metadata_cache_refresh_interval_sec=1) as refreshing: + write(stale, _insert(table, "card"), ("stale-loaded", "pt-refresh-card")) + write(refreshing, _insert(table, "card"), ("refreshing-loaded", "pt-refresh-card")) + with plain_connect() as admin: + with admin.cursor() as cur: + if is_pg(): + cur.execute(f"DROP TRIGGER {table}_card_encrypted ON {table}") + else: + cur.execute(f"DROP TRIGGER {table}_card_hmac_insert") + cur.execute(f"DROP TRIGGER {table}_card_hmac_update") + key_management_utility(admin, kms_settings, kms_schema).remove_encryption_for_column(table, "card") + admin.commit() + write(stale, _insert(table, "card"), ("removed-early", "pt-refresh-removed-early")) + (early,) = kms_verifier.stored(f"SELECT card FROM {table} WHERE name = 'removed-early'") + assert EncryptionService.key_id_from_payload(bytes(early)) == card_key_id, \ + "before its refresh, an open connection still encrypts the removed column" + + def stores_plaintext() -> bool: + write(refreshing, f"DELETE FROM {table} WHERE name = 'removed'") + write(refreshing, _insert(table, "card"), ("removed", b"pt-refresh-removed")) + return kms_verifier.stored(f"SELECT card FROM {table} WHERE name = 'removed'") == [b"pt-refresh-removed"] + assert retry_until(stores_plaintext, timeout_sec=15, delay_sec=0.25), \ + "after its refresh, an open connection writes the removed column as plain" + + +@kms_builds() +def test_missing_or_unreadable_metadata_refuses_every_statement( + build: str, new_table: Callable[..., str], kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str, kms_app_user: DbUser) -> None: + table = new_table() + late = f"{kms_schema}_late" + engine = TestEnvironment.get_current().get_engine() + drop = f"DROP SCHEMA IF EXISTS {late} CASCADE" if is_pg() else f"DROP DATABASE IF EXISTS {late}" + try: + with wrapper_connect(build, kms_app_user, kms_settings, late, kms_master_key_arn) as conn: + for sql, params in (("SELECT 1", None), (_insert(table), ("missing", "pt-missing"))): + with pytest.raises(MetadataError): + _read(conn, sql, params) if params is None else write(conn, sql, params) + conn.rollback() + with plain_connect(admin=True) as admin, admin.cursor() as cur: + if is_pg(): + cur.execute(ddl_for(engine.value, late)) + cur.execute(f"GRANT USAGE ON SCHEMA {late} TO {kms_app_user.name}") + cur.execute(f"GRANT SELECT ON ALL TABLES IN SCHEMA {late} TO {kms_app_user.name}") + else: + run_script(cur, ddl_for(engine.value, late)) + cur.execute(f"GRANT SELECT ON {late}.* TO %s", (kms_app_user.name,)) + admin.commit() + assert _read(conn, "SELECT 1") == [(1,)], "once the metadata tables exist, the same connection works" + finally: + with plain_connect(admin=True) as admin, admin.cursor() as cur: + cur.execute(drop) + admin.commit() + + grant = (f"SELECT ON ALL TABLES IN SCHEMA {kms_schema}" if is_pg() else f"SELECT ON {kms_schema}.*") + user = kms_app_user.name if is_pg() else f"'{kms_app_user.name}'" + with plain_connect(admin=True) as admin: + write(admin, f"REVOKE {grant} FROM {user}") + try: + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + for _ in range(2): + with pytest.raises(MetadataError): + _read(conn, "SELECT 1") + conn.rollback() + with plain_connect(admin=True) as admin: + write(admin, f"GRANT {grant} TO {user}") + assert _read(conn, "SELECT 1") == [(1,)], "once SELECT on the metadata is granted, the connection works" + finally: + with plain_connect(admin=True) as admin: + write(admin, f"GRANT {grant} TO {user}") + + +@kms_builds() +def test_initialize_encryption_for_column_then_round_trip( + build: str, new_table: Callable[..., str], kms_connect: Callable[..., Any], kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, kms_verifier: PayloadVerifier) -> None: + table = new_table() + client = kms_client(kms_settings) + calls: List[str] = [] + client.meta.events.register("before-call.kms.GenerateDataKey", lambda **_: calls.append("GenerateDataKey")) + with plain_connect(build=build) as conn: + utility = KeyManagementUtility(conn, client, encryption_config(kms_settings, kms_schema)) + key_id = utility.initialize_encryption_for_column(table, "ssn", kms_master_key_arn) + conn.commit() + assert calls == ["GenerateDataKey"], "setting up a column calls GenerateDataKey once" + + (encrypted_data_key,) = kms_verifier.stored( + f"SELECT encrypted_data_key FROM {kms_schema}.key_storage WHERE id = %s", (key_id,)) + (hmac_key,) = kms_verifier.stored(f"SELECT hmac_key FROM {kms_schema}.key_storage WHERE id = %s", (key_id,)) + plaintext = client.decrypt(CiphertextBlob=base64.b64decode(encrypted_data_key), + KeyId=kms_master_key_arn)["Plaintext"] + stored_text = encrypted_data_key.encode() + bytes(hmac_key) + leaked = any(form in stored_text for form in (plaintext, base64.b64encode(plaintext), plaintext.hex().encode())) + plaintext = b"" + assert not leaked, "key_storage holds only the KMS-encrypted data key, never its plaintext" + + with kms_connect() as conn: + write(conn, _insert(table), ("initialized", "pt-init")) + assert _read(conn, f"SELECT ssn FROM {table}") == [("pt-init",)], "the column round-trips through the plugin" + (payload,) = kms_verifier.stored(f"SELECT ssn FROM {table}") + assert kms_verifier.decode(table, "ssn", payload) == (key_id, "pt-init"), \ + "the stored payload names the key row the utility made" + + +@kms_builds() +def test_rotation_keeps_old_values_readable_and_new_writes_use_the_new_key( + build: str, new_table: Callable[..., str], kms_connect: Callable[..., Any], kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, kms_verifier: PayloadVerifier) -> None: + table = new_table(triggered=True) + with plain_connect() as admin: + old_key_id = encrypt_column(admin, kms_settings, kms_schema, table, "ssn", kms_master_key_arn) + with kms_connect(encryption_metadata_cache_refresh_interval_sec=1) as open_conn: + write(open_conn, _insert(table), ("old", "pt-rotate-old")) + with plain_connect() as admin: + new_key_id = key_management_utility(admin, kms_settings, kms_schema).rotate_data_key( + table, "ssn", kms_master_key_arn) + admin.commit() + with kms_connect() as fresh: + assert _read(fresh, f"SELECT ssn FROM {table} WHERE name = 'old'") == [("pt-rotate-old",)], \ + "a value written under the old key row still decrypts after the rotation" + write(fresh, _insert(table), ("new", "pt-rotate-new")) + assert retry_until(lambda: writes(open_conn, _insert(table), ("open", "pt-rotate-open")), + timeout_sec=15, delay_sec=0.25), \ + "an open plugin connection writes with the new key after its metadata refresh" + decoded = [kms_verifier.decode(table, "ssn", payload) + for payload in kms_verifier.stored(f"SELECT ssn FROM {table} ORDER BY id")] + assert decoded == [(old_key_id, "pt-rotate-old"), (new_key_id, "pt-rotate-new"), (new_key_id, "pt-rotate-open")], \ + "old values keep their key row and new writes name the new one" + + +@kms_builds("mysql-pure") +def test_create_master_key_makes_a_symmetric_key_with_the_wrapper_alias( + build: str, new_table: Callable[..., str], kms_settings: KmsSettings, kms_schema: str, + kms_app_user: DbUser) -> None: + client = kms_client(kms_settings) + with plain_connect(build=build) as conn: + arn = key_management_utility(conn, kms_settings, kms_schema).create_master_key("kms container test key") + metadata = client.describe_key(KeyId=arn)["KeyMetadata"] + assert (metadata["KeySpec"], metadata["KeyUsage"]) == ("SYMMETRIC_DEFAULT", "ENCRYPT_DECRYPT"), \ + "create_master_key makes a symmetric encrypt-and-decrypt key" + aliases = [alias["AliasName"] for page in client.get_paginator("list_aliases").paginate() + for alias in page["Aliases"] if alias.get("TargetKeyId") == metadata["KeyId"]] + assert len(aliases) == 1 and aliases[0].startswith(KeyManagementUtility.ALIAS_PREFIX), \ + "the new key gets one alias under the wrapper's prefix" + + table = new_table() + with plain_connect() as conn: + encrypt_column(conn, kms_settings, kms_schema, table, "ssn", arn) + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, arn) as conn: + write(conn, _insert(table), ("new-key", "pt-master")) + assert _read(conn, f"SELECT ssn FROM {table}") == [("pt-master",)], "a column set up on the key round-trips" + + +@kms_builds() +def test_key_management_utility_on_a_caller_connection_rollback( + build: str, new_table: Callable[..., str], kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str, kms_verifier: PayloadVerifier) -> None: + table = new_table() + key_rows = f"SELECT count(*) FROM {kms_schema}.key_storage" + (key_rows_before,) = kms_verifier.stored(key_rows) + with plain_connect(build=build) as conn: + key_management_utility(conn, kms_settings, kms_schema).initialize_encryption_for_column( + table, "ssn", kms_master_key_arn) + conn.rollback() + assert (_metadata_key_ids(kms_verifier, kms_schema, table), kms_verifier.stored(key_rows)) == \ + ([], [key_rows_before]), "the utility runs in the caller's transaction, so a rollback keeps nothing" + + +@kms_builds() +def test_key_management_utility_on_a_caller_connection_with_block( + build: str, new_table: Callable[..., str], kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str, kms_verifier: PayloadVerifier) -> None: + table = new_table() + with plain_connect(build=build) as conn: + key_management_utility(conn, kms_settings, kms_schema).initialize_encryption_for_column( + table, "ssn", kms_master_key_arn) + assert len(_metadata_key_ids(kms_verifier, kms_schema, table)) == (1 if is_pg() else 0), \ + "psycopg's with-block commits the utility's work; mysql-connector's closes without committing" + + +@kms_builds() +def test_key_management_utility_on_a_caller_connection_remove( + build: str, new_table: Callable[..., str], kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str, kms_verifier: PayloadVerifier) -> None: + table = new_table() + with plain_connect(build=build) as conn: + key_id = encrypt_column(conn, kms_settings, kms_schema, table, "ssn", kms_master_key_arn) + if is_pg(): + triggers = kms_verifier.stored("SELECT count(*) FROM pg_trigger WHERE tgrelid = %s::regclass", (table,)) + else: + triggers = kms_verifier.stored("SELECT count(*) FROM information_schema.TRIGGERS " + "WHERE EVENT_OBJECT_TABLE = %s", (table,)) + assert triggers == [0], "the utility creates no trigger" + + with plain_connect(build=build) as conn: + write(conn, f"INSERT INTO {kms_schema}.encryption_metadata (table_name, column_name, encryption_algorithm, " + "key_id) VALUES (%s, 'card', %s, %s)", (table, EncryptionAlgorithm.DEFAULT, key_id)) + utility = key_management_utility(conn, kms_settings, kms_schema) + assert utility.columns_using_key(key_id) == [f"{table}.card", f"{table}.ssn"], \ + "columns_using_key lists every column on the key row" + assert utility.remove_encryption_for_column(table, "card") is True, \ + "remove_encryption_for_column reports that it removed the column" + conn.commit() + assert kms_verifier.stored(f"SELECT column_name FROM {kms_schema}.encryption_metadata WHERE table_name = %s", + (table,)) == ["ssn"], \ + "remove_encryption_for_column deletes only that column's metadata row" + assert kms_verifier.stored(f"SELECT count(*) FROM {kms_schema}.key_storage WHERE id = %s", (key_id,)) == [1], \ + "the key row stays, so values already written still decrypt" + + +@kms_builds() +def test_key_management_utility_on_a_caller_connection_sqlalchemy_raw_connection( + build: str, new_table: Callable[..., str], kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str, kms_verifier: PayloadVerifier) -> None: + table = new_table() + url = "postgresql+psycopg://" if is_pg() else "mysql+mysqlconnector://" + engine = sqlalchemy.create_engine(url, creator=lambda: plain_connect(build=build)) + try: + raw = engine.raw_connection() + try: + key_management_utility(raw.driver_connection, kms_settings, kms_schema).initialize_encryption_for_column( + table, "ssn", kms_master_key_arn) + raw.commit() + finally: + raw.close() + finally: + engine.dispose() + assert len(_metadata_key_ids(kms_verifier, kms_schema, table)) == 1, \ + "the utility works on SQLAlchemy's raw_connection().driver_connection" + + +@kms_builds() +def test_metadata_in_another_schema(build: str, new_table: Callable[..., str], kms_connect: Callable[..., Any], + kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_verifier: PayloadVerifier) -> None: + schemas = ("SELECT nspname FROM pg_namespace WHERE nspname = %s" if is_pg() + else "SELECT SCHEMA_NAME FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = %s") + assert kms_schema != "encrypt" and kms_verifier.stored(schemas, ("encrypt",)) == [], \ + "the module's metadata is in its own schema and there is no encrypt schema to fall back on" + if not is_pg(): + (body,) = kms_verifier.stored("SELECT ROUTINE_DEFINITION FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA " + "= DATABASE() AND ROUTINE_NAME = 'validate_encrypted_data_hmac_before_insert'") + assert f"{kms_schema}.encryption_metadata" in body, "the MySQL trigger SQL reads the configured schema" + table = new_table(triggered=True) + with plain_connect() as admin: + encrypt_column(admin, kms_settings, kms_schema, table, "ssn", kms_master_key_arn) + rotated_key_id = key_management_utility(admin, kms_settings, kms_schema).rotate_data_key( + table, "ssn", kms_master_key_arn) + admin.commit() + with kms_connect() as conn: + write(conn, _insert(table), ("other-schema", "pt-schema")) + assert _read(conn, f"SELECT ssn FROM {table}") == [("pt-schema",)], \ + "the plugin reads the metadata from encryption_metadata_schema" + (payload,) = kms_verifier.stored(f"SELECT ssn FROM {table}") + assert kms_verifier.decode(table, "ssn", payload)[0] == rotated_key_id, \ + "the utility's rotation in that schema is what the plugin and the trigger use" + with plain_connect(build=build) as conn: + assert not writes(conn, _insert(table), ("plain", b"pt-schema-plain-" + b"x" * 64)), \ + "the trigger reads its key from the configured schema and rejects plaintext" + + +@kms_builds() +def test_text_or_short_column_refuses_or_fails_closed( + build: str, new_table: Callable[..., str], kms_connect: Callable[..., Any], kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, kms_verifier: PayloadVerifier) -> None: + if is_pg(): + table = new_table("TEXT") + with plain_connect() as admin: + encrypt_column(admin, kms_settings, kms_schema, table, "ssn", kms_master_key_arn) + with kms_connect() as conn: + write(conn, _insert(table), ("text", "pt-text")) + with pytest.raises(IntegrityCheckError, match="is not binary"): + _read(conn, f"SELECT ssn FROM {table}") + (stored,) = kms_verifier.stored(f"SELECT ssn FROM {table}") + assert isinstance(stored, str) and stored.startswith("\\x"), "a TEXT column stores the payload as hex text" + return + # The server's own refusal for each type, so a privilege or connection error doesn't pass for it. + for column_type, refusal in (("TEXT", "Incorrect string value"), ("VARBINARY(16)", "Data too long")): + table = new_table(column_type) + with plain_connect() as admin: + encrypt_column(admin, kms_settings, kms_schema, table, "ssn", kms_master_key_arn) + with kms_connect() as conn: + with pytest.raises(DB_ERRORS, match=refusal): + write(conn, _insert(table), (column_type, "pt-text")) + conn.rollback() + assert kms_verifier.stored(f"SELECT name FROM {table}") == [], \ + f"a MySQL {column_type} column refuses the plugin's payload" + + +@kms_builds() +def test_hand_made_key_row_is_used_by_the_plugin_and_the_trigger( + build: str, new_table: Callable[..., str], kms_connect: Callable[..., Any], kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, kms_verifier: PayloadVerifier) -> None: + table = new_table(triggered=True) + blob = kms_client(kms_settings).generate_data_key(KeyId=kms_master_key_arn, KeySpec="AES_256")["CiphertextBlob"] + wrapped = base64.encodebytes(blob).decode("ascii") + assert "\n" in wrapped.strip(), "the wrapped form spans more than one 76-column line" + key_ids = {} + with plain_connect(build=build) as conn: + for column, encoded in (("ssn", base64.b64encode(blob).decode("ascii")), ("card", wrapped)): + with conn.cursor() as cur: + cur.execute(f"INSERT INTO {kms_schema}.key_storage (key_id, name, master_key_arn, encrypted_data_key, " + "hmac_key, key_spec) VALUES (%s, %s, %s, %s, %s, 'AES_256')" + + (" RETURNING id" if is_pg() else ""), + (str(uuid.uuid4()), f"hand-made-{column}", kms_master_key_arn, encoded, + secrets.token_bytes(32))) + key_ids[column] = cur.fetchone()[0] if is_pg() else cur.lastrowid + cur.execute(f"INSERT INTO {kms_schema}.encryption_metadata (table_name, column_name, " + "encryption_algorithm, key_id) VALUES (%s, %s, %s, %s)", + (table, column, EncryptionAlgorithm.DEFAULT, key_ids[column])) + conn.commit() + with kms_connect() as conn: + write(conn, f"INSERT INTO {table} (name, ssn, card) VALUES (%s, %s, %s)", + ("hand-made", "pt-hand-ssn", "pt-hand-card")) + assert _read(conn, f"SELECT ssn, card FROM {table}") == [("pt-hand-ssn", "pt-hand-card")], \ + "the plugin reads hand-made key rows, base64 on one line and wrapped, through the driver's column types" + decoded = [kms_verifier.decode(table, column, kms_verifier.stored(f"SELECT {column} FROM {table}")[0]) + for column in ("ssn", "card")] + assert decoded == [(key_ids["ssn"], "pt-hand-ssn"), (key_ids["card"], "pt-hand-card")], \ + "the trigger accepted the payloads signed with the hand-made rows' HMAC keys" diff --git a/tests/integration/container/test_kms_encryption_kms.py b/tests/integration/container/test_kms_encryption_kms.py new file mode 100644 index 00000000..8fbf6fe4 --- /dev/null +++ b/tests/integration/container/test_kms_encryption_kms.py @@ -0,0 +1,327 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""What the KMS Encryption Plugin asks of KMS, and how it fails when KMS refuses or doesn't answer. + +The module changes the state of KMS keys it creates, so it runs on LocalStack only. KMS requests are recorded at +botocore's ``BaseClient._make_api_call``, once per attempt. +""" + +from __future__ import annotations + +import logging +import secrets +import socket +import time +from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple + +import pytest +from botocore.client import BaseClient + +from aws_advanced_python_wrapper.aws_credentials_manager import \ + AwsCredentialsManager +from .utils.conditions import enable_on_features +from .utils.kms_encryption_helper import (binary_type, create_table, + drop_table, encrypt_column, + encrypted_table, is_pg, + key_management_utility, kms_builds, + kms_client, plain_connect, raised, + read_column, wrapper_connect, write) +from .utils.test_environment_features import TestEnvironmentFeatures + +if TYPE_CHECKING: + import io + + from .utils.kms_encryption_helper import DbUser, KmsSettings + +pytestmark = [pytest.mark.kms_encryption, + enable_on_features([TestEnvironmentFeatures.RUN_KMS_ENCRYPTION_TESTS_ONLY, + TestEnvironmentFeatures.KMS_LOCALSTACK])] + +_TABLE = "kms_kms_users" +# One column per KMS key whose state a test changes; each key is the module's own. +_STATES = "kms_kms_states" +_STATE_COLUMNS = ("repointed", "disabled", "pending", "warm") +_INSERT = f"INSERT INTO {_TABLE} (name, ssn) VALUES (%s, %s)" +_SELECT = f"SELECT ssn FROM {_TABLE} WHERE name = %s ORDER BY id" +_CLOSED_PORT = "http://127.0.0.1:1" +# The guide's bound: each attempt waits up to 5 s to connect and 10 s for KMS to answer. +_ATTEMPT_BOUND_SEC = 15.0 + + +@pytest.fixture(scope="module") +def key_row(kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, kms_app_user: DbUser) -> Iterator[int]: + """The module's users table with ``ssn`` encrypted under the module's KMS key; yields the key row's id.""" + with encrypted_table(_TABLE, f"name VARCHAR(100), ssn {binary_type()}", ("ssn",), kms_settings, kms_schema, + kms_master_key_arn) as key_ids: + yield key_ids["ssn"] + + +@pytest.fixture(scope="module") +def state_keys(request: pytest.FixtureRequest, kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser) -> Iterator[Dict[str, str]]: + """A KMS key of the module's own for each column of the states table, each column encrypted under its key + and holding a value in the row named ``seed``. Then ``repointed``'s key row names the module's main KMS key + instead, ``disabled``'s key is disabled and ``pending``'s is scheduled for deletion. Yields each column's key.""" + client = kms_client(kms_settings) + keys = {column: client.create_key(Description=f"kms container test {request.module.__name__} {column}") + ["KeyMetadata"]["Arn"] for column in _STATE_COLUMNS} + create_table(_STATES, "name VARCHAR(100), " + ", ".join(f"{column} {binary_type()}" for column in _STATE_COLUMNS)) + with plain_connect() as conn: + key_rows = {column: encrypt_column(conn, kms_settings, kms_schema, _STATES, column, arn) + for column, arn in keys.items()} + allowed = ",".join([kms_master_key_arn, *keys.values()]) + with wrapper_connect("default" if is_pg() else "mysql-pure", kms_app_user, kms_settings, kms_schema, + allowed) as conn: + with conn.cursor() as cur: + cur.execute(f"INSERT INTO {_STATES} (name, {', '.join(_STATE_COLUMNS)}) VALUES (%s, %s, %s, %s, %s)", + ("seed", *(f"pt-seed-{column}" for column in _STATE_COLUMNS))) + conn.commit() + with plain_connect() as conn, conn.cursor() as cur: + cur.execute(f"UPDATE {kms_schema}.key_storage SET master_key_arn = %s WHERE id = %s", + (kms_master_key_arn, key_rows["repointed"])) + conn.commit() + client.disable_key(KeyId=keys["disabled"]) + client.schedule_key_deletion(KeyId=keys["pending"], PendingWindowInDays=7) + try: + yield keys + finally: + drop_table(_STATES) + + +@pytest.fixture +def kms_calls(monkeypatch: pytest.MonkeyPatch) -> List[Tuple[str, Optional[str]]]: + """Each KMS request this process sends while the test runs, one per attempt: the operation and its KeyId.""" + calls: List[Tuple[str, Optional[str]]] = [] + make_api_call = BaseClient._make_api_call # type: ignore[attr-defined] + + def recorded(client: Any, operation_name: str, api_params: Dict[str, Any]) -> Any: + if client.meta.service_model.service_name == "kms": + calls.append((operation_name, api_params.get("KeyId"))) + return make_api_call(client, operation_name, api_params) + + monkeypatch.setattr(BaseClient, "_make_api_call", recorded) + return calls + + +def _stored_names(name: str) -> List[Any]: + with plain_connect() as conn, conn.cursor() as cur: + cur.execute(f"SELECT name FROM {_TABLE} WHERE name = %s", (name,)) + rows = cur.fetchall() + cur.execute(f"SELECT name FROM {_STATES} WHERE name = %s", (name,)) + return rows + cur.fetchall() + + +@kms_builds() +def test_unlisted_master_key_is_refused_before_any_kms_call(build: str, key_row: int, state_keys: Dict[str, str], + kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str, kms_app_user: DbUser, + kms_calls: List[Tuple[str, Optional[str]]]) -> None: + name, refused = f"it35-{build}", f"it35-refused-{build}" + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + write(conn, _INSERT, (name, "pt-it35")) + kms_calls.clear() + outcomes = {} + for endpoint in (kms_settings.endpoint_url, _CLOSED_PORT): + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, state_keys["warm"], + encryption_kms_endpoint=endpoint) as conn: + outcomes[(endpoint, "write")] = type(raised(conn, lambda: write(conn, _INSERT, (refused, "pt-it35-x")))) + outcomes[(endpoint, "read")] = type(raised(conn, lambda: read_column(conn, _SELECT, name))) + assert {key: cls.__name__ for key, cls in outcomes.items()} == \ + {key: "UnauthorizedMasterKeyError" for key in outcomes}, \ + "a key row whose master key isn't allowed is refused on write and read, also with KMS unreachable" + assert (kms_calls, _stored_names(refused)) == ([], []), "the refusal comes before any KMS request and writes nothing" + + +@kms_builds("mysql-pure") +def test_debug_log_holds_no_data_key(build: str, key_row: int, kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str, kms_app_user: DbUser, + kms_log_capture: io.StringIO) -> None: + name = f"it36-{build}" + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + write(conn, _INSERT, (name, "pt-it36")) + start = len(kms_log_capture.getvalue()) + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: # a cold cache + value = read_column(conn, _SELECT, name) + logged = kms_log_capture.getvalue()[start:] + # Booleans only, so a failing assertion can't print a logged data key. + decrypt_logged, plaintext_logged = "TrentService.Decrypt" in logged, '"Plaintext"' in logged + parsers = logging.getLogger("botocore.parsers") + removed = [f for f in parsers.filters if type(f).__name__ == "_NoKmsPlaintext"] + for f in removed: + parsers.removeFilter(f) + try: + start = len(kms_log_capture.getvalue()) + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + read_column(conn, _SELECT, name) + control_logged = '"Plaintext"' in kms_log_capture.getvalue()[start:] + finally: + for f in removed: + parsers.addFilter(f) + assert (value, decrypt_logged, plaintext_logged) == (["pt-it36"], True, False), \ + "a cold read at DEBUG logs the Decrypt request but no response body that holds the data key" + assert (len(removed), control_logged) == (1, True), \ + "without the plugin's filter the same read logs the data key, so the check above can find one" + + +@kms_builds() +def test_kms_refusals_fail_closed(build: str, key_row: int, state_keys: Dict[str, str], kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, kms_app_user: DbUser) -> None: + name, seed = f"it37-{build}", f"it37-seed-{build}" + arns = [kms_master_key_arn, *state_keys.values()] + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + write(conn, _INSERT, (seed, "pt-it37-seed")) + errors: Dict[Tuple[str, str], Optional[Exception]] = {} + # Each case runs on new connections, so no data key is cached: a key row repointed to another allowed KMS key, + # a disabled KMS key, one pending deletion, an unlisted KMS key and an unreachable endpoint. + cases: List[Tuple[str, str, str, str, str, Dict[str, Any]]] = [ + (column, f"INSERT INTO {_STATES} (name, {column}) VALUES (%s, %s)", + f"SELECT {column} FROM {_STATES} WHERE name = %s", "seed", ",".join(arns), {}) + for column in ("repointed", "disabled", "pending")] + cases += [("unlisted", _INSERT, _SELECT, seed, state_keys["warm"], {}), + ("unreachable", _INSERT, _SELECT, seed, kms_master_key_arn, + {"encryption_kms_endpoint": _CLOSED_PORT, "encryption_key_management_max_retries": 0})] + for case, insert, select, row, allowed, props in cases: + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, allowed, **props) as conn: + errors[(case, "write")] = raised(conn, lambda: write(conn, insert, (name, f"pt-it37-{case}"))) + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, allowed, **props) as conn: + errors[(case, "read")] = raised(conn, lambda: read_column(conn, select, row)) + assert {key: type(err).__name__ for key, err in errors.items()} == \ + {key: "UnauthorizedMasterKeyError" if key[0] == "unlisted" else "KeyManagementError" for key in errors}, \ + "each KMS refusal raises KeyManagementError on write and read, and an unlisted master key is refused" + shown = [key for key, err in errors.items() if any(arn in f"{err} {err!r}" for arn in arns)] + assert (shown, _stored_names(name)) == ([], []), "no error shows a master key ARN unmasked, and nothing is written" + + +@kms_builds("mysql-pure") +def test_open_connection_keeps_its_data_key_after_key_disabled(build: str, state_keys: Dict[str, str], + kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str, kms_app_user: DbUser) -> None: + name = f"it38-{build}" + allowed = ",".join([kms_master_key_arn, *state_keys.values()]) + insert = f"INSERT INTO {_STATES} (name, warm) VALUES (%s, %s)" + select = f"SELECT warm FROM {_STATES} WHERE name = %s ORDER BY id" + client = kms_client(kms_settings) + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, allowed) as warm: + before = read_column(warm, select, "seed") # the connection now holds the column's data key + client.disable_key(KeyId=state_keys["warm"]) + try: + write(warm, insert, (name, "pt-it38")) + after = read_column(warm, select, name) + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, allowed) as cold: + refused = type(raised(cold, lambda: read_column(cold, select, "seed"))).__name__ + finally: + client.enable_key(KeyId=state_keys["warm"]) + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, allowed) as again: + recovered = read_column(again, select, "seed") + assert (before, after) == (["pt-seed-warm"], ["pt-it38"]), \ + "a connection that holds the data key keeps reading and writing after the master key is disabled" + assert (refused, recovered) == ("KeyManagementError", ["pt-seed-warm"]), \ + "a new connection fails while the key is disabled, and the failure isn't kept: after EnableKey one reads" + + +@kms_builds("mysql-pure") +def test_plugin_connections_call_only_kms_decrypt(build: str, key_row: int, kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, kms_app_user: DbUser, + kms_calls: List[Tuple[str, Optional[str]]]) -> None: + name = f"it39-{build}" + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + write(conn, _INSERT, (name, "pt-it39-1")) + read_column(conn, _SELECT, name) + sent = len(kms_calls) + with plain_connect() as plain: + key_management_utility(plain, kms_settings, kms_schema).rotate_data_key(_TABLE, "ssn", kms_master_key_arn) + plain.commit() + del kms_calls[sent:] # the utility's own requests + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + write(conn, _INSERT, (name, "pt-it39-2")) + back = read_column(conn, _SELECT, name) + assert back == ["pt-it39-1", "pt-it39-2"], "values under the old and the rotated key row read back" + assert len(kms_calls) >= 2 and set(kms_calls) == {("Decrypt", kms_master_key_arn)}, \ + f"every KMS request of a plugin connection is a Decrypt naming the row's master key: {sorted(set(kms_calls))}" + + +@kms_builds() +def test_unreachable_kms_fails_with_key_management_error_within_the_bound( + build: str, key_row: int, kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser, kms_calls: List[Tuple[str, Optional[str]]]) -> None: + seed = f"it40-{build}" + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + write(conn, _INSERT, (seed, "pt-it40")) + # A loopback socket that listens but never accepts: the kernel completes the handshake, and nothing answers. + silent = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + silent.bind(("127.0.0.1", 0)) + silent.listen(8) + results = {} + try: + endpoints = {"closed port": (_CLOSED_PORT, 1), "silent listener": (f"http://127.0.0.1:{silent.getsockname()[1]}", 0)} + for label, (endpoint, retries) in endpoints.items(): + for call in ("write", "read"): + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn, + encryption_kms_endpoint=endpoint, + encryption_key_management_max_retries=retries) as conn: + kms_calls.clear() + started = time.monotonic() + err = raised(conn, (lambda: write(conn, _INSERT, (f"{seed}-x", "pt-it40-x"))) if call == "write" + else (lambda: read_column(conn, _SELECT, seed))) + within = time.monotonic() - started <= (1 + retries) * _ATTEMPT_BOUND_SEC + results[(label, call)] = (type(err).__name__, type(getattr(err, "__cause__", None)).__name__, + len(kms_calls), within) + finally: + silent.close() + expected = {("closed port", call): ("KeyManagementError", "EndpointConnectionError", 2, True) + for call in ("write", "read")} + expected.update({("silent listener", call): ("KeyManagementError", "ReadTimeoutError", 1, True) + for call in ("write", "read")}) + assert results == expected, \ + "an unreachable KMS raises KeyManagementError from the connection or read timeout, after 1 + max_retries " \ + "attempts and within the documented bound" + + +@kms_builds("mysql-pure") +def test_debug_log_holds_no_aws_credentials(build: str, monkeypatch: pytest.MonkeyPatch, key_row: int, + kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser, kms_log_capture: io.StringIO) -> None: + name = f"it41-{build}" + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + write(conn, _INSERT, (name, "pt-it41")) + key_id, token = "AKIAKMSIT" + secrets.token_hex(6).upper(), "kmsit-token-" + secrets.token_hex(8) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", key_id) + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "kmsit-secret-" + secrets.token_hex(8)) + monkeypatch.setenv("AWS_SESSION_TOKEN", token) + AwsCredentialsManager.release_resources() # so the plugin's next session reads these marker credentials + try: + start = len(kms_log_capture.getvalue()) + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + value = read_column(conn, _SELECT, name) + logged = kms_log_capture.getvalue()[start:] + leaked = [marker for marker in (key_id, token, "Signature:") if marker in logged] + loggers = [logging.getLogger(logger_name) for logger_name in ("botocore.endpoint", "botocore.auth")] + removed = [(lg, f) for lg in loggers for f in lg.filters if type(f).__name__ == "_NoKmsRequestCredentials"] + for lg, f in removed: + lg.removeFilter(f) + try: + start = len(kms_log_capture.getvalue()) + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + read_column(conn, _SELECT, name) + shown = [marker for marker in (key_id, token) if marker in kms_log_capture.getvalue()[start:]] + finally: + for lg, f in removed: + lg.addFilter(f) + finally: + AwsCredentialsManager.release_resources() # the next session reads the module's credentials again + assert (value, leaked) == (["pt-it41"], []), \ + "a cold read at DEBUG logs neither the access key id, the session token nor a signature record" + assert (len(removed), shown) == (2, [key_id, token]), \ + "without the plugin's filter the same read logs both markers, so the check above can find them" diff --git a/tests/integration/container/test_kms_encryption_lifecycle.py b/tests/integration/container/test_kms_encryption_lifecycle.py new file mode 100644 index 00000000..6c335851 --- /dev/null +++ b/tests/integration/container/test_kms_encryption_lifecycle.py @@ -0,0 +1,438 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The KMS Encryption Plugin over a connection's life: pipelines, several cursors and threads, session character +sets, connection limits, the plugin's own connections and the wrapper's release_resources().""" + +from __future__ import annotations + +import secrets +import threading +from typing import TYPE_CHECKING, Any, Dict, Iterator, List + +import mysql.connector +import psycopg +import pytest + +import aws_advanced_python_wrapper +from aws_advanced_python_wrapper.encryption.connection_source import \ + PluginConnectionSource +from aws_advanced_python_wrapper.errors import (AwsWrapperError, + EncryptionPluginError) +from .utils.conditions import (disable_on_engines, enable_on_engines, + enable_on_features) +from .utils.database_engine import DatabaseEngine +from .utils.kms_encryption_helper import (DbUser, binary_type, + create_column_trigger, + encrypt_column, encrypted_table, + install_trigger_sql, is_pg, + key_management_utility, kms_builds, + plain_connect, raised, read_column, + statement_log_lines, wrapper_connect, + write) +from .utils.test_environment import TestEnvironment +from .utils.test_environment_features import TestEnvironmentFeatures + +if TYPE_CHECKING: + from aws_advanced_python_wrapper import AwsWrapperConnection + from .utils.kms_encryption_helper import KmsSettings, PayloadVerifier + +pytestmark = [pytest.mark.kms_encryption, + enable_on_features([TestEnvironmentFeatures.RUN_KMS_ENCRYPTION_TESTS_ONLY, + TestEnvironmentFeatures.KMS_LOCALSTACK])] + +_TABLE = "kms_lifecycle_users" +_INSERT = f"INSERT INTO {_TABLE} (name, ssn) VALUES (%s, %s)" +_SELECT = f"SELECT ssn FROM {_TABLE} WHERE name = %s ORDER BY id" +# A table whose encrypted column has the package's trigger, which refuses plaintext. +_GUARDED = "kms_lifecycle_guarded" +# A MySQL table named with \u00c9, which the swe7 character set writes as @. +_SWE7_TABLE, _SWE7_SPELLING = "kms_lifecycle_\u00c9", "kms_lifecycle_@" + + +@pytest.fixture(scope="module") +def key_rows(kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser) -> Iterator[Dict[str, int]]: + """The module's table, with ``ssn`` and ``dob`` encrypted under key rows of their own; yields their ids.""" + with encrypted_table(_TABLE, f"name VARCHAR(100), ssn {binary_type()}, dob {binary_type()}", ("ssn", "dob"), + kms_settings, kms_schema, kms_master_key_arn) as key_ids: + yield key_ids + + +@pytest.fixture(scope="module") +def guarded(kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, kms_app_user: DbUser) -> Iterator[None]: + with encrypted_table(_GUARDED, f"name VARCHAR(100), ssn {binary_type()}", ("ssn",), kms_settings, kms_schema, + kms_master_key_arn, triggered=("ssn",)): + yield + + +@pytest.fixture(scope="module") +def swe7_table(kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser) -> Iterator[str]: + """A MySQL table named with \u00c9 whose encrypted column has the package's trigger.""" + with plain_connect() as conn: + with conn.cursor() as cur: + cur.execute(f"DROP TABLE IF EXISTS `{_SWE7_TABLE}`") + cur.execute(f"CREATE TABLE `{_SWE7_TABLE}` (id INT AUTO_INCREMENT PRIMARY KEY, name VARCHAR(100), " + f"ssn VARBINARY(1024))") + conn.commit() + encrypt_column(conn, kms_settings, kms_schema, _SWE7_TABLE, "ssn", kms_master_key_arn) + install_trigger_sql(conn, kms_schema) + create_column_trigger(conn, _SWE7_TABLE, "ssn") + try: + yield _SWE7_TABLE + finally: + with plain_connect() as conn, conn.cursor() as cur: + cur.execute(f"DROP TABLE IF EXISTS `{_SWE7_TABLE}`") + conn.commit() + + +@pytest.fixture +def conn(build: str, key_rows: Dict[str, int], kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser) -> Iterator[AwsWrapperConnection]: + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + yield conn + + +def _opened(verifier: PayloadVerifier, name: str) -> List[Any]: + """The values of the rows named ``name``, opened without the plugin.""" + return [opened[1] for opened in verifier.opened(_TABLE, "ssn", name) if opened is not None] + + +@disable_on_engines([DatabaseEngine.MYSQL]) +@kms_builds() +def test_psycopg_pipeline_reads_decrypt(build: str, key_rows: Dict[str, int], kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, kms_app_user: DbUser, + kms_verifier: PayloadVerifier, kms_statement_log: str) -> None: + reads: Dict[Any, List[Any]] = {} + stored: Dict[Any, bool] = {} + pids: List[int] = [] + for metadata in ("loaded", "loaded in the pipeline"): + for read in ("fetchone", "fetchmany", "fetchall", "for"): + name, values = f"it42-{len(reads)}", [f"pt-it42-{len(reads)}-{i}" for i in range(3)] + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + pids.append(conn.target_connection.info.backend_pid) + if metadata == "loaded": + read_column(conn, _SELECT, "it42-nothing") # the connection's first statement loads the metadata + with conn.cursor() as cur: + with conn.pipeline(): + for value in values: + cur.execute(_INSERT, (name, value)) + cur.execute(_SELECT, (name,)) + if read == "fetchone": + rows = [cur.fetchone() for _ in values] + elif read == "fetchmany": + rows = cur.fetchmany(2) + cur.fetchmany(5) + elif read == "fetchall": + rows = cur.fetchall() + else: + rows = list(cur) + conn.commit() + reads[(metadata, read)] = [row[0] for row in rows] + stored[(metadata, read)] = _opened(kms_verifier, name) == values + lookups = [line for line in statement_log_lines(kms_statement_log) if f"{kms_schema}.".encode() in line] + in_pipeline = [line for line in lookups if any(f"[{pid}]".encode() in line for pid in pids)] + assert reads == {key: [f"pt-it42-{i}-{j}" for j in range(3)] for i, key in enumerate(reads)}, \ + "in a pipeline every fetch path returns the rows decrypted, with the metadata loaded before or inside it" + assert stored == {key: True for key in stored}, "the pipeline's writes store payloads of the values written" + assert len(lookups) > 0 and in_pipeline == [], \ + "the plugin reads its metadata on its own connection, never in the application's pipeline session" + + +@kms_builds() +def test_interleaved_cursors_and_threads_keep_their_own_values_cursors(build: str, conn: AwsWrapperConnection, + key_rows: Dict[str, int], + kms_verifier: PayloadVerifier) -> None: + name = f"it43-{build}" + cursor_args = {} if is_pg() else {"buffered": True} # two unread results at once on one MySQL connection + with conn.cursor(**cursor_args) as first, conn.cursor(**cursor_args) as second: + first.execute(_INSERT, (name, "pt-it43-ssn-1")) + second.execute(f"INSERT INTO {_TABLE} (dob, name, ssn) VALUES (%s, %s, %s)", + ("pt-it43-dob-2", name, "pt-it43-ssn-2")) + conn.commit() + first.execute(_SELECT, (name,)) + second.execute(f"SELECT name, dob, ssn FROM {_TABLE} WHERE name = %s ORDER BY id", (name,)) + first_rows, second_rows = [tuple(r) for r in first.fetchall()], [tuple(r) for r in second.fetchall()] + assert (first_rows, second_rows) == ([("pt-it43-ssn-1",), ("pt-it43-ssn-2",)], + [(name, None, "pt-it43-ssn-1"), (name, "pt-it43-dob-2", "pt-it43-ssn-2")]), \ + "two cursors of one connection, executed before either fetches, each decode their own columns" + assert _opened(kms_verifier, name) == ["pt-it43-ssn-1", "pt-it43-ssn-2"], "each cursor's own values were stored" + + +@enable_on_engines([DatabaseEngine.PG]) +@kms_builds() +def test_interleaved_cursors_and_threads_keep_their_own_values_psycopg_threads( + build: str, key_rows: Dict[str, int], kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser, kms_verifier: PayloadVerifier) -> None: + # Threads on one psycopg connection, half reading another column list. + wrong: List[str] = [] + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as shared: + shared.autocommit = True + + def work(n: int) -> None: + try: + with shared.cursor() as cur: + for i in range(5): + value = f"pt-it43-thread-{n}-{i}" + cur.execute(_INSERT, (f"it43-thread-{n}", value)) + if n % 2: + cur.execute(f"{_SELECT} DESC LIMIT 1", (f"it43-thread-{n}",)) + got = cur.fetchone()[0] + else: + cur.execute(f"SELECT name, ssn FROM {_TABLE} WHERE name = %s ORDER BY id DESC LIMIT 1", + (f"it43-thread-{n}",)) + got = cur.fetchone()[1] + if got != value: + wrong.append(f"thread {n} statement {i} read a {type(got).__name__}") + except Exception as err: # noqa: BLE001 - reported by class + wrong.append(f"thread {n} raised {type(err).__name__}") + + threads = [threading.Thread(target=work, args=(n,), name=f"kms-it43-{n}") for n in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(60) + assert (wrong, [t.name for t in threads if t.is_alive()]) == ([], []), \ + "threads on one psycopg connection each encrypt their own parameters and decrypt their own rows" + assert all(_opened(kms_verifier, f"it43-thread-{n}") == [f"pt-it43-thread-{n}-{i}" for i in range(5)] + for n in range(8)), "each thread's values were stored as payloads" + + +@kms_builds() +def test_mysql_session_charsets_never_store_plaintext_decoding_off( + build: str, key_rows: Dict[str, int], kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser, kms_verifier: PayloadVerifier) -> None: + name = f"it44-{build}" + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + with conn.cursor() as cur: + cur.execute(_INSERT, (name, "pt-it44-seed")) + conn.commit() + # Connection options that turn text decoding off, which the plugin's own metadata reads share. + options: List[Dict[str, Any]] = [{"client_encoding": "SQL_ASCII"}] if is_pg() else \ + [{"use_unicode": False}, {"raw": True}] + outcomes = {} + for option in options: + label = ",".join(f"{key}={value}" for key, value in option.items()) + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn, **option) as off: + write = raised(off, lambda: off.cursor().execute(_INSERT, (f"{name}-off", "pt-it44-off"))) + off.commit() + read = raised(off, lambda: read_column(off, _SELECT, name)) + outcomes[label] = (type(write).__name__, type(read).__name__) + if is_pg(): # the plugin reads the metadata's algorithm name as bytes (the guide's SQL_ASCII note) + assert outcomes == {"client_encoding=SQL_ASCII": ("InvalidAlgorithmError", "InvalidAlgorithmError")}, \ + "on a SQL_ASCII connection a write and a read raise InvalidAlgorithmError" + else: + assert [label for label, raised in outcomes.items() if "NoneType" in raised] == [], \ + f"with use_unicode=False or raw=True a write and a read raise instead of running: {outcomes}" + assert kms_verifier.stored(_SELECT, (f"{name}-off",)) == [], "nothing is stored with text decoding off" + + +@enable_on_engines([DatabaseEngine.MYSQL]) +@kms_builds() +def test_mysql_session_charsets_never_store_plaintext_swe7(build: str, swe7_table: str, kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser, + kms_verifier: PayloadVerifier) -> None: + name = f"it44-{build}" + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + with conn.cursor() as cur: # in the negotiated utf8mb4 session the plugin encrypts for the table + cur.execute(f"INSERT INTO `{swe7_table}` (name, ssn) VALUES (%s, %s)", (f"{name}-utf8", "pt-it44-utf8")) + conn.commit() + # SQL as non-ASCII bytes runs unanalyzed on MySQL, so its plaintext reaches the trigger. + with conn.cursor() as cur: + as_bytes = raised(conn, lambda: cur.execute( + f"INSERT INTO `{swe7_table}` (name, ssn) VALUES (%s, %s)".encode(), (f"{name}-bytes", "pt-it44-bytes"))) + with conn.cursor() as cur: + cur.execute("SET NAMES swe7") + # The server reads this name as the table's own; the plugin reads it as a table without encrypted columns. + with conn.cursor() as cur: + swe7 = raised(conn, lambda: cur.execute(f"INSERT INTO `{_SWE7_SPELLING}` (name, ssn) VALUES (%s, %s)", + (f"{name}-swe7", "pt-it44-swe7"))) + select = f"SELECT ssn FROM `{swe7_table}` WHERE name = %s" + utf8_stored = [kms_verifier.decode(swe7_table, "ssn", payload)[1] + for payload in kms_verifier.stored(select, (f"{name}-utf8",))] + assert utf8_stored == ["pt-it44-utf8"], "in the negotiated session the write to the table named with \u00c9 is encrypted" + assert (isinstance(as_bytes, mysql.connector.Error), kms_verifier.stored(select, (f"{name}-bytes",))) == (True, []), \ + f"a write in non-ASCII bytes SQL goes unanalyzed and the trigger rejects its plaintext: {as_bytes!r}" + assert (swe7 is not None, isinstance(swe7, EncryptionPluginError), kms_verifier.stored(select, (f"{name}-swe7",))) \ + == (True, False, []), \ + f"after SET NAMES swe7 the plaintext write fails in the driver or at the trigger and stores nothing: {swe7!r}" + + +@kms_builds("mysql-pure") +def test_connection_limit_needs_room_for_the_plugins_own_connection(build: str, key_rows: Dict[str, int], + kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str) -> None: + database = TestEnvironment.get_current().get_database_info().get_default_db_name() + outcomes = {} + for limit in (1, 2): + user = DbUser(f"kms_limit_{secrets.token_hex(4)}", secrets.token_hex(16)) + with plain_connect(admin=True) as admin, admin.cursor() as cur: + if is_pg(): + cur.execute("SET log_statement = 'none'") + cur.execute(f"CREATE ROLE {user.name} LOGIN PASSWORD '{user.password}' CONNECTION LIMIT {limit}") + for statement in (f"GRANT USAGE ON SCHEMA {kms_schema} TO {user.name}", + f"GRANT SELECT ON ALL TABLES IN SCHEMA {kms_schema} TO {user.name}", + f"GRANT SELECT, INSERT ON {_TABLE} TO {user.name}", + f"GRANT USAGE ON ALL SEQUENCES IN SCHEMA public TO {user.name}"): + cur.execute(statement) + else: + cur.execute("SET SESSION sql_log_off = 1") + cur.execute(f"CREATE USER %s IDENTIFIED BY %s WITH MAX_USER_CONNECTIONS {limit}", + (user.name, user.password)) + cur.execute(f"GRANT SELECT, INSERT ON {database}.{_TABLE} TO %s", (user.name,)) + cur.execute(f"GRANT SELECT ON {kms_schema}.* TO %s", (user.name,)) + admin.commit() + try: + with wrapper_connect(build, user, kms_settings, kms_schema, kms_master_key_arn) as limited: + err = raised(limited, lambda: read_column(limited, _SELECT, "it45-nothing")) + outcomes[limit] = None if err is None else type(err).__name__ + finally: + with plain_connect(admin=True) as admin, admin.cursor() as cur: + if is_pg(): + cur.execute(f"DROP OWNED BY {user.name}") + cur.execute(f"DROP ROLE IF EXISTS {user.name}") + else: + cur.execute("DROP USER IF EXISTS %s", (user.name,)) + admin.commit() + assert outcomes == {1: "MetadataError", 2: None}, \ + "with room for one connection the first statement raises MetadataError, and with room for two it runs" + + +@kms_builds("mysql-pure") +def test_trigger_rejection_error_class_with_and_without_failover(build: str, guarded: None, kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser) -> None: + # Without a column list the plugin passes the write through with a warning, so the trigger sees the plaintext. + unlisted = f"INSERT INTO {_GUARDED} VALUES (DEFAULT, %s, %s)" + row = (f"it46-{build}", "pt-it46") + driver_errors = (psycopg.Error, mysql.connector.Error) + chains: Dict[str, Dict[str, Any]] = { + "kms_encryption": {"plugins": "kms_encryption"}, + "failover_v2,kms_encryption": {"plugins": "failover_v2,kms_encryption", "auto_sort_wrapper_plugin_order": False}, + "failover_v2,kms_encryption preserving the driver's class": { + "plugins": "failover_v2,kms_encryption", "auto_sort_wrapper_plugin_order": False, + "preserve_driver_exception_type": True}} + seen = {} + for chain, props in chains.items(): + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn, **props) as conn: + for call in ("execute", "executemany"): + with conn.cursor() as cur: + err = raised(conn, (lambda: cur.execute(unlisted, row)) if call == "execute" + else (lambda: cur.executemany(unlisted, [row]))) + if isinstance(err, driver_errors): + seen[(chain, call)] = "the driver's error" + elif type(err) is AwsWrapperError and isinstance(err.__cause__, driver_errors): + seen[(chain, call)] = "AwsWrapperError from the driver's error" + else: + seen[(chain, call)] = type(err).__name__ + assert seen == { + ("kms_encryption", "execute"): "the driver's error", + ("kms_encryption", "executemany"): "the driver's error", + ("failover_v2,kms_encryption", "execute"): "AwsWrapperError from the driver's error", + ("failover_v2,kms_encryption", "executemany"): "the driver's error", + ("failover_v2,kms_encryption preserving the driver's class", "execute"): "the driver's error", + ("failover_v2,kms_encryption preserving the driver's class", "executemany"): "the driver's error"}, \ + "a trigger rejection reaches the application as the driver's error, except from execute() under failover_v2, " \ + "which wraps it in AwsWrapperError unless preserve_driver_exception_type is on" + + +@kms_builds() +def test_plugin_lookups_use_their_own_autocommit_connection(build: str, monkeypatch: pytest.MonkeyPatch, + key_rows: Dict[str, int], kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser) -> None: + name = f"it47-{build}" + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + with conn.cursor() as cur: + cur.execute(_INSERT, (name, "pt-it47-1")) + conn.commit() + with plain_connect() as plain: + key_management_utility(plain, kms_settings, kms_schema).rotate_data_key(_TABLE, "ssn", kms_master_key_arn) + plain.commit() + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + with conn.cursor() as cur: + cur.execute(_INSERT, (name, "pt-it47-2")) + conn.commit() + opened: List[Any] = [] + open_connection = PluginConnectionSource._open + + def counted_open(source: PluginConnectionSource) -> Any: + opened.append(source) + return open_connection(source) + + monkeypatch.setattr(PluginConnectionSource, "_open", counted_open) + session = "SET search_path TO public" if is_pg() else "SET SESSION sql_mode = CONCAT(@@SESSION.sql_mode, '')" + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn, + encryption_metadata_cache_enabled=False) as conn: + with conn.cursor() as cur: + cur.execute(session) # a session setting as the first statement, inside the application's transaction + before = len(opened) + cur.execute(_SELECT, (name,)) + at_execute = len(opened) - before + # The first row names the key row the rotation replaced, which the plugin looks up while the rest of + # the result is still unread on the application's connection. + first = cur.fetchone()[0] + rest = [row[0] for row in cur.fetchall()] + at_fetch = len(opened) - before - at_execute + names_after = read_column(conn, f"SELECT name FROM {_TABLE} WHERE name = %s", name) + conn.commit() + assert [first, *rest] == ["pt-it47-1", "pt-it47-2"], \ + "a lookup during an unread result runs on the plugin's own connection, and the rows read back" + assert (at_execute, at_fetch, names_after) == (1, 1, [name, name]), \ + "with the metadata cache off, the execute and the fetch each open one plugin connection, and the " \ + "application's transaction goes on after the session setting and the lookups" + + +@kms_builds("mysql-cext") +def test_release_resources_leaves_open_connections_working(build: str, conn: AwsWrapperConnection, + key_rows: Dict[str, int]) -> None: + name = f"it48-{build}" + with conn.cursor() as cur: + cur.execute(_INSERT, (name, "pt-it48-ssn")) # loads the metadata and ssn's data key, not dob's + conn.commit() + aws_advanced_python_wrapper.release_resources() + with conn.cursor() as cur: + cur.execute(_INSERT, (name, "pt-it48-ssn-after")) + cur.execute(f"INSERT INTO {_TABLE} (name, dob) VALUES (%s, %s)", (name, "pt-it48-dob")) + conn.commit() + assert (read_column(conn, _SELECT, name), read_column(conn, f"SELECT dob FROM {_TABLE} WHERE name = %s ORDER BY id", name)) == \ + (["pt-it48-ssn", "pt-it48-ssn-after", None], [None, None, "pt-it48-dob"]), \ + "after release_resources() an open connection still writes and reads a loaded column and a column whose " \ + "key it hadn't loaded" + + +@kms_builds() +def test_socket_timeout_hands_the_driver_the_encrypted_copy(build: str, key_rows: Dict[str, int], + monkeypatch: pytest.MonkeyPatch, + kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str, kms_app_user: DbUser, + kms_verifier: PayloadVerifier) -> None: + name = f"socket-{build}" + on_main: List[bool] = [] + # With socket_timeout the wrapper runs the driver call on a worker thread. + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn, socket_timeout=5) as conn: + with conn.target_connection.cursor() as probe: + driver_cursor = type(probe) + original = driver_cursor.execute + + def recording(self: Any, *args: Any, **kwargs: Any) -> Any: + on_main.append(threading.current_thread() is threading.main_thread()) + return original(self, *args, **kwargs) + monkeypatch.setattr(driver_cursor, "execute", recording) + write(conn, _INSERT, (name, "pt-socket")) + got = read_column(conn, _SELECT, name) + assert False in on_main, "with socket_timeout the driver's execute runs on a worker thread" + assert _opened(kms_verifier, name) == ["pt-socket"], "the worker thread gets the encrypted copy of the parameters" + assert got == ["pt-socket"], "the row read on the worker thread is decrypted" diff --git a/tests/integration/container/test_kms_encryption_real_kms.py b/tests/integration/container/test_kms_encryption_real_kms.py new file mode 100644 index 00000000..f7fc456b --- /dev/null +++ b/tests/integration/container/test_kms_encryption_real_kms.py @@ -0,0 +1,422 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The KMS Encryption Plugin against a real KMS key, which LocalStack can't stand in for. + +LocalStack ignores KeySpec and IAM, and has neither CloudTrail nor STS session expiry. +The key is ``KMS_KEY_ARN``, made once by an administrator; these tests never create, change or delete a key or a role. +Setup runs on the credentials the module starts with (the CI or maintainer role). +The Decrypt-only role (``KMS_DECRYPT_ONLY_ROLE_ARN``) is assumed inside the tests, narrowed where a test needs it by an +inline session policy, and its credentials reach the plugin through the environment for the blocks that need them. +The last three tests are manual: they run only with ``KMS_MANUAL_TESTS=1`` (CloudTrail's delivery delay, a 900-second +session, an IAM condition), never in CI. +""" + +from __future__ import annotations + +import base64 +import json +import os +import secrets +import uuid +from contextlib import contextmanager +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Optional + +import boto3 +import pytest +from botocore.exceptions import ClientError + +from aws_advanced_python_wrapper.aws_credentials_manager import \ + AwsCredentialsManager +from aws_advanced_python_wrapper.encryption.algorithm import \ + EncryptionAlgorithm +from aws_advanced_python_wrapper.encryption.key_management_utility import \ + KeyManagementUtility +from .utils.conditions import (disable_on_features, enable_on_deployments, + enable_on_engines, enable_on_features) +from .utils.database_engine import DatabaseEngine +from .utils.database_engine_deployment import DatabaseEngineDeployment +from .utils.kms_encryption_helper import (binary_type, create_table, + drop_table, encrypt_column, + encryption_config, is_pg, kms_builds, + kms_client, plain_connect, raised, + read_column, wrapper_connect, write) +from .utils.retry_helper import retry_until +from .utils.test_environment_features import TestEnvironmentFeatures + +if TYPE_CHECKING: + from .utils.kms_encryption_helper import (DbUser, KmsSettings, + PayloadVerifier) + +pytestmark = [pytest.mark.kms_encryption, + enable_on_features([TestEnvironmentFeatures.RUN_KMS_ENCRYPTION_TESTS_ONLY]), + disable_on_features([TestEnvironmentFeatures.KMS_LOCALSTACK]), + enable_on_deployments([DatabaseEngineDeployment.DOCKER])] + +manual = pytest.mark.skipif(os.environ.get("KMS_MANUAL_TESTS") != "1", + reason="manual real-KMS test: set KMS_MANUAL_TESTS=1 on a maintainer's machine") + +_TABLE = "kms_real_users" +_REFUSALS_TABLE = "kms_real_refusals" +_INSERT = f"INSERT INTO {_TABLE} (name, ssn) VALUES (%s, %s)" +_SELECT = f"SELECT ssn FROM {_TABLE} WHERE name = %s ORDER BY id" +_SEED = ("seed", "pt-real-seed") +# The request parameters a KMS event may carry: no encryption context, nothing derived from a value. +_ALLOWED_EVENT_PARAMETERS = {"encryptionAlgorithm", "keyId", "keySpec"} + + +@dataclass(frozen=True) +class RoleSession: + """Temporary credentials of an assumed role, kept out of every repr, so no report shows them.""" + access_key_id: str = field(repr=False) + secret_access_key: str = field(repr=False) + session_token: str = field(repr=False) + + +def _assume_decrypt_only(settings: KmsSettings, session_name: str, session_policy: Optional[Dict[str, Any]] = None + ) -> RoleSession: + """A 900-second session of the Decrypt-only role, narrowed by ``session_policy`` when it is given.""" + if settings.decrypt_only_role_arn is None: + pytest.fail("A real-key environment without KMS_DECRYPT_ONLY_ROLE_ARN: set it to run the real-KMS tests.") + extra = {} if session_policy is None else {"Policy": json.dumps(session_policy)} + credentials = boto3.client("sts", region_name=settings.region).assume_role( + RoleArn=settings.decrypt_only_role_arn, RoleSessionName=session_name, DurationSeconds=900, + **extra)["Credentials"] + return RoleSession(credentials["AccessKeyId"], credentials["SecretAccessKey"], credentials["SessionToken"]) + + +@contextmanager +def _plugin_credentials(role: RoleSession) -> Iterator[None]: + """Inside the block, a plugin connection's first statement builds its KMS client from ``role``. + + The wrapper keeps one boto3 session per host and region, so the sessions are released on the way in and out. + """ + AwsCredentialsManager.release_resources() + with pytest.MonkeyPatch.context() as patch: + for name in ("AWS_PROFILE", "AWS_DEFAULT_PROFILE"): + patch.delenv(name, raising=False) + patch.setenv("AWS_ACCESS_KEY_ID", role.access_key_id) + patch.setenv("AWS_SECRET_ACCESS_KEY", role.secret_access_key) + patch.setenv("AWS_SESSION_TOKEN", role.session_token) + try: + yield + finally: + AwsCredentialsManager.release_resources() + + +def _boto3_error(call: Callable[[], Any]) -> Optional[BaseException]: + """The error a boto3 ``call`` raised, or None.""" + try: + call() + except Exception as err: # noqa: BLE001 - the test asserts on the class + return err + return None + + +def _count_rows(table: str, name: str) -> int: + with plain_connect() as conn, conn.cursor() as cur: + cur.execute(f"SELECT COUNT(*) FROM {table} WHERE name = %s", (name,)) + return cur.fetchone()[0] + + +@pytest.fixture(scope="module") +def table(kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, kms_app_user: DbUser) -> Iterator[int]: + """The module's table, ``ssn`` encrypted under the real key by the setup credentials, with one seed row written + through the plugin on them; yields the ``key_storage`` id.""" + create_table(_TABLE, f"name VARCHAR(100), ssn {binary_type()}") + with plain_connect() as conn: + key_storage_id = encrypt_column(conn, kms_settings, kms_schema, _TABLE, "ssn", kms_master_key_arn) + build = "default" if is_pg() else "mysql-cext" + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + write(conn, _INSERT, _SEED) + try: + yield key_storage_id + finally: + drop_table(_TABLE) + + +@pytest.fixture +def sized_table(build: str) -> Iterator[str]: + """A table of the test's own with two unencrypted columns, ``ssn128`` and ``ssn256``.""" + name = f"kms_real_sized_{build.replace('-', '_')}" + create_table(name, f"name VARCHAR(100), ssn128 {binary_type()}, ssn256 {binary_type()}") + try: + yield name + finally: + drop_table(name) + + +@pytest.fixture(scope="module") +def missing_key_arn(kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, kms_app_user: DbUser + ) -> Iterator[str]: + """The refusals table: ``ssn`` and ``ssn_gone`` each under a key row of their own, a seed row in both written + through the plugin, and then ``ssn_gone``'s key row repointed (in the database only) to a well-formed ARN in the + key's account and region that names no key; yields that ARN.""" + create_table(_REFUSALS_TABLE, f"name VARCHAR(100), ssn {binary_type()}, ssn_gone {binary_type()}") + with plain_connect() as conn: + key_ids = {column: encrypt_column(conn, kms_settings, kms_schema, _REFUSALS_TABLE, column, kms_master_key_arn) + for column in ("ssn", "ssn_gone")} + build = "default" if is_pg() else "mysql-cext" + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + write(conn, f"INSERT INTO {_REFUSALS_TABLE} (name, ssn, ssn_gone) VALUES (%s, %s, %s)", + (_SEED[0], _SEED[1], _SEED[1])) + arn = kms_master_key_arn.rsplit("/", 1)[0] + "/" + str(uuid.uuid4()) + with plain_connect() as conn, conn.cursor() as cur: + cur.execute(f"UPDATE {kms_schema}.key_storage SET master_key_arn = %s WHERE id = %s", (arn, key_ids["ssn_gone"])) + conn.commit() + try: + yield arn + finally: + drop_table(_REFUSALS_TABLE) + + +@kms_builds() +def test_data_key_is_sized_for_the_column_algorithm(build: str, sized_table: str, kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, kms_app_user: DbUser, + kms_verifier: PayloadVerifier) -> None: + columns = {"ssn128": EncryptionAlgorithm.AES_128_GCM, "ssn256": EncryptionAlgorithm.AES_256_GCM} + with plain_connect() as conn: + utility = KeyManagementUtility(conn, kms_client(kms_settings), encryption_config(kms_settings, kms_schema)) + key_ids = {column: utility.initialize_encryption_for_column(sized_table, column, kms_master_key_arn, algorithm) + for column, algorithm in columns.items()} + conn.commit() + with conn.cursor() as cur: + rows = {} + for column, key_storage_id in key_ids.items(): + cur.execute(f"SELECT key_spec, encrypted_data_key FROM {kms_schema}.key_storage WHERE id = %s", + (key_storage_id,)) + rows[column] = cur.fetchone() + kms = kms_client(kms_settings) + sizes = {column: (key_spec, len(kms.decrypt(CiphertextBlob=base64.b64decode(encrypted_data_key), + KeyId=kms_master_key_arn)["Plaintext"])) + for column, (key_spec, encrypted_data_key) in rows.items()} + assert sizes == {"ssn128": ("AES_128", 16), "ssn256": ("AES_256", 32)}, \ + "on real KMS an AES-128-GCM column gets a 16-byte AES_128 data key and an AES-256-GCM one a 32-byte AES_256 key" + + name, value = f"it58-{build}", f"pt-it58-{build}" + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + write(conn, f"INSERT INTO {sized_table} (name, ssn128, ssn256) VALUES (%s, %s, %s)", (name, value, value)) + with conn.cursor() as cur: + cur.execute(f"SELECT ssn128, ssn256 FROM {sized_table} WHERE name = %s", (name,)) + read = list(cur.fetchall()) + stored = {column: [kms_verifier.decode(sized_table, column, payload) + for payload in kms_verifier.stored(f"SELECT {column} FROM {sized_table} WHERE name = %s", + (name,))] + for column in columns} + assert (read, stored) == ([(value, value)], {"ssn128": [(key_ids["ssn128"], value)], + "ssn256": [(key_ids["ssn256"], value)]}), \ + "a value written to both columns reads back and is stored under each column's own key row" + + +@kms_builds("mysql-pure") +def test_decrypt_only_credentials_read_and_write_every_encrypted_value( + build: str, table: int, kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser, kms_verifier: PayloadVerifier) -> None: + role = _assume_decrypt_only(kms_settings, "kms-it-decrypt-only") + written, updated = f"it59-{build}", f"it59-updated-{build}" + with _plugin_credentials(role): + caller = boto3.Session(region_name=kms_settings.region).client("sts").get_caller_identity()["Arn"] + on_the_role = ":assumed-role/" in caller and caller.endswith("/kms-it-decrypt-only") + assert on_the_role, "a session made inside the block, as the plugin's is, runs on the Decrypt-only role" + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn, + encryption_data_key_cache_enabled=False) as conn: + write(conn, _INSERT, (written, "pt-it59-written")) + write(conn, _INSERT, (updated, "pt-it59-before")) + write(conn, f"UPDATE {_TABLE} SET ssn = %s WHERE name = %s", ("pt-it59-after", updated)) + reads = {name: read_column(conn, _SELECT, name) for name in (written, updated, _SEED[0])} + generate = _boto3_error(lambda: kms_client(kms_settings).generate_data_key( + KeyId=kms_master_key_arn, KeySpec="AES_256")) + stored = {name: [kms_verifier.decode(_TABLE, "ssn", payload) + for payload in kms_verifier.stored(f"SELECT ssn FROM {_TABLE} WHERE name = %s", (name,))] + for name in (written, updated)} + assert reads == {written: ["pt-it59-written"], updated: ["pt-it59-after"], _SEED[0]: [_SEED[1]]}, \ + "on the Decrypt-only role the plugin writes, updates and reads every encrypted value, the seed included" + assert stored == {written: [(table, "pt-it59-written")], updated: [(table, "pt-it59-after")]}, \ + "what the Decrypt-only role wrote is stored encrypted under the column's key row" + # A boolean, so a failure prints no part of KMS's error, which names the account. + generate_denied = isinstance(generate, ClientError) and generate.response["Error"]["Code"] == "AccessDeniedException" + assert generate_denied, "the role really is Decrypt-only: it can't generate a data key" + + +@kms_builds("mysql-pure") +def test_real_kms_refusals_fail_closed_with_arns_masked(build: str, missing_key_arn: str, kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser) -> None: + select = f"SELECT ssn FROM {_REFUSALS_TABLE} WHERE name = %s" + denied = _assume_decrypt_only(kms_settings, "kms-it-denied", {"Version": "2012-10-17", "Statement": [ + {"Effect": "Allow", "Action": "kms:DescribeKey", "Resource": "*"}]}) + other_region = "us-west-2" if kms_settings.region != "us-west-2" else "us-east-1" + errors: Dict[str, Optional[BaseException]] = {} + with _plugin_credentials(denied): + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + errors["access denied"] = raised(conn, lambda: read_column(conn, select, _SEED[0])) + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, f"{kms_master_key_arn},{missing_key_arn}" + ) as conn: + errors["no such key"] = raised( + conn, lambda: read_column(conn, f"SELECT ssn_gone FROM {_REFUSALS_TABLE} WHERE name = %s", _SEED[0])) + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn, + encryption_kms_region=other_region) as conn: + errors["other region"] = raised(conn, lambda: read_column(conn, select, _SEED[0])) + + assert {case: type(err).__name__ for case, err in errors.items()} == dict.fromkeys(errors, "KeyManagementError"), \ + "each real-KMS refusal fails the read with KeyManagementError, returning no value" + assert all("after 1 attempt(s)" in str(err) for err in errors.values()), \ + "a refused Decrypt isn't retried" + account, key_id = kms_master_key_arn.split(":")[4], kms_master_key_arn.rsplit("/", 1)[1] + parts = {"account": account, "key id": key_id, "missing key's ARN": missing_key_arn} + unmasked = {case: [label for label, part in parts.items() if part in f"{err} {err!r}"] + for case, err in errors.items()} + assert unmasked == dict.fromkeys(errors, []), \ + "the plugin's message masks the key ARN and the role session ARN that KMS's error names" + # A boolean, so a failure prints neither the account nor KMS's error. + cause_names_account = account in str(getattr(errors["access denied"], "__cause__", None)) + assert cause_names_account, \ + "KMS's own error, kept as the cause, names the session and the key, so there was something to mask" + + +@manual +@enable_on_engines([DatabaseEngine.PG]) +@kms_builds() +def test_via_service_condition_makes_every_statement_fail_closed(build: str, table: int, kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser) -> None: + via_rds = _assume_decrypt_only(kms_settings, "kms-it-via-service", {"Version": "2012-10-17", "Statement": [ + {"Effect": "Allow", "Action": "kms:Decrypt", "Resource": "*", + "Condition": {"StringEquals": {"kms:ViaService": f"rds.{kms_settings.region}.amazonaws.com"}}}]}) + name = f"it62-{build}" + with _plugin_credentials(via_rds): + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + read_error = raised(conn, lambda: read_column(conn, _SELECT, _SEED[0])) + write_error = raised(conn, lambda: write(conn, _INSERT, (name, "pt-it62"))) + assert (type(read_error).__name__, type(write_error).__name__) == ("KeyManagementError", "KeyManagementError"), \ + "with Decrypt allowed only through another service, every encrypted read and write fails closed" + assert _count_rows(_TABLE, name) == 0, "the refused write stores nothing" + + +@manual +@enable_on_engines([DatabaseEngine.PG]) +@kms_builds() +def test_cloudtrail_records_decrypt_calls_without_values(build: str, kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str, kms_app_user: DbUser) -> None: + table_name, session_name = "kms_real_cloudtrail", f"kms-it-ct-{secrets.token_hex(4)}" + values = [f"pt-it63-{i}-{secrets.token_hex(4)}" for i in range(2)] + start = datetime.now(timezone.utc) + utility_requests: List[str] = [] + create_table(table_name, f"name VARCHAR(100), ssn {binary_type()}") + try: + kms = kms_client(kms_settings) + kms.meta.events.register("after-call.kms.GenerateDataKey", + lambda parsed, **_: utility_requests.append(parsed["ResponseMetadata"]["RequestId"])) + with plain_connect() as conn: + KeyManagementUtility(conn, kms, encryption_config(kms_settings, kms_schema)).initialize_encryption_for_column( + table_name, "ssn", kms_master_key_arn) + conn.commit() + with _plugin_credentials(_assume_decrypt_only(kms_settings, session_name)): + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn, + encryption_data_key_cache_enabled=False) as conn: + for value in values: + write(conn, f"INSERT INTO {table_name} (name, ssn) VALUES (%s, %s)", ("it63", value)) + read = read_column(conn, f"SELECT ssn FROM {table_name} WHERE name = %s ORDER BY id", "it63") + finally: + drop_table(table_name) + assert read == values, "the workload ran through the plugin" + + cloudtrail = boto3.client("cloudtrail", region_name=kms_settings.region) + events: Dict[str, Dict[str, Any]] = {} + + def delivered() -> bool: + pages = cloudtrail.get_paginator("lookup_events").paginate( + LookupAttributes=[{"AttributeKey": "ResourceName", "AttributeValue": kms_master_key_arn}], + StartTime=start - timedelta(minutes=5), EndTime=datetime.now(timezone.utc) + timedelta(minutes=5)) + for page in pages: + for event in page["Events"]: + body = json.loads(event["CloudTrailEvent"]) + events[body["requestID"]] = body + plugin = [body for body in events.values() if body["userIdentity"].get("arn", "").endswith(session_name)] + return set(utility_requests) <= set(events) and len(plugin) >= len(values) + 1 + + found = retry_until(delivered, timeout_sec=25 * 60, delay_sec=60) + plugin_events = [body for body in events.values() if body["userIdentity"].get("arn", "").endswith(session_name)] + ours = plugin_events + [events[request] for request in utility_requests if request in events] + assert found and utility_requests, "CloudTrail delivered the plugin's and the utility's events within 25 minutes" + assert sorted({body["eventName"] for body in plugin_events}) == ["Decrypt"], \ + "the plugin's only KMS calls are Decrypt" + assert [events[request]["eventName"] for request in utility_requests] == ["GenerateDataKey"], \ + "the utility's setup is one GenerateDataKey" + shapes = [(body["requestParameters"].get("keyId") == kms_master_key_arn, + set(body["requestParameters"]) <= _ALLOWED_EVENT_PARAMETERS, + body.get("responseElements") is None, any(value in json.dumps(body) for value in values)) + for body in ours] + assert shapes == [(True, True, True, False)] * len(ours), \ + "each event names the row's key ARN, carries only encryptionAlgorithm, keyId and keySpec, and holds no " \ + "response elements and no value" + + +@manual +@enable_on_engines([DatabaseEngine.PG]) +@kms_builds() +def test_expired_credentials_refresh_or_the_next_kms_call_fails_closed( + build: str, table: int, tmp_path: Any, kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser) -> None: + # The refreshing source is botocore's assume-role provider, reading the base credentials from the environment; + # the static one is a single AssumeRole result in the environment, which botocore never refreshes. + config_file = tmp_path / "aws-config" + config_file.write_text(f"[profile kms-it-refreshing]\nrole_arn = {kms_settings.decrypt_only_role_arn}\n" + "credential_source = Environment\nduration_seconds = 900\n" + "role_session_name = kms-it-refreshing\n") + credentials = boto3.Session(region_name=kms_settings.region).get_credentials() + assert credentials is not None, "the module runs with AWS credentials" + base = credentials.get_frozen_credentials() + base_role = RoleSession(base.access_key or "", base.secret_key or "", base.token or "") + static = _assume_decrypt_only(kms_settings, "kms-it-static") + props = {"encryption_data_key_cache_enabled": False} + connections = {} + try: + with _plugin_credentials(static): + connections["static"] = wrapper_connect(build, kms_app_user, kms_settings, kms_schema, + kms_master_key_arn, **props) + before = {"static": read_column(connections["static"], _SELECT, _SEED[0])} + with _plugin_credentials(base_role), pytest.MonkeyPatch.context() as patch: + patch.setenv("AWS_CONFIG_FILE", str(config_file)) + connections["refreshing"] = wrapper_connect(build, kms_app_user, kms_settings, kms_schema, + kms_master_key_arn, aws_profile="kms-it-refreshing", **props) + before["refreshing"] = read_column(connections["refreshing"], _SELECT, _SEED[0]) + assert before == {"static": [_SEED[1]], "refreshing": [_SEED[1]]}, \ + "both sources' credentials read before the expiry" + + def static_expired() -> bool: + error = _boto3_error(lambda: boto3.client( + "sts", region_name=kms_settings.region, aws_access_key_id=static.access_key_id, + aws_secret_access_key=static.secret_access_key, + aws_session_token=static.session_token).get_caller_identity()) + return isinstance(error, ClientError) and "Expired" in error.response["Error"]["Code"] + + assert retry_until(static_expired, timeout_sec=900 + 600, delay_sec=30), \ + "the static credentials expire within their 900-second lifetime" + after = {} + for kind, conn in connections.items(): + name = f"it64-{kind}-{build}" + write_error = raised(conn, lambda: write(conn, _INSERT, (name, f"pt-it64-{kind}"))) + read_error = raised(conn, lambda: read_column(conn, _SELECT, _SEED[0])) + after[kind] = (None if write_error is None else type(write_error).__name__, + None if read_error is None else type(read_error).__name__, _count_rows(_TABLE, name)) + assert after == {"refreshing": (None, None, 1), + "static": ("KeyManagementError", "KeyManagementError", 0)}, \ + "after the expiry the refreshing source keeps KMS calls working, and the static one fails the next " \ + "cold KMS call closed, storing and returning nothing" + finally: + for conn in connections.values(): + conn.close() diff --git a/tests/integration/container/test_kms_encryption_round_trip.py b/tests/integration/container/test_kms_encryption_round_trip.py new file mode 100644 index 00000000..39830dfc --- /dev/null +++ b/tests/integration/container/test_kms_encryption_round_trip.py @@ -0,0 +1,465 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Round trips through the KMS Encryption Plugin on a real server. + +Each write is checked against the bytes the server stored, opened by a decoder outside the plugin with the column's +keys, and each read against the values written. +""" + +from __future__ import annotations + +import base64 +from datetime import date, datetime, time, timedelta, timezone +from decimal import Decimal +from enum import Enum, IntEnum +from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Tuple + +import mysql.connector +import pytest + +from aws_advanced_python_wrapper.encryption.connection_source import \ + PluginConnectionSource +from aws_advanced_python_wrapper.encryption.encryption_service import \ + EncryptionService +from aws_advanced_python_wrapper.encryption.type_marker import TypeMarker +from aws_advanced_python_wrapper.errors import (EncryptionError, + EncryptionFailedError, + IntegrityCheckError, + InvalidKeyMetadataError) +from .utils.conditions import disable_on_engines, enable_on_features +from .utils.database_engine import DatabaseEngine +from .utils.kms_encryption_helper import (binary_type, encrypted_table, is_pg, + key_management_utility, kms_builds, + kms_client, plain_connect, raised, + read_column, statement_log_scan, + wrapper_connect, write) +from .utils.test_environment_features import TestEnvironmentFeatures + +if TYPE_CHECKING: + from aws_advanced_python_wrapper import AwsWrapperConnection + from .utils.kms_encryption_helper import (DbUser, KmsSettings, + PayloadVerifier) + +pytestmark = [pytest.mark.kms_encryption, + enable_on_features([TestEnvironmentFeatures.RUN_KMS_ENCRYPTION_TESTS_ONLY, + TestEnvironmentFeatures.KMS_LOCALSTACK])] + +_TABLE = "kms_round_trip_users" +_INSERT = f"INSERT INTO {_TABLE} (name, ssn) VALUES (%s, %s)" +_INSERT_DOB = f"INSERT INTO {_TABLE} (name, dob) VALUES (%s, %s)" +_SELECT = f"SELECT ssn FROM {_TABLE} WHERE name = %s ORDER BY id" +_SELECT_DOB = f"SELECT dob FROM {_TABLE} WHERE name = %s ORDER BY id" +_FETCHMANY_DEFECT = ("AwsWrapperCursor.fetchmany passes size=0 to the driver when no size is given " + "(aws_advanced_python_wrapper/wrapper.py:433-435): the C extension's fetchmany(0) returns no " + "rows, where its fetchmany() returns one") +# Added to the module's last key_storage id, so the payload names an id no key row has. +_UNKNOWN_KEY_ID_OFFSET = 1000 +# A payload's type marker follows the 32-byte HMAC tag and the 4-byte key id (encryption_service.py's layout). +_TYPE_MARKER_OFFSET = 36 +# An int too large for the encryption service to serialize. +_TOO_LARGE_INT = 2 ** 70 + + +class _Size(IntEnum): + LARGE = 7 + + +class _Color(str, Enum): + RED = "pt-strenum" + + +# Values the plugin stores in an exact form of another type: (the value written, the value read back). +_EXACT_FORM: Dict[str, Tuple[Any, Any]] = { + "none": (None, None), "int-enum": (_Size.LARGE, 7), "decimal": (Decimal("12.50"), Decimal("12.50")), + "str-enum": (_Color.RED, "pt-strenum"), + "mysql-binary": (mysql.connector.Binary(b"pt-mysql-binary"), b"pt-mysql-binary")} + + +@pytest.fixture(scope="module") +def key_rows(kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser) -> Iterator[Dict[str, int]]: + """The module's table, with ``ssn`` and ``dob`` encrypted under key rows of their own; yields each column's + ``key_storage`` id.""" + with encrypted_table(_TABLE, f"name VARCHAR(100), ssn {binary_type()}, dob {binary_type()}", ("ssn", "dob"), + kms_settings, kms_schema, kms_master_key_arn) as key_ids: + yield key_ids + + +@pytest.fixture +def conn(build: str, key_rows: Dict[str, int], kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser) -> Iterator[AwsWrapperConnection]: + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + yield conn + + +def _fake_payload(key_id: int) -> bytes: + """81 bytes laid out as a payload that names ``key_id``; the test chose every byte, and none is key material.""" + return bytes(32) + key_id.to_bytes(4, "big") + bytes([1]) + bytes(44) + + +@kms_builds() +def test_bound_write_stores_a_payload(build: str, conn: AwsWrapperConnection, key_rows: Dict[str, int], + kms_verifier: PayloadVerifier) -> None: + one, multi, mapping, keyword, updated = (f"it1-{shape}-{build}" + for shape in ("one", "multi", "mapping", "keyword", "updated")) + with conn.cursor() as cur: + cur.execute(_INSERT, (one, "pt-it1-one")) + cur.execute(f"INSERT INTO {_TABLE} (name, ssn) VALUES (%s, %s), (%s, %s)", + (multi, "pt-it1-multi-1", multi, "pt-it1-multi-2")) + cur.execute(f"INSERT INTO {_TABLE} (ssn, name) VALUES (%(ssn)s, %(name)s)", + {"ssn": "pt-it1-mapping", "name": mapping}) + cur.execute(_INSERT, params=(keyword, "pt-it1-keyword")) + cur.execute(_INSERT, (updated, "pt-it1-before")) + cur.execute(f"UPDATE {_TABLE} SET ssn = %s WHERE name = %s", ("pt-it1-after", updated)) + conn.commit() + ssn = key_rows["ssn"] + stored = {name: kms_verifier.opened(_TABLE, "ssn", name) for name in (one, multi, mapping, keyword, updated)} + assert stored == {one: [(ssn, "pt-it1-one")], multi: [(ssn, "pt-it1-multi-1"), (ssn, "pt-it1-multi-2")], + mapping: [(ssn, "pt-it1-mapping")], keyword: [(ssn, "pt-it1-keyword")], + updated: [(ssn, "pt-it1-after")]}, \ + "each bound INSERT and UPDATE stores, in the row its other values name, a payload under ssn's key row " \ + "that opens to the bound value" + + +@kms_builds() +def test_every_fetch_path_decrypts_rows_in_order(build: str, conn: AwsWrapperConnection, key_rows: Dict[str, int], + kms_verifier: PayloadVerifier) -> None: + name = f"it2-{build}" + values = [f"pt-it2-{i}" for i in range(4)] + with conn.cursor() as cur: + cur.executemany(_INSERT, [(name, value) for value in values]) + conn.commit() + reads: Dict[str, List[Any]] = {} + for path in ("fetchone", "fetchmany", "fetchall", "next", "for"): + with conn.cursor() as cur: + cur.execute(_SELECT, (name,)) + if path == "fetchone": + rows = list(iter(cur.fetchone, None)) + elif path == "fetchmany": + rows = cur.fetchmany(3) + cur.fetchmany(3) + elif path == "fetchall": + rows = cur.fetchall() + elif path == "next": + rows = list(iter(lambda: next(cur, None), None)) + else: + rows = [row for row in cur] + reads[path] = [row[0] for row in rows] + assert reads == {path: values for path in reads}, \ + "fetchone, fetchmany(n), fetchall, next() and a for loop each return every row decrypted, once and in order" + assert [value for _, value in kms_verifier.opened(_TABLE, "ssn", name)] == values, "the rows read are the rows stored" + + +@disable_on_engines([DatabaseEngine.PG]) +@kms_builds() +def test_mysql_cursor_kinds_and_binary_prefix(build: str, conn: AwsWrapperConnection, key_rows: Dict[str, int], + kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser, kms_verifier: PayloadVerifier, + kms_statement_log: str) -> None: + kinds, strict, prepared, nulls = (f"it3-{part}-{build}" for part in ("kinds", "strict", "prepared", "nulls")) + with conn.cursor() as cur: + cur.executemany(_INSERT, [(kinds, "pt-it3-kinds-1"), (kinds, "pt-it3-kinds-2")]) + cur.execute(_INSERT, (nulls, None)) + cur.executemany(_INSERT, [(nulls, None), (nulls, "pt-it3-nulls")]) + with conn.cursor(prepared=True) as cur: + cur.execute(f"INSERT INTO {_TABLE} (ssn, name) VALUES (%s, %s)", ("pt-it3-prepared", prepared)) + conn.commit() + reads: Dict[str, List[Any]] = {} + for kind in ("dictionary", "buffered", "raw"): + with conn.cursor(**{kind: True}) as cur: + cur.execute(_SELECT, (kinds,)) + reads[kind] = [row["ssn"] if kind == "dictionary" else row[0] for row in cur.fetchall()] + assert reads == {kind: ["pt-it3-kinds-1", "pt-it3-kinds-2"] for kind in reads}, \ + "dictionary, buffered and raw cursors return the stored values decrypted" + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn, + raise_on_warnings=True) as warned: + with warned.cursor() as cur: + cur.execute(_INSERT, (strict, "pt-it3-strict-1")) + cur.executemany(_INSERT, [(strict, "pt-it3-strict-2"), (strict, "pt-it3-strict-3")]) + warned.commit() + ssn = key_rows["ssn"] + stored = {name: kms_verifier.opened(_TABLE, "ssn", name) for name in (strict, prepared, nulls)} + assert stored == {strict: [(ssn, "pt-it3-strict-1"), (ssn, "pt-it3-strict-2"), (ssn, "pt-it3-strict-3")], + prepared: [(ssn, "pt-it3-prepared")], nulls: [None, None, (ssn, "pt-it3-nulls")]}, \ + "writes under raise_on_warnings raise no Warning 1300, a prepared cursor's write stores a payload, and a " \ + "NULL or a batch mixing NULL and a value is stored as written" + # A prepared cursor's SQL reaches the server with ? markers (the Prepare entry). The Execute entry is the server's + # rendering of the values, and the other writes' entries carry _binary before quoted values. + logged = statement_log_scan({"prepared statement": f"INSERT INTO {_TABLE} (ssn, name) VALUES (?, ?)", + "_binary before a ? marker": "_bin" + "ary ?"}, kms_statement_log) + assert logged == {"prepared statement": 1, "_binary before a ? marker": 0}, \ + "a prepared cursor's statement reaches the server once, with no _binary before its markers" + + +@kms_builds() +def test_executemany_reads_a_generator_once(build: str, conn: AwsWrapperConnection, key_rows: Dict[str, int], + kms_verifier: PayloadVerifier) -> None: + listed, generated = f"it4-list-{build}", f"it4-generator-{build}" + with conn.cursor() as cur: + cur.executemany(_INSERT, [(listed, f"pt-it4-list-{i}") for i in range(3)]) + cur.executemany(_INSERT, ((generated, f"pt-it4-generator-{i}") for i in range(3))) + cur.executemany(_INSERT, iter(())) + conn.commit() + ssn = key_rows["ssn"] + stored = {name: kms_verifier.opened(_TABLE, "ssn", name) for name in (listed, generated)} + assert stored == {listed: [(ssn, f"pt-it4-list-{i}") for i in range(3)], + generated: [(ssn, f"pt-it4-generator-{i}") for i in range(3)]}, \ + "executemany stores every row of a list and of a one-shot generator encrypted, and an empty batch raises nothing" + + +@kms_builds() +def test_values_of_each_python_type_round_trip(build: str, conn: AwsWrapperConnection, key_rows: Dict[str, int], + kms_verifier: PayloadVerifier) -> None: + ist = timezone(timedelta(hours=5, minutes=30)) + # Each value written, and what it reads back as: an aware datetime as UTC milliseconds, a time as its ISO text. + cases = [("nul\x00 and non-BMP \U0001F600\U00010348", "nul\x00 and non-BMP \U0001F600\U00010348"), + (b"\x00\x01\xff", b"\x00\x01\xff"), (bytearray(b"bytearray\x00"), b"bytearray\x00"), + (memoryview(b"memoryview\x80"), b"memoryview\x80"), (-2 ** 63, -2 ** 63), (2 ** 63 - 1, 2 ** 63 - 1), + (0.1, 0.1), (-0.0, -0.0), (5e-324, 5e-324), (True, True), (False, False), + (Decimal("12.50"), Decimal("12.50")), + (datetime(2026, 8, 11, 12, 30, 45, 123456, tzinfo=ist), + datetime(2026, 8, 11, 7, 0, 45, 123000, tzinfo=timezone.utc)), + (datetime(2026, 8, 11, 12, 30, 45, 123456), datetime(2026, 8, 11, 12, 30, 45, 123456)), + (date(2024, 2, 29), date(2024, 2, 29)), (time(9, 26, 53, 120000), "09:26:53.120000")] + names = [f"it5-{i}-{build}" for i in range(len(cases))] + with conn.cursor() as cur: + for name, (value, _) in zip(names, cases): + cur.execute(_INSERT, (name, value)) + conn.commit() + read = [read_column(conn, _SELECT, name)[0] for name in names] + opened = [kms_verifier.opened(_TABLE, "ssn", name)[0][1] for name in names] + expected = [repr(back) for _, back in cases] + assert [repr(value) for value in read] == expected, \ + "each Python type reads back through the plugin with its type and value (repr shows both)" + assert [repr(value) for value in opened] == expected, "each stored payload opens to the same typed value" + + +@kms_builds() +def test_two_encrypted_columns_use_their_own_key_rows(build: str, conn: AwsWrapperConnection, + kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str, kms_app_user: DbUser, + kms_verifier: PayloadVerifier) -> None: + before, after = f"it6-before-{build}", f"it6-after-{build}" + insert_both = f"INSERT INTO {_TABLE} (name, ssn, dob) VALUES (%s, %s, %s)" + with conn.cursor() as cur: + cur.execute(insert_both, (before, "pt-it6-ssn-1", "pt-it6-dob-1")) + conn.commit() + ssn_before, dob_before = kms_verifier.opened(_TABLE, "ssn", before), kms_verifier.opened(_TABLE, "dob", before) + with plain_connect() as plain: + rotated = key_management_utility(plain, kms_settings, kms_schema).rotate_data_key(_TABLE, "dob", kms_master_key_arn) + plain.commit() + # A new connection, so the plugin reads the metadata the rotation changed. + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as later: + old_back = (read_column(later, _SELECT, before), read_column(later, _SELECT_DOB, before)) + with later.cursor() as cur: + cur.execute(insert_both, (after, "pt-it6-ssn-2", "pt-it6-dob-2")) + later.commit() + ssn_id, dob_id = ssn_before[0][0], dob_before[0][0] + assert ssn_id != dob_id and [value for _, value in ssn_before + dob_before] == ["pt-it6-ssn-1", "pt-it6-dob-1"], \ + "each encrypted column's payload records its own key_storage id and opens to its own value" + assert old_back == (["pt-it6-ssn-1"], ["pt-it6-dob-1"]), \ + "after dob's data key is rotated, the values written under the old key rows still read back" + assert (kms_verifier.opened(_TABLE, "ssn", after), kms_verifier.opened(_TABLE, "dob", after)) == \ + ([(ssn_id, "pt-it6-ssn-2")], [(rotated, "pt-it6-dob-2")]), \ + "after the rotation, a new dob value uses the new key row and a new ssn value keeps ssn's" + + +@kms_builds() +def test_later_result_sets_use_their_own_columns(build: str, conn: AwsWrapperConnection, key_rows: Dict[str, int], + kms_verifier: PayloadVerifier) -> None: + name = f"it7-{build}" + with conn.cursor() as cur: + cur.execute(_INSERT, (name, "pt-it7")) + conn.commit() + # No parameters: psycopg's default cursor refuses several statements with parameters. Each set is read to its end + # before nextset(), as mysql-connector's pure build requires. + with conn.cursor() as cur: + cur.execute(f"SELECT ssn FROM {_TABLE} WHERE name = '{name}'; SELECT name FROM {_TABLE} WHERE name = '{name}'") + first = cur.fetchall() + moved = bool(cur.nextset()) + second = cur.fetchall() + assert (first, moved, second) == ([("pt-it7",)], True, [(name,)]), \ + "a later result set of one execute() is never decoded with the first set's columns" + want: List[List[Any]] + with conn.cursor() as cur: + if is_pg(): + values = [f"pt-it7-returning-{i}" for i in range(3)] + cur.executemany(f"{_INSERT} RETURNING ssn", [(name, value) for value in values], returning=True) + sets = [cur.fetchall()] + while cur.nextset(): + sets.append(cur.fetchall()) + want = [[(value,)] for value in values] + else: + # mysql-connector runs a string that isn't an INSERT through execute() for each parameter set. + cur.executemany(f"SELECT ssn FROM {_TABLE} WHERE name = %s; SELECT name, ssn FROM {_TABLE} WHERE name = %s", + [(name, name)]) + cur.fetchall() # the first set, whose rows the driver has already read when executemany returns + sets = [] + while cur.nextset(): + sets.append(cur.fetchall()) + want = [[(name, "pt-it7")]] + conn.commit() + assert sets == want, "each result set of an executemany is decoded by its own columns" + + +@kms_builds() +def test_fetchmany_without_a_size_reads_arraysize_rows(build: str, request: pytest.FixtureRequest, + conn: AwsWrapperConnection, key_rows: Dict[str, int]) -> None: + """fetchmany() without a size returns arraysize decrypted rows, as a plain connection does.""" + if build == "mysql-cext": + request.applymarker(pytest.mark.xfail(strict=True, reason=_FETCHMANY_DEFECT)) + name = f"it8-{build}" + values = [f"pt-it8-{i}" for i in range(5)] + with conn.cursor() as cur: + cur.executemany(_INSERT, [(name, value) for value in values]) + conn.commit() + arraysize = 3 + with plain_connect(build=build) as plain, plain.cursor() as cur: + cur.arraysize = arraysize + cur.execute(_SELECT, (name,)) + plain_page = len(cur.fetchmany()) + cur.fetchall() + # psycopg and the pure build read arraysize rows; the C extension's fetchmany() reads one, whatever arraysize says. + assert plain_page == (1 if build == "mysql-cext" else arraysize), \ + "the plain driver's fetchmany() with no size reads arraysize rows, or one on the C extension" + with conn.cursor() as cur: + driver_cursor: Any = cur.target_cursor # the wrapper's arraysize is read-only + driver_cursor.arraysize = arraysize + cur.execute(_SELECT, (name,)) + page = [row[0] for row in cur.fetchmany()] + cur.fetchall() + assert (len(page), page) == (plain_page, values[:plain_page]), \ + "fetchmany() with no size returns as many rows as the plain driver's fetchmany(), decrypted" + + +@kms_builds() +def test_legacy_and_foreign_key_id_values_strict_and_lenient(build: str, monkeypatch: pytest.MonkeyPatch, + key_rows: Dict[str, int], kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser) -> None: + foreign = f"it9-foreign-{build}" + with plain_connect() as plain, plain.cursor() as cur: + cur.execute(f"SELECT max(id) FROM {kms_schema}.key_storage") + (last_key_id,) = cur.fetchone() + payload = _fake_payload(last_key_id + _UNKNOWN_KEY_ID_OFFSET) + cur.execute(_INSERT_DOB, (foreign, payload)) + plain.commit() + opened: List[Any] = [] + open_connection = PluginConnectionSource._open + + def counted_open(source: PluginConnectionSource) -> Any: + opened.append(source) + return open_connection(source) + + monkeypatch.setattr(PluginConnectionSource, "_open", counted_open) + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as strict: + read_column(strict, _SELECT, foreign) # loads the column metadata, so the read below opens a plugin connection only to look up key_storage + before = len(opened) + with pytest.raises(IntegrityCheckError): + read_column(strict, _SELECT_DOB, foreign) + strict_lookups = len(opened) - before + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn, + encryption_return_unverified_data=True) as lenient: + read_column(lenient, _SELECT, foreign) + before = len(opened) + lenient_back = read_column(lenient, _SELECT_DOB, foreign) + lenient_lookups = len(opened) - before + assert [bytes(value) for value in lenient_back] == [payload], \ + "in lenient mode a value naming a key_storage id that doesn't exist comes back as stored" + assert (strict_lookups, lenient_lookups) == (1, 1), "each lookup of a stored key_storage id opens one plugin connection" + + +@kms_builds() +def test_legacy_and_foreign_key_id_values_unusable_key_row(build: str, key_rows: Dict[str, int], + kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str, kms_app_user: DbUser) -> None: + own = f"it9-own-{build}" + with plain_connect() as plain, plain.cursor() as cur: + # The column's current key row, which an earlier test may have rotated away from key_rows["dob"]. + cur.execute(f"SELECT key_id FROM {kms_schema}.encryption_metadata WHERE table_name = %s AND column_name = %s", + (_TABLE, "dob")) + (dob_key_id,) = cur.fetchone() + cur.execute(f"SELECT master_key_arn FROM {kms_schema}.key_storage WHERE id = %s", (dob_key_id,)) + (dob_arn,) = cur.fetchone() + cur.execute(_INSERT_DOB, (own, _fake_payload(dob_key_id))) + cur.execute(f"UPDATE {kms_schema}.key_storage SET master_key_arn = %s WHERE id = %s", ("", dob_key_id)) + plain.commit() + try: + for lenient_mode in (False, True): + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn, + encryption_return_unverified_data=lenient_mode) as unusable: + with pytest.raises(InvalidKeyMetadataError): + read_column(unusable, _SELECT_DOB, own) + finally: + with plain_connect() as plain, plain.cursor() as cur: + cur.execute(f"UPDATE {kms_schema}.key_storage SET master_key_arn = %s WHERE id = %s", (dob_arn, dob_key_id)) + plain.commit() + + +@kms_builds() +@pytest.mark.parametrize("call", ["execute", "executemany"]) +def test_unserializable_value_keeps_the_encryption_service_error(build: str, call: str, conn: AwsWrapperConnection, + key_rows: Dict[str, int], + kms_verifier: PayloadVerifier) -> None: + with pytest.raises(EncryptionError) as direct: # what the encryption service raises for it on its own + EncryptionService.encrypt(_TOO_LARGE_INT, bytearray(32), bytearray(32), key_id=1) + name = f"big-{build}-{call}" + with conn.cursor() as cur: + err = raised(conn, (lambda: cur.execute(_INSERT, (name, _TOO_LARGE_INT))) if call == "execute" + else (lambda: cur.executemany(_INSERT, [(name, _TOO_LARGE_INT)]))) + assert (type(direct.value), type(err)) == (EncryptionFailedError, EncryptionFailedError), \ + "an int too large to encrypt fails with the encryption service's own EncryptionFailedError" + assert kms_verifier.stored(_SELECT, (name,)) == [], "nothing is stored" + + +@kms_builds() +@pytest.mark.parametrize("kind", [kind for kind in _EXACT_FORM if kind != "mysql-binary" or not is_pg()]) +def test_value_with_an_exact_stored_form_is_still_encrypted(build: str, kind: str, conn: AwsWrapperConnection, + key_rows: Dict[str, int], + kms_verifier: PayloadVerifier) -> None: + value, want = _EXACT_FORM[kind] + name = f"exact-{build}-{kind}" + write(conn, _INSERT, (name, value)) + got = read_column(conn, _SELECT, name) + assert kms_verifier.opened(_TABLE, "ssn", name) == [None if want is None else (key_rows["ssn"], want)], \ + "the value is stored as a payload of its exact form (and None as NULL)" + assert (got, [type(v) for v in got]) == ([want], [type(want)]), "it reads back as that form, with its type" + + +@kms_builds() +def test_stored_generic_payload_reads_back_as_text(build: str, conn: AwsWrapperConnection, key_rows: Dict[str, int], + kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str) -> None: + value = {"k": "pt-generic"} + name = f"generic-{build}" + with plain_connect() as plain, plain.cursor() as cur: + # The column's current key row, which an earlier test may have rotated away from key_rows["dob"]. + cur.execute(f"SELECT key_id FROM {kms_schema}.encryption_metadata WHERE table_name = %s AND column_name = %s", + (_TABLE, "dob")) + (dob_key_id,) = cur.fetchone() + cur.execute(f"SELECT encrypted_data_key, hmac_key FROM {kms_schema}.key_storage WHERE id = %s", (dob_key_id,)) + encrypted_data_key, hmac_key = cur.fetchone() + # The encryption service with dob's keys, as a writer other than the plugin would store the value. + data_key = bytearray(kms_client(kms_settings).decrypt(CiphertextBlob=base64.b64decode(encrypted_data_key), + KeyId=kms_master_key_arn)["Plaintext"]) + hmac = bytearray(hmac_key) + try: + payload = EncryptionService.encrypt(value, data_key, hmac, key_id=dob_key_id) + finally: + EncryptionService.wipe(data_key) + EncryptionService.wipe(hmac) + assert payload is not None and payload[_TYPE_MARKER_OFFSET] == TypeMarker.GENERIC, "the service stores a dict under the GENERIC type marker" + with plain_connect() as plain: + write(plain, _INSERT_DOB, (name, payload)) + assert read_column(conn, _SELECT_DOB, name) == [str(value)], "a GENERIC payload reads back as its text" diff --git a/tests/integration/container/test_kms_encryption_server_logs.py b/tests/integration/container/test_kms_encryption_server_logs.py new file mode 100644 index 00000000..bf92edba --- /dev/null +++ b/tests/integration/container/test_kms_encryption_server_logs.py @@ -0,0 +1,294 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""What the servers' statement logs hold after the KMS Encryption Plugin's statements. + +PostgreSQL logs every statement (``log_statement=all``) and MySQL's general log is switched on by the +``kms_statement_log`` fixture, which first proves the scan finds its own marker. Needles are built by concatenation, +so the test's source never holds them whole. +""" + +from __future__ import annotations + +import base64 +import logging +import secrets +from typing import (TYPE_CHECKING, Any, Callable, Dict, Iterator, List, + Optional, Set, Tuple) + +import pytest + +from aws_advanced_python_wrapper.encryption.key_manager import KeyManager +from aws_advanced_python_wrapper.errors import AwsWrapperError +from aws_advanced_python_wrapper.pep249_methods import DbApiMethod +from aws_advanced_python_wrapper.plugin import Plugin, PluginFactory +from aws_advanced_python_wrapper.plugin_service import PluginManager +from .utils.conditions import enable_on_features +from .utils.kms_encryption_helper import (TRIGGER_REJECTION, binary_type, + create_table, drop_table, + encrypt_column, encrypted_table, + key_management_utility, kms_builds, + plain_connect, raised, + statement_log_scan, wrapper_connect) +from .utils.test_environment_features import TestEnvironmentFeatures + +if TYPE_CHECKING: + import io + + from aws_advanced_python_wrapper import AwsWrapperConnection + from aws_advanced_python_wrapper.plugin_service import PluginService + from aws_advanced_python_wrapper.utils.properties import Properties + from .utils.kms_encryption_helper import DbUser, KmsSettings + +pytestmark = [pytest.mark.kms_encryption, + enable_on_features([TestEnvironmentFeatures.RUN_KMS_ENCRYPTION_TESTS_ONLY, + TestEnvironmentFeatures.KMS_LOCALSTACK])] + +_TABLE = "kms_logs_users" +_INSERT = f"INSERT INTO {_TABLE} (name, ssn) VALUES (%s, %s)" +_SELECT = f"SELECT ssn FROM {_TABLE} WHERE name = %s ORDER BY id" +_COLUMNS = f"name VARCHAR(100), ssn {binary_type()}" +# Just below kms_encryption's weight (390), so the logging plugin would come before it even in a sorted chain; IT-52 +# turns sorting off and lists the chain itself. +_ARGS_LOGGER_WEIGHT = 385 + + +class _ArgsLoggerPlugin(Plugin): + """IT-52's positive control: a plugin that logs, at DEBUG, the arguments it receives.""" + + @property + def subscribed_methods(self) -> Set[str]: + return {DbApiMethod.CURSOR_EXECUTE.method_name, DbApiMethod.CURSOR_EXECUTEMANY.method_name} + + def execute(self, target: type, method_name: str, execute_func: Callable, *args: Any, **kwargs: Any) -> Any: + logging.getLogger(__name__).debug("args_logger %s %r", method_name, args) + return execute_func() + + +class _ArgsLoggerFactory(PluginFactory): + @staticmethod + def get_instance(plugin_service: PluginService, props: Properties) -> Plugin: + return _ArgsLoggerPlugin() + + +@pytest.fixture(scope="module") +def key_row(kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, kms_app_user: DbUser) -> Iterator[int]: + """The module's users table with ``ssn`` encrypted; yields the key row's id.""" + with encrypted_table(_TABLE, _COLUMNS, ("ssn",), kms_settings, kms_schema, kms_master_key_arn) as key_ids: + yield key_ids["ssn"] + + +@pytest.fixture +def args_logger() -> Iterator[str]: + """The code of a plugin, registered for the test, that logs the arguments it receives.""" + PluginManager.register_plugin("kms_args_logger", _ArgsLoggerFactory, _ARGS_LOGGER_WEIGHT) + try: + yield "kms_args_logger" + finally: + PluginManager.PLUGIN_FACTORIES.pop("kms_args_logger", None) + PluginManager.PLUGIN_FACTORY_WEIGHTS.pop(_ArgsLoggerFactory, None) + + +@pytest.fixture +def conn(build: str, key_row: int, kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser) -> Iterator[AwsWrapperConnection]: + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + yield conn + + +@pytest.fixture +def data_keys(monkeypatch: pytest.MonkeyPatch) -> List[bytes]: + """Every plaintext data key KMS hands out while the test runs, through KeyManager, in memory only.""" + keys: List[bytes] = [] + generate, decrypt = KeyManager.generate_data_key, KeyManager.decrypt_data_key + + def generated(self: KeyManager, *args: Any, **kwargs: Any) -> Any: + out = generate(self, *args, **kwargs) + keys.append(bytes(out.plaintext)) + return out + + def decrypted(self: KeyManager, *args: Any, **kwargs: Any) -> Any: + out = decrypt(self, *args, **kwargs) + keys.append(bytes(out)) + return out + + monkeypatch.setattr(KeyManager, "generate_data_key", generated) + monkeypatch.setattr(KeyManager, "decrypt_data_key", decrypted) + return keys + + +def _key_forms(keys: List[bytes]) -> Dict[str, str]: + """The text forms each data key would take in a statement log, by a label that shows no key: base64 and hex.""" + return {f"data key {i} {kind}": form for i, key in enumerate(keys) + for kind, form in (("base64", base64.b64encode(key).decode()), ("hex", key.hex()), + ("HEX", key.hex().upper()))} + + +def _error_class(err: Optional[Exception]) -> str: + """The class of ``err``, or of the error it wraps when failover_v2's execute() wrapped it in AwsWrapperError.""" + if type(err) is AwsWrapperError and err.__cause__ is not None: + return f"AwsWrapperError from {type(err.__cause__).__name__}" + return type(err).__name__ + + +@kms_builds() +def test_statement_logs_hold_no_written_plaintext(build: str, conn: AwsWrapperConnection, key_row: int, + kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_statement_log: str, data_keys: List[bytes]) -> None: + tag = build.replace("-", "") + values = {shape: "pt-it49-" + shape + tag for shape in ("one", "many-1", "many-2", "multi-1", "multi-2", + "update", "compared")} + name = "it49-name-" + tag + with conn.cursor() as cur: + cur.execute(_INSERT, (name, values["one"])) + cur.executemany(_INSERT, [(name, values["many-1"]), (name, values["many-2"])]) + cur.execute(f"INSERT INTO {_TABLE} (name, ssn) VALUES (%s, %s), (%s, %s)", + (name, values["multi-1"], name, values["multi-2"])) + cur.execute(f"UPDATE {_TABLE} SET ssn = %s WHERE name = %s AND ssn IS NULL", (values["update"], name)) + cur.execute(f"SELECT name FROM {_TABLE} WHERE ssn = %s", (values["compared"],)) + cur.fetchall() + conn.commit() + setup_table = f"kms_logs_setup_{tag}" + create_table(setup_table, _COLUMNS) + try: + with plain_connect() as plain: # the utility's setup of a column, under the same log + encrypt_column(plain, kms_settings, kms_schema, setup_table, "ssn", kms_master_key_arn) + key_management_utility(plain, kms_settings, kms_schema).rotate_data_key(setup_table, "ssn", + kms_master_key_arn) + plain.commit() + finally: + drop_table(setup_table) + key_needles = _key_forms(list(dict.fromkeys(data_keys))) + seen_keys = len(set(data_keys)) # counts only, so no assertion can print a key + # PostgreSQL logs a parameter sent as bytes in hex. + hexed = {f"{label} hex": value.encode().hex() for label, value in values.items()} + counts = statement_log_scan({**values, **hexed, "name": name, **key_needles}, kms_statement_log) + assert {label: counts[label] for label in [*values, *hexed]} == {label: 0 for label in [*values, *hexed]}, \ + "no written or compared plaintext value reaches the server's statement log" + assert seen_keys >= 2 and [label for label in key_needles if counts[label]] == [], \ + "no plaintext data key, of the plugin's or the utility's, reaches the statement log" + assert counts["name"] >= 4, "the plain name the statements bound does reach the log, so the scan reads it" + + +@kms_builds() +def test_compared_parameter_in_statement_logs(build: str, conn: AwsWrapperConnection, kms_statement_log: str) -> None: + tag = build.replace("-", "") + direct, through = "pt-it50-direct-" + tag, "pt-it50-coalesce-" + tag + with conn.cursor() as cur: + cur.execute(f"SELECT name FROM {_TABLE} WHERE ssn = %s", (direct,)) + direct_rows = cur.fetchall() + # The plugin can't tie a value compared with an expression to the column, so it sends it as written. + cur.execute(f"SELECT name FROM {_TABLE} WHERE COALESCE(ssn, ssn) = %s", (through,)) + through_rows = cur.fetchall() + conn.commit() + # PostgreSQL logs a parameter sent as bytes in hex. + counts = statement_log_scan({"direct": direct, "direct hex": direct.encode().hex(), + "coalesce": through, "coalesce hex": through.encode().hex()}, kms_statement_log) + assert (direct_rows, through_rows) == ([], []), "neither comparison matches a row" + assert counts["coalesce"] + counts["coalesce hex"] >= 1, \ + "a value compared through COALESCE is sent as written and reaches the log" + assert counts["direct"] + counts["direct hex"] == 0, \ + "a value compared with the encrypted column is encrypted before it is sent" + + +@kms_builds("mysql-cext") +def test_no_plaintext_data_key_reaches_the_database(build: str, key_row: int, kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, kms_app_user: DbUser, + kms_statement_log: str, data_keys: List[bytes]) -> None: + tag = build.replace("-", "") + table = f"kms_logs_keys_{tag}" + create_table(table, _COLUMNS) + try: + with plain_connect() as plain: # setup and rotation: GenerateDataKey twice + encrypt_column(plain, kms_settings, kms_schema, table, "ssn", kms_master_key_arn) + for i in range(2): # use: Decrypt, under the first key row and then under the rotated one + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + with conn.cursor() as cur: + cur.execute(f"INSERT INTO {table} (name, ssn) VALUES (%s, %s)", ("it51", f"pt-it51-{i}")) + cur.execute(f"SELECT ssn FROM {table} WHERE name = %s ORDER BY id", ("it51",)) + back = [row[0] for row in cur.fetchall()] + conn.commit() + if i == 0: + with plain_connect() as plain: + key_management_utility(plain, kms_settings, kms_schema).rotate_data_key(table, "ssn", + kms_master_key_arn) + plain.commit() + keys = list(dict.fromkeys(data_keys)) + needles = _key_forms(keys) + # The positive control: a random key of a data key's size, written on purpose in base64, is found by the + # same scan; no real data key is written. + control = {f"control {label}": form for label, form in _key_forms([secrets.token_bytes(32)]).items()} + with plain_connect() as plain, plain.cursor() as cur: + cur.execute(f"INSERT INTO {table} (name) VALUES (%s)", + ("it51-control-" + control["control data key 0 base64"],)) + plain.commit() + with plain_connect(admin=True) as admin, admin.cursor() as cur: + cur.execute(f"SELECT id, key_id, name, master_key_arn, encrypted_data_key, hmac_key, key_spec " + f"FROM {kms_schema}.key_storage") + rows = cur.fetchall() + finally: + drop_table(table) + # Counts and labels only, so no assertion can print a key. + cells = [bytes(c) if isinstance(c, (bytes, bytearray, memoryview)) else str(c).encode() for r in rows for c in r] + stored = sum(any(key in cell for cell in cells) for key in keys) + \ + sum(any(form.encode() in cell for cell in cells) for form in needles.values()) + in_wrapped = sum(key in base64.b64decode("".join(str(r[4]).split())) for key in keys for r in rows) + seen_keys = len(keys) + logged = {label: count for label, count in statement_log_scan({**needles, **control}, kms_statement_log).items() + if count} + assert back == ["pt-it51-0", "pt-it51-1"] and seen_keys >= 2, \ + "setup and rotation handed out two data keys, use decrypted them, and the values read back" + assert (stored, in_wrapped) == (0, 0), "key_storage holds no plaintext data key, only KMS's encrypted form" + assert logged == {"control data key 0 base64": 1}, \ + "no plaintext data key reaches the statement log; the control key written on purpose is found once" + + +@kms_builds() +def test_failover_debug_log_holds_no_bound_value(build: str, key_row: int, kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str, kms_app_user: DbUser, + kms_log_capture: io.StringIO, data_keys: List[bytes], + args_logger: str) -> None: + tag = build.replace("-", "") + guarded = f"kms_logs_guarded_{tag}" + values = {"refused": "pt-it52-refused-" + tag, "rejected": "pt-it52-rejected-" + tag} + logged: Dict[str, int] = {} + outcomes: Dict[str, Tuple[str, str]] = {} + with encrypted_table(guarded, _COLUMNS, ("ssn",), kms_settings, kms_schema, kms_master_key_arn, + triggered=("ssn",)): + for chain in ("failover_v2,kms_encryption", f"{args_logger},failover_v2,kms_encryption"): + start = len(kms_log_capture.getvalue()) + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn, plugins=chain, + auto_sort_wrapper_plugin_order=False) as conn: + with conn.cursor() as cur: + # Refused by the plugin before anything is sent. + refused = raised(conn, lambda: cur.execute(f"DELETE FROM {_TABLE} WHERE ssn <> %s", + (values["refused"],))) + # No column list: passed through, and the trigger rejects the plaintext. + rejected = raised(conn, lambda: cur.execute(f"INSERT INTO {guarded} VALUES (DEFAULT, %s, %s)", + ("it52", values["rejected"]))) + outcomes[chain] = (_error_class(refused), "rejected" if TRIGGER_REJECTION in str(rejected) else + f"{type(rejected).__name__}: {rejected}") + text = kms_log_capture.getvalue()[start:] + keys = _key_forms(list(dict.fromkeys(data_keys))) + logged[chain] = sum(value in text for value in values.values()) + logged[f"{chain} data keys"] = sum(form in text for form in keys.values()) + assert outcomes == {"failover_v2,kms_encryption": ("AwsWrapperError from MetadataError", "rejected"), + f"{args_logger},failover_v2,kms_encryption": ("AwsWrapperError from MetadataError", "rejected")}, \ + "the plugin refuses the DELETE (failover_v2's execute() wraps the MetadataError) and the trigger rejects the " \ + "unlisted INSERT, so the log covers both paths" + assert logged == {"failover_v2,kms_encryption": 0, "failover_v2,kms_encryption data keys": 0, + f"{args_logger},failover_v2,kms_encryption": 2, + f"{args_logger},failover_v2,kms_encryption data keys": 0}, \ + "with failover_v2 before kms_encryption, no DEBUG record holds a bound plaintext or a data key, while a " \ + "plugin that logs its arguments shows the scan finds both values" diff --git a/tests/integration/container/test_kms_encryption_statements.py b/tests/integration/container/test_kms_encryption_statements.py new file mode 100644 index 00000000..3f1f65c9 --- /dev/null +++ b/tests/integration/container/test_kms_encryption_statements.py @@ -0,0 +1,519 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Statements whose text decides what the KMS Encryption Plugin does, on a real server. + +The plugin's reading of the SQL has to agree with the server's, so each test checks what the server stored or +returned: a plain read of the table, or the stored bytes opened by a decoder outside the plugin. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Tuple + +import pytest +from django.db import connections, models +from psycopg import sql as pgsql +from sqlalchemy import (Column, Integer, MetaData, String, Table, + create_engine, select) +from sqlalchemy.engine import URL +from sqlalchemy.types import UserDefinedType + +from aws_advanced_python_wrapper.errors import AwsWrapperError, MetadataError +from .utils.conditions import disable_on_engines, enable_on_features +from .utils.database_engine import DatabaseEngine +from .utils.kms_encryption_helper import (DB_ERRORS, TRIGGER_REJECTION, + binary_type, create_table, + drop_table, encrypted_table, is_pg, + kms_builds, plain_connect, raised, + wrapper_connect, write) +from .utils.test_environment import TestEnvironment +from .utils.test_environment_features import TestEnvironmentFeatures + +if TYPE_CHECKING: + import io + + from aws_advanced_python_wrapper import AwsWrapperConnection + from .utils.kms_encryption_helper import (DbUser, KmsSettings, + PayloadVerifier) + +pytestmark = [pytest.mark.kms_encryption, + enable_on_features([TestEnvironmentFeatures.RUN_KMS_ENCRYPTION_TESTS_ONLY, + TestEnvironmentFeatures.KMS_LOCALSTACK])] + +_TABLE = "kms_statements_users" +# Two encrypted tables and a plain one for the recursive CTE and the outer join. +_FIRST, _SECOND, _PLAIN = "kms_statements_first", "kms_statements_second", "kms_statements_plain" +_INSERT = f"INSERT INTO {_TABLE} (name, ssn) VALUES (%s, %s)" +_SELECT = f"SELECT ssn FROM {_TABLE} WHERE name = %s ORDER BY id" + + +class _EncryptedValue(UserDefinedType): + """The guide's pass-through type on PostgreSQL, where SQLAlchemy would cast a typed parameter that bytea + rejects.""" + cache_ok = True + + def get_col_spec(self, **kw: Any) -> str: + return "BYTEA" + + +_USERS = Table(_TABLE, MetaData(), Column("id", Integer, primary_key=True), Column("name", String(100)), + Column("ssn", _EncryptedValue().with_variant(String(1024), "mysql"))) + + +@pytest.fixture(scope="module") +def key_rows(kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser) -> Iterator[Dict[str, int]]: + """The module's tables: ``ssn`` encrypted in the users table, whose trigger refuses plaintext, and in the two CTE + tables, under key rows of their own; yields each table's ``key_storage`` id.""" + binary = binary_type() + with encrypted_table(_TABLE, f"name VARCHAR(100) UNIQUE, note VARCHAR(100), ssn {binary}, alt {binary}", ("ssn",), + kms_settings, kms_schema, kms_master_key_arn, triggered=("ssn",)) as users, \ + encrypted_table(_FIRST, f"ssn {binary}", ("ssn",), kms_settings, kms_schema, kms_master_key_arn) as first, \ + encrypted_table(_SECOND, f"ssn {binary}", ("ssn",), kms_settings, kms_schema, kms_master_key_arn) as second: + create_table(_PLAIN, "ssn VARCHAR(100)") + try: + yield {_TABLE: users["ssn"], _FIRST: first["ssn"], _SECOND: second["ssn"]} + finally: + drop_table(_PLAIN) + + +@pytest.fixture(scope="module") +def django_users(kms_django_settings: None, key_rows: Dict[str, int], kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str, kms_app_user: DbUser) -> Iterator[Dict[str, Any]]: + """A Django model of the users table and, for each MySQL build, a database alias of the module's own with the + plugin; ``default`` is never changed.""" + env = TestEnvironment.get_current() + info, writer = env.get_database_info(), env.get_writer() + database = {"ENGINE": "aws_advanced_python_wrapper.django.backends.mysql_connector", + "NAME": info.get_default_db_name(), "HOST": writer.get_host(), "PORT": writer.get_port()} + + class StatementsUser(models.Model): + name: models.CharField[str, str] = models.CharField(max_length=100) + ssn: models.CharField[str, str] = models.CharField(max_length=100) + + class Meta: + app_label = "kms_statements" + db_table = _TABLE + managed = False + + aliases = {build: f"kms_statements_{build.replace('-', '_')}" for build in ("mysql-cext", "mysql-pure")} + for build, alias in aliases.items(): + connections.databases[alias] = { + **connections["default"].settings_dict, **database, "USER": kms_app_user.name, + "PASSWORD": kms_app_user.password, + "OPTIONS": {"plugins": "kms_encryption", "encryption_metadata_schema": kms_schema, + "encryption_allowed_master_key_arns": kms_master_key_arn, "use_pure": build == "mysql-pure", + **kms_settings.props()}} + try: + yield {"model": StatementsUser, "aliases": aliases} + finally: + for alias in aliases.values(): + connections[alias].close() + del connections[alias] + del connections.databases[alias] + + +@pytest.fixture +def conn(build: str, key_rows: Dict[str, int], kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser) -> Iterator[AwsWrapperConnection]: + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as conn: + yield conn + + +def _table_rows(build: str) -> List[Tuple[Any, ...]]: + """Every row of the users table as the server stores it.""" + with plain_connect(build=build) as plain, plain.cursor() as cur: + cur.execute(f"SELECT id, name, note, ssn FROM {_TABLE} ORDER BY id") + return [tuple(bytes(v) if isinstance(v, (bytes, bytearray, memoryview)) else v for v in row) + for row in cur.fetchall()] + + +def _plaintext_stored(build: str, values: List[str]) -> List[str]: + """The values whose plaintext a stored ``ssn`` holds.""" + stored = [bytes(row[3]) for row in _table_rows(build) if row[3] is not None] + return [value for value in values if any(value.encode() in payload for payload in stored)] + + +def _literal_shapes() -> Dict[str, str]: + """Literals and comments to put before a parameter, each ended by the server's own lexing rules.""" + shapes = {"doubled-quote": "'it''s'", "semicolon": "'a;b'", "block-comment": "/* it's; */ 'c'", + "line-comment": "-- it's;\n 'd'"} + if is_pg(): + shapes.update({"escape-string": "E'it\\'s'", "dollar": "$$it's$$", "tagged-dollar": "$tag$a;b$tag$"}) + else: + shapes.update({"backslash-escape": "'it\\'s'", "hash-comment": "# it's;\n 'e'"}) + return shapes + + +def _upserts(build: str) -> List[Tuple[str, str, Any, str]]: + """Upserts: (name, SQL, parameters, the value the row ends with); a name ending in -existing is written first.""" + t = _TABLE + if is_pg(): + return [(f"it10-conflict-new-{build}", f"INSERT INTO {t} (name, ssn) VALUES (%s, %s) " + f"ON CONFLICT (name) DO UPDATE SET ssn = %s", ("pt-it10-ins", "pt-it10-upd"), "pt-it10-ins"), + (f"it10-conflict-existing-{build}", f"INSERT INTO {t} (name, ssn) VALUES (%s, %s) " + f"ON CONFLICT (name) DO UPDATE SET ssn = %s", ("pt-it10-ins", "pt-it10-upd"), "pt-it10-upd"), + (f"it10-merge-new-{build}", f"MERGE INTO {t} USING (SELECT %s::text AS n) s ON {t}.name = s.n " + f"WHEN MATCHED THEN UPDATE SET ssn = %s WHEN NOT MATCHED THEN INSERT (name, ssn) VALUES (s.n, %s)", + ("pt-it10-upd", "pt-it10-ins"), "pt-it10-ins"), + (f"it10-merge-existing-{build}", f"MERGE INTO {t} USING (SELECT %s::text AS n) s ON {t}.name = s.n " + f"WHEN MATCHED THEN UPDATE SET ssn = %s WHEN NOT MATCHED THEN INSERT (name, ssn) VALUES (s.n, %s)", + ("pt-it10-upd", "pt-it10-ins"), "pt-it10-upd"), + (f"it10-cte-{build}", f"WITH w AS (INSERT INTO {t} (name, ssn) VALUES (%s, %s) RETURNING id) " + f"SELECT id FROM w", ("pt-it10-ins",), "pt-it10-ins")] + odku = f"INSERT INTO {t} (name, ssn) VALUES (%s, %s) ON DUPLICATE KEY UPDATE ssn = %s" + return [(f"it10-odku-new-{build}", odku, ("pt-it10-ins", "pt-it10-upd"), "pt-it10-ins"), + (f"it10-odku-existing-{build}", odku, ("pt-it10-ins", "pt-it10-upd"), "pt-it10-upd")] + + +def _non_default_quoting() -> Dict[str, Tuple[str, List[str]]]: + """Each session setting that changes how the server reads quotes: the statement that sets it, and writes that the + default rules read differently (each binds name and ssn). Under ANSI_QUOTES, ``"name"`` in the value list is the + name column, set by the value before it, so the server runs the write.""" + t = _TABLE + if is_pg(): + return {"standard_conforming_strings=off": ( + "SET standard_conforming_strings = off", + [f"INSERT INTO {t} (name, note, ssn) VALUES (%s, 'it\\'s', %s)", + f"INSERT INTO {t} (name, note, ssn) VALUES (%s, 'a\\\\', %s)"])} + return {"NO_BACKSLASH_ESCAPES": ( + "SET SESSION sql_mode = CONCAT(@@SESSION.sql_mode, ',NO_BACKSLASH_ESCAPES')", + [f"INSERT INTO {t} (name, note, ssn) VALUES (%s, 'a\\', %s)", + f"INSERT INTO {t} (name, note, ssn) VALUES (%s, 'a\\', %s) /* ' */"]), + "ANSI_QUOTES": ( + "SET SESSION sql_mode = CONCAT(@@SESSION.sql_mode, ',ANSI_QUOTES')", + [f'INSERT INTO {t} ("name", "ssn") VALUES (%s, %s)', f'INSERT INTO {t} (name, note, ssn) VALUES (%s, "name", %s)'])} + + +@kms_builds() +def test_statement_text_the_server_reads_its_own_way(build: str, conn: AwsWrapperConnection, key_rows: Dict[str, int], + kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str, kms_app_user: DbUser, + kms_verifier: PayloadVerifier) -> None: + want: Dict[str, str] = {} + with conn.cursor() as cur: + for label, literal in _literal_shapes().items(): + name, value = f"it10-{label}-{build}", f"pt-it10-{label}" + cur.execute(f"INSERT INTO {_TABLE} (name, note, ssn) VALUES (%s, {literal}, %s)", (name, value)) + want[name] = value + if is_pg(): + for label, sql in (("b", f"INSERT INTO {_TABLE} (name, ssn, note) VALUES (%s, %b, %s)"), + ("t", f"INSERT INTO {_TABLE} (name, ssn, note) VALUES (%t, %t, %b)")): + name, value = f"it10-placeholder-{label}-{build}", f"pt-it10-placeholder-{label}" + cur.execute(sql, (name, value, "note")) + want[name] = value + for name, sql, params, ends_with in _upserts(build): + if name.endswith(f"existing-{build}"): + cur.execute(_INSERT, (name, "pt-it10-before")) + cur.execute(sql, (name, *params)) + if cur.description is not None: + cur.fetchall() + want[name] = ends_with + conn.commit() + opened = {name: [value for _, value in kms_verifier.opened(_TABLE, "ssn", name)] for name in want} + assert opened == {name: [value] for name, value in want.items()}, \ + "a parameter after a literal or comment the server ends its own way, a %b or %t placeholder, and an " \ + "upsert's values are each stored as a payload of the value bound" + + +@kms_builds() +def test_write_under_non_default_quoting_never_stores_plaintext(build: str, key_rows: Dict[str, int], + kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str, kms_app_user: DbUser) -> None: + # The plugin reads SQL with the default quoting rules, so it can miss these writes' values (the guide lists the + # settings as unsupported); the trigger refuses the plaintext they would store. + sent: List[str] = [] + outcomes: Dict[str, str] = {} + for setting, (statement, shapes) in _non_default_quoting().items(): + with wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn) as quoted: + quoted.autocommit = True # so a failed write keeps the session's setting on PostgreSQL + with quoted.cursor() as cur: + cur.execute(statement) + for i, sql in enumerate(shapes): + name, value = f"it10-{setting}-{i}-{build}", f"pt-it10-{setting}-{i}-{build}" + sent.append(value) + try: + cur.execute(sql, (name, value)) + outcomes[name] = "stored" + except Exception as err: # noqa: BLE001 - the outcome is checked below + if isinstance(err, AwsWrapperError): + outcomes[name] = "refused" + elif TRIGGER_REJECTION in str(err): + outcomes[name] = "rejected" + else: + outcomes[name] = f"{type(err).__name__}: {err}" + assert _plaintext_stored(build, sent) == [], \ + f"a write under non-default quoting never stores plaintext (outcomes: {outcomes})" + assert set(outcomes.values()) <= {"stored", "refused", "rejected"}, \ + f"each write is stored, refused by the plugin or rejected by the trigger, never failed otherwise: {outcomes}" + + +@kms_builds() +def test_refused_statements_leave_no_row(build: str, conn: AwsWrapperConnection, key_rows: Dict[str, int]) -> None: + """Statements refused before sending leave no row written, changed or deleted.""" + with conn.cursor() as cur: + cur.executemany(_INSERT, [(f"it11-1-{build}", "pt-it11-1"), (f"it11-2-{build}", "pt-it11-2")]) + conn.commit() + t, before = _TABLE, _table_rows(build) + statements = [(f"INSERT INTO {t} (name, ssn) VALUES (%s, 'pt-it11-literal')", (f"it11-literal-{build}",)), + (f"DELETE FROM {t} WHERE ssn <> %s", ("pt-it11-neq",)), + (f"DELETE FROM {t} WHERE ssn != %s", ("pt-it11-bang",)), + (f"UPDATE {t} SET note = %s WHERE ssn < %s", ("it11-changed", "pt-it11-lt")), + (f"DELETE FROM {t} WHERE ssn > %s", ("pt-it11-gt",)), + (f"DELETE FROM {t} WHERE ssn BETWEEN %s AND %s", ("pt-it11-low", "pt-it11-high")), + (f"DELETE FROM {t} WHERE ssn NOT IN (%s)", ("pt-it11-not-in",)), + (f"DELETE FROM {t} WHERE ssn NOT LIKE %s", ("pt-it11-not-like",)), + (f"DELETE FROM {t} WHERE NOT (ssn = %s)", ("pt-it11-not",)), + (f"UPDATE {t} SET note = %s WHERE id IN (SELECT id FROM (SELECT u.id FROM {_PLAIN} p " + f"RIGHT JOIN {t} u ON u.ssn = %s) j)", ("it11-changed", "pt-it11-outer-join"))] + not_refused: List[str] = [] + for sql, params in statements: + try: + with conn.cursor() as cur: + cur.execute(sql, params) + except MetadataError: + conn.rollback() + continue + conn.commit() + not_refused.append(sql) + assert _table_rows(build) == before, "a refused statement leaves no row written, changed or deleted" + assert not_refused == [], "a literal in an encrypted column and each comparison the plugin can't make safe raise MetadataError" + + +@kms_builds() +def test_equality_on_an_encrypted_column_matches_no_row(build: str, conn: AwsWrapperConnection, + key_rows: Dict[str, int], kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, + kms_app_user: DbUser) -> None: + name, value = f"it12-{build}", "pt-it12" + with conn.cursor() as cur: + cur.execute(_INSERT, (name, value)) + conn.commit() + cur.execute(f"SELECT name FROM {_TABLE} WHERE ssn = %s", (value,)) + dbapi = cur.fetchall() + env = TestEnvironment.get_current() + info, writer = env.get_database_info(), env.get_writer() + url = URL.create("postgresql+aws_wrapper_psycopg" if is_pg() else "mysql+aws_wrapper_mysqlconnector", + username=kms_app_user.name, password=kms_app_user.password, host=writer.get_host(), + port=writer.get_port(), database=info.get_default_db_name()) + props: Dict[str, Any] = {"plugins": "kms_encryption", "encryption_metadata_schema": kms_schema, + "encryption_allowed_master_key_arns": kms_master_key_arn, **kms_settings.props()} + if not is_pg(): + props["use_pure"] = build == "mysql-pure" + engine = create_engine(url, connect_args=props) # a ?plugins= in the URL would be dropped + try: + with engine.connect() as sa_conn: + orm = sa_conn.execute(select(_USERS.c.name).filter_by(ssn=value)).all() + by_name = sa_conn.execute(select(_USERS.c.ssn).filter_by(name=name)).all() + finally: + engine.dispose() + assert (dbapi, orm) == ([], []), \ + "an equality on an encrypted column, through DB-API and SQLAlchemy filter_by, is sent encrypted and matches no row" + assert [tuple(row) for row in by_name] == [(value,)], "the row is there, and SQLAlchemy reads it decrypted" + + +@disable_on_engines([DatabaseEngine.PG]) +@kms_builds() +def test_django_equality_on_an_encrypted_column_matches_no_row(build: str, conn: AwsWrapperConnection, + django_users: Dict[str, Any]) -> None: + name, value = f"it12-django-{build}", "pt-it12-django" + with conn.cursor() as cur: + cur.execute(_INSERT, (name, value)) + conn.commit() + users = django_users["model"].objects.using(django_users["aliases"][build]) + assert (users.filter(ssn=value).count(), users.filter(name=name).count()) == (0, 1), \ + "a Django filter on an encrypted column is sent encrypted and matches no row, while the row is there" + + +@kms_builds() +def test_annotated_placeholder_is_stored_encrypted(build: str, conn: AwsWrapperConnection, key_rows: Dict[str, int], + kms_verifier: PayloadVerifier) -> None: + plain, mapped, overriding = (f"it13-{shape}-{build}" for shape in ("plain", "mapped", "overriding")) + t = _TABLE + with conn.cursor() as cur: + cur.execute(f"INSERT INTO {t} (name, ssn) VALUES (%s, /*@encrypt:{t}.ssn*/ %s)", (plain, "pt-it13-plain")) + cur.execute(f"INSERT INTO {t} (name, note, ssn) VALUES ('{mapped}', %s, /*@encrypt:{t}.ssn*/ %s)", + ("it13-note", "pt-it13-mapped")) + # The column list maps this value to the plain column alt; the annotation names ssn. + cur.execute(f"INSERT INTO {t} (name, alt) VALUES (%s, /*@encrypt:{t}.ssn*/ %s)", + (overriding, "pt-it13-overriding")) + conn.commit() + ssn = key_rows[_TABLE] + # alt has no metadata row of its own, so its payload is opened as ssn's. + overriding_stored = [kms_verifier.decode(_TABLE, "ssn", payload) for payload in + kms_verifier.stored(f"SELECT alt FROM {_TABLE} WHERE name = %s ORDER BY id", (overriding,))] + assert (kms_verifier.opened(_TABLE, "ssn", plain), kms_verifier.opened(_TABLE, "ssn", mapped), + overriding_stored) == \ + ([(ssn, "pt-it13-plain")], [(ssn, "pt-it13-mapped")], [(ssn, "pt-it13-overriding")]), \ + "a placeholder annotated /*@encrypt:table.ssn*/ is stored as a payload under ssn's key row, also where the " \ + "column list names a plain column" + + +@kms_builds() +def test_prepared_statements_and_other_placeholder_styles(build: str, conn: AwsWrapperConnection, + key_rows: Dict[str, int], + kms_verifier: PayloadVerifier) -> None: + prepared, numbered = f"it14-prepared-{build}", f"it14-numbered-{build}" + if is_pg(): + with conn.cursor() as cur: + cur.execute(_INSERT, (prepared, "pt-it14-prepared"), prepare=True) + numbered_sql = f"INSERT INTO {_TABLE} (name, ssn) VALUES ($1, $2)" + numbered_cursor: Dict[str, Any] = {} + else: + with conn.cursor(prepared=True) as cur: + cur.execute(_INSERT, (prepared, "pt-it14-prepared")) + numbered_sql = f"INSERT INTO {_TABLE} (name, ssn) VALUES (?, ?)" + numbered_cursor = {"prepared": True} + conn.commit() + with pytest.raises(MetadataError), conn.cursor(**numbered_cursor) as cur: + cur.execute(numbered_sql, (numbered, "pt-it14-numbered")) + conn.rollback() + assert kms_verifier.opened(_TABLE, "ssn", prepared) == [(key_rows[_TABLE], "pt-it14-prepared")], \ + "a server-side prepared write with %s placeholders stores a payload, as a plain execute does" + assert kms_verifier.stored(_SELECT, (numbered,)) == [], \ + "a write naming an encrypted column with $1 or ? placeholders is refused before anything is sent" + + +@kms_builds() +def test_recursive_cte_named_like_a_table(build: str, conn: AwsWrapperConnection, key_rows: Dict[str, int]) -> None: + with plain_connect(build=build) as plain, plain.cursor() as cur: + for table in (_FIRST, _SECOND, _PLAIN): + cur.execute(f"DELETE FROM {table}") + cur.execute(f"INSERT INTO {_PLAIN} (ssn) VALUES ('it15-plain')") + plain.commit() + with conn.cursor() as cur: + cur.execute(f"INSERT INTO {_FIRST} (ssn) VALUES (%s)", ("pt-it15-first",)) + cur.execute(f"INSERT INTO {_SECOND} (ssn) VALUES (%s)", ("pt-it15-second",)) + conn.commit() + with plain_connect(build=build) as plain, plain.cursor() as cur: + cur.execute(f"SELECT ssn FROM {_FIRST}") + (first_stored,) = [bytes(value) for (value,) in cur.fetchall()] + cur.execute(f"SELECT ssn FROM {_SECOND}") + (second_stored,) = [bytes(value) for (value,) in cur.fetchall()] + # a reads the relation named like the later CTE: the CTE on PostgreSQL, the table on MySQL. + reads: Dict[str, List[Any]] = {} + for source in (_SECOND, _PLAIN): + with conn.cursor() as cur: + cur.execute(f"WITH RECURSIVE a AS (SELECT ssn FROM {_FIRST}), {_FIRST} AS (SELECT ssn FROM {source}) " + f"SELECT ssn FROM a") + reads[source] = [bytes(v) if isinstance(v, (bytes, bytearray)) else v for (v,) in cur.fetchall()] + # Nested under an outer CTE of the same name, c reads the inner, later one on PostgreSQL and the outer one on MySQL; + # the statement also reads the users table, so by name alone ssn matches two encrypted columns. + with conn.cursor() as cur: + cur.execute(f"WITH {_FIRST} AS (SELECT ssn FROM {_SECOND}) SELECT z.ssn FROM (WITH RECURSIVE c AS " + f"(SELECT ssn FROM {_FIRST}), {_FIRST} AS (SELECT ssn FROM {_PLAIN}) SELECT ssn FROM c) z " + f"WHERE NOT EXISTS (SELECT 1 FROM {_TABLE} WHERE 1 = 0)") + reads["nested"] = [bytes(v) if isinstance(v, (bytes, bytearray)) else v for (v,) in cur.fetchall()] + if is_pg(): + assert reads == {_SECOND: ["pt-it15-second"], _PLAIN: ["it15-plain"], "nested": ["it15-plain"]}, \ + "on PostgreSQL a reads the later CTE, and its value comes back decrypted, or as written from a plain " \ + "table; nested, the plugin doesn't take c for the outer CTE" + else: + assert reads == {_SECOND: ["pt-it15-first"], _PLAIN: [first_stored], "nested": [second_stored]}, \ + "on MySQL a reads the table: its value decrypts by name when exactly one other table read has an " \ + "encrypted ssn, and comes back as stored bytes when none has or two have" + + +# target_cursor is the driver's cursor, which the plugin doesn't see. + +@kms_builds() +def test_target_cursor_is_the_driver_cursor(build: str, conn: AwsWrapperConnection) -> None: + with conn.cursor() as cur, conn.target_connection.cursor() as own: + assert type(cur.target_cursor) is type(own), "a wrapper cursor's target_cursor is the driver's own cursor" + + +@kms_builds() +@pytest.mark.parametrize("how", ["close", "leave-with"]) +def test_closing_the_cursor_closes_the_driver_cursor(build: str, how: str, conn: AwsWrapperConnection) -> None: + if how == "close": + cur = conn.cursor() + target: Any = cur.target_cursor + cur.close() + else: + with conn.cursor() as cur: + target = cur.target_cursor + closed = isinstance(raised(conn, lambda: target.execute("SELECT 1")), DB_ERRORS) + assert closed, "closing the wrapper cursor, or leaving its with-block, closes the driver cursor" + + +@kms_builds() +def test_target_cursor_is_not_seen(build: str, conn: AwsWrapperConnection, key_rows: Dict[str, int], + kms_verifier: PayloadVerifier) -> None: + name, unseen = f"target-{build}", f"target-unseen-{build}" + write(conn, _INSERT, (name, "pt-target")) + with conn.cursor() as cur: + target: Any = cur.target_cursor + target.execute(_SELECT, (name,)) + stored = target.fetchone()[0] + target.fetchall() + with pytest.raises(DB_ERRORS, match=TRIGGER_REJECTION): + target.execute(_INSERT, (unseen, "pt-target-unseen-" + "x" * 64)) + conn.rollback() + assert kms_verifier.opened(_TABLE, "ssn", name) == [(key_rows[_TABLE], "pt-target")], \ + "the wrapper cursor's write is stored as a payload" + assert bytes(stored) == bytes(kms_verifier.stored(_SELECT, (name,))[0]), \ + "a read on the driver cursor returns the stored payload, which the plugin doesn't decrypt" + assert kms_verifier.stored(_SELECT, (unseen,)) == [], \ + "a write on the driver cursor isn't encrypted, so the trigger rejects its plaintext with the driver's error" + + +# SQL that isn't str. + +@disable_on_engines([DatabaseEngine.MYSQL]) +@kms_builds() +def test_composable_insert_and_select(build: str, conn: AwsWrapperConnection, key_rows: Dict[str, int], + kms_verifier: PayloadVerifier) -> None: + name = f"composable-{build}" + with conn.cursor() as cur: + cur.execute(pgsql.SQL("INSERT INTO {} (name, ssn) VALUES ({}, {})").format( + pgsql.Identifier(_TABLE), pgsql.Placeholder(), pgsql.Placeholder()), (name, "pt-composable")) + conn.commit() + select = pgsql.SQL("SELECT ssn FROM {} WHERE name = {}").format(pgsql.Identifier(_TABLE), pgsql.Placeholder()) + cur.execute(select, (name,)) + got = cur.fetchone()[0] + assert kms_verifier.opened(_TABLE, "ssn", name) == [(key_rows[_TABLE], "pt-composable")], \ + "a psycopg Composable INSERT stores a payload" + assert got == "pt-composable", "a psycopg Composable SELECT reads the value decrypted" + + +@disable_on_engines([DatabaseEngine.MYSQL]) +@kms_builds() +def test_composed_insert_with_percent_placeholders_is_encrypted(build: str, conn: AwsWrapperConnection, + key_rows: Dict[str, int], + kms_verifier: PayloadVerifier) -> None: + name = f"composed-{build}" + with conn.cursor() as cur: + cur.execute(pgsql.SQL("INSERT INTO {} (name, ssn) VALUES (%s, %s)").format(pgsql.Identifier(_TABLE)), + (name, "pt-composed")) + conn.commit() + assert kms_verifier.opened(_TABLE, "ssn", name) == [(key_rows[_TABLE], "pt-composed")], \ + "a Composed statement with %s placeholders stores a payload" + + +@disable_on_engines([DatabaseEngine.MYSQL]) +@kms_builds() +def test_bytes_sql_is_analyzed_and_encrypted(build: str, conn: AwsWrapperConnection, key_rows: Dict[str, int], + kms_verifier: PayloadVerifier, kms_log_capture: io.StringIO) -> None: + names = [f"bytes-{build}-{i}" for i in range(2)] + start = len(kms_log_capture.getvalue()) + for i, name in enumerate(names): + with conn.cursor() as cur: + cur.execute(_INSERT.encode(), (name, f"pt-bytes-{i}")) + conn.commit() + plugin_records = kms_log_capture.getvalue()[start:].count("[KmsEncryptionPlugin]") + assert [kms_verifier.opened(_TABLE, "ssn", name) for name in names] == \ + [[(key_rows[_TABLE], f"pt-bytes-{i}")] for i in range(2)], \ + "SQL given as bytes is analyzed, so each write stores a payload" + assert plugin_records == 0, "the plugin logs no pass-through warning for SQL given as ASCII bytes" diff --git a/tests/integration/container/test_kms_encryption_triggers.py b/tests/integration/container/test_kms_encryption_triggers.py new file mode 100644 index 00000000..a079aa1e --- /dev/null +++ b/tests/integration/container/test_kms_encryption_triggers.py @@ -0,0 +1,346 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The KMS Encryption Plugin's database-side trigger, installed from the package's trigger SQL, on a live server.""" + +from __future__ import annotations + +import os +import secrets +from typing import Any, Callable, Iterator, NamedTuple + +import mysql.connector +import psycopg +import pytest + +from .utils.conditions import enable_on_engines, enable_on_features +from .utils.database_engine import DatabaseEngine +from .utils.kms_encryption_helper import (DB_ERRORS, TRIGGER_REJECTION, + KmsSettings, PayloadVerifier, + binary_type, create_column_trigger, + encrypt_column, install_trigger_sql, + is_pg, key_management_utility, + kms_builds, plain_connect, write, + writes) +from .utils.retry_helper import retry_until +from .utils.test_environment_features import TestEnvironmentFeatures + +pytestmark = [pytest.mark.kms_encryption, + enable_on_features([TestEnvironmentFeatures.RUN_KMS_ENCRYPTION_TESTS_ONLY, + TestEnvironmentFeatures.KMS_LOCALSTACK])] + +# The trigger's other rejection (TRIGGER_REJECTION is the first). PostgreSQL's messages go on to name the table and +# column; MySQL's name neither. +_NO_KEY = "No HMAC key is configured" + + +class EncryptedTable(NamedTuple): + """A table whose ``ssn`` and ``card`` columns are encrypted, each with its own key row, and triggered.""" + name: str + ssn_key_id: int + card_key_id: int + + def insert(self, column: str = "ssn") -> str: + return f"INSERT INTO {self.name} (name, {column}) VALUES (%s, %s)" + + +def _plain_write(build: str, sql: str, params: Any = None) -> None: + with plain_connect(build=build) as conn: + write(conn, sql, params) + + +def _assert_rejected(write: Callable[[], Any], message: str, conn: Any = None) -> str: + """Runs ``write``, which the trigger must reject with ``message``; rolls ``conn`` back and returns the error.""" + with pytest.raises(DB_ERRORS, match=message) as caught: + write() + if conn is not None: + conn.rollback() + return str(caught.value) + + +def _stored(verifier: PayloadVerifier, table: str, column: str, name: str) -> list: + return verifier.stored(f"SELECT {column} FROM {table} WHERE name = %s ORDER BY id", (name,)) + + +def _session_id(conn: Any) -> int: + with conn.cursor() as cur: + cur.execute("SELECT pg_backend_pid()" if is_pg() else "SELECT CONNECTION_ID()") + return cur.fetchone()[0] + + +def _drop_column_trigger(conn: Any, table: str, column: str) -> None: + if is_pg(): + statements = [f"DROP TRIGGER {table}_{column}_encrypted ON {table}"] + else: + statements = [f"DROP TRIGGER {table}_{column}_hmac_insert", f"DROP TRIGGER {table}_{column}_hmac_update"] + _run(conn, *statements) + + +def _run(conn: Any, *statements: str) -> None: + with conn.cursor() as cur: + for statement in statements: + cur.execute(statement) + conn.commit() + + +@pytest.fixture +def table(kms_trigger_installed: None, kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str) -> Iterator[EncryptedTable]: + """A new table: ``ssn`` and ``card`` triggered and encrypted, ``pin`` binary and plain, ``note`` text.""" + name = f"kms_trg_{secrets.token_hex(4)}" + id_type = "SERIAL PRIMARY KEY" if is_pg() else "INT AUTO_INCREMENT PRIMARY KEY" + binary = binary_type() + with plain_connect() as conn: + _run(conn, f"CREATE TABLE {name} (id {id_type}, name VARCHAR(100), ssn {binary}, card {binary}, " + f"pin {binary}, note VARCHAR(100))") + for column in ("ssn", "card"): + create_column_trigger(conn, name, column) + created = EncryptedTable(name, encrypt_column(conn, kms_settings, kms_schema, name, "ssn", kms_master_key_arn), + encrypt_column(conn, kms_settings, kms_schema, name, "card", kms_master_key_arn)) + try: + yield created + finally: + with plain_connect() as conn: + _run(conn, f"DROP TABLE IF EXISTS {name}") + + +@kms_builds() +def test_trigger_after_rotation(build: str, table: EncryptedTable, kms_connect: Callable[..., Any], + kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str, + kms_verifier: PayloadVerifier) -> None: + stale = kms_connect() + refreshing = kms_connect(encryption_metadata_cache_refresh_interval_sec=1) + try: + write(stale, table.insert(), ("before", "pt-rot-before")) + write(refreshing, table.insert(), ("refreshing-before", "pt-rot-refreshing")) + (old_payload,) = _stored(kms_verifier, table.name, "ssn", "before") + with plain_connect() as admin: + new_key_id = key_management_utility(admin, kms_settings, kms_schema).rotate_data_key( + table.name, "ssn", kms_master_key_arn) + admin.commit() + assert new_key_id != table.ssn_key_id, "rotation adds a key row" + + _assert_rejected(lambda: write(stale, table.insert(), ("stale", "pt-rot-stale")), TRIGGER_REJECTION, stale) + write(stale, f"UPDATE {table.name} SET note = %s WHERE name = %s", ("touched", "before")) + assert _stored(kms_verifier, table.name, "ssn", "before") == [old_payload], \ + "an UPDATE of another column on a row written before the rotation is accepted and keeps its value" + + assert retry_until(lambda: writes(refreshing, table.insert(), ("refreshed", "pt-rot-after")), + timeout_sec=15, delay_sec=0.25), \ + "a connection with a short refresh interval writes with the new key after its next refresh" + (new_payload,) = _stored(kms_verifier, table.name, "ssn", "refreshed") + assert kms_verifier.decode(table.name, "ssn", new_payload) == (new_key_id, "pt-rot-after"), \ + "the refreshed connection's write names the new key row" + with refreshing.cursor() as cur: + cur.execute(f"SELECT ssn FROM {table.name} WHERE name = %s", ("before",)) + assert cur.fetchall() == [("pt-rot-before",)], "a value written before the rotation still decrypts" + finally: + stale.close() + refreshing.close() + + +@kms_builds() +def test_writes_outside_the_plugin_reach_the_trigger(build: str, table: EncryptedTable, + kms_connect: Callable[..., Any], + kms_verifier: PayloadVerifier) -> None: + with kms_connect() as conn: + write(conn, table.insert(), ("seed", "pt-seen-seed")) + cur = conn.cursor() + no_list = f"INSERT INTO {table.name} VALUES ({'DEFAULT' if is_pg() else 'NULL'}, %s, %s, NULL, NULL, NULL)" + _assert_rejected(lambda: cur.execute(no_list, ("no-list", b"pt-unseen-no-list")), TRIGGER_REJECTION, conn) + if is_pg(): + def copy_in() -> None: + with cur.copy(f"COPY {table.name} (name, ssn) FROM STDIN") as copy: + copy.write_row(("copy", b"pt-unseen-copy")) + _assert_rejected(copy_in, TRIGGER_REJECTION, conn) + _assert_rejected(lambda: list(cur.stream(table.insert(), ("stream", b"pt-unseen-stream"))), TRIGGER_REJECTION, conn) + conn.target_connection.cursor_factory = psycopg.RawCursor + raw = conn.cursor() + conn.target_connection.cursor_factory = psycopg.Cursor + _assert_rejected(lambda: raw.execute(f"INSERT INTO {table.name} VALUES (DEFAULT, $1, $2, NULL, NULL, NULL)", + ("raw-no-list", b"pt-unseen-raw")), TRIGGER_REJECTION, conn) + cur.execute(f"PREPARE kms_insert (text, bytea) AS INSERT INTO {table.name} (name, ssn) VALUES ($1, $2)") + _assert_rejected(lambda: cur.execute("EXECUTE kms_insert ('prepared', 'pt-unseen-prepared')"), + TRIGGER_REJECTION, conn) + else: + prepared = conn.cursor(prepared=True) + _assert_rejected(lambda: prepared.execute(f"INSERT INTO {table.name} VALUES (NULL, ?, ?, NULL, NULL, NULL)", + ("prepared-no-list", b"pt-unseen-prepared")), TRIGGER_REJECTION, conn) + cur.execute(f"PREPARE kms_insert FROM 'INSERT INTO {table.name} (name, ssn) " + f"VALUES (''sql-prepare'', ''pt-unseen-sql'')'") + _assert_rejected(lambda: cur.execute("EXECUTE kms_insert"), TRIGGER_REJECTION, conn) + _assert_rejected(lambda: cur.execute(f"INSERT INTO {table.name} (name, ssn) VALUES ('bytes-é', %s)" + .encode("utf-8"), (b"pt-unseen-bytes",)), TRIGGER_REJECTION, conn) + cur.close() + names = kms_verifier.stored(f"SELECT name FROM {table.name} ORDER BY id") + assert names == ["seed"], "no write the plugin can't see or doesn't analyze stored its plaintext" + + +@kms_builds() +def test_packaged_trigger_accepts_payloads_and_rejects_plaintext(build: str, table: EncryptedTable, + kms_connect: Callable[..., Any], + kms_verifier: PayloadVerifier) -> None: + with plain_connect() as conn, conn.cursor() as cur: + if is_pg(): + cur.execute("SELECT obj_description('enforce_encrypted_column'::regproc, 'pg_proc')") + else: + cur.execute("SELECT ROUTINE_COMMENT FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA = DATABASE() " + "AND ROUTINE_NAME = 'validate_encrypted_data_hmac_before_insert'") + (stamp,) = cur.fetchone() + assert stamp.startswith("aws-advanced-python-wrapper kms_encryption trigger v"), \ + "the trigger SQL installed from the package records its version" + + with kms_connect() as conn: + write(conn, f"INSERT INTO {table.name} (name, ssn, card) VALUES (%s, %s, %s)", + ("good", "pt-trg-ssn", "pt-trg-card")) + _plain_write(build, table.insert(), ("null", None)) + (good_ssn,) = _stored(kms_verifier, table.name, "ssn", "good") + (good_card,) = _stored(kms_verifier, table.name, "card", "good") + good_ssn, good_card = bytes(good_ssn), bytes(good_card) + rejected = {"plaintext": b"pt-trg-plaintext-" + b"x" * 64, + "short": os.urandom(64), + "random": os.urandom(80), + "tag-flipped": bytes([good_ssn[0] ^ 1]) + good_ssn[1:], + "ciphertext-flipped": good_ssn[:-1] + bytes([good_ssn[-1] ^ 1]), + "other-column": good_card} + with plain_connect(build=build) as conn: + for label, value in rejected.items(): + inserted = _assert_rejected(lambda: write(conn, table.insert(), (label, value)), TRIGGER_REJECTION, conn) + updated = _assert_rejected(lambda: write(conn, f"UPDATE {table.name} SET ssn = %s WHERE name = 'null'", + (value,)), TRIGGER_REJECTION, conn) + leaks = any("pt-trg" in message for message in (inserted, updated)) + assert not leaks, f"the trigger's error for the {label} value carries none of the statement's values" + if is_pg(): + assert f"{table.name}.ssn" in inserted, "PostgreSQL's error names the table and column" + create_column_trigger(conn, table.name, "pin") + _assert_rejected(lambda: write(conn, f"INSERT INTO {table.name} (name, pin) VALUES (%s, %s)", + ("no-key", good_ssn)), _NO_KEY, conn) + write(conn, f"INSERT INTO {table.name} (name, pin) VALUES (%s, %s)", ("no-key-null", None)) + assert kms_verifier.stored(f"SELECT name FROM {table.name} ORDER BY id") == ["good", "null", "no-key-null"], \ + "only the plugin's payload and NULL are stored" + assert _stored(kms_verifier, table.name, "ssn", "null") == [None], "a rejected UPDATE leaves the row as it was" + + +@enable_on_engines([DatabaseEngine.PG]) +@kms_builds() +def test_pg_trigger_ignores_a_hostile_search_path(build: str, table: EncryptedTable, + kms_verifier: PayloadVerifier) -> None: + hostile = f"kms_hostile_{secrets.token_hex(4)}" + with plain_connect() as conn: + _run(conn, f"CREATE SCHEMA {hostile}", + f"CREATE FUNCTION {hostile}.never_differs(bytea, bytea) RETURNS boolean LANGUAGE sql IMMUTABLE " + "AS 'SELECT false'", + f"CREATE OPERATOR {hostile}.<> (LEFTARG = bytea, RIGHTARG = bytea, FUNCTION = {hostile}.never_differs)", + f"CREATE FUNCTION {hostile}.hmac(bytea, bytea, text) RETURNS bytea LANGUAGE sql IMMUTABLE " + "AS 'SELECT substring($1 from 1 for 32)'") + try: + with plain_connect() as conn: + _run(conn, f"SET search_path = {hostile}, pg_catalog, public") + _assert_rejected(lambda: write(conn, table.insert(), ("hostile", b"pt-hostile-" + b"x" * 80)), + TRIGGER_REJECTION, conn) + finally: + with plain_connect() as conn: + _run(conn, f"DROP SCHEMA {hostile} CASCADE") + assert kms_verifier.stored(f"SELECT name FROM {table.name}") == [], \ + "a session's own hmac() and <> ahead of pg_catalog get no plaintext past the trigger" + + +@kms_builds("mysql-pure") +def test_removing_a_column_without_dropping_its_trigger_fails_every_write( + build: str, table: EncryptedTable, kms_connect: Callable[..., Any], kms_settings: KmsSettings, + kms_schema: str, kms_verifier: PayloadVerifier) -> None: + with plain_connect() as conn: + _drop_column_trigger(conn, table.name, "ssn") + assert key_management_utility(conn, kms_settings, kms_schema).remove_encryption_for_column( + table.name, "ssn"), "the utility removes ssn's metadata row" + conn.commit() + assert key_management_utility(conn, kms_settings, kms_schema).remove_encryption_for_column( + table.name, "card"), "the utility removes card's metadata row" + conn.commit() + _plain_write(build, table.insert(), ("removed", b"pt-removed-ssn")) + assert _stored(kms_verifier, table.name, "ssn", "removed") == [b"pt-removed-ssn"], \ + "the guide's removal (drop the triggers, remove the metadata, commit) leaves the column writable" + + with plain_connect(build=build) as conn: + _assert_rejected(lambda: write(conn, table.insert("card"), ("kept-trigger", b"pt-kept-trigger")), + _NO_KEY, conn) + with kms_connect() as conn: + _assert_rejected(lambda: write(conn, table.insert("card"), ("kept-plugin", "pt-kept-plugin")), _NO_KEY, conn) + assert kms_verifier.stored(f"SELECT name FROM {table.name} WHERE card IS NOT NULL") == [], \ + "with the metadata removed and the trigger kept, every non-NULL write fails with 'No HMAC key'" + + +@kms_builds() +def test_trigger_across_transactions_and_rotation(build: str, table: EncryptedTable, + kms_connect: Callable[..., Any], kms_settings: KmsSettings, + kms_schema: str, kms_master_key_arn: str, + kms_verifier: PayloadVerifier) -> None: + # Without the metadata cache, each statement reads the column's current key row, so the writer is never stale. + with kms_connect(encryption_metadata_cache_enabled=False) as writer: + session_before = _session_id(writer) + write(writer, table.insert(), ("tx1", "pt-tx1")) + write(writer, table.insert(), ("tx2", "pt-tx2")) + with plain_connect() as admin: + new_key_id = key_management_utility(admin, kms_settings, kms_schema).rotate_data_key( + table.name, "ssn", kms_master_key_arn) + admin.commit() + write(writer, table.insert(), ("tx3", "pt-tx3")) + session_after = _session_id(writer) + assert session_before == session_after, "the three transactions ran in one server session" + decoded = [kms_verifier.decode(table.name, "ssn", payload) + for payload in kms_verifier.stored(f"SELECT ssn FROM {table.name} ORDER BY id")] + assert decoded == [(table.ssn_key_id, "pt-tx1"), (table.ssn_key_id, "pt-tx2"), (new_key_id, "pt-tx3")], \ + "the trigger accepted every transaction's write, the last one under the rotated key" + + +@enable_on_engines([DatabaseEngine.MYSQL]) +@kms_builds() +def test_mysql_trigger_install_without_trust_fails_with_1419(build: str, kms_trigger_installed: None, + kms_schema: str) -> None: + # Imported here, so a host without mysql-connector's C extension skips this test instead of failing the module. + _mysql_connector = pytest.importorskip("_mysql_connector") + with plain_connect(admin=True) as root, root.cursor() as cur: + cur.execute("SELECT @@log_bin") + assert cur.fetchone()[0] == 1, "the server writes a binary log, which is what makes the trust setting count" + cur.execute("SET GLOBAL log_bin_trust_function_creators = 0") + try: + with plain_connect(build=build) as conn: + # The C extension raises its own MySQLInterfaceError from inside the multi-statement script. + with pytest.raises((mysql.connector.Error, _mysql_connector.MySQLInterfaceError), + match="SUPER privilege and binary logging") as caught: + install_trigger_sql(conn, kms_schema) + assert getattr(caught.value, "errno", None) == 1419, \ + "a user without SUPER can't install the routines while log_bin_trust_function_creators is 0" + finally: + with plain_connect(admin=True) as root, root.cursor() as cur: + cur.execute("SET GLOBAL log_bin_trust_function_creators = 1") + with plain_connect(build=build) as conn: + install_trigger_sql(conn, kms_schema) # succeeds once the setting is 1 again + + +@kms_builds("mysql-pure") +def test_value_copied_to_another_row_passes_and_to_another_column_fails( + build: str, table: EncryptedTable, kms_connect: Callable[..., Any]) -> None: + with kms_connect() as conn: + write(conn, f"INSERT INTO {table.name} (name, ssn, card) VALUES (%s, %s, %s)", + ("source", "pt-copy-ssn", "pt-copy-card")) + with plain_connect(build=build) as conn: + write(conn, f"INSERT INTO {table.name} (name, ssn) SELECT 'copy', ssn FROM {table.name} WHERE name = 'source'") + _assert_rejected(lambda: write(conn, f"UPDATE {table.name} SET card = ssn WHERE name = 'source'"), + TRIGGER_REJECTION, conn) + with kms_connect() as conn, conn.cursor() as cur: + cur.execute(f"SELECT name, ssn, card FROM {table.name} ORDER BY id") + rows = cur.fetchall() + assert rows == [("source", "pt-copy-ssn", "pt-copy-card"), ("copy", "pt-copy-ssn", None)], \ + "a value copied to another row of its column decrypts there; one copied to another column is rejected" diff --git a/tests/integration/container/utils/kms_encryption_helper.py b/tests/integration/container/utils/kms_encryption_helper.py new file mode 100644 index 00000000..7779048b --- /dev/null +++ b/tests/integration/container/utils/kms_encryption_helper.py @@ -0,0 +1,690 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Fixtures and helpers for the KMS Encryption Plugin container modules. + +Each KMS module starts with:: + + pytestmark = [pytest.mark.kms_encryption, + enable_on_features([TestEnvironmentFeatures.RUN_KMS_ENCRYPTION_TESTS_ONLY, + TestEnvironmentFeatures.KMS_LOCALSTACK])] + +so that every other lane skips it, and ``conftest.py`` skips every other module in a KMS environment; a module that +needs a real key or Aurora names those features instead of ``KMS_LOCALSTACK``. +``conftest.py`` registers this module as a pytest plugin, so a module only names the fixtures as arguments. +The test user is the PostgreSQL superuser; on Docker MySQL the administrator is root, which has the test password, +and on Aurora it is the test user, the cluster's admin user. +Plugin connections use a DML-only application user (``kms_app_user``); the administrator only sets up and reads logs. +Connection failures become ``pytest.fail`` messages, because pytest's report prints a raising frame's arguments, and +the driver's connect frame holds the password. +The module's report hook prints the test password, each application user's password and a real key's account id and +key id as ``***`` in every report, captured output included, because CI logs are public. +""" + +from __future__ import annotations + +import io +import logging +import os +import re +import secrets +from contextlib import contextmanager +from dataclasses import dataclass, field +from importlib.resources import files +from typing import (Any, Callable, Dict, Generator, Iterator, List, Optional, + Sequence, Set, Tuple) + +import boto3 +import django +import mysql.connector +import psycopg +import pytest +from django.conf import settings as django_settings + +from aws_advanced_python_wrapper import AwsWrapperConnection +from aws_advanced_python_wrapper.encryption.config import EncryptionConfig +from aws_advanced_python_wrapper.encryption.connection_source import \ + CallerConnectionSource +from aws_advanced_python_wrapper.encryption.data_key_cache import DataKeyCache +from aws_advanced_python_wrapper.encryption.encryption_service import \ + EncryptionService +from aws_advanced_python_wrapper.encryption.key_management_utility import \ + KeyManagementUtility +from aws_advanced_python_wrapper.encryption.key_manager import KeyManager +from aws_advanced_python_wrapper.encryption.metadata_manager import \ + MetadataManager +from aws_advanced_python_wrapper.encryption.sql_dialect import SqlDialect +from aws_advanced_python_wrapper.errors import AwsWrapperError +from aws_advanced_python_wrapper.utils.properties import Properties +from .database_engine import DatabaseEngine +from .database_engine_deployment import DatabaseEngineDeployment +from .kms_encryption_schema import ddl_for +from .retry_helper import retry_until +from .test_environment import TestEnvironment +from .test_environment_features import TestEnvironmentFeatures + +# The class of the driver connection each MySQL build must produce, so a missing C extension fails the run instead of +# running the pure build twice. +_MYSQL_CONNECTION_CLASS = {"mysql-cext": "CMySQLConnection", "mysql-pure": "MySQLConnection"} +_MYSQL_ROUTINES = ("FUNCTION hmac_sha256", "FUNCTION verify_encrypted_data_hmac", + "PROCEDURE validate_encrypted_data_hmac_before_insert", + "PROCEDURE validate_encrypted_data_hmac_before_update") +# The only AWS_* variables a LocalStack module keeps, so the plugin's session finds no credentials but the dummy ones. +_KEPT_AWS_VARIABLES = ("AWS_REGION", "AWS_DEFAULT_REGION") +# The trigger's rejection of a value that isn't a payload of the column's key, in PostgreSQL's and MySQL's messages. +TRIGGER_REJECTION = "does not carry a valid HMAC tag" +DB_ERRORS = (psycopg.Error, mysql.connector.Error) +# The values every test report prints as ``***``, added by the fixtures that come to know them. +_REDACTED: Set[str] = set() +# MySQL's "doesn't yet support" error, which Aurora MySQL raises for a session-level sql_log_off. +_ER_NOT_SUPPORTED_YET = 1235 + + +@dataclass(frozen=True) +class KmsSettings: + """Where KMS is: LocalStack (``endpoint_url`` set) or a real key (``key_arn`` set).""" + region: str + endpoint_url: Optional[str] = None + key_arn: Optional[str] = None + decrypt_only_role_arn: Optional[str] = None + + def props(self) -> Dict[str, str]: + """The plugin's KMS connection properties.""" + props = {"encryption_kms_region": self.region} + if self.endpoint_url: + props["encryption_kms_endpoint"] = self.endpoint_url + return props + + +@dataclass(frozen=True) +class DbUser: + name: str + password: str = field(repr=False) + + +def kms_builds(*excluded: str) -> pytest.MarkDecorator: + """Parametrizes ``build`` with the current engine's builds: ``default`` on PostgreSQL, ``mysql-cext`` and + ``mysql-pure`` on MySQL, less ``excluded``.""" + engine = TestEnvironment.get_current().get_engine() + builds = ["default"] if engine == DatabaseEngine.PG else ["mysql-cext", "mysql-pure"] + return pytest.mark.parametrize("build", [build for build in builds if build not in excluded]) + + +def is_pg() -> bool: + return TestEnvironment.get_current().get_engine() == DatabaseEngine.PG + + +def kms_client(settings: KmsSettings) -> Any: + """A KMS client built the way the plugin builds its own.""" + config = EncryptionConfig.from_props(Properties(settings.props()), require_allowed_master_keys=False) + return KeyManager.create_kms_client(boto3.Session(region_name=settings.region), config) + + +def encryption_config(settings: KmsSettings, schema: str, allowed_master_key_arns: Optional[str] = None + ) -> EncryptionConfig: + props = Properties({**settings.props(), "encryption_metadata_schema": schema}) + if allowed_master_key_arns is None: + return EncryptionConfig.from_props(props, require_allowed_master_keys=False) + props["encryption_allowed_master_key_arns"] = allowed_master_key_arns + return EncryptionConfig.from_props(props) + + +def plain_connect(user: Optional[DbUser] = None, *, admin: bool = False, build: str = "default") -> Any: + """A driver connection without the wrapper: as ``user``, as the administrator, or else as the test user. + + The administrator is the test user, except on Docker MySQL, where it is root; on Aurora the test user is the + cluster's admin user. + """ + env = TestEnvironment.get_current() + info, writer = env.get_database_info(), env.get_writer() + if user is not None: + name = user.name + elif admin and not is_pg() and env.get_deployment() == DatabaseEngineDeployment.DOCKER: + name = "root" + else: + name = info.get_username() + try: + if is_pg(): + return psycopg.connect(host=writer.get_host(), port=writer.get_port(), dbname=info.get_default_db_name(), + user=name, password=user.password if user else info.get_password(), + connect_timeout=10) + return mysql.connector.connect(host=writer.get_host(), port=writer.get_port(), + database=info.get_default_db_name(), user=name, + password=user.password if user else info.get_password(), + use_pure=build == "mysql-pure", connection_timeout=10) + except (psycopg.Error, mysql.connector.Error) as err: + failure = f"Couldn't connect as {name} to {writer.get_host()}:{writer.get_port()}: {type(err).__name__}: {err}" + pytest.fail(failure) + + +def wrapper_connect(build: str, user: DbUser, settings: KmsSettings, schema: str, allowed_master_key_arns: str, + **props: Any) -> AwsWrapperConnection: + """A connection through the wrapper with ``kms_encryption`` as its only plugin unless ``props`` says otherwise. + + On MySQL it checks that the driver connection is the build's class. + """ + env = TestEnvironment.get_current() + info, writer = env.get_database_info(), env.get_writer() + props = {"plugins": "kms_encryption", "encryption_metadata_schema": schema, + "encryption_allowed_master_key_arns": allowed_master_key_arns, **settings.props(), **props} + try: + if is_pg(): + conn = AwsWrapperConnection.connect(psycopg.Connection.connect, host=writer.get_host(), + port=writer.get_port(), dbname=info.get_default_db_name(), + user=user.name, password=user.password, **props) + else: + conn = AwsWrapperConnection.connect(mysql.connector.connect, host=writer.get_host(), + port=writer.get_port(), database=info.get_default_db_name(), + user=user.name, password=user.password, + use_pure=build == "mysql-pure", **props) + except (psycopg.Error, mysql.connector.Error, AwsWrapperError) as err: + failure = (f"Couldn't connect as {user.name} through the wrapper ({build}) to " + f"{writer.get_host()}:{writer.get_port()}: {type(err).__name__}: {err}") + else: + driver_class = type(conn.target_connection).__name__ + if build not in _MYSQL_CONNECTION_CLASS or driver_class == _MYSQL_CONNECTION_CLASS[build]: + return conn + conn.close() + failure = f"The {build} build connected with a {driver_class}, not a {_MYSQL_CONNECTION_CLASS[build]}." + pytest.fail(failure) + + +def run_script(cur: Any, script: str) -> None: + """Runs a MySQL script, DELIMITER lines included, in one execute(); the C extension needs every result read.""" + cur.execute(script) + while cur.nextset(): + pass + + +def trigger_sql(schema: str) -> str: + """The package's trigger SQL for the current engine, with its metadata schema replaced by ``schema``.""" + sql_folder = files("aws_advanced_python_wrapper.encryption") / "sql" + if is_pg(): + return re.sub(r"\bencrypt(?=\.)", schema, (sql_folder / "trigger_pg.sql").read_text(encoding="utf-8")) + return (sql_folder / "trigger_mysql.sql").read_text(encoding="utf-8").replace("SCHEMA_NAME", schema) + + +def install_trigger_sql(conn: Any, schema: str) -> None: + """Installs the package's trigger SQL on ``conn`` (the test user's) and commits; on MySQL it first drops the + routines a previous module installed.""" + with conn.cursor() as cur: + if is_pg(): + cur.execute(trigger_sql(schema)) # no parameters, or psycopg parses the file's % signs + else: + for routine in _MYSQL_ROUTINES: + cur.execute(f"DROP {routine.replace(' ', ' IF EXISTS ', 1)}") + run_script(cur, trigger_sql(schema)) + conn.commit() + + +def create_column_trigger(conn: Any, table: str, column: str) -> None: + """Creates the per-column trigger that calls the installed trigger SQL, and commits.""" + if is_pg(): + statements = [f"CREATE TRIGGER {table}_{column}_encrypted BEFORE INSERT OR UPDATE ON {table} " + f"FOR EACH ROW EXECUTE FUNCTION enforce_encrypted_column('{column}')"] + else: + statements = [f"CREATE TRIGGER {table}_{column}_hmac_insert BEFORE INSERT ON {table} FOR EACH ROW " + f"CALL validate_encrypted_data_hmac_before_insert('{table}', '{column}', NEW.{column})", + f"CREATE TRIGGER {table}_{column}_hmac_update BEFORE UPDATE ON {table} FOR EACH ROW " + f"CALL validate_encrypted_data_hmac_before_update('{table}', '{column}', NEW.{column}, " + f"OLD.{column})"] + with conn.cursor() as cur: + for statement in statements: + cur.execute(statement) + conn.commit() + + +def key_management_utility(conn: Any, settings: KmsSettings, schema: str) -> KeyManagementUtility: + return KeyManagementUtility(conn, kms_client(settings), encryption_config(settings, schema)) + + +def encrypt_column(conn: Any, settings: KmsSettings, schema: str, table: str, column: str, + master_key_arn: str) -> int: + """Turns encryption on for ``table.column`` through KeyManagementUtility, commits, and returns the + ``key_storage`` id.""" + key_storage_id = key_management_utility(conn, settings, schema).initialize_encryption_for_column( + table, column, master_key_arn) + conn.commit() + return key_storage_id + + +def binary_type() -> str: + """The binary column type the modules store payloads in.""" + return "BYTEA" if is_pg() else "VARBINARY(1024)" + + +def write(conn: Any, sql: Any, params: Any = None) -> None: + """Runs one statement on ``conn`` and commits.""" + with conn.cursor() as cur: + cur.execute(sql, params) + conn.commit() + + +def writes(conn: Any, sql: str, params: Any) -> bool: + """Whether ``conn`` writes ``params`` (the trigger accepts them); a rejected write is rolled back.""" + try: + write(conn, sql, params) + except DB_ERRORS: + conn.rollback() + return False + return True + + +def read_column(conn: Any, sql: str, name: str) -> List[Any]: + """The first column of each row ``sql`` returns for the rows named ``name``.""" + with conn.cursor() as cur: + cur.execute(sql, (name,)) + return [row[0] for row in cur.fetchall()] + + +def raised(conn: Any, call: Callable[[], Any]) -> Optional[Exception]: + """The error ``call`` raised, after a rollback that also discards an unread result, or None.""" + try: + call() + except Exception as err: # noqa: BLE001 - the caller checks the class + conn.rollback() + return err + return None + + +def create_table(table: str, columns: str) -> None: + """Creates ``table`` with an id primary key and ``columns``, as the test user; a table a previous run left is + dropped first.""" + id_type = "SERIAL PRIMARY KEY" if is_pg() else "INT AUTO_INCREMENT PRIMARY KEY" + with plain_connect() as conn, conn.cursor() as cur: + cur.execute(f"DROP TABLE IF EXISTS {table}") + cur.execute(f"CREATE TABLE {table} (id {id_type}, {columns})") + conn.commit() + + +def drop_table(table: str) -> None: + with plain_connect() as conn, conn.cursor() as cur: + cur.execute(f"DROP TABLE IF EXISTS {table}") + conn.commit() + + +@contextmanager +def encrypted_table(table: str, columns: str, encrypted: Sequence[str], settings: KmsSettings, schema: str, + master_key_arn: str, *, triggered: Sequence[str] = ()) -> Iterator[Dict[str, int]]: + """Creates ``table`` (see ``create_table``), turns encryption on for each column in ``encrypted`` and puts the + package's trigger on each column in ``triggered``; yields each encrypted column's ``key_storage`` id and drops the + table at the end.""" + create_table(table, columns) + try: + with plain_connect() as conn: + key_ids = {column: encrypt_column(conn, settings, schema, table, column, master_key_arn) + for column in encrypted} + if triggered: + install_trigger_sql(conn, schema) + for column in triggered: + create_column_trigger(conn, table, column) + yield key_ids + finally: + drop_table(table) + + +class PayloadVerifier: + """Opens stored payloads without the plugin: an administrator connection reads the key rows, and + EncryptionService and a KeyManager of the verifier's own decrypt.""" + + def __init__(self, settings: KmsSettings, schema: str, allowed_master_key_arns: str): + self._conn = plain_connect(admin=True) + self._conn.autocommit = True + config = encryption_config(settings, schema, allowed_master_key_arns) + source, dialect = CallerConnectionSource(self._conn), SqlDialect.for_connection(self._conn) + # Never started, so every lookup reads the database. + self._metadata = MetadataManager(source, dialect, config) + self._cache = DataKeyCache(10, 60.0, start_cleanup_thread=False) + self._keys = KeyManager(kms_client(settings), source, dialect, config, self._cache) + + def stored(self, sql: str, params: Any = None) -> List[Any]: + """The first column of each row ``sql`` returns, as the server stores it.""" + with self._conn.cursor() as cur: + cur.execute(sql, params) + return [row[0] for row in cur.fetchall()] + + def decode(self, table: str, column: str, payload: Any) -> Tuple[Optional[int], Any]: + """The ``key_storage`` id that ``payload`` records and the value it decrypts to under that key row.""" + column_config = self._metadata.column_config(table, column) + assert column_config is not None, f"{table}.{column} has no encryption_metadata row" + data = bytes(payload) + key_storage_id = EncryptionService.key_id_from_payload(data) + key = None if key_storage_id is None else self._keys.key_metadata_by_id(key_storage_id) + assert key is not None and key.hmac_key is not None, ( + f"the {table}.{column} payload records key_storage id {key_storage_id}, which has no usable key row") + data_key = self._keys.decrypt_data_key(key.encrypted_data_key, key.master_key_arn) + hmac_key = bytearray(key.hmac_key) + try: + return key_storage_id, EncryptionService.decrypt(data, data_key, hmac_key, column_config.algorithm) + finally: + EncryptionService.wipe(data_key) + EncryptionService.wipe(hmac_key) + + def opened(self, table: str, column: str, name: str) -> List[Any]: + """Each stored value of ``table.column`` in the rows named ``name``, opened: the ``key_storage`` id its payload + records and the value, or None for a NULL.""" + sql = f"SELECT {column} FROM {table} WHERE name = %s ORDER BY id" + return [None if payload is None else self.decode(table, column, payload) + for payload in self.stored(sql, (name,))] + + def close(self) -> None: + self._metadata.close() + self._cache.close() + self._conn.close() + + +def _statement_log(admin_conn: Any) -> List[bytes]: + """The server's statement log, one entry per line, read once. + + The read's own statements stay out of the log: PostgreSQL's session sets ``log_statement = 'none'`` and MySQL's + ``sql_log_off = 1``. + """ + with admin_conn.cursor() as cur: + if is_pg(): + cur.execute("SET log_statement = 'none'") + cur.execute("SELECT pg_read_file(pg_current_logfile())") + text = cur.fetchone()[0] + if text is None: + pytest.fail("PostgreSQL has no current log file; start it with logging_collector=on.") + return [line.encode() for line in text.splitlines()] + cur.execute("SET SESSION sql_log_off = 1") + cur.execute("SELECT argument FROM mysql.general_log") + return [value.encode() if isinstance(value, str) else bytes(value) for (value,) in cur.fetchall()] + + +def _log_marker(prefix: str) -> str: + """Runs ``SELECT `` on a new administrator connection and returns the marker.""" + marker = prefix + secrets.token_hex(8) + with plain_connect(admin=True) as writer, writer.cursor() as cur: + cur.execute("SELECT %s", (marker,)) + cur.fetchall() + return marker + + +def _wait_for_log_line(admin_conn: Any, marker: str) -> List[bytes]: + """The statement log once it holds ``marker``, polled for up to 10 s: PostgreSQL's logging collector writes its + file after the statement returns.""" + lines: List[bytes] = [] + + def logged() -> bool: + lines[:] = _statement_log(admin_conn) + return any(marker.encode() in line for line in lines) + if not retry_until(logged, timeout_sec=10, delay_sec=0.2): + pytest.fail("The statement log doesn't hold the marker statement, so the log is off or unreadable.") + return lines + + +def statement_log_lines(after: str) -> List[bytes]: + """The statement log's lines after the last one that holds ``after``, once every statement before this call is + in the log: it runs an end marker, waits for the log to show it, and stops there.""" + with plain_connect(admin=True) as reader: + reader.autocommit = True + end = _log_marker("kms_log_end_") + lines = _wait_for_log_line(reader, end) + starts = [i for i, line in enumerate(lines) if after.encode() in line] + stops = [i for i, line in enumerate(lines) if end.encode() in line] + if not starts: + pytest.fail("The statement log no longer holds the test's start marker, so the scan can't cover the test.") + return lines[starts[-1] + 1:stops[-1]] + + +def statement_log_scan(needles: Dict[str, str], after: str) -> Dict[str, int]: + """For each needle, by its label, how many statement-log lines after the marker ``after`` hold it; the log is read + once. Build each needle by concatenation, so the test's own source never holds it whole.""" + lines = statement_log_lines(after) + return {label: sum(needle.encode() in line for line in lines) for label, needle in needles.items()} + + +@pytest.hookimpl(wrapper=True) +def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo[None] + ) -> Generator[None, pytest.TestReport, pytest.TestReport]: + """Prints each value in ``_REDACTED`` as ``***`` in the report: in the failure, whose traceback shows each frame's + arguments (a connect frame's password, a fixture's key ARN), and in the captured output.""" + report = yield + hidden = [value for value in _REDACTED if value] + + def redacted(text: str) -> str: + for value in hidden: + text = text.replace(value, "***") + return text + if hidden and report.longrepr is not None and not isinstance(report.longrepr, tuple): + text = str(report.longrepr) + if any(value in text for value in hidden): + report.longrepr = redacted(text) + if hidden: + report.sections = [(title, redacted(content)) for title, content in report.sections] + return report + + +# ---- Fixtures, registered by conftest.py ---- + +@pytest.fixture(scope="module") +def kms_settings() -> Iterator[KmsSettings]: + """The KMS settings of a KMS environment, from ``KMS_ENDPOINT_URL``, ``KMS_REGION``, ``KMS_KEY_ARN`` and + ``KMS_DECRYPT_ONLY_ROLE_ARN``. A missing value fails the module instead of skipping it. + + With LocalStack the module runs with dummy ``test`` keys and every other ``AWS_*`` variable removed. + """ + localstack = TestEnvironmentFeatures.KMS_LOCALSTACK in TestEnvironment.get_current().get_features() + required = ("KMS_ENDPOINT_URL", "KMS_REGION") if localstack else ("KMS_KEY_ARN", "KMS_REGION") + missing = [name for name in required if not os.environ.get(name, "").strip()] + if missing: + pytest.fail(f"KMS environment without {', '.join(missing)}: set it to run the KMS encryption tests.") + settings = KmsSettings(region=os.environ["KMS_REGION"].strip(), + endpoint_url=os.environ["KMS_ENDPOINT_URL"].strip() if localstack else None, + key_arn=None if localstack else os.environ["KMS_KEY_ARN"].strip(), + decrypt_only_role_arn=os.environ.get("KMS_DECRYPT_ONLY_ROLE_ARN", "").strip() or None) + _REDACTED.add(TestEnvironment.get_current().get_database_info().get_password()) + if settings.key_arn is not None: + # A real key's ARN, the role's and the role session ARNs KMS's errors name all hold the account id. + _REDACTED.add(settings.key_arn.rsplit("/", 1)[-1]) + _REDACTED.update(arn.split(":")[4] for arn in (settings.key_arn, settings.decrypt_only_role_arn) + if arn is not None and arn.count(":") >= 5) + with pytest.MonkeyPatch.context() as patch: + if localstack: + for name in [name for name in os.environ if name.startswith("AWS_") and name not in _KEPT_AWS_VARIABLES]: + patch.delenv(name) + for name, value in (("AWS_ACCESS_KEY_ID", "test"), ("AWS_SECRET_ACCESS_KEY", "test"), + ("AWS_CONFIG_FILE", os.devnull), ("AWS_SHARED_CREDENTIALS_FILE", os.devnull), + ("AWS_EC2_METADATA_DISABLED", "true")): + patch.setenv(name, value) + yield settings + + +@pytest.fixture(scope="module") +def kms_master_key_arn(request: pytest.FixtureRequest, kms_settings: KmsSettings) -> str: + """The module's KMS key: on LocalStack a new key per module, on real KMS the administrator's key.""" + if kms_settings.key_arn is not None: + return kms_settings.key_arn + created = kms_client(kms_settings).create_key(Description=f"kms container test {request.module.__name__}") + return created["KeyMetadata"]["Arn"] + + +@pytest.fixture(scope="module") +def kms_schema(request: pytest.FixtureRequest) -> Iterator[str]: + """A metadata schema of the module's own (``kms__``), made from the guide's DDL by the + administrator, dropped before it is created and again at the end; on MySQL the test user gets DML on it.""" + module = request.module.__name__.rsplit(".", 1)[-1] + short = module.replace("test_", "", 1).replace("kms_encryption", "").strip("_") or "module" + schema = f"kms_{short[:40]}_{secrets.token_hex(4)}" + drop = f"DROP SCHEMA IF EXISTS {schema} CASCADE" if is_pg() else f"DROP DATABASE IF EXISTS {schema}" + with plain_connect(admin=True) as admin, admin.cursor() as cur: + cur.execute(drop) + if is_pg(): + cur.execute(ddl_for(DatabaseEngine.PG.value, schema)) + else: + run_script(cur, ddl_for(DatabaseEngine.MYSQL.value, schema)) + cur.execute(f"GRANT SELECT, INSERT, UPDATE, DELETE ON {schema}.* TO %s", + (TestEnvironment.get_current().get_database_info().get_username(),)) + admin.commit() + try: + yield schema + finally: + with plain_connect(admin=True) as admin, admin.cursor() as cur: + cur.execute(drop) + admin.commit() + + +@pytest.fixture(scope="module") +def kms_app_user(kms_schema: str) -> Iterator[DbUser]: + """A DML-only application user for plugin connections: DML on the database's tables, SELECT on the metadata + schema. Its password is created here and kept out of both servers' statement logs and of every report. + Aurora MySQL refuses a session-level ``sql_log_off``, so there the user is created only while the cluster keeps + no general or slow query log; Aurora PostgreSQL's admin user is no superuser, so it takes the role before + ``DROP OWNED BY``.""" + user = DbUser(f"kms_app_{secrets.token_hex(4)}", secrets.token_hex(16)) + _REDACTED.add(user.password) + database = TestEnvironment.get_current().get_database_info().get_default_db_name() + with plain_connect(admin=True) as admin, admin.cursor() as cur: + if is_pg(): + cur.execute("SET log_statement = 'none'") + cur.execute(f"CREATE ROLE {user.name} LOGIN PASSWORD '{user.password}'") # the password is hex only + for statement in (f"GRANT USAGE ON SCHEMA {kms_schema} TO {user.name}", + f"GRANT SELECT ON ALL TABLES IN SCHEMA {kms_schema} TO {user.name}", + f"GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO {user.name}", + f"GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO {user.name}", + f"ALTER DEFAULT PRIVILEGES IN SCHEMA public " + f"GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO {user.name}", + f"ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO {user.name}"): + cur.execute(statement) + else: + try: + cur.execute("SET SESSION sql_log_off = 1") + except mysql.connector.Error as err: + if err.errno != _ER_NOT_SUPPORTED_YET: + raise + cur.execute("SELECT @@GLOBAL.general_log, @@GLOBAL.slow_query_log") + if any(int(on) for on in cur.fetchall()[0]): + pytest.fail("The cluster keeps a general or slow query log, which would record the application " + "user's CREATE USER: turn both off to run the KMS Aurora tests.") + cur.execute("CREATE USER %s IDENTIFIED BY %s", (user.name, user.password)) + cur.execute(f"GRANT SELECT, INSERT, UPDATE, DELETE ON {database}.* TO %s", (user.name,)) + cur.execute(f"GRANT SELECT ON {kms_schema}.* TO %s", (user.name,)) + admin.commit() + try: + yield user + finally: + with plain_connect(admin=True) as admin, admin.cursor() as cur: + if is_pg(): + cur.execute(f"GRANT {user.name} TO CURRENT_USER") + cur.execute(f"DROP OWNED BY {user.name}") + cur.execute(f"DROP ROLE IF EXISTS {user.name}") + else: + cur.execute("DROP USER IF EXISTS %s", (user.name,)) + admin.commit() + + +@pytest.fixture(scope="module") +def kms_trigger_installed(kms_schema: str) -> None: + """The package's trigger SQL, installed once for the module's schema.""" + with plain_connect() as conn: + install_trigger_sql(conn, kms_schema) + + +@pytest.fixture +def kms_connect(build: str, kms_app_user: DbUser, kms_settings: KmsSettings, kms_schema: str, + kms_master_key_arn: str) -> Callable[..., AwsWrapperConnection]: + """Opens a plugin connection of the test's build as the application user, with extra connection properties.""" + def connect(**props: Any) -> AwsWrapperConnection: + return wrapper_connect(build, kms_app_user, kms_settings, kms_schema, kms_master_key_arn, **props) + return connect + + +@pytest.fixture(scope="session") +def kms_django_settings() -> None: + """Django's settings for every KMS module that uses Django, configured once per process as + ``test_django_basic.py`` does; the modules run the plugin only in aliases of their own.""" + if not django_settings.configured: + env = TestEnvironment.get_current() + info, writer = env.get_database_info(), env.get_writer() + django_settings.configure( + DEBUG=False, + DATABASES={"default": { + "ENGINE": "aws_advanced_python_wrapper.django.backends.mysql_connector", + "NAME": info.get_default_db_name(), + "USER": info.get_username(), + "PASSWORD": info.get_password(), + "HOST": writer.get_host(), # the writer instance: a Docker server has no cluster endpoint + "PORT": writer.get_port(), + "OPTIONS": {"plugins": "failover_v2,aurora_connection_tracker", "connect_timeout": 10, + "autocommit": True}, + }}, + INSTALLED_APPS=["django.contrib.contenttypes", "django.contrib.auth"], + DEFAULT_AUTO_FIELD="django.db.models.AutoField", + SECRET_KEY="test-secret-key-for-django-tests", + USE_TZ=True, + ) + django.setup() + + +@pytest.fixture(scope="module") +def kms_verifier(kms_settings: KmsSettings, kms_schema: str, kms_master_key_arn: str) -> Iterator[PayloadVerifier]: + verifier = PayloadVerifier(kms_settings, kms_schema, kms_master_key_arn) + try: + yield verifier + finally: + verifier.close() + + +@pytest.fixture +def kms_statement_log() -> Iterator[str]: + """Yields a marker that a statement wrote after the fixture's negative check, once the log is seen to hold + it. On MySQL it turns the general log on and restores its settings afterwards; PostgreSQL must start with + ``logging_collector=on`` and ``log_statement=all``.""" + reader = plain_connect(admin=True) + reader.autocommit = True + saved = None + try: + if not is_pg(): + with reader.cursor() as cur: + cur.execute("SELECT @@log_output, @@general_log") + saved = cur.fetchone() + cur.execute("SET GLOBAL log_output = 'TABLE', general_log = 'ON'") + marker = "kms_log_" + secrets.token_hex(8) + if any(marker.encode() in line for line in _statement_log(reader)): + pytest.fail("The statement log holds the marker before its statement ran, so the scan matches itself.") + with plain_connect(admin=True) as writer, writer.cursor() as cur: + cur.execute("SELECT %s", (marker,)) + cur.fetchall() + _wait_for_log_line(reader, marker) + yield marker + finally: + if saved is not None: + with reader.cursor() as cur: + cur.execute("SET GLOBAL log_output = %s, general_log = %s", saved) + reader.close() + + +@pytest.fixture +def kms_log_capture() -> Iterator[io.StringIO]: + """Everything logged at DEBUG while the test runs, through the fixture's own handler on the root logger. + + It is the root logger's only handler meanwhile, so a data key that a positive control logs reaches no report. + The harness runs pytest with ``-p no:logging``, so ``caplog`` doesn't exist there. + """ + root = logging.getLogger() + level, others = root.level, list(root.handlers) + buffer = io.StringIO() + handler = logging.StreamHandler(buffer) + for other in others: + root.removeHandler(other) + root.addHandler(handler) + root.setLevel(logging.DEBUG) + try: + yield buffer + finally: + root.removeHandler(handler) + root.setLevel(level) + for other in others: + root.addHandler(other) diff --git a/tests/integration/container/utils/kms_encryption_schema.py b/tests/integration/container/utils/kms_encryption_schema.py new file mode 100644 index 00000000..ba88472f --- /dev/null +++ b/tests/integration/container/utils/kms_encryption_schema.py @@ -0,0 +1,83 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The metadata DDL of the KMS Encryption Plugin, as the plugin's guide gives it, for the KMS container modules. + +Standard library only. +""" + +import re + +PG_METADATA_DDL = """ +CREATE SCHEMA IF NOT EXISTS encrypt; + +CREATE TABLE encrypt.key_storage ( + id SERIAL PRIMARY KEY, + key_id VARCHAR(255) NOT NULL, + name VARCHAR(255) NOT NULL, + master_key_arn VARCHAR(512) NOT NULL, + encrypted_data_key TEXT NOT NULL, + hmac_key BYTEA NOT NULL, + key_spec VARCHAR(50) DEFAULT 'AES_256', + created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + last_used_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE encrypt.encryption_metadata ( + table_name VARCHAR(255) NOT NULL, + column_name VARCHAR(255) NOT NULL, + encryption_algorithm VARCHAR(50) NOT NULL, + key_id INTEGER NOT NULL, + created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (table_name, column_name), + FOREIGN KEY (key_id) REFERENCES encrypt.key_storage(id) +); +""" + +MYSQL_METADATA_DDL = """ +CREATE SCHEMA IF NOT EXISTS encrypt; + +CREATE TABLE encrypt.key_storage ( + id INT AUTO_INCREMENT PRIMARY KEY, + key_id VARCHAR(255) NOT NULL, + name VARCHAR(255) NOT NULL, + master_key_arn VARCHAR(512) NOT NULL, + encrypted_data_key TEXT NOT NULL, + hmac_key VARBINARY(32) NOT NULL, + key_spec VARCHAR(50) DEFAULT 'AES_256', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + last_used_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE encrypt.encryption_metadata ( + table_name VARCHAR(255) NOT NULL, + column_name VARCHAR(255) NOT NULL, + encryption_algorithm VARCHAR(50) NOT NULL, + key_id INT NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (table_name, column_name), + FOREIGN KEY (key_id) REFERENCES encrypt.key_storage(id) +); +""" + + +def ddl_for(engine: str, schema: str) -> str: + """The metadata DDL for ``engine`` (``"PG"`` or ``"MYSQL"``) with ``encrypt`` replaced by ``schema``. + + Only a name followed by ``.`` or ``;`` is replaced, so the ``encryption_metadata`` table keeps its name. + """ + ddl = PG_METADATA_DDL if engine == "PG" else MYSQL_METADATA_DDL + return re.sub(r"\bencrypt(?=[.;])", schema, ddl) diff --git a/tests/integration/container/utils/test_environment.py b/tests/integration/container/utils/test_environment.py index f5d31530..ee532f92 100644 --- a/tests/integration/container/utils/test_environment.py +++ b/tests/integration/container/utils/test_environment.py @@ -179,7 +179,7 @@ def get_proxy_info(self, instance_name: str) -> ProxyInfo: return p def get_proxy_infos(self) -> List[ProxyInfo]: - return list(self._proxies.values()) # type: ignore[union-attr] + return list(self._proxies.values()) if self._proxies else [] def get_info(self) -> TestEnvironmentInfo: return self._info diff --git a/tests/integration/container/utils/test_environment_features.py b/tests/integration/container/utils/test_environment_features.py index 95a3f1c9..5341ea28 100644 --- a/tests/integration/container/utils/test_environment_features.py +++ b/tests/integration/container/utils/test_environment_features.py @@ -33,3 +33,5 @@ class TestEnvironmentFeatures(Enum): SKIP_ASYNC_DRIVER_TESTS = "SKIP_ASYNC_DRIVER_TESTS" TELEMETRY_TRACES_ENABLED = "TELEMETRY_TRACES_ENABLED" TELEMETRY_METRICS_ENABLED = "TELEMETRY_METRICS_ENABLED" + RUN_KMS_ENCRYPTION_TESTS_ONLY = "RUN_KMS_ENCRYPTION_TESTS_ONLY" + KMS_LOCALSTACK = "KMS_LOCALSTACK" diff --git a/tests/integration/host/build.gradle.kts b/tests/integration/host/build.gradle.kts index c84e90f8..0abb5cf3 100644 --- a/tests/integration/host/build.gradle.kts +++ b/tests/integration/host/build.gradle.kts @@ -434,6 +434,55 @@ tasks.register("test-autoscaling") { } } +val kmsCommon = mapOf( + "test-kms-encryption" to "true", + "exclude-multi-az-cluster" to "true", + "exclude-multi-az-instance" to "true", + "exclude-bg" to "true", + "exclude-performance" to "true", + "exclude-async-drivers" to "true", + "exclude-traces-telemetry" to "true", + "exclude-metrics-telemetry" to "true", + "exclude-python-3-10" to "true", + "exclude-python-3-11" to "true", + "exclude-python-3-13" to "true", + "exclude-python-3-14" to "true", +) + +tasks.register("test-kms-encryption-docker") { + group = "verification" + filter.includeTestsMatching("integration.host.TestRunner.runTests") + doFirst { + kmsCommon.forEach { (k, v) -> systemProperty(k, v) } + systemProperty("exclude-aurora", "true") + systemProperty("exclude-secrets-manager", "true") + } +} + +tasks.register("test-kms-encryption-docker-kms") { + group = "verification" + filter.includeTestsMatching("integration.host.TestRunner.runTests") + doFirst { + kmsCommon.forEach { (k, v) -> systemProperty(k, v) } + systemProperty("exclude-aurora", "true") + systemProperty("exclude-secrets-manager", "true") + systemProperty("exclude-kms-localstack", "true") + } +} + +tasks.register("test-kms-encryption-aurora") { + group = "verification" + filter.includeTestsMatching("integration.host.TestRunner.runTests") + doFirst { + kmsCommon.forEach { (k, v) -> systemProperty(k, v) } + systemProperty("exclude-docker", "true") + systemProperty("exclude-kms-localstack", "true") + systemProperty("exclude-instances-1", "true") + systemProperty("exclude-instances-3", "true") + systemProperty("exclude-instances-5", "true") + } +} + tasks.register("test-pg-aurora-performance") { group = "verification" filter.includeTestsMatching("integration.host.TestRunner.runTests") diff --git a/tests/integration/host/src/test/java/integration/TestEnvironmentFeatures.java b/tests/integration/host/src/test/java/integration/TestEnvironmentFeatures.java index 6ee1d210..2df8bfda 100644 --- a/tests/integration/host/src/test/java/integration/TestEnvironmentFeatures.java +++ b/tests/integration/host/src/test/java/integration/TestEnvironmentFeatures.java @@ -31,5 +31,7 @@ public enum TestEnvironmentFeatures { RUN_AUTOSCALING_TESTS_ONLY, TELEMETRY_TRACES_ENABLED, TELEMETRY_METRICS_ENABLED, - BLUE_GREEN_DEPLOYMENT + BLUE_GREEN_DEPLOYMENT, + RUN_KMS_ENCRYPTION_TESTS_ONLY, + KMS_LOCALSTACK } diff --git a/tests/integration/host/src/test/java/integration/host/TestEnvironment.java b/tests/integration/host/src/test/java/integration/host/TestEnvironment.java index fcd32a3a..a4f015b2 100644 --- a/tests/integration/host/src/test/java/integration/host/TestEnvironment.java +++ b/tests/integration/host/src/test/java/integration/host/TestEnvironment.java @@ -40,7 +40,9 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Random; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -65,6 +67,7 @@ public class TestEnvironment implements AutoCloseable { private static final String TEST_CONTAINER_NAME = "test-container"; private static final String TELEMETRY_XRAY_CONTAINER_NAME = "xray-daemon"; private static final String TELEMETRY_OTLP_CONTAINER_NAME = "otlp-daemon"; + private static final String LOCALSTACK_CONTAINER_NAME = "localstack"; private static final String PROXIED_DOMAIN_NAME_SUFFIX = ".proxied"; protected static final int PROXY_CONTROL_PORT = 8474; protected static final int PROXY_PORT = 8666; @@ -95,6 +98,7 @@ public class TestEnvironment implements AutoCloseable { private ArrayList proxyContainers; private GenericContainer telemetryXRayContainer; private GenericContainer telemetryOtlpContainer; + private GenericContainer localStackContainer; private String runnerIP; @@ -134,6 +138,15 @@ public static TestEnvironment build(TestEnvironmentRequest request) throws IOExc TestEnvironmentFeatures.BLUE_GREEN_DEPLOYMENT.toString()); } + if (request.getFeatures().contains(TestEnvironmentFeatures.KMS_LOCALSTACK)) { + env.localStackContainer = + new ContainerHelper().createLocalStackContainer(env.network, LOCALSTACK_CONTAINER_NAME); + env.localStackContainer.start(); + } else if (request.getFeatures().contains(TestEnvironmentFeatures.RUN_KMS_ENCRYPTION_TESTS_ONLY)) { + // The real-key lane; TestEnvironmentProvider has checked KMS_KEY_ARN and KMS_REGION. + initAwsCredentials(env); + } + break; case AURORA: case RDS_MULTI_AZ_CLUSTER: @@ -437,7 +450,9 @@ private static void createDatabaseContainers(TestEnvironment env) { DATABASE_CONTAINER_NAME_PREFIX + i, env.info.getDatabaseInfo().getDefaultDbName(), env.info.getDatabaseInfo().getUsername(), - env.info.getDatabaseInfo().getPassword())); + env.info.getDatabaseInfo().getPassword(), + env.info.getRequest().getFeatures() + .contains(TestEnvironmentFeatures.RUN_KMS_ENCRYPTION_TESTS_ONLY))); env.databaseContainers.get(0).start(); env.info @@ -1069,6 +1084,22 @@ private static void createTestContainer(TestEnvironment env) { .withEnv("AWS_SESSION_TOKEN", env.awsSessionToken); } + // After the credentials block, so the LocalStack dummy keys win. + final Set features = env.info.getRequest().getFeatures(); + if (features.contains(TestEnvironmentFeatures.KMS_LOCALSTACK)) { + env.testContainer + .withEnv("KMS_ENDPOINT_URL", "http://" + LOCALSTACK_CONTAINER_NAME + ":4566") + .withEnv("KMS_REGION", "us-east-1") + .withEnv("AWS_ACCESS_KEY_ID", "test") + .withEnv("AWS_SECRET_ACCESS_KEY", "test"); + } else if (features.contains(TestEnvironmentFeatures.RUN_KMS_ENCRYPTION_TESTS_ONLY)) { + env.testContainer + .withEnv("KMS_KEY_ARN", config.kmsKeyArn) + .withEnv("KMS_REGION", config.kmsRegion) + .withEnv("KMS_DECRYPT_ONLY_ROLE_ARN", Objects.toString(config.kmsDecryptOnlyRoleArn, "")) + .withEnv("KMS_MANUAL_TESTS", Objects.toString(config.kmsManualTests, "")); + } + env.testContainer.start(); } @@ -1279,6 +1310,11 @@ public void close() throws Exception { this.testContainer = null; } + if (this.localStackContainer != null) { + this.localStackContainer.stop(); + this.localStackContainer = null; + } + if (this.proxyContainers != null) { for (ToxiproxyContainer proxyContainer : this.proxyContainers) { proxyContainer.stop(); diff --git a/tests/integration/host/src/test/java/integration/host/TestEnvironmentConfiguration.java b/tests/integration/host/src/test/java/integration/host/TestEnvironmentConfiguration.java index 4bf93e2d..8057081e 100644 --- a/tests/integration/host/src/test/java/integration/host/TestEnvironmentConfiguration.java +++ b/tests/integration/host/src/test/java/integration/host/TestEnvironmentConfiguration.java @@ -54,6 +54,10 @@ public class TestEnvironmentConfiguration { Boolean.parseBoolean(System.getProperty("exclude-secrets-manager", "false")); public boolean testAutoscalingOnly = Boolean.parseBoolean(System.getProperty("test-autoscaling", "false")); + public boolean testKmsEncryptionOnly = + Boolean.parseBoolean(System.getProperty("test-kms-encryption", "false")); + public boolean excludeKmsLocalstack = + Boolean.parseBoolean(System.getProperty("exclude-kms-localstack", "false")); public boolean excludeInstances1 = Boolean.parseBoolean(System.getProperty("exclude-instances-1", "false")); @@ -113,6 +117,11 @@ public class TestEnvironmentConfiguration { public String iamUser = System.getenv("IAM_USER"); + public String kmsKeyArn = System.getenv("KMS_KEY_ARN"); + public String kmsRegion = System.getenv("KMS_REGION"); + public String kmsDecryptOnlyRoleArn = System.getenv("KMS_DECRYPT_ONLY_ROLE_ARN"); + public String kmsManualTests = System.getenv("KMS_MANUAL_TESTS"); + final String numInstancesVar = System.getenv("NUM_INSTANCES"); final Integer numInstances = numInstancesVar == null ? null : Integer.parseInt(numInstancesVar); diff --git a/tests/integration/host/src/test/java/integration/host/TestEnvironmentProvider.java b/tests/integration/host/src/test/java/integration/host/TestEnvironmentProvider.java index 1ef433b6..937f0856 100644 --- a/tests/integration/host/src/test/java/integration/host/TestEnvironmentProvider.java +++ b/tests/integration/host/src/test/java/integration/host/TestEnvironmentProvider.java @@ -23,6 +23,7 @@ import integration.TargetPythonVersion; import integration.TestEnvironmentFeatures; import integration.TestEnvironmentRequest; +import integration.util.StringUtils; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -90,6 +91,9 @@ public Stream provideTestTemplateInvocationContex if (deployment == DatabaseEngineDeployment.RDS_MULTI_AZ_INSTANCE && config.excludeMultiAzInstance) { continue; } + final boolean kmsLocalstack = config.testKmsEncryptionOnly + && deployment == DatabaseEngineDeployment.DOCKER + && !config.excludeKmsLocalstack; for (DatabaseEngine engine : DatabaseEngine.values()) { if (engine == DatabaseEngine.PG && config.excludePgEngine) { @@ -183,9 +187,11 @@ public Stream provideTestTemplateInvocationContex targetPythonVersion, TestEnvironmentFeatures.NETWORK_OUTAGES_ENABLED, engine == DatabaseEngine.PG ? TestEnvironmentFeatures.ABORT_CONNECTION_SUPPORTED : null, + // A KMS-only Docker environment without LocalStack calls real KMS, so it needs credentials. deployment == DatabaseEngineDeployment.DOCKER && config.excludeTracesTelemetry && config.excludeMetricsTelemetry + && (!config.testKmsEncryptionOnly || kmsLocalstack) ? null : TestEnvironmentFeatures.AWS_CREDENTIALS_ENABLED, deployment == DatabaseEngineDeployment.DOCKER || config.excludeFailover @@ -205,7 +211,9 @@ public Stream provideTestTemplateInvocationContex config.testAutoscalingOnly ? TestEnvironmentFeatures.RUN_AUTOSCALING_TESTS_ONLY : null, config.excludeTracesTelemetry ? null : TestEnvironmentFeatures.TELEMETRY_TRACES_ENABLED, config.excludeMetricsTelemetry ? null : TestEnvironmentFeatures.TELEMETRY_METRICS_ENABLED, - withBlueGreenFeature ? TestEnvironmentFeatures.BLUE_GREEN_DEPLOYMENT : null))); + withBlueGreenFeature ? TestEnvironmentFeatures.BLUE_GREEN_DEPLOYMENT : null, + config.testKmsEncryptionOnly ? TestEnvironmentFeatures.RUN_KMS_ENCRYPTION_TESTS_ONLY : null, + kmsLocalstack ? TestEnvironmentFeatures.KMS_LOCALSTACK : null))); } } } @@ -213,6 +221,18 @@ public Stream provideTestTemplateInvocationContex } } + // Fail before any environment starts, so a KMS Aurora run never creates a cluster without a key. + for (EnvPreCreateInfo preCreateInfo : preCreateInfos) { + final Set features = preCreateInfo.request.getFeatures(); + if (features.contains(TestEnvironmentFeatures.RUN_KMS_ENCRYPTION_TESTS_ONLY) + && !features.contains(TestEnvironmentFeatures.KMS_LOCALSTACK) + && (StringUtils.isNullOrEmpty(config.kmsKeyArn) || StringUtils.isNullOrEmpty(config.kmsRegion))) { + throw new RuntimeException( + "Environment variables KMS_KEY_ARN and KMS_REGION are required to run the KMS encryption tests " + + "without LocalStack: " + preCreateInfo.request.getDisplayName()); + } + } + int index = 1; for (TestTemplateInvocationContext testTemplateInvocationContext : resultContextList) { LOGGER.finest( diff --git a/tests/integration/host/src/test/java/integration/util/ContainerHelper.java b/tests/integration/host/src/test/java/integration/util/ContainerHelper.java index 4d7c7b02..b209fb46 100644 --- a/tests/integration/host/src/test/java/integration/util/ContainerHelper.java +++ b/tests/integration/host/src/test/java/integration/util/ContainerHelper.java @@ -60,6 +60,11 @@ public class ContainerHelper { private static final String XRAY_TELEMETRY_IMAGE_NAME = "amazon/aws-xray-daemon"; private static final String OTLP_TELEMETRY_IMAGE_NAME = "amazon/aws-otel-collector"; + // LocalStack 3.8.1 community (tag 3), which needs no auth token. Pinned by digest alone: Testcontainers + // rejects a reference that has both a tag and a digest. + private static final String LOCALSTACK_IMAGE = "public.ecr.aws/localstack/localstack@sha256:" + + "b279c01f4cfb8f985a482e4014cabc1e2697b9d7a6c8c8db2e40f4d9f93687c7"; + public void runTest(GenericContainer container, String testFolder, String primaryInfo, TestEnvironmentConfiguration config) throws IOException, InterruptedException { System.out.println("==== Container console feed ==== >>>>"); @@ -71,8 +76,10 @@ public void runTest(GenericContainer container, String testFolder, String pri String reportSetting = String.format( "--html=./tests/integration/container/reports/%s.html", primaryInfo); + String junitReportSetting = String.format( + "--junitxml=./tests/integration/container/reports/%s.xml", primaryInfo); Long exitCode = execInContainer(container, consumer, - "poetry", "run", "pytest", "-vvvvv", reportSetting, "-k", config.testFilter, + "poetry", "run", "pytest", "-vvvvv", reportSetting, junitReportSetting, "-k", config.testFilter, "--timeout=600", "--tb=long", "--timeout-method=thread", "-p", "no:logging", "--capture=tee-sys",testFolder); @@ -324,14 +331,37 @@ public MySQLContainer createMysqlContainer( } public PostgreSQLContainer createPostgresContainer( - Network network, String networkAlias, String testDbName, String username, String password) { + Network network, + String networkAlias, + String testDbName, + String username, + String password, + boolean logStatements) { - return new PostgreSQLContainer<>(POSTGRES_CONTAINER_IMAGE_NAME) + final PostgreSQLContainer container = new PostgreSQLContainer<>(POSTGRES_CONTAINER_IMAGE_NAME) .withNetwork(network) .withNetworkAliases(networkAlias) .withDatabaseName(testDbName) .withUsername(username) .withPassword(password); + if (logStatements) { + // The logging collector writes the server log to a file that tests read over SQL. It also takes the + // "ready to accept connections" lines that the default wait strategy reads from the container output, + // so wait until the server accepts TCP connections; the server that initializes the database accepts none. + container + .withCommand("postgres", "-c", "fsync=off", "-c", "logging_collector=on", "-c", "log_statement=all") + .waitingFor(Wait.forSuccessfulCommand("pg_isready -h 127.0.0.1 -p 5432")); + } + return container; + } + + // No exposed port: only containers on the test network reach LocalStack. + public GenericContainer createLocalStackContainer(Network network, String networkAlias) { + return new GenericContainer<>(DockerImageName.parse(LOCALSTACK_IMAGE)) + .withNetwork(network) + .withNetworkAliases(networkAlias) + .withEnv("SERVICES", "kms") + .waitingFor(Wait.forLogMessage(".*Ready\\.\\s*", 1)); } public ToxiproxyContainer createAndStartProxyContainer( diff --git a/tests/integration/kms_local/compose.yaml b/tests/integration/kms_local/compose.yaml new file mode 100644 index 00000000..0b7a7d06 --- /dev/null +++ b/tests/integration/kms_local/compose.yaml @@ -0,0 +1,27 @@ +# The compose lane of the KMS encryption tests: the harness's database images and server flags, and LocalStack's KMS. +# Every port is published on 127.0.0.1 only, and the credentials come from variables; run it through run.sh. +services: + postgres: + image: public.ecr.aws/docker/library/postgres:latest + environment: + POSTGRES_USER: ${DB_USER:?export DB_USER} + POSTGRES_PASSWORD: ${DB_PASSWORD:?export DB_PASSWORD} + POSTGRES_DB: ${DB_NAME:?export DB_NAME} + command: ["postgres", "-c", "fsync=off", "-c", "logging_collector=on", "-c", "log_statement=all"] + ports: ["127.0.0.1:${PG_PORT:-59210}:5432"] + mysql: + image: public.ecr.aws/docker/library/mysql:lts + environment: + MYSQL_ROOT_PASSWORD: ${DB_PASSWORD:?export DB_PASSWORD} + MYSQL_DATABASE: ${DB_NAME:?export DB_NAME} + MYSQL_USER: ${DB_USER:?export DB_USER} + MYSQL_PASSWORD: ${DB_PASSWORD:?export DB_PASSWORD} + command: ["--local_infile=1", "--max_allowed_packet=40M", "--max-connections=2048", + "--log_bin_trust_function_creators=1", "--character-set-server=utf8mb4", + "--collation-server=utf8mb4_0900_as_cs", "--log-error-verbosity=3"] + ports: ["127.0.0.1:${MYSQL_PORT:-59211}:3306"] + localstack: + image: public.ecr.aws/localstack/localstack:3@sha256:b279c01f4cfb8f985a482e4014cabc1e2697b9d7a6c8c8db2e40f4d9f93687c7 + environment: + SERVICES: kms + ports: ["127.0.0.1:${KMS_PORT:-59212}:4566"] diff --git a/tests/integration/kms_local/run.sh b/tests/integration/kms_local/run.sh new file mode 100755 index 00000000..45008420 --- /dev/null +++ b/tests/integration/kms_local/run.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# The compose lane of the KMS encryption tests: PostgreSQL, MySQL and LocalStack's KMS on 127.0.0.1, and the KMS +# container modules run on this host with a hand-written TEST_ENV_INFO_JSON, without the Gradle harness. +# +# Usage: run.sh up | test [pytest arguments] | real [pytest arguments] | down +# +# Export DB_USER, DB_NAME and a throwaway DB_PASSWORD; optionally PG_PORT, MYSQL_PORT and KMS_PORT (59210 to 59212), +# COMPOSE ("docker compose", the default, or "finch compose") and COMPOSE_PROJECT_NAME (kms-local). +# finch compose doesn't read exported variables, so each compose command gets them in an env file under $HOME that is +# deleted when the command ends. Finch hands that file to every container, LocalStack included. +# real runs the real-key module against KMS_KEY_ARN in KMS_REGION, with the caller's AWS credentials (for example +# AWS_PROFILE) and KMS_DECRYPT_ONLY_ROLE_ARN; KMS_MANUAL_TESTS=1 adds its manual tests. +# The JUnit report goes to tests/integration/container/reports/kms-local-.xml (kms-real-.xml for real), +# and up's output to kms-local-up.log there (mode 600). +set -euo pipefail + +here="$(cd "$(dirname "$0")" && pwd)" +root="$(cd "${here}/../../.." && pwd)" +: "${DB_USER:?export DB_USER}" "${DB_PASSWORD:?export DB_PASSWORD}" "${DB_NAME:?export DB_NAME}" +export PG_PORT="${PG_PORT:-59210}" MYSQL_PORT="${MYSQL_PORT:-59211}" KMS_PORT="${KMS_PORT:-59212}" +read -r -a compose <<< "${COMPOSE:-docker compose}" +project="${COMPOSE_PROJECT_NAME:-kms-local}" +env_file="" +trap 'rm -f "${env_file}"' EXIT + +run_compose() { + local rc=0 + env_file="$(mktemp "${HOME}/.kms-local-env.XXXXXX")" + chmod 600 "${env_file}" + printf '%s=%s\n' DB_USER "${DB_USER}" DB_PASSWORD "${DB_PASSWORD}" DB_NAME "${DB_NAME}" \ + PG_PORT "${PG_PORT}" MYSQL_PORT "${MYSQL_PORT}" KMS_PORT "${KMS_PORT}" > "${env_file}" + (cd "${HOME}" && "${compose[@]}" -p "${project}" -f "${here}/compose.yaml" --env-file "${env_file}" "$@") || rc=$? + rm -f "${env_file}" + return "${rc}" +} + +# finch compose ignores healthcheck:, so each service is probed from this host until it answers, for up to 240 s. +wait_for_services() { + (cd "${root}" && poetry run python - <<'PY' +import json +import os +import time +import urllib.request + +import mysql.connector +import psycopg + +e = os.environ + + +def postgres_answers(): + psycopg.connect(host="127.0.0.1", port=int(e["PG_PORT"]), user=e["DB_USER"], password=e["DB_PASSWORD"], + dbname=e["DB_NAME"], connect_timeout=3).close() + + +def mysql_answers(): + mysql.connector.connect(host="127.0.0.1", port=int(e["MYSQL_PORT"]), user="root", password=e["DB_PASSWORD"], + database=e["DB_NAME"], connection_timeout=3).close() + + +def kms_answers(): + with urllib.request.urlopen(f"http://127.0.0.1:{e['KMS_PORT']}/_localstack/health", timeout=3) as response: + if json.load(response)["services"]["kms"] not in ("available", "running"): + raise RuntimeError("kms isn't available yet") + + +pending = {"postgres": postgres_answers, "mysql": mysql_answers, "localstack": kms_answers} +deadline = time.monotonic() + 240 +while pending and time.monotonic() < deadline: + for name, probe in list(pending.items()): + try: + probe() + except Exception: + continue + print(f"{name} is ready", flush=True) + del pending[name] + if pending: + time.sleep(2) +if pending: + raise SystemExit(f"Not ready after 240 s: {', '.join(sorted(pending))}") +PY + ) +} + +case "${1:-}" in + up) + # Compose may print each container's environment, the password included, so up's output goes to a mode-600 file. + up_log="${root}/tests/integration/container/reports/kms-local-up.log" + mkdir -p "$(dirname "${up_log}")" + : > "${up_log}" + chmod 600 "${up_log}" + if ! run_compose up -d > "${up_log}" 2>&1; then + echo "compose up failed; its output is in ${up_log}" >&2 + exit 1 + fi + echo "compose up done; its output is in ${up_log}" + wait_for_services + ;; + test | real) + mode="$1" + engine="${2:-}" + if [[ "${engine}" != PG && "${engine}" != MYSQL ]]; then + echo "Usage: run.sh ${mode} [pytest arguments]" >&2 + exit 2 + fi + shift 2 + cd "${root}" + # By path, never with -k kms, which also matches folder names. + modules=() + for module in tests/integration/container/test_kms_encryption_*.py \ + tests/integration/container/sqlalchemy/test_sqlalchemy_kms_encryption.py; do + if [[ -f "${module}" ]]; then + modules+=("${module}") + fi + done + if [[ "${engine}" == MYSQL && -f tests/integration/container/django/test_django_kms_encryption.py ]]; then + modules+=(tests/integration/container/django/test_django_kms_encryption.py) + fi + if [[ ${#modules[@]} -eq 0 ]]; then + echo "No KMS encryption test modules under tests/integration/container." >&2 + exit 2 + fi + export ENGINE="${engine}" FEATURE="KMS_LOCALSTACK" + if [[ "${mode}" == real ]]; then + : "${KMS_KEY_ARN:?export KMS_KEY_ARN}" "${KMS_REGION:?export KMS_REGION}" + modules=(tests/integration/container/test_kms_encryption_real_kms.py) + FEATURE="AWS_CREDENTIALS_ENABLED" + fi + # rdsEndpoint is LocalStack's port in both modes, so conftest's RDS prewarm never reaches AWS. + TEST_ENV_INFO_JSON="$(python3 -c 'import json, os; e = os.environ; port = int(e["PG_PORT"] if e["ENGINE"] == "PG" else e["MYSQL_PORT"]); print(json.dumps({"request": {"engine": e["ENGINE"], "instances": "SINGLE_INSTANCE", "deployment": "DOCKER", "targetPythonVersion": "PYTHON_3_12", "numOfInstances": 1, "features": ["RUN_KMS_ENCRYPTION_TESTS_ONLY", e["FEATURE"]]}, "region": e.get("KMS_REGION", "us-east-1"), "rdsEndpoint": "http://127.0.0.1:" + e["KMS_PORT"], "databaseInfo": {"username": e["DB_USER"], "password": e["DB_PASSWORD"], "defaultDbName": e["DB_NAME"], "instances": [{"instanceId": e["ENGINE"].lower(), "host": "127.0.0.1", "port": port}]}}))')" + export TEST_ENV_INFO_JSON + if [[ "${mode}" == real ]]; then + env -u KMS_ENDPOINT_URL AWS_REGION="${KMS_REGION}" \ + poetry run python -m pytest -p no:cacheprovider -p no:logging \ + --junitxml="tests/integration/container/reports/kms-real-${engine}.xml" "${modules[@]}" "$@" + exit + fi + # The harness's pytest options that change what a test sees: -p no:logging, so no test can rely on caplog. + env -u AWS_SESSION_TOKEN -u AWS_SECURITY_TOKEN -u AWS_PROFILE -u AWS_DEFAULT_PROFILE \ + AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test AWS_REGION=us-east-1 \ + AWS_CONFIG_FILE=/dev/null AWS_SHARED_CREDENTIALS_FILE=/dev/null AWS_EC2_METADATA_DISABLED=true \ + KMS_ENDPOINT_URL="http://127.0.0.1:${KMS_PORT}" KMS_REGION=us-east-1 \ + poetry run python -m pytest -p no:cacheprovider -p no:logging \ + --junitxml="tests/integration/container/reports/kms-local-${engine}.xml" "${modules[@]}" "$@" + ;; + down) + run_compose down -v + ;; + *) + echo "Usage: run.sh up | test [pytest arguments] | real [pytest arguments] | down" >&2 + exit 2 + ;; +esac diff --git a/tests/integration/test_kms_encryption_live.py b/tests/integration/test_kms_encryption_live.py deleted file mode 100644 index a0467539..00000000 --- a/tests/integration/test_kms_encryption_live.py +++ /dev/null @@ -1,1912 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Live checks of the KMS Encryption Plugin on PostgreSQL 16, MySQL 8 and LocalStack's KMS, all on 127.0.0.1. - -The module seeds both databases with the metadata tables, the trigger SQL that the package ships in -``aws_advanced_python_wrapper/encryption/sql`` and two encrypted columns with key rows of their own: ``users.ssn``, -which has the trigger, and ``users.dob``, which has none. -It first drops ``users``, ``addresses``, the ``encrypt`` schema and the trigger SQL's routines, and on MySQL turns on -``log_bin_trust_function_creators``, so point it only at servers it may reset. It runs the typical driver and ORM -shapes through the plugin, reads every stored value back with a plain driver, and counts plaintext in both servers' -own logs. It fails, never skips, when a server or a setting is missing, and reaches KMS only on LocalStack, with dummy -credentials. Give one run the two servers to itself: the log checks count every statement each server logs during the -run, and MySQL has one ``encrypt`` database per server, which every run drops and recreates whatever its ``DB_NAME``. -Run the module with:: - - pytest tests/integration/test_kms_encryption_live.py - -All of these environment variables are required. ``PG_PORT``, ``MYSQL_PORT`` and ``KMS_PORT`` are the local ports. -``DB_USER``, ``DB_PASSWORD`` and ``DB_NAME`` name a PostgreSQL superuser and a MySQL user with that password and -database; MySQL's root account uses ``DB_PASSWORD`` too. ``PG_LOG_COMMAND`` and -``MYSQL_LOG_COMMAND`` each hold a command that prints that server's statement log (PostgreSQL started with -``log_statement=all``, MySQL with its general log on); each is split into arguments and run without a shell. If -``AWS_PROFILE`` or ``AWS_DEFAULT_PROFILE`` is set, it must name a profile that exists, because importing the wrapper -reads it; the module removes both for its run. If SQLAlchemy or Django can't connect, pytest's failure report shows the -wrapper's connection properties, ``DB_PASSWORD`` included, so use a password that belongs to these local servers only. -""" - -from __future__ import annotations - -import contextlib -import copy -import io -import logging -import os -import re -import shlex -import subprocess -import time -import uuid -from dataclasses import dataclass, field -from datetime import datetime, timedelta -from decimal import Decimal -from enum import Enum, IntEnum -from importlib.resources import files -from ipaddress import IPv4Address -from typing import (TYPE_CHECKING, Any, Callable, Dict, Iterator, List, - Optional, Set, Tuple, Type) - -import boto3 -import django -import mysql.connector -import psycopg -import pytest -from botocore.exceptions import (BotoCoreError, ClientError, - EndpointConnectionError) -from django.conf import settings as django_settings -from django.db import connection as django_connection -from django.db import models -from django.test.utils import override_settings -from psycopg import sql as pgsql -from psycopg.rows import class_row, dict_row, namedtuple_row, scalar_row -from psycopg.types.json import Jsonb -from sqlalchemy import (Column, ForeignKey, Integer, MetaData, String, Table, - create_engine) -from sqlalchemy import exc as sa_exc -from sqlalchemy import insert, select, text -from sqlalchemy.engine import URL -from sqlalchemy.orm import (DeclarativeBase, Session, joinedload, - mapped_column, relationship) -from sqlalchemy.types import UserDefinedType - -from aws_advanced_python_wrapper import AwsWrapperConnection -from aws_advanced_python_wrapper.encryption.config import EncryptionConfig -from aws_advanced_python_wrapper.encryption.connection_source import ( - CallerConnectionSource, PluginConnectionSource) -from aws_advanced_python_wrapper.encryption.data_key_cache import DataKeyCache -from aws_advanced_python_wrapper.encryption.encryption_service import \ - EncryptionService -from aws_advanced_python_wrapper.encryption.key_management_utility import \ - KeyManagementUtility -from aws_advanced_python_wrapper.encryption.key_manager import KeyManager -from aws_advanced_python_wrapper.encryption.metadata_manager import \ - MetadataManager -from aws_advanced_python_wrapper.encryption.sql_dialect import SqlDialect -from aws_advanced_python_wrapper.errors import (AwsWrapperError, - EncryptionError, - EncryptionFailedError, - IntegrityCheckError, - InvalidKeyMetadataError, - KeyManagementError, - MetadataError, - UnauthorizedMasterKeyError) -from aws_advanced_python_wrapper.pep249_methods import DbApiMethod -from aws_advanced_python_wrapper.plugin import Plugin, PluginFactory -from aws_advanced_python_wrapper.plugin_service import PluginManager -from aws_advanced_python_wrapper.utils.messages import Messages -from aws_advanced_python_wrapper.utils.properties import (Properties, - WrapperProperties) - -if TYPE_CHECKING: - from sqlalchemy.engine import Engine - - from aws_advanced_python_wrapper.plugin_service import PluginService - -_HOST = "127.0.0.1" -_REGION = "us-east-1" -_REQUIRED = ("PG_PORT", "MYSQL_PORT", "KMS_PORT", "DB_USER", "DB_PASSWORD", "DB_NAME", "PG_LOG_COMMAND", - "MYSQL_LOG_COMMAND") -_PORTS = ("PG_PORT", "MYSQL_PORT", "KMS_PORT") -_LOG_TIMEOUT = 120 # seconds for a log command -_STAMP = re.compile(r"kms_encryption trigger v(\d+)") -_INSERT = "INSERT INTO users (name, ssn) VALUES (%s, %s)" -_SELECT = "SELECT ssn FROM users WHERE name = %s ORDER BY id" -_INSERT_DOB = "INSERT INTO users (name, dob) VALUES (%s, %s)" -_SELECT_DOB = "SELECT dob FROM users WHERE name = %s ORDER BY id" -_COLUMN_SELECT = {"ssn": _SELECT, "dob": _SELECT_DOB} -_BUILDS = pytest.mark.parametrize("build", ["pg", "mysql-cext", "mysql-pure"]) -_MYSQL_BUILDS = pytest.mark.parametrize("build", ["mysql-cext", "mysql-pure"]) -_ENGINES = pytest.mark.parametrize("engine", ["pg", "mysql"]) -_CURSOR_CLASS = {"mysql-cext": "CMySQLCursor", "mysql-pure": "MySQLCursor"} -_ENGINE = {"pg": "pg", "mysql-cext": "mysql", "mysql-pure": "mysql"} -_SERVER = {"pg": "PostgreSQL", "mysql": "MySQL"} - -# The metadata DDL as the guide's "Create the metadata tables" gives it. -_PG_METADATA_DDL = """ -CREATE SCHEMA IF NOT EXISTS encrypt; - -CREATE TABLE encrypt.key_storage ( - id SERIAL PRIMARY KEY, - key_id VARCHAR(255) NOT NULL, - name VARCHAR(255) NOT NULL, - master_key_arn VARCHAR(512) NOT NULL, - encrypted_data_key TEXT NOT NULL, - hmac_key BYTEA NOT NULL, - key_spec VARCHAR(50) DEFAULT 'AES_256', - created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, - last_used_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP -); - -CREATE TABLE encrypt.encryption_metadata ( - table_name VARCHAR(255) NOT NULL, - column_name VARCHAR(255) NOT NULL, - encryption_algorithm VARCHAR(50) NOT NULL, - key_id INTEGER NOT NULL, - created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (table_name, column_name), - FOREIGN KEY (key_id) REFERENCES encrypt.key_storage(id) -); -""" - -_MYSQL_METADATA_DDL = """ -CREATE SCHEMA IF NOT EXISTS encrypt; - -CREATE TABLE encrypt.key_storage ( - id INT AUTO_INCREMENT PRIMARY KEY, - key_id VARCHAR(255) NOT NULL, - name VARCHAR(255) NOT NULL, - master_key_arn VARCHAR(512) NOT NULL, - encrypted_data_key TEXT NOT NULL, - hmac_key VARBINARY(32) NOT NULL, - key_spec VARCHAR(50) DEFAULT 'AES_256', - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - last_used_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP -); - -CREATE TABLE encrypt.encryption_metadata ( - table_name VARCHAR(255) NOT NULL, - column_name VARCHAR(255) NOT NULL, - encryption_algorithm VARCHAR(50) NOT NULL, - key_id INT NOT NULL, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (table_name, column_name), - FOREIGN KEY (key_id) REFERENCES encrypt.key_storage(id) -); -""" - -_PG_TABLES = ( - "CREATE TABLE users (id SERIAL PRIMARY KEY, name TEXT, ssn BYTEA, dob BYTEA)", - """CREATE TRIGGER users_ssn_encrypted -BEFORE INSERT OR UPDATE ON users -FOR EACH ROW EXECUTE FUNCTION enforce_encrypted_column('ssn')""", - "CREATE TABLE addresses (id SERIAL PRIMARY KEY, user_id INTEGER, email TEXT)", -) - -_MYSQL_TABLES = ( - "CREATE TABLE users (id INT AUTO_INCREMENT PRIMARY KEY, name VARCHAR(100), ssn VARBINARY(1024), dob VARBINARY(1024))", - """CREATE TRIGGER users_ssn_hmac_check -BEFORE INSERT ON users -FOR EACH ROW -BEGIN - CALL validate_encrypted_data_hmac_before_insert('users', 'ssn', NEW.ssn); -END""", - """CREATE TRIGGER users_ssn_hmac_check_update -BEFORE UPDATE ON users -FOR EACH ROW -BEGIN - CALL validate_encrypted_data_hmac_before_update('users', 'ssn', NEW.ssn, OLD.ssn); -END""", - "CREATE TABLE addresses (id INT AUTO_INCREMENT PRIMARY KEY, user_id INT, email VARCHAR(100))", -) - -_MYSQL_RESET = ("DROP TABLE IF EXISTS addresses, users", "DROP DATABASE IF EXISTS encrypt", - "DROP FUNCTION IF EXISTS hmac_sha256", "DROP FUNCTION IF EXISTS verify_encrypted_data_hmac", - "DROP PROCEDURE IF EXISTS validate_encrypted_data_hmac_before_insert", - "DROP PROCEDURE IF EXISTS validate_encrypted_data_hmac_before_update") - - -@dataclass(frozen=True) -class _Env: - pg_port: int - mysql_port: int - kms_port: int - user: str - password: str = field(repr=False) - dbname: str - pg_log_command: str = field(repr=False) - mysql_log_command: str = field(repr=False) - - -@dataclass(frozen=True) -class _Bed: - """The seeded databases: per engine, the master key ARN and ``key_storage`` id of ``users.ssn``, and what the - standalone utility's DEBUG log showed while it gave ``users.dob`` its key rows (booleans only).""" - env: _Env - arns: Dict[str, str] - key_ids: Dict[str, int] - marker: str - utility_log: Dict[str, Dict[str, bool]] - - -@pytest.fixture(scope="module", autouse=True) -def _localstack_only() -> Iterator[None]: - # The plugin's AWS session reads the process environment, so this is what keeps ambient credentials out. - with pytest.MonkeyPatch.context() as patch: - for name, value in (("AWS_ACCESS_KEY_ID", "test"), ("AWS_SECRET_ACCESS_KEY", "test"), - ("AWS_CONFIG_FILE", os.devnull), ("AWS_SHARED_CREDENTIALS_FILE", os.devnull)): - patch.setenv(name, value) - for name in ("AWS_PROFILE", "AWS_DEFAULT_PROFILE", "AWS_SESSION_TOKEN", "AWS_SECURITY_TOKEN"): - patch.delenv(name, raising=False) - yield - - -@pytest.fixture(scope="module") -def env() -> _Env: - # Every value is read from os.environ by name, so no local holds the password and pytest's report can't show it. - missing = [name for name in _REQUIRED if not os.environ.get(name, "").strip()] - if missing: - pytest.fail(f"Set {', '.join(missing)}: the module docstring says what each one holds.") - not_ports = [name for name in _PORTS if not os.environ[name].strip().isdecimal()] - if not_ports: - pytest.fail(f"Set {', '.join(not_ports)} to a port number.") - return _Env(pg_port=int(os.environ["PG_PORT"]), mysql_port=int(os.environ["MYSQL_PORT"]), - kms_port=int(os.environ["KMS_PORT"]), user=os.environ["DB_USER"], password=os.environ["DB_PASSWORD"], - dbname=os.environ["DB_NAME"], pg_log_command=os.environ["PG_LOG_COMMAND"], mysql_log_command=os.environ["MYSQL_LOG_COMMAND"]) - - -def _plain(engine: str, env: _Env, *, root: bool = False) -> Any: - """A driver connection without the wrapper.""" - # The driver's connect frame holds the password, and pytest's report prints a raising frame's arguments, so the - # error is turned into a message here and the test fails outside the except block. - port = env.pg_port if engine == "pg" else env.mysql_port - try: - if engine == "pg": - return psycopg.connect(host=_HOST, port=port, user=env.user, password=env.password, dbname=env.dbname, - connect_timeout=5) - return mysql.connector.connect(host=_HOST, port=port, user="root" if root else env.user, - password=env.password, database=env.dbname, connection_timeout=5) - except (psycopg.Error, mysql.connector.Error) as err: - failure = f"Couldn't connect to {_SERVER[engine]} on {_HOST}:{port}: {type(err).__name__}: {err}" - pytest.fail(failure) - - -def _kms_props(env: _Env) -> Properties: - return Properties({"encryption_kms_region": _REGION, "encryption_kms_endpoint": f"http://{_HOST}:{env.kms_port}"}) - - -def _kms_client(env: _Env) -> Any: - config = EncryptionConfig.from_props(_kms_props(env), require_allowed_master_keys=False) - session = boto3.Session(aws_access_key_id="test", aws_secret_access_key="test", region_name=_REGION) - return KeyManager.create_kms_client(session, config) - - -@pytest.fixture(scope="module", autouse=True) -def _services(env: _Env, _localstack_only: None) -> None: - for engine in ("pg", "mysql"): - _plain(engine, env).close() # fails naming the server and its port - try: - _kms_client(env).list_keys() - reached = True - except (BotoCoreError, ClientError): - reached = False - if not reached: - pytest.fail(f"LocalStack's KMS isn't reachable on {_HOST}:{env.kms_port}; start it before running this module.") - - -def _trigger_sql(name: str) -> str: - """The trigger SQL as the installed package ships it.""" - return (files("aws_advanced_python_wrapper.encryption") / "sql" / name).read_text(encoding="utf-8") - - -def _check_stamp(name: str, row: Any) -> None: - if not _STAMP.search((row or [None])[0] or ""): - pytest.fail(f"{name} installed without its version stamp.") - - -def _utility(conn: Any, env: _Env, allowed: Optional[str] = None) -> KeyManagementUtility: - """A utility on a plain connection, with its KMS client built by ``create_kms_client`` as KeyManagementUtility's - docstring shows.""" - props = _kms_props(env) - if allowed is not None: - props["encryption_allowed_master_key_arns"] = allowed - config = EncryptionConfig.from_props(props, require_allowed_master_keys=False) - return KeyManagementUtility(conn, _kms_client(env), config) - - -def _encrypt_column(conn: Any, env: _Env) -> Tuple[str, int]: - """Turns encryption on for ``users.ssn`` under a new master key, then commits: the utility never does.""" - utility = _utility(conn, env) - arn = utility.create_master_key("live module users.ssn") - key_id = utility.initialize_encryption_for_column("users", "ssn", arn) - conn.commit() - return arn, key_id - - -def _encrypt_dob(engine: str, env: _Env, arn: str) -> Dict[str, bool]: - """Gives ``users.dob`` its own key row under ``users.ssn``'s master key, with the root logger at DEBUG and no - plugin connection opened yet in the process, then rotates it once with the plaintext filter removed. Returns - booleans only: the captured text holds a data key, so it is never kept.""" - parsers = logging.getLogger("botocore.parsers") - with _plain(engine, env) as conn: - utility = _utility(conn, env) - with _debug_log() as log: - utility.initialize_encryption_for_column("users", "dob", arn) - conn.commit() - seen = {"plaintext": '"Plaintext"' in log.getvalue(), - "kms-request": "TrentService.GenerateDataKey" in log.getvalue()} - # The positive control: without the filter, generating the next data key logs it. - removed = [f for f in parsers.filters if type(f).__name__ == "_NoKmsPlaintext"] - for f in removed: - parsers.removeFilter(f) - try: - with _debug_log() as log: - utility.rotate_data_key("users", "dob", arn) - conn.commit() - seen.update({"filter-found": bool(removed), "control-plaintext": '"Plaintext"' in log.getvalue()}) - finally: - for f in removed: - parsers.addFilter(f) - return seen - - -def _seed_pg(env: _Env) -> Tuple[str, int]: - with _plain("pg", env) as conn: - with conn.cursor() as cur: - for statement in ("DROP TABLE IF EXISTS addresses, users CASCADE", "DROP SCHEMA IF EXISTS encrypt CASCADE", - "DROP FUNCTION IF EXISTS enforce_encrypted_column() CASCADE", _PG_METADATA_DDL): - cur.execute(statement) - cur.execute(_trigger_sql("trigger_pg.sql")) # no parameters, or psycopg parses the file's % signs - cur.execute("SELECT obj_description('enforce_encrypted_column'::regproc, 'pg_proc')") - _check_stamp("trigger_pg.sql", cur.fetchone()) - for statement in _PG_TABLES: - cur.execute(statement) - conn.commit() - return _encrypt_column(conn, env) - - -def _run_script(cur: Any, script: str) -> None: - # mysql-connector 9.7 runs a script, DELIMITER lines included, in one execute(); on its C extension every result - # has to be read before the next statement. - cur.execute(script) - while cur.nextset(): - pass - - -def _seed_mysql(env: _Env) -> Tuple[str, int]: - # Root runs only the administrator's steps; DB_USER does the rest. Without log_bin_trust_function_creators, a user - # without SUPER can't create the routines or the triggers while binary logging is on. - with _plain("mysql", env, root=True) as root, root.cursor() as cur: - for statement in _MYSQL_RESET: - cur.execute(statement) - _run_script(cur, _MYSQL_METADATA_DDL) - cur.execute("SET GLOBAL log_bin_trust_function_creators = 1") - cur.execute("GRANT SELECT, INSERT, UPDATE, DELETE ON encrypt.* TO %s", (env.user,)) - with _plain("mysql", env) as conn: - with conn.cursor() as cur: - # The file asks for SCHEMA_NAME to be replaced with the metadata schema. - _run_script(cur, _trigger_sql("trigger_mysql.sql").replace("SCHEMA_NAME", "encrypt")) - cur.execute("SELECT ROUTINE_COMMENT FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA = DATABASE() " - "AND ROUTINE_NAME = 'validate_encrypted_data_hmac_before_insert'") - _check_stamp("trigger_mysql.sql", cur.fetchone()) - for statement in _MYSQL_TABLES: - cur.execute(statement) - conn.commit() - return _encrypt_column(conn, env) - - -@pytest.fixture(scope="module") -def bed(env: _Env, _services: None) -> _Bed: - seeds = {"pg": _seed_pg(env), "mysql": _seed_mysql(env)} - utility_log = {engine: _encrypt_dob(engine, env, seed[0]) for engine, seed in seeds.items()} - # The log checks count only what each server logs after this statement. - marker = f"kms_live_{uuid.uuid4().hex}" - for engine in seeds: - with _plain(engine, env) as conn, conn.cursor() as cur: - cur.execute("SELECT %s", (marker,)) - cur.fetchall() - return _Bed(env, {e: s[0] for e, s in seeds.items()}, {e: s[1] for e, s in seeds.items()}, marker, utility_log) - - -def _plugin_props(engine: str, bed: _Bed) -> Dict[str, Any]: - chain = WrapperProperties.DEFAULT_PLUGINS if engine == "pg" else WrapperProperties.MYSQL_CONNECTOR_DEFAULT_PLUGINS - return {"plugins": f"kms_encryption,{chain}", "wrapper_dialect": engine, - "encryption_allowed_master_key_arns": bed.arns[engine], **_kms_props(bed.env)} - - -def _connect(build: str, bed: _Bed, **overrides: Any) -> AwsWrapperConnection: - # As in _plain: the wrapper's connect frames hold the password, so the test fails outside the except block. - env, engine = bed.env, _ENGINE[build] - props = {**_plugin_props(engine, bed), **overrides} - port = env.pg_port if engine == "pg" else env.mysql_port - try: - if engine == "pg": - return AwsWrapperConnection.connect(psycopg.Connection.connect, host=_HOST, port=port, user=env.user, - password=env.password, dbname=env.dbname, **props) - return AwsWrapperConnection.connect(mysql.connector.Connect, host=_HOST, port=port, user=env.user, - password=env.password, database=env.dbname, - use_pure=build == "mysql-pure", **props) - except (psycopg.Error, mysql.connector.Error, AwsWrapperError) as err: - failure = (f"Couldn't connect to {_SERVER[engine]} on {_HOST}:{port} through the wrapper ({build}): " - f"{type(err).__name__}: {err}") - pytest.fail(failure) - - -class _Verifier: - """Reads stored values with a plain driver and opens them with EncryptionService and a KeyManager of its own: a - value counts as written only if a reader without the plugin finds a payload of the column's current key that - decrypts to it.""" - - def __init__(self, engine: str, bed: _Bed, column: str = "ssn"): - self._conn = _plain(engine, bed.env) - self._conn.autocommit = True - self._select = _COLUMN_SELECT[column] - config = EncryptionConfig.from_props(Properties(_kms_props(bed.env), - encryption_allowed_master_key_arns=bed.arns[engine])) - source, dialect = CallerConnectionSource(self._conn), SqlDialect.for_connection(self._conn) - metadata = MetadataManager(source, dialect, config) - found = metadata.column_config("users", column) - metadata.close() - if (found is None or found.key_id is None or found.key_metadata is None - or found.key_metadata.hmac_key is None): - pytest.fail(f"users.{column} has no usable key row after seeding.") - self._key_id = bed.key_ids[engine] if column == "ssn" else found.key_id - self._key = found.key_metadata - self._hmac_key = found.key_metadata.hmac_key - self._algorithm = found.algorithm - self._cache = DataKeyCache(10, 60.0, start_cleanup_thread=False) - self._keys = KeyManager(_kms_client(bed.env), source, dialect, config, self._cache) - - def stored(self, name: str) -> List[Any]: - with self._conn.cursor() as cur: - cur.execute(self._select, (name,)) - return [row[0] for row in cur.fetchall()] - - def check(self, name: str, expected: List[Any]) -> None: - stored = self.stored(name) - assert len(stored) == len(expected) - for value, want in zip(stored, expected): - if want is None: - assert value is None - continue - payload = bytes(value) - assert len(payload) >= 65 and EncryptionService.key_id_from_payload(payload) == self._key_id - data_key = self._keys.decrypt_data_key(self._key.encrypted_data_key, self._key.master_key_arn) - hmac_key = bytearray(self._hmac_key) - try: - plain = EncryptionService.decrypt(payload, data_key, hmac_key, self._algorithm) - finally: - EncryptionService.wipe(data_key) - EncryptionService.wipe(hmac_key) - assert plain == want and type(plain) is type(want) - - def seal(self, value: Any) -> Optional[bytes]: - """The payload a writer other than the plugin would store for ``value`` under this column's key.""" - data_key = self._keys.decrypt_data_key(self._key.encrypted_data_key, self._key.master_key_arn) - hmac_key = bytearray(self._hmac_key) - try: - return EncryptionService.encrypt(value, data_key, hmac_key, self._algorithm, key_id=self._key_id) - finally: - EncryptionService.wipe(data_key) - EncryptionService.wipe(hmac_key) - - def empty(self) -> None: - with self._conn.cursor() as cur: - cur.execute("DELETE FROM addresses") - cur.execute("DELETE FROM users") - - def close(self) -> None: - self._cache.close() - self._conn.close() - - -@pytest.fixture(scope="module") -def verifiers(bed: _Bed) -> Iterator[Dict[str, _Verifier]]: - opened = {engine: _Verifier(engine, bed) for engine in ("pg", "mysql")} - yield opened - for verifier in opened.values(): - verifier.close() - - -@pytest.fixture(autouse=True) -def _empty_tables(verifiers: Dict[str, _Verifier]) -> None: - # Each test writes the rows it reads, so no test depends on another one's. - for verifier in verifiers.values(): - verifier.empty() - - -@dataclass(frozen=True) -class _Db: - conn: AwsWrapperConnection - verifier: _Verifier - - -@pytest.fixture -def db(build: str, bed: _Bed, verifiers: Dict[str, _Verifier]) -> Iterator[_Db]: - with _connect(build, bed) as conn: - if build in _CURSOR_CLASS: # so a missing C extension fails instead of running the pure build twice - with conn.cursor() as cur: - driver_class = type(cur.target_cursor).__name__ - if driver_class != _CURSOR_CLASS[build]: - pytest.fail(f"On {build} the driver's cursor is a {driver_class}, not a {_CURSOR_CLASS[build]}.") - yield _Db(conn, verifiers[_ENGINE[build]]) - - -def _write(conn: AwsWrapperConnection, sql: str, params: Any) -> None: - with conn.cursor() as cur: - cur.execute(sql, params) - conn.commit() - - -def _read_one(conn: AwsWrapperConnection, name: str) -> Any: - with conn.cursor() as cur: - cur.execute(_SELECT, (name,)) - row = cur.fetchone() - cur.fetchall() # an unread row on mysql-connector's default cursor would fail the next statement - return None if row is None else row[0] - - -def _write_many(conn: AwsWrapperConnection, name: str, as_generator: bool = False) -> List[str]: - values = [f"pt-{name}-{i}" for i in range(3)] - rows = [(name, value) for value in values] - with conn.cursor() as cur: - cur.executemany(_INSERT, (row for row in rows) if as_generator else rows) - conn.commit() - return values - - -# ---- DB-API: psycopg, and mysql-connector on its C extension and with use_pure=True ---- - -@_BUILDS -@pytest.mark.parametrize("sql, params, by_keyword, want", [ - pytest.param(_INSERT, ("d1", "pt-tuple-1"), False, "pt-tuple-1", id="insert-tuple"), - pytest.param(_INSERT, ["d1", "pt-tuple-1"], False, "pt-tuple-1", id="caller-params-unchanged"), - pytest.param("INSERT INTO users (name, ssn) VALUES (%(n)s, %(s)s)", {"n": "d1", "s": "pt-dict-2"}, False, - "pt-dict-2", id="insert-dict"), - pytest.param(_INSERT, ("d1", "pt-kw-3"), True, "pt-kw-3", id="params-keyword"), - pytest.param(_INSERT, ("d1", 42), False, 42, id="insert-int"), - pytest.param(_INSERT, ("d1", None), False, None, id="insert-null")]) -def test_write_stores_a_payload(db: _Db, sql: str, params: Any, by_keyword: bool, want: Any) -> None: - caller = copy.copy(params) - with db.conn.cursor() as cur: - if by_keyword: - cur.execute(sql, params=params) - else: - cur.execute(sql, params) - db.conn.commit() - assert params == caller - value = _read_one(db.conn, "d1") - assert value == want and type(value) is type(want) - db.verifier.check("d1", [want]) - - -@_BUILDS -@pytest.mark.parametrize("as_generator", [False, True], ids=["list", "generator"]) -def test_executemany_stores_every_row(db: _Db, as_generator: bool) -> None: - name = "g" if as_generator else "m" - db.verifier.check(name, _write_many(db.conn, name, as_generator)) - - -@_BUILDS -def test_update_stores_a_new_payload(db: _Db) -> None: - _write(db.conn, _INSERT, ("d1", "pt-tuple-1")) - _write(db.conn, "UPDATE users SET ssn = %s WHERE name = %s", ("pt-upd-1", "d1")) - assert _read_one(db.conn, "d1") == "pt-upd-1" - db.verifier.check("d1", ["pt-upd-1"]) - - -@_BUILDS -@pytest.mark.parametrize("read", ["fetchall", "fetchmany", "fetchone", "next", "iterate"]) -def test_reads_decrypt_rows_in_order(db: _Db, read: str) -> None: - want = _write_many(db.conn, "m") - with db.conn.cursor() as cur: - cur.execute(_SELECT, ("m",)) - if read == "fetchall": - got = [row[0] for row in cur.fetchall()] - elif read == "fetchmany": - got = [row[0] for row in cur.fetchmany(2)] + [row[0] for row in cur.fetchmany(5)] - elif read == "fetchone": - got = [cur.fetchone()[0] for _ in want] - assert cur.fetchone() is None - elif read == "next": # psycopg 3.3's cursor is its own iterator; mysql-connector's isn't - got = [next(cur)[0] for _ in want] - with pytest.raises(StopIteration): - next(cur) - else: - got = [row[0] for row in cur] - assert got == want - db.verifier.check("m", want) - - -@_BUILDS -@pytest.mark.parametrize("sql, params, at, want", [ - pytest.param("SELECT * FROM users WHERE name = %s", ("d1",), 2, "pt-tuple-1", id="select-star"), - pytest.param("SELECT count(*) FROM users WHERE ssn = %s", ("pt-tuple-1",), 0, 0, id="where-never-matches"), - pytest.param("SELECT s.ssn FROM (SELECT ssn FROM users WHERE name = %s) s", ("d1",), 0, "pt-tuple-1", - id="derived-table"), - pytest.param("WITH w AS (SELECT ssn FROM users WHERE name = %s) SELECT ssn FROM w", ("d1",), 0, "pt-tuple-1", - id="cte")]) -def test_one_row_reads(db: _Db, sql: str, params: Tuple[str], at: int, want: Any) -> None: - # An equality on the encrypted column compares payloads, so it never matches; the others decrypt. - _write(db.conn, _INSERT, ("d1", "pt-tuple-1")) - with db.conn.cursor() as cur: - cur.execute(sql, params) - assert cur.fetchone()[at] == want - db.verifier.check("d1", ["pt-tuple-1"]) - - -@pytest.mark.parametrize("build", ["pg"]) -@pytest.mark.parametrize("read", ["chained-fetchone", "chained-iterate", "named-cursor", "dict-row"]) -def test_psycopg_chained_named_and_dict_row_reads(db: _Db, read: str) -> None: - values = _write_many(db.conn, "m") - want = values - cursor_args: Dict[str, Any] = {"named-cursor": {"name": "live_named"}, "dict-row": {"row_factory": dict_row}} - with db.conn.cursor(**cursor_args.get(read, {})) as cur: - chained = cur.execute(_SELECT, ("m",)) - if read == "chained-fetchone": - got = [chained.fetchone()[0]] - want = values[:1] - elif read == "dict-row": - got = [cur.fetchone()["ssn"]] - want = values[:1] - else: - got = [row[0] for row in (chained if read == "chained-iterate" else cur)] - db.conn.commit() - assert got == want - db.verifier.check("m", values) - - -@_MYSQL_BUILDS -@pytest.mark.parametrize("cursor_args", [{"dictionary": True}, {"buffered": True}], ids=["dictionary", "buffered"]) -def test_mysql_dictionary_and_buffered_cursors(db: _Db, cursor_args: Dict[str, bool]) -> None: - _write(db.conn, _INSERT, ("d2", "pt-dict-2")) - with db.conn.cursor(**cursor_args) as cur: - cur.execute(_SELECT, ("d2",)) - row = cur.fetchone() - assert (row["ssn"] if "dictionary" in cursor_args else row[0]) == "pt-dict-2" - db.verifier.check("d2", ["pt-dict-2"]) - - -@_BUILDS -def test_literal_into_the_encrypted_column_is_refused(db: _Db) -> None: - with db.conn.cursor() as cur, pytest.raises(MetadataError) as raised: - cur.execute("INSERT INTO users (name, ssn) VALUES (%s, 'net-literal')", ("refused",)) - db.conn.rollback() - err = raised.value - assert type(err).__module__ == "aws_advanced_python_wrapper" - assert (err.table_name, err.column_name) == ("users", "ssn") - assert err.__cause__ is None and err.__context__ is None - assert db.verifier.stored("refused") == [] - - -@_BUILDS -def test_insert_without_a_column_list_reaches_the_trigger(db: _Db, build: str) -> None: - # Passed through with a warning; the server log checks find each control value once. - control = {"pg": "net-unanalysed", "mysql-cext": "net-unanalysed-cext", "mysql-pure": "net-unanalysed-pure"}[build] - with db.conn.cursor() as cur, pytest.raises(AwsWrapperError) as raised: - cur.execute("INSERT INTO users VALUES (DEFAULT, %s, %s, NULL)", ("netrow", control)) - db.conn.rollback() - assert type(raised.value) is AwsWrapperError - assert isinstance(raised.value.__cause__, (psycopg.Error, mysql.connector.Error)) - assert db.verifier.stored("netrow") == [] - - -# ---- SQLAlchemy Core and ORM ---- - -class EncryptedValue(UserDefinedType): - cache_ok = True - - def get_col_spec(self, **kw: Any) -> str: - return "BYTEA" - - -# The guide's pass-through type on PostgreSQL, where SQLAlchemy would cast a typed parameter that bytea rejects. -_SSN = EncryptedValue().with_variant(String(1024), "mysql") -_USERS = Table("users", MetaData(), Column("id", Integer, primary_key=True), Column("name", String(100)), - Column("ssn", _SSN)) - - -class _Base(DeclarativeBase): - pass - - -class _User(_Base): - __tablename__ = "users" - id = mapped_column(Integer, primary_key=True) - name = mapped_column(String(100)) - ssn = mapped_column(_SSN) - - -class _JoinedBase(DeclarativeBase): - pass - - -class _Owner(_JoinedBase): - """``users`` with its addresses loaded by a join, so a LIMIT reads the encrypted column through a subquery.""" - __tablename__ = "users" - id = mapped_column(Integer, primary_key=True) - name = mapped_column(String(100)) - ssn = mapped_column(_SSN) - addresses = relationship("_Address", lazy="joined", order_by="_Address.id") - - -class _Address(_JoinedBase): - __tablename__ = "addresses" - id = mapped_column(Integer, primary_key=True) - user_id = mapped_column(ForeignKey("users.id")) - email = mapped_column(String(100)) - - -@dataclass(frozen=True) -class _Orm: - engine: Engine - verifier: _Verifier - - -@pytest.fixture -def sa(engine: str, bed: _Bed, verifiers: Dict[str, _Verifier]) -> Iterator[_Orm]: - env, pg = bed.env, engine == "pg" - url = URL.create("postgresql+aws_wrapper_psycopg" if pg else "mysql+aws_wrapper_mysqlconnector", - username=env.user, password=env.password, host=_HOST, - port=env.pg_port if pg else env.mysql_port, database=env.dbname) - sa_engine = create_engine(url, connect_args=_plugin_props(engine, bed)) # a ?plugins= in the URL would be dropped - yield _Orm(sa_engine, verifiers[engine]) - sa_engine.dispose() - - -@_ENGINES -@pytest.mark.parametrize("shape", ["insert-values-select-scalar", "executemany-select-all", "result-iteration"]) -def test_core_writes_and_reads(sa: _Orm, shape: str) -> None: - want = ["pt-sa-core-1"] if shape == "insert-values-select-scalar" else ["pt-sa-many-0", "pt-sa-many-1"] - with sa.engine.begin() as conn: - if len(want) == 1: - conn.execute(insert(_USERS).values(name="sa", ssn=want[0])) - else: - conn.execute(insert(_USERS), [{"name": "sa", "ssn": value} for value in want]) - query = select(_USERS.c.ssn).where(_USERS.c.name == "sa").order_by(_USERS.c.id) - with sa.engine.connect() as conn: - if len(want) == 1: - got = [conn.execute(query).scalar()] - else: - got = [row.ssn if shape == "executemany-select-all" else row[0] for row in conn.execute(query)] - assert got == want - sa.verifier.check("sa", want) - - -def _orm_users(sa: _Orm, values: List[str]) -> List[int]: - with Session(sa.engine) as session: - session.add_all([_User(name="orm", ssn=value) for value in values]) - session.commit() - return [user.id for user in session.scalars(select(_User).where(_User.name == "orm").order_by(_User.id))] - - -@_ENGINES -@pytest.mark.parametrize("read", ["add-all-scalars", "session-get", "legacy-filter-by-first"]) -def test_orm_reads(sa: _Orm, read: str) -> None: - values = [f"pt-orm-{i}" for i in range(3)] - ids = _orm_users(sa, values) - with Session(sa.engine) as session: - if read == "add-all-scalars": - users: List[Optional[_User]] = list(session.scalars(select(_User).where(_User.name == "orm") - .order_by(_User.id))) - want = values - elif read == "session-get": - users = [session.get(_User, ids[1])] - want = values[1:2] - else: - users = [session.query(_User).filter_by(id=ids[2]).first()] - want = values[2:] - assert [user.ssn if user else None for user in users] == want - sa.verifier.check("orm", values) - - -@_ENGINES -def test_orm_attribute_update(sa: _Orm) -> None: - ids = _orm_users(sa, [f"pt-orm-{i}" for i in range(3)]) - with Session(sa.engine) as session: - user = session.get(_User, ids[0]) - assert user is not None - user.ssn = "pt-orm-upd" - session.commit() - with Session(sa.engine) as session: - user = session.get(_User, ids[0]) - assert user is not None and user.ssn == "pt-orm-upd" - sa.verifier.check("orm", ["pt-orm-upd", "pt-orm-1", "pt-orm-2"]) - - -@_ENGINES -@pytest.mark.parametrize("read", ["legacy-query-first", "joinedload-limit"]) -def test_joined_eager_reads(sa: _Orm, read: str) -> None: - with Session(sa.engine) as session: - owner = _Owner(name="ow", ssn="pt-owner-1", addresses=[_Address(email="a1@example.com"), - _Address(email="a2@example.com")]) - session.add(owner) - session.commit() - owner_id = owner.id - with Session(sa.engine) as session: - if read == "legacy-query-first": - found = session.query(_Owner).filter(_Owner.id == owner_id).first() - else: - found = session.scalars(select(_Owner).options(joinedload(_Owner.addresses)).limit(1)).unique().first() - assert found is not None and found.ssn == "pt-owner-1" and len(found.addresses) == 2 - sa.verifier.check("ow", ["pt-owner-1"]) - - -# ---- botocore at DEBUG ---- - -@contextlib.contextmanager -def _debug_log() -> Iterator[io.StringIO]: - # While the root logger is at DEBUG, this handler is its only one, so the data key that the positive control logs - # reaches no other handler, pytest's included, and so no report. - root = logging.getLogger() - level, others = root.level, list(root.handlers) - buffer = io.StringIO() - handler = logging.StreamHandler(buffer) - for other in others: - root.removeHandler(other) - root.addHandler(handler) - root.setLevel(logging.DEBUG) - try: - yield buffer - finally: - root.removeHandler(handler) - root.setLevel(level) - for other in others: - root.addHandler(other) - - -@_ENGINES -def test_debug_log_holds_no_data_key(engine: str, bed: _Bed, verifiers: Dict[str, _Verifier]) -> None: - build = "pg" if engine == "pg" else "mysql-cext" - with _connect(build, bed) as conn: - _write(conn, _INSERT, ("dbg", "pt-debug-1")) - verifiers[engine].check("dbg", ["pt-debug-1"]) - with _debug_log() as log, _connect(build, bed) as conn: # a new connection decrypts its data key through KMS - value = _read_one(conn, "dbg") - holds_plaintext, logged_decrypt = '"Plaintext"' in log.getvalue(), "TrentService.Decrypt" in log.getvalue() - assert value == "pt-debug-1" and not holds_plaintext and logged_decrypt - # The positive control: without the filter the same read logs the data key. - parsers = logging.getLogger("botocore.parsers") - removed = [f for f in parsers.filters if type(f).__name__ == "_NoKmsPlaintext"] - for f in removed: - parsers.removeFilter(f) - try: - with _debug_log() as log, _connect(build, bed) as conn: - _read_one(conn, "dbg") - control_logged = '"Plaintext"' in log.getvalue() - finally: - for f in removed: - parsers.addFilter(f) - assert removed and control_logged - - -# ---- Django on MySQL ---- - -@pytest.fixture(scope="module") -def django_model(bed: _Bed) -> Iterator[Any]: - env = bed.env - if django_settings.configured: # another module's settings would point these shapes at another database - pytest.fail("Django settings are already configured in this process; run this module on its own.") - django_settings.configure( - DEBUG=False, USE_TZ=False, INSTALLED_APPS=[], DEFAULT_AUTO_FIELD="django.db.models.AutoField", - DATABASES={"default": {"ENGINE": "aws_advanced_python_wrapper.django.backends.mysql_connector", - "NAME": env.dbname, "USER": env.user, "PASSWORD": env.password, "HOST": _HOST, - "PORT": env.mysql_port, "OPTIONS": _plugin_props("mysql", bed)}}) - django.setup() - - class LiveUser(models.Model): - name: models.CharField[str, str] = models.CharField(max_length=100) - ssn: models.CharField[Optional[str], Optional[str]] = models.CharField(max_length=100, null=True) - - class Meta: - app_label = "kms_live" - db_table = "users" - managed = False - - yield LiveUser - django_connection.close() - - -@pytest.mark.parametrize("write", ["create", "bulk-create", "save", "queryset-update"]) -def test_django_writes(django_model: Any, verifiers: Dict[str, _Verifier], write: str) -> None: - objects = django_model.objects - if write == "bulk-create": - want = ["pt-djb-0", "pt-djb-1"] - assert len(objects.bulk_create([django_model(name="dj", ssn=value) for value in want])) == 2 - else: - want = [{"create": "pt-dj-1", "save": "pt-dj-upd", "queryset-update": "pt-dj-qsupd"}[write]] - assert objects.create(name="dj", ssn="pt-dj-1").ssn == "pt-dj-1" - if write == "save": - user = objects.get(name="dj") - user.ssn = want[0] - user.save() - elif write == "queryset-update": - objects.filter(name="dj").update(ssn=want[0]) - assert objects.get(name="dj").ssn == want[0] - verifiers["mysql"].check("dj", want) - - -@pytest.mark.parametrize("read", ["get", "values-list", "iterator", "raw", "connection-cursor"]) -def test_django_reads(django_model: Any, verifiers: Dict[str, _Verifier], read: str) -> None: - want = ["pt-djb-0", "pt-djb-1"] - django_model.objects.bulk_create([django_model(name="djb", ssn=value) for value in want]) - rows = django_model.objects.filter(name="djb").order_by("id") - if read == "get": - got = [django_model.objects.get(pk=pk).ssn for pk in rows.values_list("id", flat=True)] - elif read == "values-list": - got = list(rows.values_list("ssn", flat=True)) - elif read == "iterator": - got = [user.ssn for user in rows.iterator(chunk_size=1)] - elif read == "raw": - got = [user.ssn for user in django_model.objects.raw("SELECT id, name, ssn FROM users WHERE name = %s " - "ORDER BY id", ["djb"])] - else: - with django_connection.cursor() as cur: - cur.execute(_SELECT, ["djb"]) - got = [row[0] for row in cur] - assert got == want - verifiers["mysql"].check("djb", want) - - -# ---- The servers' own logs (after every test above, so they cover them) ---- - -def _log_lines(bed: _Bed, variable: str) -> List[bytes]: - """The lines a server logged after this run's marker statement, from the command in ``variable``. - - A failure names the variable and holds neither the command nor its output; the tests report counts, never a line. - """ - command = bed.env.pg_log_command if variable == "PG_LOG_COMMAND" else bed.env.mysql_log_command - failure = "" - try: - done = subprocess.run(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - timeout=_LOG_TIMEOUT) - except subprocess.TimeoutExpired: # its text holds the command - failure = f"{variable} didn't finish within {_LOG_TIMEOUT} seconds." - except (OSError, ValueError): # no such program, one that can't run, or an unclosed quote - failure = f"{variable} couldn't be started; check its program and its quoting." - if failure: - pytest.fail(failure) - if done.returncode != 0: - pytest.fail(f"{variable} exited with status {done.returncode}.") - lines = done.stdout.splitlines() - starts = [i for i, line in enumerate(lines) if bed.marker.encode() in line] - if not starts: - pytest.fail(f"The output of {variable} has no line with this run's marker statement.") - return lines[starts[-1] + 1:] - - -def _counts(lines: List[bytes], needles: Tuple[str, ...]) -> Dict[str, int]: - return {needle: sum(needle.encode() in line for line in lines) for needle in needles} - - -def test_postgresql_log_holds_no_plaintext(bed: _Bed) -> None: - lines = _log_lines(bed, "PG_LOG_COMMAND") - total = len(lines) - # PostgreSQL logs a bytea parameter in hex, so a "pt-" value bound as bytes would show as '\x70742d. - # A read of PostgreSQL's container log now and then comes back without some of the lines the log holds. When a - # PostgreSQL log test is the run's only failure and its only count off is one expected above 0 that came out lower, - # re-run the module once: a wrong command misses the line again, and a missed read almost never does. A count - # expected to be 0 that comes out above 0 is a leak even when it is the run's only failure. - counts = _counts(lines, ("pt-", r"'\x70742d", "net-literal", r"'\x6e65742d")) - assert total > 0 - assert counts == {"pt-": 0, r"'\x70742d": 0, "net-literal": 0, r"'\x6e65742d": 1} - - -def test_mysql_general_log_holds_no_plaintext(bed: _Bed) -> None: - lines = _log_lines(bed, "MYSQL_LOG_COMMAND") - total, writes = len(lines), sum(bool(re.search(rb"(?i)(INSERT INTO|UPDATE) `?users", line)) for line in lines) - # The general log holds each encrypted value as raw bytes, so even a run without a leak can show "pt-" by chance, - # though rarely. When this test is the run's only failure and "pt-" its only count off, re-run the module once: a - # leak repeats and a chance match almost never does, so treat a second hit as a leak. - counts = _counts(lines, ("pt-", "net-literal", "net-unanalysed-cext", "net-unanalysed-pure")) - assert total > 0 and writes > 0 - assert counts == {"pt-": 0, "net-literal": 0, "net-unanalysed-cext": 1, "net-unanalysed-pure": 1} - - -# ---- Further checks: error classes, target_cursor, key rows, logging and driver quirks ---- -# They run after the two log tests above, so those tests' counts don't include them. Plaintexts written through the -# plugin here start with "pt-x-", and values meant to reach a server unencrypted with "net-x-"; the file's last two -# tests count them, and count _NAIVE_DATETIME, which can't carry a prefix, by its time of day. - -_LITERAL = "INSERT INTO users (name, ssn) VALUES (%s, 'net-x-literal')" -_SET_ARN = "UPDATE encrypt.key_storage SET master_key_arn = %s WHERE id = %s" -_FOREIGN_KEY_ID = 987654321 # no key_storage row has it -_READS = ("fetchone", "fetchmany", "fetchall", "iterate", "chained") # mysql-connector's execute returns None - - -def _per_build(options: Tuple[str, ...], pg_only: Tuple[str, ...] = (), mysql_only: Tuple[str, ...] = ()) -> List[Any]: - return [pytest.param(build, option, id=f"{build}-{option}") for build in _ENGINE for option in options - if option not in (mysql_only if build == "pg" else pg_only)] - - -def _plain_rows(engine: str, env: _Env, sql: str, params: Any) -> List[Any]: - """The first column of every row a plain driver reads.""" - with _plain(engine, env) as conn, conn.cursor() as cur: - cur.execute(sql, params) - return [row[0] for row in cur.fetchall()] - - -def _plain_write(engine: str, env: _Env, sql: str, params: Any) -> None: - with _plain(engine, env) as conn: - with conn.cursor() as cur: - cur.execute(sql, params) - conn.commit() - - -def _key_id(engine: str, env: _Env, column: str) -> int: - """The column's current ``key_storage`` id, read when the test runs, since a rotation changes it.""" - return int(_plain_rows(engine, env, "SELECT key_id FROM encrypt.encryption_metadata WHERE table_name = %s " - "AND column_name = %s", ("users", column))[0]) - - -@contextlib.contextmanager -def _dob_verifier(engine: str, bed: _Bed) -> Iterator[_Verifier]: - verifier = _Verifier(engine, bed, "dob") - try: - yield verifier - finally: - verifier.close() - - -def _read_all(conn: AwsWrapperConnection, sql: str, name: str) -> List[Any]: - with conn.cursor() as cur: - cur.execute(sql, (name,)) - return [row[0] for row in cur.fetchall()] - - -def _write_tampered(build: str, bed: _Bed) -> None: - """Row ``x-tamper``: ``pt-x-dob`` written through the plugin, then its last byte flipped with a plain driver. The - payload keeps the column's key id, so reading it fails the HMAC check and no other key row is looked up.""" - engine = _ENGINE[build] - with _connect(build, bed) as conn: - _write(conn, _INSERT_DOB, ("x-tamper", "pt-x-dob")) - stored = bytes(_plain_rows(engine, bed.env, _SELECT_DOB, ("x-tamper",))[0]) - tampered = stored[:-1] + bytes([stored[-1] ^ 0x01]) - _plain_write(engine, bed.env, "UPDATE users SET dob = %s WHERE name = %s", (tampered, "x-tamper")) - - -def _read_tampered(cur: Any, read: str) -> Any: - if read == "chained": - return cur.execute(_SELECT_DOB, ("x-tamper",)).fetchone() - cur.execute(_SELECT_DOB, ("x-tamper",)) - if read == "fetchone": - return cur.fetchone() - if read == "fetchmany": - return cur.fetchmany(2) - return cur.fetchall() if read == "fetchall" else list(cur) - - -@contextlib.contextmanager -def _refused(conn: AwsWrapperConnection, cls: Type[BaseException], kms_requests: Optional[int] = None) -> Iterator[Any]: - """Expects exactly ``cls`` from the block, run with the root logger at DEBUG: no failover record names it, and if - ``kms_requests`` is given, that many captured lines name a KMS operation (one request logs more than one; the - captured text is only counted). After a rollback the connection still runs ``SELECT 1``. The rollback also - discards rows a failed read left unread on mysql-connector's default cursor.""" - with _debug_log() as log: - with pytest.raises(cls) as raised: - yield raised - lines = log.getvalue().splitlines() - named = sum("failover" in line.lower() and cls.__name__ in line for line in lines) - sent = sum("TrentService." in line for line in lines) - assert type(raised.value) is cls and named == 0 - if kms_requests is not None: - assert sent == kms_requests - conn.rollback() - with conn.cursor() as cur: - cur.execute("SELECT 1") - assert cur.fetchone()[0] == 1 - - -# Error classes as the application sees them through the default chains, failover_v2 included. - -@pytest.mark.parametrize("build, read", _per_build(_READS, pg_only=("chained",))) -def test_tampered_value_raises_integrity_check_error(db: _Db, bed: _Bed, build: str, read: str) -> None: - _write_tampered(build, bed) - with _refused(db.conn, IntegrityCheckError), db.conn.cursor() as cur: - _read_tampered(cur, read) - - -@_ENGINES -def test_tampered_value_through_sqlalchemy_raises_integrity_error(sa: _Orm, bed: _Bed, engine: str) -> None: - _write_tampered("pg" if engine == "pg" else "mysql-cext", bed) - with sa.engine.connect() as conn, pytest.raises(sa_exc.IntegrityError) as raised: - conn.execute(text("SELECT dob FROM users WHERE name = :n"), {"n": "x-tamper"}).scalar() - assert type(raised.value) is sa_exc.IntegrityError and type(raised.value.orig) is IntegrityCheckError - - -@pytest.mark.parametrize("build, call", _per_build(("execute", "executemany"))) -def test_unserializable_value_keeps_the_encryption_service_error(db: _Db, call: str) -> None: - with pytest.raises(EncryptionError) as direct: # what the encryption service raises for it on its own - EncryptionService.encrypt(2 ** 70, bytearray(32), bytearray(32), "AES-256-GCM", key_id=1) - expected = type(direct.value) - with _refused(db.conn, expected), db.conn.cursor() as cur: - if call == "execute": - cur.execute(_INSERT, ("x-big", 2 ** 70)) - else: - cur.executemany(_INSERT, [("x-big", 2 ** 70)]) - assert expected is EncryptionFailedError and db.verifier.stored("x-big") == [] - - -@_BUILDS -def test_literal_through_executemany_is_refused(db: _Db) -> None: - with _refused(db.conn, MetadataError) as raised, db.conn.cursor() as cur: - cur.executemany(_LITERAL, [("x-lit",)]) - err = raised.value - assert (err.table_name, err.column_name) == ("users", "ssn") - assert err.__cause__ is None and err.__context__ is None - assert db.verifier.stored("x-lit") == [] - - -@_BUILDS -def test_hidden_placeholder_on_an_encrypted_table_is_refused(db: _Db) -> None: - # Both drivers bind the %s in the comment too, so the plugin can't tell which value goes to ssn. Refused before - # anything is sent, so the "pt-x-" count of the last two tests stays 0. - with _refused(db.conn, MetadataError) as raised, db.conn.cursor() as cur: - cur.execute("INSERT INTO users (name, ssn) VALUES (%s, %s) /* %s */", ("x-hidden", "pt-x-hidden", "x-comment")) - err = raised.value - assert str(err) == Messages.get_formatted("KmsEncryptionPlugin.HiddenPlaceholder", "users") - assert err.__cause__ is None and err.__context__ is None - assert db.verifier.stored("x-hidden") == [] - - -@_BUILDS -def test_unsafe_comparison_on_an_encrypted_column_is_refused(db: _Db, bed: _Bed, build: str) -> None: - # An encrypted value equals no stored value, so these comparisons would match every row or arbitrary rows (the - # first would delete every row). They are refused before anything is sent, so no "pt-x-" value reaches a server. - rows = {"x-cmp-1": "pt-x-cmp-1", "x-cmp-2": "pt-x-cmp-2"} - for row in rows.items(): - _write(db.conn, _INSERT, row) - names = "SELECT name FROM users ORDER BY name" - for sql, params in (("DELETE FROM users WHERE ssn <> %s", ("pt-x-cmp-neq",)), - ("DELETE FROM users WHERE id NOT IN (SELECT id FROM users WHERE ssn = %s)", ("pt-x-cmp-not-in",)), - ("UPDATE users SET name = %s WHERE ssn > %s", ("x-cmp-renamed", "pt-x-cmp-gt")), - ("SELECT name FROM users WHERE ssn NOT LIKE %s", ("pt-x-cmp-not-like",))): - with _refused(db.conn, MetadataError) as raised, db.conn.cursor() as cur: - cur.execute(sql, params) - err = raised.value - assert str(err) == Messages.get_formatted("KmsEncryptionPlugin.UnsafeComparison", "users.ssn") - assert (err.table_name, err.column_name) == ("users", "ssn") - assert err.__cause__ is None and err.__context__ is None - assert _plain_rows(_ENGINE[build], bed.env, names, ()) == list(rows) - # An equality is encrypted and still runs: it matches nothing. - with db.conn.cursor() as cur: - cur.execute("DELETE FROM users WHERE ssn = %s", ("pt-x-cmp-eq",)) - deleted = cur.rowcount - cur.execute("WITH w AS (SELECT id FROM users WHERE ssn = %s) SELECT id FROM w", ("pt-x-cmp-cte",)) - found = cur.fetchall() - db.conn.commit() - assert deleted == 0 and found == [] - assert _plain_rows(_ENGINE[build], bed.env, names, ()) == list(rows) - for name, value in rows.items(): - db.verifier.check(name, [value]) - - -def _refused_write_or_read(build: str, bed: _Bed, call: str, label: str, cls: Type[BaseException], - kms_requests: Optional[int] = None, **overrides: Any) -> Any: - """A write, or a read of a row a normal connection wrote first, on a connection with ``overrides``.""" - name, value = f"x-{label}", f"pt-x-{label}" - if call == "read": - with _connect(build, bed) as conn: - _write(conn, _INSERT, (name, value)) - with _connect(build, bed, **overrides) as conn: - with _refused(conn, cls, kms_requests) as raised, conn.cursor() as cur: - if call == "write": - cur.execute(_INSERT, (name, value)) - else: - cur.execute(_SELECT, (name,)) - cur.fetchone() - assert len(_plain_rows(_ENGINE[build], bed.env, _SELECT, (name,))) == (1 if call == "read" else 0) - return raised.value - - -@pytest.mark.parametrize("build, call", _per_build(("write", "read"))) -def test_master_key_outside_the_allow_list_is_refused(bed: _Bed, build: str, call: str) -> None: - other = bed.arns["mysql" if _ENGINE[build] == "pg" else "pg"] # a LocalStack key this run made for the other engine - # Refused before any KMS request: the captured DEBUG text names no TrentService operation. - _refused_write_or_read(build, bed, call, "unauth", UnauthorizedMasterKeyError, kms_requests=0, - encryption_allowed_master_key_arns=other) - - -@pytest.mark.parametrize("build, call", _per_build(("write", "read"))) -def test_unreachable_kms_raises_key_management_error(bed: _Bed, build: str, call: str) -> None: - err = _refused_write_or_read(build, bed, call, "down", KeyManagementError, - encryption_kms_endpoint=f"http://{_HOST}:1", encryption_key_management_max_retries=0) - assert type(err.__cause__) is EndpointConnectionError - - -# target_cursor is the driver's cursor, which the plugin doesn't see. - -@_BUILDS -def test_target_cursor_is_the_driver_cursor(db: _Db) -> None: - with db.conn.cursor() as cur, db.conn.target_connection.cursor() as own: - assert type(cur.target_cursor) is type(own) - - -def _closed(cursor: Any) -> bool: - if isinstance(cursor, psycopg.Cursor): - return cursor.closed - return cursor._connection is None # both mysql-connector builds clear it in close() - - -@_BUILDS -@pytest.mark.parametrize("how", ["close", "leave-with"]) -def test_closing_the_cursor_closes_the_driver_cursor(db: _Db, how: str) -> None: - if how == "close": - cur = db.conn.cursor() - target: Any = cur.target_cursor - cur.close() - else: - with db.conn.cursor() as cur: - target = cur.target_cursor - assert _closed(target) - - -class _ArgsLoggerPlugin(Plugin): - """The parameter-logging row's controls: a plugin that logs the arguments it receives.""" - - @property - def subscribed_methods(self) -> Set[str]: - return {DbApiMethod.CURSOR_EXECUTE.method_name, DbApiMethod.CURSOR_EXECUTEMANY.method_name} - - def execute(self, target: type, method_name: str, execute_func: Callable, *args: Any, **kwargs: Any) -> Any: - # stdlib logging: a test can't add a message key, and the record only has to reach the root handler. - logging.getLogger(__name__).debug("args_logger %s %r", method_name, args) - return execute_func() - - -class _ArgsLoggerFactory(PluginFactory): - @staticmethod - def get_instance(plugin_service: PluginService, props: Properties) -> Plugin: - return _ArgsLoggerPlugin() - - -class _LateArgsLoggerFactory(_ArgsLoggerFactory): - """The same plugin at another weight, since a weight belongs to the factory type.""" - - -@pytest.fixture -def args_loggers() -> Iterator[None]: - # Weight 385 puts args_logger before kms_encryption (390), and 395 puts late_args_logger after it and before - # failover_v2 (410). - loggers = {"args_logger": (_ArgsLoggerFactory, 385), "late_args_logger": (_LateArgsLoggerFactory, 395)} - for code, (factory, weight) in loggers.items(): - PluginManager.register_plugin(code, factory, weight) - try: - yield - finally: - for code, (factory, _) in loggers.items(): - PluginManager.PLUGIN_FACTORIES.pop(code, None) - PluginManager.PLUGIN_FACTORY_WEIGHTS.pop(factory, None) - - -@_BUILDS -@pytest.mark.parametrize("extra", ["execute_time", "args_logger", "late_args_logger"]) -def test_other_plugins_log_no_bound_value(bed: _Bed, verifiers: Dict[str, _Verifier], args_loggers: None, build: str, - extra: str) -> None: - # The plugins before kms_encryption receive the caller's values, and the plugins after it the payloads; the - # built-in ones log neither. args_logger is the positive control; late_args_logger logs only payloads. - value = f"pt-x-r7-{build}-{extra}" - plugins = f"{_plugin_props(_ENGINE[build], bed)['plugins']},{extra}" - with _debug_log() as log, _connect(build, bed, plugins=plugins) as conn: - with conn.cursor() as cur: - cur.execute(_INSERT, ("x-r7", value)) - cur.executemany(_INSERT, [("x-r7", f"{value}-m")]) - conn.commit() - back = _read_all(conn, _SELECT, "x-r7") - verifiers[_ENGINE[build]].check("x-r7", [value, f"{value}-m"]) - lines = log.getvalue().splitlines() - logged = sum(value in line for line in lines) - if extra == "args_logger": - assert logged > 0 - else: - assert logged == 0 and back == [value, f"{value}-m"] - if extra == "late_args_logger": - writes = [line for line in lines if line.startswith("args_logger ") and "INSERT INTO users" in line] - assert len(writes) == 2 and all("b'" in line or 'b"' in line for line in writes) - - -@_BUILDS -def test_target_cursor_is_not_seen(db: _Db) -> None: - # Its reads return the stored bytes, and the trigger rejects its plaintext write with the driver's own error. - _write(db.conn, _INSERT, ("x-tc", "pt-x-tc")) - with db.conn.cursor() as cur: - target: Any = cur.target_cursor - target.execute(_SELECT, ("x-tc",)) - stored = target.fetchone()[0] - target.fetchall() - with pytest.raises((psycopg.Error, mysql.connector.Error)): - target.execute(_INSERT, ("x-tc-net", "net-x-target-cursor")) - db.conn.rollback() - db.verifier.check("x-tc", ["pt-x-tc"]) - assert bytes(stored) == bytes(db.verifier.stored("x-tc")[0]) and db.verifier.stored("x-tc-net") == [] - - -# SQL that isn't str, and psycopg row factories. - -@pytest.mark.parametrize("build", ["pg"]) -def test_composable_insert_and_select(db: _Db) -> None: - with db.conn.cursor() as cur: - cur.execute(pgsql.SQL("INSERT INTO {} (name, ssn) VALUES ({}, {})").format( - pgsql.Identifier("users"), pgsql.Placeholder(), pgsql.Placeholder()), ("x-comp", "pt-x-comp")) - db.conn.commit() - cur.execute(pgsql.SQL("SELECT ssn FROM users WHERE name = {}").format(pgsql.Placeholder()), ("x-comp",)) - got = cur.fetchone()[0] - db.verifier.check("x-comp", ["pt-x-comp"]) - assert got == "pt-x-comp" - - -@pytest.mark.parametrize("build", ["pg"]) -def test_composed_insert_with_percent_placeholders_is_encrypted(db: _Db) -> None: - with db.conn.cursor() as cur: - cur.execute(pgsql.SQL("INSERT INTO {} (name, ssn) VALUES (%s, %s)").format(pgsql.Identifier("users")), - ("x-composed", "pt-x-composed")) - db.conn.commit() - db.verifier.check("x-composed", ["pt-x-composed"]) - - -@pytest.mark.parametrize("build", ["pg"]) -def test_bytes_sql_is_analyzed_and_encrypted(db: _Db) -> None: - with _debug_log() as log: - for i in range(2): - with db.conn.cursor() as cur: - cur.execute(b"INSERT INTO users (name, ssn) VALUES (%s, %s)", (f"x-bytes{i}", f"pt-x-bytes{i}")) - db.conn.commit() - plugin_records = log.getvalue().count("[KmsEncryptionPlugin]") - for i in range(2): - db.verifier.check(f"x-bytes{i}", [f"pt-x-bytes{i}"]) - assert plugin_records == 0 - - -@_BUILDS -def test_unanalysed_insert_control_reaches_the_trigger(db: _Db, build: str) -> None: - # The positive control of the last two tests: passed through with a warning, found once in each server's log. - control = {"pg": "net-x-unanalysed", "mysql-cext": "net-x-unanalysed-cext", "mysql-pure": "net-x-unanalysed-pure"} - with db.conn.cursor() as cur, pytest.raises(AwsWrapperError) as raised: - cur.execute("INSERT INTO users VALUES (DEFAULT, %s, %s, NULL)", ("x-netrow", control[build])) - db.conn.rollback() - assert type(raised.value) is AwsWrapperError - assert isinstance(raised.value.__cause__, (psycopg.Error, mysql.connector.Error)) - assert db.verifier.stored("x-netrow") == [] - - -def _first_row(cur: Any, read: str) -> Any: - if read == "chained": - return cur.execute(_SELECT, ("x-rf",)).fetchone() - cur.execute(_SELECT, ("x-rf",)) - return cur.fetchone() if read == "fetchone" else list(cur)[0] - - -@dataclass(frozen=True) -class _NameRow: - name: str - - -@pytest.mark.parametrize("build", ["pg"]) -@pytest.mark.parametrize("factory", ["class_row", "scalar_row", "dict_row", "namedtuple_row"]) -@pytest.mark.parametrize("read", ["fetchone", "iterate", "chained"]) -def test_row_factories(db: _Db, factory: str, read: str) -> None: - _write(db.conn, _INSERT, ("x-rf", "pt-x-rf")) - db.verifier.check("x-rf", ["pt-x-rf"]) - made: List[int] = [] - - class Counted: - def __init__(self, **columns: Any) -> None: - made.append(1) - - row_factory = {"class_row": class_row(Counted), "scalar_row": scalar_row, "dict_row": dict_row, - "namedtuple_row": namedtuple_row}[factory] - with db.conn.cursor(row_factory=row_factory) as cur: - if factory in ("class_row", "scalar_row"): # refused before any row is built - with pytest.raises(MetadataError) as raised: - _first_row(cur, read) - assert type(raised.value) is MetadataError and made == [] - return - row = _first_row(cur, read) - if factory == "dict_row": - assert type(row) is dict and row["ssn"] == "pt-x-rf" - else: - assert type(row).__name__ == "Row" and row.ssn == "pt-x-rf" - - -@pytest.mark.parametrize("build", ["pg"]) -def test_class_row_without_an_encrypted_column_is_not_refused(db: _Db) -> None: - _write(db.conn, _INSERT, ("x-rf", "pt-x-rf")) - db.verifier.check("x-rf", ["pt-x-rf"]) - with db.conn.cursor(row_factory=class_row(_NameRow)) as cur: - cur.execute("SELECT name FROM users WHERE name = %s", ("x-rf",)) - row = cur.fetchone() - assert row == _NameRow("x-rf") - - -# Key rows: rotation, two columns, and values whose key row is missing or unusable. - -@pytest.mark.parametrize("build", ["pg"]) -def test_rotation_keeps_values_under_the_old_key_readable(db: _Db, bed: _Bed) -> None: - env = bed.env - old = [f"pt-x-old-{i}" for i in range(200)] - old_id = _key_id("pg", env, "dob") - with db.conn.cursor() as cur: - cur.executemany(_INSERT_DOB, [("x-old", value) for value in old]) - db.conn.commit() - with _plain("pg", env) as conn: - new_id = _utility(conn, env, allowed=bed.arns["pg"]).rotate_data_key("users", "dob") - conn.commit() - new = [f"pt-x-new-{i}" for i in range(200)] - with _connect("pg", bed) as conn: # an open connection keeps the old key row until its metadata refreshes - old_back = _read_all(conn, _SELECT_DOB, "x-old") - with conn.cursor() as cur: - cur.executemany(_INSERT_DOB, [("x-new", value) for value in new]) - conn.commit() - new_back = _read_all(conn, _SELECT_DOB, "x-new") - with _dob_verifier("pg", bed) as verifier: # built after the rotation, so it checks the new key id - verifier.check("x-new", new) - old_ids = {EncryptionService.key_id_from_payload(bytes(value)) for value in verifier.stored("x-old")} - assert new_id != old_id and old_ids == {old_id} and old_back == old and new_back == new - - -@_BUILDS -def test_two_encrypted_columns_use_their_own_key_rows(db: _Db, bed: _Bed, build: str) -> None: - engine = _ENGINE[build] - _write(db.conn, "INSERT INTO users (name, ssn, dob) VALUES (%s, %s, %s)", ("x-two", "pt-x-two-ssn", "pt-x-two-dob")) - db.verifier.check("x-two", ["pt-x-two-ssn"]) - with _dob_verifier(engine, bed) as verifier: - verifier.check("x-two", ["pt-x-two-dob"]) - assert _key_id(engine, bed.env, "dob") != bed.key_ids[engine] - - -def _fake_payload(key_id: int) -> bytes: - """81 bytes in the payload layout naming ``key_id``; the test owns every byte, and none of them is a key.""" - return bytes(32) + key_id.to_bytes(4, "big") + bytes([1]) + bytes(44) - - -@_BUILDS -def test_foreign_key_id_is_refused_in_strict_mode(bed: _Bed, build: str) -> None: - payload = _fake_payload(_FOREIGN_KEY_ID) - _plain_write(_ENGINE[build], bed.env, _INSERT_DOB, ("x-foreign", payload)) - with _connect(build, bed) as conn, conn.cursor() as cur, pytest.raises(IntegrityCheckError) as raised: - cur.execute(_SELECT_DOB, ("x-foreign",)) - cur.fetchone() - err = raised.value - shown = " ".join((str(err), repr(err), repr(err.args))) - forms = (str(_FOREIGN_KEY_ID), payload[32:36].hex(), repr(payload[32:36])[2:-1], payload.hex()) - leaked = [form for form in forms if form in shown] - assert type(err) is IntegrityCheckError and "users.dob" in str(err) and leaked == [] - assert err.__cause__ is None and err.__context__ is None - - -@_BUILDS -def test_foreign_key_id_comes_back_as_stored_in_lenient_mode(bed: _Bed, build: str) -> None: - payload = _fake_payload(_FOREIGN_KEY_ID) - _plain_write(_ENGINE[build], bed.env, _INSERT_DOB, ("x-foreign", payload)) - with _connect(build, bed, encryption_return_unverified_data=True) as conn: - back = _read_all(conn, _SELECT_DOB, "x-foreign") - as_stored = [bytes(value) for value in back] == [payload] - assert as_stored - - -@pytest.mark.parametrize("build, mode", _per_build(("strict", "lenient"))) -def test_unusable_key_row_of_the_column_raises_in_both_modes(bed: _Bed, build: str, mode: str) -> None: - engine, env = _ENGINE[build], bed.env - key_id = _key_id(engine, env, "dob") - _plain_write(engine, env, _INSERT_DOB, ("x-own", _fake_payload(key_id))) - arn = _plain_rows(engine, env, "SELECT master_key_arn FROM encrypt.key_storage WHERE id = %s", (key_id,))[0] - _plain_write(engine, env, _SET_ARN, ("", key_id)) # a key row with no master key ARN is unusable - try: - with _connect(build, bed, encryption_return_unverified_data=mode == "lenient") as conn: - with conn.cursor() as cur, pytest.raises(InvalidKeyMetadataError) as raised: - cur.execute(_SELECT_DOB, ("x-own",)) - cur.fetchone() - finally: - _plain_write(engine, env, _SET_ARN, (arn, key_id)) - assert type(raised.value) is InvalidKeyMetadataError - - -@_ENGINES -def test_standalone_utility_at_debug_logs_no_data_key(bed: _Bed, engine: str) -> None: - # The bed fixture gave users.dob its key row with the root logger at DEBUG, before any plugin connection existed. - assert bed.utility_log[engine] == {"plaintext": False, "kms-request": True, "filter-found": True, - "control-plaintext": True} - - -# The worker thread socket_timeout uses, named-cursor paging, and kms_encryption listed after failover_v2. - -@_BUILDS -def test_socket_timeout_hands_the_driver_the_encrypted_copy(bed: _Bed, verifiers: Dict[str, _Verifier], - build: str) -> None: - with _connect(build, bed, socket_timeout=5) as conn: # the driver call then runs on a worker thread - _write(conn, _INSERT, ("x-sock", "pt-x-sock")) - got = _read_one(conn, "x-sock") - verifiers[_ENGINE[build]].check("x-sock", ["pt-x-sock"]) - assert got == "pt-x-sock" - - -@pytest.mark.parametrize("build", ["pg"]) -def test_named_cursor_pages_through_250_rows(db: _Db) -> None: - # Its three FETCH FORWARD 100 lines are counted by the PostgreSQL log test at the end of the file, once the - # container's log has had time to arrive. - values = [f"pt-x-page-{i}" for i in range(250)] - with db.conn.cursor() as cur: - cur.executemany(_INSERT, [("x-page", value) for value in values]) - db.conn.commit() - with db.conn.cursor(name="live_paging") as cur: - cur.execute(_SELECT, ("x-page",)) - got = [row[0] for row in cur] - db.conn.commit() - db.verifier.check("x-page", values) - assert got == values - - -_FAILOVER_FIRST: Dict[str, Type[AwsWrapperError]] = { - "execute": AwsWrapperError, "fetchone": AwsWrapperError, "fetchmany": AwsWrapperError, - "fetchall": AwsWrapperError, "executemany": MetadataError, "iterate": IntegrityCheckError, - "chained": AwsWrapperError} # psycopg's chained fetch is the wrapper cursor's fetchone, which failover_v2 handles - - -@pytest.mark.parametrize("build, call", _per_build(tuple(_FAILOVER_FIRST), pg_only=("chained",))) -def test_failover_first_order_wraps_execute_and_fetch_errors(bed: _Bed, build: str, call: str) -> None: - engine = _ENGINE[build] - chain = WrapperProperties.DEFAULT_PLUGINS if engine == "pg" else WrapperProperties.MYSQL_CONNECTOR_DEFAULT_PLUGINS - if call not in ("execute", "executemany"): - _write_tampered(build, bed) - with _connect(build, bed, plugins=f"{chain},kms_encryption", auto_sort_wrapper_plugin_order=False) as conn: - with conn.cursor() as cur, pytest.raises(AwsWrapperError) as raised: - if call == "execute": - cur.execute(_LITERAL, ("x-lit",)) - elif call == "executemany": - cur.executemany(_LITERAL, [("x-lit",)]) - else: - _read_tampered(cur, call) - conn.rollback() - err = raised.value - assert type(err) is _FAILOVER_FIRST[call] - if type(err) is AwsWrapperError: # the failover plugin's wrapping keeps the plugin's error as its cause - assert type(err.__cause__) is (MetadataError if call == "execute" else IntegrityCheckError) - - -# psycopg's pipeline mode, where a result and its cursor.description arrive only when a fetch syncs the pipeline. - -def _pipelined_reads(cur: Any, read: str, count: int) -> List[Any]: - if read == "fetchone": - got = [cur.fetchone()[0] for _ in range(count)] - assert cur.fetchone() is None - return got - if read == "fetchmany": - return [row[0] for row in cur.fetchmany(2)] + [row[0] for row in cur.fetchmany(5)] - return [row[0] for row in (cur.fetchall() if read == "fetchall" else cur)] - - -@pytest.mark.parametrize("build", ["pg"]) -@pytest.mark.parametrize("metadata", ["loaded", "new-connection"]) -@pytest.mark.parametrize("read", ["fetchone", "fetchmany", "fetchall", "iterate"]) -def test_psycopg_pipeline_reads_decrypt(db: _Db, metadata: str, read: str) -> None: - values = [f"pt-x-pipe-{i}" for i in range(3)] - if metadata == "loaded": - _write(db.conn, _INSERT, ("x-pipe-warm", "pt-x-pipe-warm")) # the connection's first statement loads it - with db.conn.cursor() as cur: - with db.conn.pipeline(): - for value in values: - cur.execute(_INSERT, ("x-pipe", value)) - cur.execute(_SELECT, ("x-pipe",)) - got = _pipelined_reads(cur, read, len(values)) - db.conn.commit() - assert got == values - assert _read_all(db.conn, _SELECT, "x-pipe") == values # the same rows, read outside a pipeline - db.verifier.check("x-pipe", values) - - -# Result sets that nextset() moves to. - -@_BUILDS -def test_later_result_set_is_never_decoded_with_the_first_sets_mapping(db: _Db) -> None: - # Without parameters: psycopg's default cursor refuses several statements with parameters. Each set is read to its - # end before nextset(), as mysql-connector's pure build requires. - _write(db.conn, _INSERT, ("x-sets", "pt-x-sets")) - with db.conn.cursor() as cur: - cur.execute("SELECT ssn FROM users; SELECT name FROM users") - first = cur.fetchall() - moved = cur.nextset() - second = cur.fetchall() - last = cur.nextset() - assert first == [("pt-x-sets",)] and moved - assert second == [("x-sets",)] and not last - - -@pytest.mark.parametrize("build", ["pg"]) -def test_psycopg_executemany_returning_later_sets_are_decrypted(db: _Db) -> None: - values = [f"pt-x-returning-{i}" for i in range(3)] - with db.conn.cursor() as cur: - cur.executemany(f"{_INSERT} RETURNING ssn", [("x-returning", value) for value in values], returning=True) - sets = [cur.fetchall()] - while cur.nextset(): - sets.append(cur.fetchall()) - db.conn.commit() - assert sets == [[(value,)] for value in values] # one set per parameter set, all from the one statement - db.verifier.check("x-returning", values) - - -@_MYSQL_BUILDS -def test_mysql_executemany_later_result_set_is_matched_by_its_own_columns(db: _Db) -> None: - # mysql-connector runs a string that isn't an INSERT through execute() for each parameter set. With one, - # nextset() moves to the next statement's set. psycopg refuses several statements in executemany(). - _write(db.conn, _INSERT, ("x-many-sets", "pt-x-many-sets")) - with db.conn.cursor() as cur: - cur.executemany("SELECT ssn FROM users WHERE name = %s; SELECT name, ssn FROM users WHERE name = %s", - [("x-many-sets", "x-many-sets")]) - cur.fetchall() # builds the decoder from the first set's columns; the driver has already read its rows - moved = cur.nextset() - second = cur.fetchall() - assert moved and second == [("x-many-sets", "pt-x-many-sets")] - - -# MySQL's _binary introducer, on both builds and under Django. - -def _warnings_1300(cur: Any) -> int: - cur.execute("SHOW WARNINGS") - return sum(row[1] == 1300 for row in cur.fetchall()) - - -@_MYSQL_BUILDS -def test_mysql_binary_prefix_keeps_payloads_binary(db: _Db, bed: _Bed, build: str) -> None: - one, many = f"x-bin-{build}", f"x-binm-{build}" - values = [f"pt-x-binm-{i}" for i in range(3)] - with db.conn.cursor() as cur: - cur.execute(_INSERT, (one, "pt-x-bin")) - warned = _warnings_1300(cur) - cur.executemany(_INSERT, [(many, value) for value in values]) - warned += _warnings_1300(cur) - db.conn.commit() - db.verifier.check(one, ["pt-x-bin"]) - db.verifier.check(many, values) - lines = [line for line in _log_lines(bed, "MYSQL_LOG_COMMAND") if b"INSERT INTO users" in line] - prefixed = {name: sum(line.count(b"_binary '") for line in lines if f"'{name}'".encode() in line) - for name in (one, many)} - assert warned == 0 and prefixed == {one: 1, many: 3} - - -@_MYSQL_BUILDS -def test_mysql_writes_succeed_with_raise_on_warnings(bed: _Bed, verifiers: Dict[str, _Verifier], build: str) -> None: - values = [f"pt-x-strict-{i}" for i in range(4)] - with _connect(build, bed, raise_on_warnings=True) as conn: - _write(conn, _INSERT, ("x-strict", values[0])) - with conn.cursor() as cur: - cur.executemany(_INSERT, [("x-strict", value) for value in values[1:]]) - conn.commit() - verifiers["mysql"].check("x-strict", values) - - -@_MYSQL_BUILDS -def test_mysql_null_and_mixed_batches_are_stored(db: _Db) -> None: - _write(db.conn, _INSERT, ("x-null", None)) - with db.conn.cursor() as cur: # a set with None keeps that placeholder unprefixed in every set (Warning 1300) - cur.executemany(_INSERT, [("x-mixed", None), ("x-mixed", "pt-x-mixed")]) - db.conn.commit() - db.verifier.check("x-null", [None]) - db.verifier.check("x-mixed", [None, "pt-x-mixed"]) - - -@_MYSQL_BUILDS -def test_mysql_prepared_cursor_gets_no_binary_prefix(db: _Db, bed: _Bed) -> None: - with db.conn.cursor(prepared=True) as cur: - cur.execute("INSERT INTO users (ssn, name) VALUES (%s, %s)", ("pt-x-prep", "x-prep")) - db.conn.commit() - db.verifier.check("x-prep", ["pt-x-prep"]) - # The Prepare line is the SQL text the server received; the Execute line shows the server's own rendering. - prepared = [line for line in _log_lines(bed, "MYSQL_LOG_COMMAND") - if b"Prepare" in line and b"INSERT INTO users (ssn, name)" in line] - assert prepared and not any(b"_binary" in line for line in prepared) - - -@pytest.fixture(scope="module") -def django_binary_model(django_model: Any) -> Any: - class LiveBinaryUser(models.Model): # a second unmanaged model over users needs a class name of its own - name: models.CharField[str, str] = models.CharField(max_length=100) - ssn: models.BinaryField[Optional[bytes], Optional[bytes]] = models.BinaryField(null=True) - - class Meta: - app_label = "kms_live" - db_table = "users" - managed = False - - return LiveBinaryUser - - -def test_django_binary_field_under_debug_gets_one_binary_prefix(django_model: Any, django_binary_model: Any, bed: _Bed, - verifiers: Dict[str, _Verifier]) -> None: - with override_settings(DEBUG=True): - # The backend sets raise_on_warnings from DEBUG when it connects, so the row needs a connection of its own. - django_connection.close() - try: - django_connection.ensure_connection() - strict = django_connection.connection.target_connection.raise_on_warnings is True - logged = django_connection.queries_logged - django_model.objects.create(name="x-djchar", ssn="pt-x-djchar") # the plugin adds the prefix - django_binary_model.objects.create(name="x-djbin", ssn=b"pt-x-djbin") # Django writes "_binary %s" - got = django_binary_model.objects.get(name="x-djbin").ssn - finally: - django_connection.close() # later statements reconnect under DEBUG=False - lines = _log_lines(bed, "MYSQL_LOG_COMMAND") - doubled = sum(b"_binary _binary" in line for line in lines) - prefixes = {name: [line.count(b"_binary ") for line in lines if b"INSERT INTO" in line and name in line] - for name in (b"'x-djchar'", b"'x-djbin'")} - verifiers["mysql"].check("x-djchar", ["pt-x-djchar"]) - verifiers["mysql"].check("x-djbin", [b"pt-x-djbin"]) - assert strict and logged and bytes(got) == b"pt-x-djbin" - assert doubled == 0 and prefixes == {b"'x-djchar'": [1], b"'x-djbin'": [1]} - - -# Values with no exact stored form are refused; the rest still encrypt. - -class _Shade(Enum): - TEAL = "pt-x-enum" - - -class _Size(IntEnum): - LARGE = 7 - - -class _Color(str, Enum): - RED = "pt-x-strenum" - - -# Each refused value, and text of it that no part of the error may show. -_NO_EXACT_FORM: Dict[str, Tuple[Any, str]] = { - "uuid": (uuid.UUID("0c0ffee0-0000-4000-8000-00000000c0de"), "0c0ffee0-0000-4000-8000-00000000c0de"), - "dict": ({"k": "pt-x-dict"}, "pt-x-dict"), "list": (["pt-x-list"], "pt-x-list"), - "ipv4": (IPv4Address("192.0.2.7"), "192.0.2.7"), "timedelta": (timedelta(hours=1, minutes=2, seconds=3), "1:02:03"), - "enum": (_Shade.TEAL, "pt-x-enum"), "jsonb": (Jsonb({"k": "pt-x-jsonb"}), "pt-x-jsonb"), - "psycopg-binary": (psycopg.Binary(b"pt-x-binary"), "pt-x-binary")} - -# Each accepted value, and what it reads back as. -_EXACT_FORM: Dict[str, Tuple[Any, Any]] = { - "none": (None, None), "int-enum": (_Size.LARGE, 7), "decimal": (Decimal("12.50"), Decimal("12.50")), - "str-enum": (_Color.RED, "pt-x-strenum"), - "mysql-binary": (mysql.connector.Binary(b"pt-x-mysql-binary"), b"pt-x-mysql-binary")} - -# A naive datetime with microseconds. A datetime can't carry the "pt-x-" prefix, so the file's last two tests count -# this one's time of day. -_NAIVE_DATETIME = datetime(2026, 8, 11, 12, 30, 45, 123456) - - -@pytest.mark.parametrize("build, kind", _per_build(tuple(_NO_EXACT_FORM), pg_only=("jsonb", "psycopg-binary"))) -def test_value_with_no_exact_stored_form_is_refused(db: _Db, kind: str) -> None: - value, value_text = _NO_EXACT_FORM[kind] - with db.conn.cursor() as cur, pytest.raises(EncryptionFailedError) as raised: - cur.execute(_INSERT, ("x-generic", value)) - db.conn.rollback() - err = raised.value - shown = " ".join((str(err), repr(err), repr(err.args))) - assert type(err) is EncryptionFailedError and err.data_type == "GENERIC" - assert type(value).__name__ in str(err) and "users.ssn" in str(err) - leaked = [text for text in (value_text, str(value), repr(value)) if text in shown] - assert leaked == [] and err.__cause__ is None and err.__context__ is None - assert db.verifier.stored("x-generic") == [] - - -@pytest.mark.parametrize("build, kind", _per_build(tuple(_EXACT_FORM), mysql_only=("mysql-binary",))) -def test_value_with_an_exact_stored_form_is_still_encrypted(db: _Db, kind: str) -> None: - value, want = _EXACT_FORM[kind] - _write(db.conn, _INSERT, ("x-exact", value)) - got = _read_one(db.conn, "x-exact") - db.verifier.check("x-exact", [want]) - assert got == want and type(got) is type(want) - - -@_BUILDS -def test_naive_datetime_reads_back_with_its_microseconds(db: _Db) -> None: - _write(db.conn, _INSERT, ("x-naive", _NAIVE_DATETIME)) - got = _read_one(db.conn, "x-naive") - assert got == _NAIVE_DATETIME and type(got) is datetime and got.tzinfo is None - db.verifier.check("x-naive", [_NAIVE_DATETIME]) - assert bytes(db.verifier.stored("x-naive")[0])[36] == 13 # LOCAL_DATE_TIME, not an instant under TIMESTAMP - - -@_BUILDS -def test_stored_generic_payload_reads_back_as_text(db: _Db, bed: _Bed, build: str) -> None: - value = {"k": "pt-x-generic"} - with _dob_verifier(_ENGINE[build], bed) as verifier: - payload = verifier.seal(value) # the encryption service with users.dob's keys, as another writer would - is_generic = payload is not None and payload[36] == 99 # the GENERIC type marker - assert is_generic - _plain_write(_ENGINE[build], bed.env, _INSERT_DOB, ("x-generic", payload)) - assert _read_all(db.conn, _SELECT_DOB, "x-generic") == [str(value)] - - -# The plugin's own connections with the metadata cache off, or once the cached copy has expired. - -def _yielded_connections(monkeypatch: pytest.MonkeyPatch) -> List[Any]: - """Lists each connection ``PluginConnectionSource.connection`` yields, once per lookup that gets it; the list keeps - every one alive, so two of them never share an ``id``.""" - yielded: List[Any] = [] - original = PluginConnectionSource.connection - - @contextlib.contextmanager - def listed(source: PluginConnectionSource) -> Iterator[Any]: - with original(source) as conn: - yielded.append(conn) - yield conn - - monkeypatch.setattr(PluginConnectionSource, "connection", listed) - return yielded - - -def _driver_closed(conn: Any) -> bool: - return conn.closed if isinstance(conn, psycopg.Connection) else not conn.is_connected() - - -@_BUILDS -@pytest.mark.parametrize("cache", ["off", "expired"]) -def test_cache_off_opens_at_most_one_metadata_connection_per_statement(bed: _Bed, verifiers: Dict[str, _Verifier], - monkeypatch: pytest.MonkeyPatch, build: str, - cache: str) -> None: - # expired: the copy expires 0.2 s after each load and no background refresh comes during the test, so the copy has - # expired at each pause's end, before a statement and before the fetch of its result. - props: Dict[str, Any] = {"encryption_metadata_cache_enabled": False} if cache == "off" else { - "encryption_metadata_cache_expiration_sec": 0.2, "encryption_metadata_cache_refresh_interval_sec": 300.0} - pause = 0.3 if cache == "expired" else 0.0 - yielded = _yielded_connections(monkeypatch) - with _connect(build, bed, **props) as conn: - _write(conn, _INSERT, ("x-scope-warm", "pt-x-scope-warm")) # builds the plugin and caches the data key - at_insert = len(yielded) - closed: List[bool] = [] - with conn.cursor() as cur: - time.sleep(pause) - cur.execute(_INSERT, ("x-scope", "pt-x-scope")) - closed.append(all(map(_driver_closed, yielded))) - conn.commit() - at_select = len(yielded) - time.sleep(pause) - cur.execute(_SELECT, ("x-scope",)) - closed.append(all(map(_driver_closed, yielded))) - time.sleep(pause) - got = cur.fetchall() - closed.append(all(map(_driver_closed, yielded))) - conn.commit() - verifiers[_ENGINE[build]].check("x-scope", ["pt-x-scope"]) - assert got == [("pt-x-scope",)] - assert closed == [True, True, True], "a plugin connection was open when its statement returned" - counts = {"INSERT": len({id(c) for c in yielded[at_insert:at_select]}), - "SELECT": len({id(c) for c in yielded[at_select:]})} - assert counts == {"INSERT": 1, "SELECT": 1}, f"plugin connections per statement: {counts}; each reads the tables on one" - - -# ---- The servers' own logs for the further checks (last, so they cover every test above) ---- - -def _lines_with(lines: List[bytes], *needles: str) -> int: - return sum(any(needle.encode() in line for needle in needles) for line in lines) - - -def test_mysql_general_log_holds_no_further_plaintext(bed: _Bed) -> None: - lines = _log_lines(bed, "MYSQL_LOG_COMMAND") - writes = sum(bool(re.search(rb"(?i)(INSERT INTO|UPDATE) `?users", line)) for line in lines) - naive = _NAIVE_DATETIME.time().isoformat() - counts = _counts(lines, ("pt-x-", "net-x-literal", "net-x-unanalysed-cext", "net-x-unanalysed-pure", naive)) - assert len(lines) > 0 and writes > 0 - assert counts == {"pt-x-": 0, "net-x-literal": 0, "net-x-unanalysed-cext": 1, "net-x-unanalysed-pure": 1, naive: 0} - - -def test_postgresql_log_holds_no_further_plaintext(bed: _Bed) -> None: - # The file's last test: PostgreSQL's container log can lag its newest lines by a moment, and here even the - # named-cursor row's FETCH lines have had the rest of the run to arrive. A count expected above 0 that comes out - # lower gets the one re-run described in test_postgresql_log_holds_no_plaintext. - lines = _log_lines(bed, "PG_LOG_COMMAND") - naive = _NAIVE_DATETIME.time().isoformat() - counts = {"pt-x-": _lines_with(lines, "pt-x-", r"'\x70742d782d"), - "net-x-literal": _lines_with(lines, "net-x-literal", r"'\x6e65742d782d6c"), - "net-x-unanalysed": _lines_with(lines, "net-x-unanalysed", r"'\x6e65742d782d756e616e616c79736564"), - "naive-datetime": _lines_with(lines, naive, naive.encode().hex()), - "live_paging-fetches": sum(b"FETCH FORWARD 100" in line and b"live_paging" in line for line in lines)} - assert len(lines) > 0 - assert counts == {"pt-x-": 0, "net-x-literal": 0, "net-x-unanalysed": 1, "naive-datetime": 0, "live_paging-fetches": 3} diff --git a/tests/unit/test_kms_encryption_schema_matches_guide.py b/tests/unit/test_kms_encryption_schema_matches_guide.py new file mode 100644 index 00000000..9370ea68 --- /dev/null +++ b/tests/unit/test_kms_encryption_schema_matches_guide.py @@ -0,0 +1,69 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The KMS integration tests install the metadata DDL from ``kms_encryption_schema.py``, which must stay equal to the +two blocks in the KMS Encryption Plugin guide that administrators run.""" + +from __future__ import annotations + +import importlib.util +import re +from pathlib import Path +from typing import TYPE_CHECKING, Dict, List + +import pytest + +if TYPE_CHECKING: + from types import ModuleType + +_ROOT = Path(__file__).resolve().parents[2] +_GUIDE = _ROOT / "docs" / "using-the-python-wrapper" / "using-plugins" / "UsingTheKmsEncryptionPlugin.md" +_SCHEMA = _ROOT / "tests" / "integration" / "container" / "utils" / "kms_encryption_schema.py" + + +def _statement_lines(sql: str) -> List[str]: + return [line for line in sql.splitlines() if line.strip() and not line.lstrip().startswith("--")] + + +@pytest.fixture(scope="module") +def schema() -> ModuleType: + # Loaded from its file, because importing the tests.integration.container package sets up the wrapper's loggers. + spec = importlib.util.spec_from_file_location("kms_encryption_schema", _SCHEMA) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def guide_ddl() -> Dict[str, List[str]]: + """The guide's metadata DDL blocks, keyed by the engine their first comment line names.""" + blocks = re.findall(r"^```sql\n(.*?)^```", _GUIDE.read_text(encoding="utf-8"), flags=re.MULTILINE | re.DOTALL) + return {block.splitlines()[0].removeprefix("--").strip(): _statement_lines(block) + for block in blocks if "CREATE TABLE encrypt.key_storage" in block} + + +def test_guide_has_one_metadata_ddl_block_per_engine(guide_ddl): + assert sorted(guide_ddl) == ["MySQL", "PostgreSQL"], \ + "The guide should have one metadata DDL block for each engine, each starting with a comment naming it." + + +def test_postgresql_metadata_ddl_matches_the_guide(guide_ddl, schema): + assert _statement_lines(schema.PG_METADATA_DDL) == guide_ddl["PostgreSQL"], \ + "PG_METADATA_DDL in kms_encryption_schema.py differs from the guide's PostgreSQL block." + + +def test_mysql_metadata_ddl_matches_the_guide(guide_ddl, schema): + assert _statement_lines(schema.MYSQL_METADATA_DDL) == guide_ddl["MySQL"], \ + "MYSQL_METADATA_DDL in kms_encryption_schema.py differs from the guide's MySQL block."