From 5f39913115cc3eef273c76cb0433fb74e84d4f8d Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Tue, 2 Jun 2020 16:40:59 +0200 Subject: [PATCH 1/2] Find unused stubtest whitelist entries This is currently a GitHub workflow that runs daily and lists all unused whitelist entries found by running stubtest against current Python versions on Linux and Windows. The workflow run will succeed if there are no such entries, and fail otherwise. In a second step, this should collate the output of the various runs and create a PR to remove the entries. In that case, the workflow should probably only run weekly or even monthly to keep the noise down. Cf. #3728 --- .../workflows/stubtest-unused-whitelist.yml | 26 +++++++++ .travis.yml | 16 +++--- tests/stubtest_test.py | 7 ++- tests/stubtest_unused.py | 56 +++++++++++++++++++ 4 files changed, 94 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/stubtest-unused-whitelist.yml create mode 100755 tests/stubtest_unused.py diff --git a/.github/workflows/stubtest-unused-whitelist.yml b/.github/workflows/stubtest-unused-whitelist.yml new file mode 100644 index 000000000000..1f8e423fd1ed --- /dev/null +++ b/.github/workflows/stubtest-unused-whitelist.yml @@ -0,0 +1,26 @@ +name: Find unused stubtest whitelist entries + +on: + schedule: + - cron: '0 4 * * *' + +jobs: + find: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["ubuntu-latest", "windows-latest"] + python-version: [3.5, 3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install -U pip + pip install -U git+git://github.com/python/mypy@b3d43984 + - name: Run stubtest + run: ./tests/stubtest_unused.py diff --git a/.travis.yml b/.travis.yml index bb70c9fdb4ad..a0833912e014 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,44 +42,44 @@ jobs: - name: "stubtest py38" python: 3.8 install: pip install -U git+git://github.com/python/mypy@b3d43984 - script: ./tests/stubtest_test.py + script: ./tests/stubtest_test.py --ignore-unused-whitelist - name: "stubtest py37" python: 3.7 install: pip install -U git+git://github.com/python/mypy@b3d43984 - script: ./tests/stubtest_test.py + script: ./tests/stubtest_test.py --ignore-unused-whitelist - name: "stubtest py36" python: 3.6 install: pip install -U git+git://github.com/python/mypy@b3d43984 - script: ./tests/stubtest_test.py + script: ./tests/stubtest_test.py --ignore-unused-whitelist - name: "stubtest py35" python: 3.5 install: pip install -U git+git://github.com/python/mypy@b3d43984 - script: ./tests/stubtest_test.py + script: ./tests/stubtest_test.py --ignore-unused-whitelist - name: "stubtest py38 (Windows)" <<: *test_windows env: <<: *env_windows PYTHON_VERSION: 3.8.3 install: pip install -U git+git://github.com/python/mypy@b3d43984 - script: python ./tests/stubtest_test.py + script: python ./tests/stubtest_test.py --ignore-unused-whitelist - name: "stubtest py37 (Windows)" <<: *test_windows env: <<: *env_windows PYTHON_VERSION: 3.7.7 install: pip install -U git+git://github.com/python/mypy@b3d43984 - script: python ./tests/stubtest_test.py + script: python ./tests/stubtest_test.py --ignore-unused-whitelist - name: "stubtest py36 (Windows)" <<: *test_windows env: <<: *env_windows PYTHON_VERSION: 3.6.8 install: pip install -U git+git://github.com/python/mypy@b3d43984 - script: python ./tests/stubtest_test.py + script: python ./tests/stubtest_test.py --ignore-unused-whitelist - name: "stubtest py35 (Windows)" <<: *test_windows env: <<: *env_windows PYTHON_VERSION: 3.5.4 install: pip install -U git+git://github.com/python/mypy@b3d43984 - script: python ./tests/stubtest_test.py + script: python ./tests/stubtest_test.py --ignore-unused-whitelist diff --git a/tests/stubtest_test.py b/tests/stubtest_test.py index fba0634febc9..31605e1dfcc9 100755 --- a/tests/stubtest_test.py +++ b/tests/stubtest_test.py @@ -21,6 +21,8 @@ def run_stubtest(typeshed_dir: Path) -> int: platform_whitelist = "{}.txt".format(sys.platform) combined_whitelist = "{}-py{}{}.txt".format(sys.platform, sys.version_info.major, sys.version_info.minor) + ignore_unused_whitelist = "--ignore-unused-whitelist" in sys.argv[1:] + cmd = [ sys.executable, "-m", @@ -37,6 +39,8 @@ def run_stubtest(typeshed_dir: Path) -> int: "--whitelist", str(whitelist_dir / version_whitelist), ] + if ignore_unused_whitelist: + cmd += ["--ignore-unused-whitelist"] if (whitelist_dir / platform_whitelist).exists(): cmd += [ "--whitelist", @@ -59,9 +63,6 @@ def run_stubtest(typeshed_dir: Path) -> int: "\nNB: stubtest output depends on the Python version (and system) it is run with. " "See README.md for more details.\n" "NB: We only check positional-only arg accuracy for Python 3.9.\n" - "If stubtest is complaining about 'unused whitelist entry' after your fix, please " - "remove the entry from the whitelist file. Note you may have to do this for other " - "version-specific whitelists as well. Thanks for helping burn the backlog of errors!\n" "\nCommand run was: {}\n".format(" ".join(cmd)), file=sys.stderr, ) diff --git a/tests/stubtest_unused.py b/tests/stubtest_unused.py new file mode 100755 index 000000000000..27c5c1260c56 --- /dev/null +++ b/tests/stubtest_unused.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +# Runs stubtest and prints each unused whitelist entry with filename. + +from typing import List, Tuple +import os.path +import subprocess +import sys + +_UNUSED_NOTE = "note: unused whitelist entry " +_WHITELIST_PATH = os.path.join("tests", "stubtest_whitelists") + + +def main() -> int: + unused = run_stubtest() + with_filenames = [] + for uu in unused: + with_filenames.extend(unused_files(uu)) + for file, uu in with_filenames: + print(file + ":" + uu) + return 1 if with_filenames else 0 + + +def run_stubtest() -> List[str]: + popen = subprocess.Popen( + ["./tests/stubtest_test.py"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + ) + assert popen.stdout is not None + unused = [] + for line in popen.stdout: + if line.startswith(_UNUSED_NOTE): + unused.append(line[len(_UNUSED_NOTE):].strip()) + popen.wait() + return unused + + +def unused_files(unused: str) -> List[Tuple[str, str]]: + version = "py{}{}".format(sys.version_info[0], sys.version_info[1]) + files = ["py3_common.txt", version + ".txt", sys.platform + ".txt", sys.platform + "-" + version + ".txt"] + found = [] + for file in files: + path = os.path.join(_WHITELIST_PATH, file) + if find_unused_in_file(unused, path): + found.append((path, unused)) + if not found: + raise ValueError("unused item {} not found in any whitelist file".format(unused)) + return found + + +def find_unused_in_file(unused: str, path: str) -> bool: + with open(path) as f: + return any(line.strip().split(" ")[0] == unused for line in f) + + +if __name__ == "__main__": + sys.exit(main()) From 911884bba769dc69df0ecb8bdc521f79cf770fef Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Tue, 2 Jun 2020 23:16:02 +0200 Subject: [PATCH 2/2] Handle missing whitelist files correctly --- tests/stubtest_unused.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/stubtest_unused.py b/tests/stubtest_unused.py index 27c5c1260c56..df26e72663ee 100755 --- a/tests/stubtest_unused.py +++ b/tests/stubtest_unused.py @@ -48,8 +48,11 @@ def unused_files(unused: str) -> List[Tuple[str, str]]: def find_unused_in_file(unused: str, path: str) -> bool: - with open(path) as f: - return any(line.strip().split(" ")[0] == unused for line in f) + try: + with open(path) as f: + return any(line.strip().split(" ")[0] == unused for line in f) + except FileNotFoundError: + return False if __name__ == "__main__":