Add more linting

.pre-commit-config.yaml

@@ -1,10 +1,17 @@
 repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.3.5
+    hooks:
+      - id: ruff
+        args: [--exit-non-zero-on-fix]
+
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
       - id: check-merge-conflict
+      - id: check-toml
       - id: check-yaml
         exclude: windows-release/azure-pipelines.yml
       - id: debug-statements

.ruff.toml

@@ -0,0 +1,23 @@
+fix = true
+
+[lint]
+select = [
+  "C4",     # flake8-comprehensions
+  "E",      # pycodestyle errors
+  "F",      # pyflakes errors
+  "I",      # isort
+  "ISC",    # flake8-implicit-str-concat
+  "LOG",    # flake8-logging
+  "PGH",    # pygrep-hooks
+  "RUF100", # unused noqa (yesqa)
+  "UP",     # pyupgrade
+  "W",      # pycodestyle warnings
+  "YTT",    # flake8-2020
+]
+ignore = [
+  "E203",   # Whitespace before ':'
+  "E221",   # Multiple spaces before operator
+  "E226",   # Missing whitespace around arithmetic operator
+  "E241",   # Multiple spaces after ','
+  "E501",   # Line too long
+]

add-to-pydotorg.py

@@ -22,19 +22,18 @@
 
 Georg Brandl, March 2014.
 """
-from __future__ import print_function
 
 import hashlib
 import json
 import os
-from os import path
 import re
-import sys
-import time
 import subprocess
+import sys
+from os import path
 
 import requests
 
+
 # Copied from release.py
 def error(*msgs):
     print('**ERROR**', file=sys.stderr)
@@ -138,7 +137,7 @@ def slug_for(release):
         ('-' +  release[len(base_version(release)):] if release[len(base_version(release)):] else '')
 
 def sigfile_for(release, rfile):
-    return download_root + '%s/%s.asc' % (release, rfile)
+    return download_root + f'{release}/{rfile}.asc'
 
 def md5sum_for(release, rfile):
     return hashlib.md5(open(ftp_root + base_version(release) + '/' + rfile, 'rb').read()).hexdigest()
@@ -164,33 +163,33 @@ def minor_version_tuple(release):
 def build_file_dict(release, rfile, rel_pk, file_desc, os_pk,
                     add_download, add_desc):
     """Return a dictionary with all needed fields for a ReleaseFile object."""
-    d = dict(
-        name = file_desc,
-        slug = slug_for(release) + '-' + make_slug(file_desc)[:40],
-        os = '/api/v1/downloads/os/%s/' % os_pk,
-        release = '/api/v1/downloads/release/%s/' % rel_pk,
-        description = add_desc,
-        is_source = os_pk == 3,
-        url = download_root + '%s/%s' % (base_version(release), rfile),
-        md5_sum = md5sum_for(release, rfile),
-        filesize = filesize_for(release, rfile),
-        download_button = add_download,
-    )
+    d = {
+        'name': file_desc,
+        'slug': slug_for(release) + '-' + make_slug(file_desc)[:40],
+        'os': '/api/v1/downloads/os/%s/' % os_pk,
+        'release': '/api/v1/downloads/release/%s/' % rel_pk,
+        'description': add_desc,
+        'is_source': os_pk == 3,
+        'url': download_root + f'{base_version(release)}/{rfile}',
+        'md5_sum': md5sum_for(release, rfile),
+        'filesize': filesize_for(release, rfile),
+        'download_button': add_download,
+    }
     # Upload GPG signature
-    if os.path.exists(ftp_root + "%s/%s.asc" % (base_version(release), rfile)):
+    if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.asc"):
         d["gpg_signature_file"] = sigfile_for(base_version(release), rfile)
     # Upload Sigstore signature
-    if os.path.exists(ftp_root + "%s/%s.sig" % (base_version(release), rfile)):
-        d["sigstore_signature_file"] = download_root + '%s/%s.sig' % (base_version(release), rfile)
+    if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.sig"):
+        d["sigstore_signature_file"] = download_root + f'{base_version(release)}/{rfile}.sig'
     # Upload Sigstore certificate
-    if os.path.exists(ftp_root + "%s/%s.crt" % (base_version(release), rfile)):
-        d["sigstore_cert_file"] = download_root + '%s/%s.crt' % (base_version(release), rfile)
+    if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.crt"):
+        d["sigstore_cert_file"] = download_root + f'{base_version(release)}/{rfile}.crt'
     # Upload Sigstore bundle
-    if os.path.exists(ftp_root + "%s/%s.sigstore" % (base_version(release), rfile)):
-        d["sigstore_bundle_file"] = download_root + '%s/%s.sigstore' % (base_version(release), rfile)
+    if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.sigstore"):
+        d["sigstore_bundle_file"] = download_root + f'{base_version(release)}/{rfile}.sigstore'
     # Upload SPDX SBOM file
-    if os.path.exists(ftp_root + "%s/%s.spdx.json" % (base_version(release), rfile)):
-        d["sbom_spdx2_file"] = download_root + '%s/%s.spdx.json' % (base_version(release), rfile)
+    if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.spdx.json"):
+        d["sbom_spdx2_file"] = download_root + f'{base_version(release)}/{rfile}.spdx.json'
 
     return d
 
@@ -206,36 +205,36 @@ def list_files(release):
             if rfile.startswith(prefix):
                 break
         else:
-            print('    File %s/%s has wrong prefix' % (reldir, rfile))
+            print(f'    File {reldir}/{rfile} has wrong prefix')
             continue
         if rfile.endswith('.chm'):
             if rfile[:-4] != 'python' + release.replace('.', ''):
-                print('    File %s/%s has a different version' % (reldir, rfile))
+                print(f'    File {reldir}/{rfile} has a different version')
                 continue
         else:
             try:
                 prefix, rest = rfile.split('-', 1)
-            except:
+            except:  # noqa: E722
                 prefix, rest = rfile, ''
             if not rest.startswith((release + '-', release + '.')):
-                print('    File %s/%s has a different version' % (reldir, rfile))
+                print(f'    File {reldir}/{rfile} has a different version')
                 continue
         for rx, info in get_file_descriptions(release):
             if rx.search(rfile):
                 file_desc, os_pk, add_download, add_desc = info
                 yield rfile, file_desc, os_pk, add_download, add_desc
                 break
         else:
-            print('    File %s/%s not recognized' % (reldir, rfile))
+            print(f'    File {reldir}/{rfile} not recognized')
             continue
 
 def query_object(objtype, **params):
     """Find an API object by query parameters."""
     uri = base_url + 'downloads/%s/' % objtype
-    uri += '?' + '&'.join('%s=%s' % v for v in params.items())
+    uri += '?' + '&'.join(f'{k}={v}' for k, v in params.items())
     resp = requests.get(uri, headers=headers)
     if resp.status_code != 200 or not json.loads(resp.text)['objects']:
-        raise RuntimeError('no object for %s params=%r' % (objtype, params))
+        raise RuntimeError(f'no object for {objtype} params={params!r}')
     obj = json.loads(resp.text)['objects'][0]
     return int(obj['resource_uri'].strip('/').split('/')[-1])
 
@@ -248,17 +247,17 @@ def post_object(objtype, datadict):
             info = json.loads(resp.text)
             print(info.get('error_message', 'No error message.'))
             print(info.get('traceback', ''))
-        except:
+        except:  # noqa: E722
             pass
-        print('Creating %s failed: %s' % (objtype, resp.status_code))
+        print(f'Creating {objtype} failed: {resp.status_code}')
         return -1
     newloc = resp.headers['Location']
     pk = int(newloc.strip('/').split('/')[-1])
     return pk
 
 def sign_release_files_with_sigstore(release, release_files):
     filenames = [
-        ftp_root + "%s/%s" % (base_version(release), rfile)
+        ftp_root + f"{base_version(release)}/{rfile}"
         for rfile, file_desc, os_pk, add_download, add_desc in release_files
     ]
 

release.py

@@ -12,12 +12,11 @@
 import optparse
 import os
 import re
-import readline  # noqa
+import readline  # noqa: F401
 import shutil
 import subprocess
 import sys
 import tempfile
-
 from contextlib import contextmanager
 
 COMMASPACE = ', '
@@ -72,7 +71,7 @@ def chdir_to_repo_root():
         def test_first_line(filename, test):
             if not os.path.exists(filename):
                 return False
-            with open(filename, "rt") as f:
+            with open(filename) as f:
                 lines = f.read().split('\n')
                 if not (lines and test(lines[0])):
                     return False
@@ -168,12 +167,12 @@ def tweak_patchlevel(tag, done=False):
 
 /* Version as a string */
 #define PY_VERSION      \t\"{tag.text}{plus}"'''.strip()
-    level_def = dict(
-        a   = 'PY_RELEASE_LEVEL_ALPHA',
-        b   = 'PY_RELEASE_LEVEL_BETA',
-        rc  = 'PY_RELEASE_LEVEL_GAMMA',
-        f   = 'PY_RELEASE_LEVEL_FINAL',
-        )[tag.level]
+    level_def = {
+        'a': 'PY_RELEASE_LEVEL_ALPHA',
+        'b': 'PY_RELEASE_LEVEL_BETA',
+        'rc': 'PY_RELEASE_LEVEL_GAMMA',
+        'f': 'PY_RELEASE_LEVEL_FINAL',
+        }[tag.level]
     new_constants = template.format(tag=tag, level_def=level_def,
                                     plus=done and '+' or '')
     if tag.as_tuple() >= (3, 7, 0, 'a', 3):
@@ -254,7 +253,7 @@ def tarball(source, clamp_mtime):
         # Sorts the entries in the tarball by name.
         "--sort=name",
         # Sets a maximum 'modified time' of entries in tarball.
-        "--mtime=%s" % (clamp_mtime,), "--clamp-mtime",
+        f"--mtime={clamp_mtime}", "--clamp-mtime",
         # Sets the owner uid and gid to 0.
         "--owner=0", "--group=0", "--numeric-owner",
         # Omits process ID, file access, and status change times.
@@ -370,7 +369,7 @@ def build_docs():
     with tempfile.TemporaryDirectory() as venv:
         run_cmd(['python3', '-m', 'venv', venv])
         pip = os.path.join(venv, 'bin', 'pip')
-        run_cmd([pip, 'install', '-r' 'Doc/requirements.txt'])
+        run_cmd([pip, 'install', '-r', 'Doc/requirements.txt'])
         sphinx_build = os.path.join(venv, 'bin', 'sphinx-build')
         blurb = os.path.join(venv, 'bin', 'blurb')
         with pushd('Doc'):
@@ -389,11 +388,11 @@ def scp(from_loc, to_loc):
         scp('src', '/data/python-releases/%s' % tag.nickname)
         print("Upload doc tarballs")
         scp('docs', '/data/python-releases/doc/%s' % tag.nickname)
-        print("* Now change the permissions on the tarballs so they are " \
-            "writable by the webmaster group. *")
+        print("* Now change the permissions on the tarballs so they are "
+              "writable by the webmaster group. *")
 
 
-class Tag(object):
+class Tag:
 
     def __init__(self, tag_name):
         # if tag is ".", use current directory name as tag
@@ -424,13 +423,13 @@ def __init__(self, tag_name):
         self.text = self.normalized()
         if self.level != "f":
             self.text += self.level + str(self.serial)
-        self.basic_version = '%s.%s' % (self.major, self.minor)
+        self.basic_version = f'{self.major}.{self.minor}'
 
     def __str__(self):
         return self.text
 
     def normalized(self):
-        return "{}.{}.{}".format(self.major, self.minor, self.patch)
+        return f"{self.major}.{self.minor}.{self.patch}"
 
     @property
     def branch(self):
@@ -467,7 +466,7 @@ def committed_at(self):
         # Fetch the epoch of the tagged commit for build reproducibility.
         proc = subprocess.run(["git", "log", self.gitname, "-1", "--pretty=%ct"], stdout=subprocess.PIPE)
         if proc.returncode != 0:
-            error("Couldn't fetch the epoch of tag %s" % (self.gitname,))
+            error(f"Couldn't fetch the epoch of tag {self.gitname}")
         return datetime.datetime.fromtimestamp(int(proc.stdout.decode().strip()), tz=datetime.timezone.utc)
 
 

run_release.py

@@ -11,21 +11,19 @@
 import contextlib
 import functools
 import getpass
-import itertools
 import json
 import os
 import pathlib
 import re
 import shelve
 import shutil
 import subprocess
-import tempfile
+import sys
 import time
 import urllib.request
-import sys
 from dataclasses import dataclass
 from shelve import DbfilenameShelf
-from typing import Callable, Iterator, List, Optional, final
+from typing import Callable, Iterator, List, Optional
 
 import aiohttp
 import gnupg
@@ -34,8 +32,8 @@
 from alive_progress import alive_bar
 
 import release as release_mod
-from buildbotapi import BuildBotAPI
 import sbom
+from buildbotapi import BuildBotAPI
 
 API_KEY_REGEXP = re.compile(r"(?P<major>\w+):(?P<minor>\w+)")
 
@@ -443,7 +441,7 @@ def run_autoconf(db: DbfilenameShelf) -> None:
 
 def check_pyspecific(db):
     with open(
-        db["git_repo"] / "Doc" / "tools" / "extensions" / "pyspecific.py", "r"
+        db["git_repo"] / "Doc" / "tools" / "extensions" / "pyspecific.py"
     ) as pyspecific:
         for line in pyspecific:
             if "SOURCE_URI =" in line:
@@ -557,20 +555,20 @@ def put_dir(self, source, target, progress=None):
         for item in os.listdir(source):
             if os.path.isfile(os.path.join(source, item)):
                 progress.text(item)
-                self.put(os.path.join(source, item), "%s/%s" % (target, item))
+                self.put(os.path.join(source, item), f"{target}/{item}")
                 progress()
             else:
-                self.mkdir("%s/%s" % (target, item), ignore_existing=True)
+                self.mkdir(f"{target}/{item}", ignore_existing=True)
                 self.put_dir(
                     os.path.join(source, item),
-                    "%s/%s" % (target, item),
+                    f"{target}/{item}",
                     progress=progress,
                 )
 
     def mkdir(self, path, mode=511, ignore_existing=False):
         try:
             super().mkdir(path, mode)
-        except IOError:
+        except OSError:
             if ignore_existing:
                 pass
             else:

sbom.py

@@ -21,9 +21,9 @@
 import subprocess
 import sys
 import tarfile
+import typing
 import zipfile
 from urllib.request import urlopen
-import typing
 
 
 def spdx_id(value: str) -> str:

size.py

@@ -2,16 +2,16 @@
 
 """Dump the sizes for the given files, suitable for pasting into content.ht"""
 
+import hashlib
 import os
 import sys
-import hashlib
 
 DOT = '.'
 
 
 # For consistency with historical use.
-sort_order = dict((ext, i) for i, ext in enumerate(
-    ('tgz', 'tar.bz2', 'tar.xz', 'pdb.zip', 'amd64.msi', 'msi', 'chm', 'dmg')))
+sort_order = {ext: i for i, ext in enumerate(
+    ('tgz', 'tar.bz2', 'tar.xz', 'pdb.zip', 'amd64.msi', 'msi', 'chm', 'dmg'))}
 
 
 def ignore(filename):
@@ -35,4 +35,4 @@ def key(filename):
     with open(filename, 'rb') as fp:
         md5.update(fp.read())
     size = os.stat(filename).st_size
-    print('  {}  {:8}  {}'.format(md5.hexdigest(), size, filename))
+    print(f'  {md5.hexdigest()}  {size:8}  {filename}')