From 7d94d9ee38d0b6844185a7caaedb91b821684d0f Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sat, 8 Jun 2024 17:46:39 +0300 Subject: [PATCH 1/3] Add Black to pre-commit --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6123425e..bffe66a5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,6 +5,11 @@ repos: - id: ruff args: [--exit-non-zero-on-fix] + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.4.2 + hooks: + - id: black + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 hooks: From f523b7141c20c36ef8a979d9d5e2f7148758e6c7 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sat, 8 Jun 2024 18:08:48 +0300 Subject: [PATCH 2/3] Format with Black --- add-to-pydotorg.py | 338 ++++++++++++++++++------------ buildbotapi.py | 4 +- release.py | 441 ++++++++++++++++++++++++--------------- run_release.py | 127 +++++++---- sbom.py | 195 +++++++++-------- size.py | 16 +- tests/test_sbom.py | 96 +++++---- windows-release/purge.py | 16 +- 8 files changed, 736 insertions(+), 497 deletions(-) diff --git a/add-to-pydotorg.py b/add-to-pydotorg.py index e76447f4..a33fe330 100755 --- a/add-to-pydotorg.py +++ b/add-to-pydotorg.py @@ -36,7 +36,7 @@ # Copied from release.py def error(*msgs): - print('**ERROR**', file=sys.stderr) + print("**ERROR**", file=sys.stderr) for msg in msgs: print(msg, file=sys.stderr) sys.exit(1) @@ -45,47 +45,50 @@ def error(*msgs): # Copied from release.py def run_cmd(cmd, silent=False, shell=True, **kwargs): if shell: - cmd = ' '.join(cmd) + cmd = " ".join(cmd) if not silent: - print('Executing %s' % cmd) + print("Executing %s" % cmd) try: if silent: subprocess.check_call(cmd, shell=shell, stdout=subprocess.PIPE, **kwargs) else: subprocess.check_call(cmd, shell=shell, **kwargs) except subprocess.CalledProcessError: - error('%s failed' % cmd) + error("%s failed" % cmd) try: - auth_info = os.environ['AUTH_INFO'] + auth_info = os.environ["AUTH_INFO"] except KeyError: - print('Please set an environment variable named AUTH_INFO ' - 'containing "username:api_key".') + print( + "Please set an environment variable named AUTH_INFO " + 'containing "username:api_key".' + ) sys.exit() -base_url = 'https://www.python.org/api/v1/' -ftp_root = '/srv/www.python.org/ftp/python/' -download_root = 'https://www.python.org/ftp/python/' +base_url = "https://www.python.org/api/v1/" +ftp_root = "/srv/www.python.org/ftp/python/" +download_root = "https://www.python.org/ftp/python/" -tag_cre = re.compile(r'(\d+)(?:\.(\d+)(?:\.(\d+))?)?(?:([ab]|rc)(\d+))?$') +tag_cre = re.compile(r"(\d+)(?:\.(\d+)(?:\.(\d+))?)?(?:([ab]|rc)(\d+))?$") -headers = {'Authorization': 'ApiKey %s' % auth_info, 'Content-Type': 'application/json'} +headers = {"Authorization": "ApiKey %s" % auth_info, "Content-Type": "application/json"} -github_oidc_provider = 'https://github.com/login/oauth' -google_oidc_provider = 'https://accounts.google.com' +github_oidc_provider = "https://github.com/login/oauth" +google_oidc_provider = "https://accounts.google.com" # Update this list when new release managers are added. release_to_sigstore_identity_and_oidc_issuer = { - '3.7': ('nad@python.org', github_oidc_provider), - '3.8': ('lukasz@langa.pl', github_oidc_provider), - '3.9': ('lukasz@langa.pl', github_oidc_provider), - '3.10': ('pablogsal@python.org', google_oidc_provider), - '3.11': ('pablogsal@python.org', google_oidc_provider), - '3.12': ('thomas@python.org', google_oidc_provider), - '3.13': ('thomas@python.org', google_oidc_provider), + "3.7": ("nad@python.org", github_oidc_provider), + "3.8": ("lukasz@langa.pl", github_oidc_provider), + "3.9": ("lukasz@langa.pl", github_oidc_provider), + "3.10": ("pablogsal@python.org", google_oidc_provider), + "3.11": ("pablogsal@python.org", google_oidc_provider), + "3.12": ("thomas@python.org", google_oidc_provider), + "3.13": ("thomas@python.org", google_oidc_provider), } + def get_file_descriptions(release): v = minor_version_tuple(release) rx = re.compile @@ -93,131 +96,173 @@ def get_file_descriptions(release): # OS=0 means no ReleaseFile object. Only one matching *file* (not regex) # per OS can have download=True. return [ - (rx(r'\.tgz$'), - ('Gzipped source tarball', 3, False, '')), - (rx(r'\.tar\.xz$'), - ('XZ compressed source tarball', 3, True, '')), - (rx(r'-webinstall\.exe$'), - ('', 0, False, '')), - (rx(r'-embed-amd64\.zip$'), - ('Windows embeddable package (64-bit)', 1, False, '')), - (rx(r'-embed-arm64\.zip$'), - ('Windows embeddable package (ARM64)', 1, False, '')), - (rx(r'-arm64\.exe$'), - ('Windows installer (ARM64)', 1, False, 'Experimental')), - (rx(r'-amd64\.exe$'), - ('Windows installer (64-bit)', 1, v >= (3, 9), 'Recommended')), - (rx(r'-embed-win32\.zip$'), - ('Windows embeddable package (32-bit)', 1, False, '')), - (rx(r'\.exe$'), - ('Windows installer (32 -bit)', 1, v < (3, 9), '')), - (rx(r'\.chm$'), - ('Windows help file', 1, False, '')), - (rx(r'-macosx10\.5(_rev\d)?\.(dm|pk)g$'), - ('macOS 32-bit i386/PPC installer', 2, False, - 'for Mac OS X 10.5 and later')), - (rx(r'-macosx10\.6(_rev\d)?\.(dm|pk)g$'), - ('macOS 64-bit/32-bit Intel installer', 2, False, - 'for Mac OS X 10.6 and later')), - (rx(r'-macos(x)?10\.9\.(dm|pk)g$'), - ('macOS 64-bit Intel-only installer', 2, False, - 'for macOS 10.9 and later, deprecated')), - (rx(r'-macos(x)?1[1-9](\.[0-9]*)?\.pkg$'), - ('macOS 64-bit universal2 installer', 2, True, - 'for macOS 10.9 and later')), + (rx(r"\.tgz$"), ("Gzipped source tarball", 3, False, "")), + (rx(r"\.tar\.xz$"), ("XZ compressed source tarball", 3, True, "")), + (rx(r"-webinstall\.exe$"), ("", 0, False, "")), + ( + rx(r"-embed-amd64\.zip$"), + ("Windows embeddable package (64-bit)", 1, False, ""), + ), + ( + rx(r"-embed-arm64\.zip$"), + ("Windows embeddable package (ARM64)", 1, False, ""), + ), + (rx(r"-arm64\.exe$"), ("Windows installer (ARM64)", 1, False, "Experimental")), + ( + rx(r"-amd64\.exe$"), + ("Windows installer (64-bit)", 1, v >= (3, 9), "Recommended"), + ), + ( + rx(r"-embed-win32\.zip$"), + ("Windows embeddable package (32-bit)", 1, False, ""), + ), + (rx(r"\.exe$"), ("Windows installer (32 -bit)", 1, v < (3, 9), "")), + (rx(r"\.chm$"), ("Windows help file", 1, False, "")), + ( + rx(r"-macosx10\.5(_rev\d)?\.(dm|pk)g$"), + ( + "macOS 32-bit i386/PPC installer", + 2, + False, + "for Mac OS X 10.5 and later", + ), + ), + ( + rx(r"-macosx10\.6(_rev\d)?\.(dm|pk)g$"), + ( + "macOS 64-bit/32-bit Intel installer", + 2, + False, + "for Mac OS X 10.6 and later", + ), + ), + ( + rx(r"-macos(x)?10\.9\.(dm|pk)g$"), + ( + "macOS 64-bit Intel-only installer", + 2, + False, + "for macOS 10.9 and later, deprecated", + ), + ), + ( + rx(r"-macos(x)?1[1-9](\.[0-9]*)?\.pkg$"), + ("macOS 64-bit universal2 installer", 2, True, "for macOS 10.9 and later"), + ), ] + def changelog_for(release): - new_url = 'http://docs.python.org/release/%s/whatsnew/changelog.html' % release + new_url = "http://docs.python.org/release/%s/whatsnew/changelog.html" % release if requests.head(new_url).status_code != 200: - return 'http://hg.python.org/cpython/file/v%s/Misc/NEWS' % release + return "http://hg.python.org/cpython/file/v%s/Misc/NEWS" % release + def slug_for(release): - return base_version(release).replace(".", "") + \ - ('-' + release[len(base_version(release)):] if release[len(base_version(release)):] else '') + return base_version(release).replace(".", "") + ( + "-" + release[len(base_version(release)) :] + if release[len(base_version(release)) :] + else "" + ) + def sigfile_for(release, rfile): - return download_root + f'{release}/{rfile}.asc' + return download_root + f"{release}/{rfile}.asc" + def md5sum_for(release, rfile): - return hashlib.md5(open(ftp_root + base_version(release) + '/' + rfile, 'rb').read()).hexdigest() + return hashlib.md5( + open(ftp_root + base_version(release) + "/" + rfile, "rb").read() + ).hexdigest() + def filesize_for(release, rfile): - return path.getsize(ftp_root + base_version(release) + '/' + rfile) + return path.getsize(ftp_root + base_version(release) + "/" + rfile) + def make_slug(text): - return re.sub('[^a-zA-Z0-9_-]', '', text.replace(' ', '-')) + return re.sub("[^a-zA-Z0-9_-]", "", text.replace(" ", "-")) + def base_version(release): m = tag_cre.match(release) return ".".join(m.groups()[:3]) + def minor_version(release): m = tag_cre.match(release) return ".".join(m.groups()[:2]) + def minor_version_tuple(release): m = tag_cre.match(release) return (int(m.groups()[0]), int(m.groups()[1])) -def build_file_dict(release, rfile, rel_pk, file_desc, os_pk, - add_download, add_desc): + +def build_file_dict(release, rfile, rel_pk, file_desc, os_pk, add_download, add_desc): """Return a dictionary with all needed fields for a ReleaseFile object.""" d = { - 'name': file_desc, - 'slug': slug_for(release) + '-' + make_slug(file_desc)[:40], - 'os': '/api/v1/downloads/os/%s/' % os_pk, - 'release': '/api/v1/downloads/release/%s/' % rel_pk, - 'description': add_desc, - 'is_source': os_pk == 3, - 'url': download_root + f'{base_version(release)}/{rfile}', - 'md5_sum': md5sum_for(release, rfile), - 'filesize': filesize_for(release, rfile), - 'download_button': add_download, + "name": file_desc, + "slug": slug_for(release) + "-" + make_slug(file_desc)[:40], + "os": "/api/v1/downloads/os/%s/" % os_pk, + "release": "/api/v1/downloads/release/%s/" % rel_pk, + "description": add_desc, + "is_source": os_pk == 3, + "url": download_root + f"{base_version(release)}/{rfile}", + "md5_sum": md5sum_for(release, rfile), + "filesize": filesize_for(release, rfile), + "download_button": add_download, } # Upload GPG signature if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.asc"): d["gpg_signature_file"] = sigfile_for(base_version(release), rfile) # Upload Sigstore signature if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.sig"): - d["sigstore_signature_file"] = download_root + f'{base_version(release)}/{rfile}.sig' + d["sigstore_signature_file"] = ( + download_root + f"{base_version(release)}/{rfile}.sig" + ) # Upload Sigstore certificate if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.crt"): - d["sigstore_cert_file"] = download_root + f'{base_version(release)}/{rfile}.crt' + d["sigstore_cert_file"] = download_root + f"{base_version(release)}/{rfile}.crt" # Upload Sigstore bundle if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.sigstore"): - d["sigstore_bundle_file"] = download_root + f'{base_version(release)}/{rfile}.sigstore' + d["sigstore_bundle_file"] = ( + download_root + f"{base_version(release)}/{rfile}.sigstore" + ) # Upload SPDX SBOM file if os.path.exists(ftp_root + f"{base_version(release)}/{rfile}.spdx.json"): - d["sbom_spdx2_file"] = download_root + f'{base_version(release)}/{rfile}.spdx.json' + d["sbom_spdx2_file"] = ( + download_root + f"{base_version(release)}/{rfile}.spdx.json" + ) return d + def list_files(release): """List all of the release's download files.""" reldir = base_version(release) for rfile in os.listdir(path.join(ftp_root, reldir)): if not path.isfile(path.join(ftp_root, reldir, rfile)): continue - if rfile.endswith(('.asc', '.sig', '.crt', '.sigstore', '.spdx.json')): + if rfile.endswith((".asc", ".sig", ".crt", ".sigstore", ".spdx.json")): continue - for prefix in ('python', 'Python'): + for prefix in ("python", "Python"): if rfile.startswith(prefix): break else: - print(f' File {reldir}/{rfile} has wrong prefix') + print(f" File {reldir}/{rfile} has wrong prefix") continue - if rfile.endswith('.chm'): - if rfile[:-4] != 'python' + release.replace('.', ''): - print(f' File {reldir}/{rfile} has a different version') + if rfile.endswith(".chm"): + if rfile[:-4] != "python" + release.replace(".", ""): + print(f" File {reldir}/{rfile} has a different version") continue else: try: - prefix, rest = rfile.split('-', 1) + prefix, rest = rfile.split("-", 1) except: # noqa: E722 - prefix, rest = rfile, '' - if not rest.startswith((release + '-', release + '.')): - print(f' File {reldir}/{rfile} has a different version') + prefix, rest = rfile, "" + if not rest.startswith((release + "-", release + ".")): + print(f" File {reldir}/{rfile} has a different version") continue for rx, info in get_file_descriptions(release): if rx.search(rfile): @@ -225,36 +270,42 @@ def list_files(release): yield rfile, file_desc, os_pk, add_download, add_desc break else: - print(f' File {reldir}/{rfile} not recognized') + print(f" File {reldir}/{rfile} not recognized") continue + def query_object(objtype, **params): """Find an API object by query parameters.""" - uri = base_url + 'downloads/%s/' % objtype - uri += '?' + '&'.join(f'{k}={v}' for k, v in params.items()) + uri = base_url + "downloads/%s/" % objtype + uri += "?" + "&".join(f"{k}={v}" for k, v in params.items()) resp = requests.get(uri, headers=headers) - if resp.status_code != 200 or not json.loads(resp.text)['objects']: - raise RuntimeError(f'no object for {objtype} params={params!r}') - obj = json.loads(resp.text)['objects'][0] - return int(obj['resource_uri'].strip('/').split('/')[-1]) + if resp.status_code != 200 or not json.loads(resp.text)["objects"]: + raise RuntimeError(f"no object for {objtype} params={params!r}") + obj = json.loads(resp.text)["objects"][0] + return int(obj["resource_uri"].strip("/").split("/")[-1]) + def post_object(objtype, datadict): """Create a new API object.""" - resp = requests.post(base_url + 'downloads/' + objtype + '/', - data=json.dumps(datadict), headers=headers) + resp = requests.post( + base_url + "downloads/" + objtype + "/", + data=json.dumps(datadict), + headers=headers, + ) if resp.status_code != 201: try: info = json.loads(resp.text) - print(info.get('error_message', 'No error message.')) - print(info.get('traceback', '')) + print(info.get("error_message", "No error message.")) + print(info.get("traceback", "")) except: # noqa: E722 pass - print(f'Creating {objtype} failed: {resp.status_code}') + print(f"Creating {objtype} failed: {resp.status_code}") return -1 - newloc = resp.headers['Location'] - pk = int(newloc.strip('/').split('/')[-1]) + newloc = resp.headers["Location"] + pk = int(newloc.strip("/").split("/")[-1]) return pk + def sign_release_files_with_sigstore(release, release_files): filenames = [ ftp_root + f"{base_version(release)}/{rfile}" @@ -262,9 +313,8 @@ def sign_release_files_with_sigstore(release, release_files): ] def has_sigstore_signature(filename): - return ( - os.path.exists(filename + '.sigstore') or - (os.path.exists(filename + '.sig') and os.path.exists(filename + '.crt')) + return os.path.exists(filename + ".sigstore") or ( + os.path.exists(filename + ".sig") and os.path.exists(filename + ".crt") ) # Skip files that already have a signature (likely source distributions) @@ -273,77 +323,89 @@ def has_sigstore_signature(filename): ] if unsigned_files: - print('Signing release files with Sigstore') - run_cmd(['python3', '-m', 'sigstore', 'sign', '--oidc-disable-ambient-providers'] + unsigned_files) + print("Signing release files with Sigstore") + run_cmd( + ["python3", "-m", "sigstore", "sign", "--oidc-disable-ambient-providers"] + + unsigned_files + ) for file in unsigned_files: - run_cmd(['chmod', '644', file + '.sig']) - run_cmd(['chmod', '644', file + '.crt']) - run_cmd(['chmod', '644', file + '.sigstore']) + run_cmd(["chmod", "644", file + ".sig"]) + run_cmd(["chmod", "644", file + ".crt"]) + run_cmd(["chmod", "644", file + ".sigstore"]) else: - print('All release files already signed with Sigstore') + print("All release files already signed with Sigstore") # Verify all the files we expect to be signed with sigstore # against the documented release manager identities and providers. try: - sigstore_identity_and_oidc_issuer = release_to_sigstore_identity_and_oidc_issuer[minor_version(release)] + sigstore_identity_and_oidc_issuer = ( + release_to_sigstore_identity_and_oidc_issuer[minor_version(release)] + ) except KeyError: error(["No release manager defined for Python release " + release]) sigstore_identity, sigstore_oidc_issuer = sigstore_identity_and_oidc_issuer - print('Verifying release files were signed correctly with Sigstore') + print("Verifying release files were signed correctly with Sigstore") sigstore_verify_argv = [ - 'python3', '-m', 'sigstore', 'verify', 'identity', - '--cert-identity', sigstore_identity, - '--cert-oidc-issuer', sigstore_oidc_issuer, + "python3", + "-m", + "sigstore", + "verify", + "identity", + "--cert-identity", + sigstore_identity, + "--cert-oidc-issuer", + sigstore_oidc_issuer, ] for filename in filenames: - filename_crt = filename + '.crt' - filename_sig = filename + '.sig' - filename_sigstore = filename + '.sigstore' + filename_crt = filename + ".crt" + filename_sig = filename + ".sig" + filename_sigstore = filename + ".sigstore" if os.path.exists(filename_sigstore): - run_cmd( - sigstore_verify_argv - + ['--bundle', filename_sigstore, filename] - ) + run_cmd(sigstore_verify_argv + ["--bundle", filename_sigstore, filename]) # We use an 'or' here to error out if one of the files is missing. if os.path.exists(filename_sig) or os.path.exists(filename_crt): run_cmd( sigstore_verify_argv - + ['--certificate', filename_crt, '--signature', filename_sig, filename] + + ["--certificate", filename_crt, "--signature", filename_sig, filename] ) + def main(): rel = sys.argv[1] - print('Querying python.org for release', rel) - rel_pk = query_object('release', name='Python+' + rel) - print('Found Release object: id =', rel_pk) + print("Querying python.org for release", rel) + rel_pk = query_object("release", name="Python+" + rel) + print("Found Release object: id =", rel_pk) release_files = list(list_files(rel)) sign_release_files_with_sigstore(rel, release_files) n = 0 file_dicts = {} for rfile, file_desc, os_pk, add_download, add_desc in release_files: - file_dict = build_file_dict(rel, rfile, rel_pk, file_desc, os_pk, - add_download, add_desc) - key = file_dict['slug'] + file_dict = build_file_dict( + rel, rfile, rel_pk, file_desc, os_pk, add_download, add_desc + ) + key = file_dict["slug"] if not os_pk: continue - print('Creating ReleaseFile object for', rfile, key) + print("Creating ReleaseFile object for", rfile, key) if key in file_dicts: - raise RuntimeError('duplicate slug generated: %s' % key) + raise RuntimeError("duplicate slug generated: %s" % key) file_dicts[key] = file_dict - print('Deleting previous release files') - resp = requests.delete(base_url + 'downloads/release_file/?release=%s' % rel_pk, - headers=headers) + print("Deleting previous release files") + resp = requests.delete( + base_url + "downloads/release_file/?release=%s" % rel_pk, headers=headers + ) if resp.status_code != 204: - raise RuntimeError('deleting previous releases failed: %s' % resp.status_code) + raise RuntimeError("deleting previous releases failed: %s" % resp.status_code) for file_dict in file_dicts.values(): - file_pk = post_object('release_file', file_dict) + file_pk = post_object("release_file", file_dict) if file_pk >= 0: - print('Created as id =', file_pk) + print("Created as id =", file_pk) n += 1 - print('Done - %d files added' % n) + print("Done - %d files added" % n) + if not sys.flags.interactive: main() diff --git a/buildbotapi.py b/buildbotapi.py index dbb894e4..6e6c0c89 100644 --- a/buildbotapi.py +++ b/buildbotapi.py @@ -52,9 +52,7 @@ async def _fetch_text(self, url: str) -> str: async with self._session.get(url) as resp: return await resp.text() - async def _fetch_json( - self, url: str - ) -> Dict[ + async def _fetch_json(self, url: str) -> Dict[ str, Union[ List[Dict[str, Union[int, bool, str]]], diff --git a/release.py b/release.py index da9d289e..e6f02ea5 100755 --- a/release.py +++ b/release.py @@ -19,15 +19,16 @@ import tempfile from contextlib import contextmanager -COMMASPACE = ', ' -SPACE = ' ' -tag_cre = re.compile(r'(\d+)(?:\.(\d+)(?:\.(\d+))?)?(?:([ab]|rc)(\d+))?$') +COMMASPACE = ", " +SPACE = " " +tag_cre = re.compile(r"(\d+)(?:\.(\d+)(?:\.(\d+))?)?(?:([ab]|rc)(\d+))?$") # Ideas stolen from Mailman's release script, Lib/tokens.py and welease + def error(*msgs): - print('**ERROR**', file=sys.stderr) + print("**ERROR**", file=sys.stderr) for msg in msgs: print(msg, file=sys.stderr) sys.exit(1) @@ -37,19 +38,21 @@ def run_cmd(cmd, silent=False, shell=False, **kwargs): if shell: cmd = SPACE.join(cmd) if not silent: - print('Executing %s' % cmd) + print("Executing %s" % cmd) try: if silent: subprocess.check_call(cmd, shell=shell, stdout=subprocess.PIPE, **kwargs) else: subprocess.check_call(cmd, shell=shell, **kwargs) except subprocess.CalledProcessError: - error('%s failed' % cmd) + error("%s failed" % cmd) + readme_re = re.compile(r"This is Python version [23]\.\d").match root = None + def chdir_to_repo_root(): global root @@ -63,7 +66,7 @@ def chdir_to_repo_root(): while True: next_path = os.path.dirname(path) if next_path == path: - sys.exit('You\'re not inside a CPython repo right now!') + sys.exit("You're not inside a CPython repo right now!") path = next_path os.chdir(path) @@ -72,16 +75,18 @@ def test_first_line(filename, test): if not os.path.exists(filename): return False with open(filename) as f: - lines = f.read().split('\n') + lines = f.read().split("\n") if not (lines and test(lines[0])): return False return True - if not (test_first_line("README", readme_re) - or test_first_line("README.rst", readme_re)): + if not ( + test_first_line("README", readme_re) + or test_first_line("README.rst", readme_re) + ): continue - if not test_first_line("LICENSE", "A. HISTORY OF THE SOFTWARE".__eq__): + if not test_first_line("LICENSE", "A. HISTORY OF THE SOFTWARE".__eq__): continue if not os.path.exists("Include/Python.h"): continue @@ -99,46 +104,72 @@ def get_output(args): def check_env(): - if 'EDITOR' not in os.environ: - error('editor not detected.', - 'Please set your EDITOR environment variable') - if not os.path.exists('.git'): - error('CWD is not a git clone') + if "EDITOR" not in os.environ: + error("editor not detected.", "Please set your EDITOR environment variable") + if not os.path.exists(".git"): + error("CWD is not a git clone") + def get_arg_parser(): - usage = '%prog [options] tagname' + usage = "%prog [options] tagname" p = optparse.OptionParser(usage=usage) - p.add_option('-b', '--bump', - default=False, action='store_true', - help='bump the revision number in important files') - p.add_option('-e', '--export', - default=False, action='store_true', - help='Export the git tag to a tarball and build docs') - p.add_option('-u', '--upload', metavar="username", - help='Upload the tarballs and docs to dinsdale') - p.add_option('-m', '--branch', - default=False, action='store_true', - help='Create a maintenance branch to go along with the release') - p.add_option('-t', '--tag', - default=False, action='store_true', - help='Tag the release in Subversion') - p.add_option('-d', '--done', - default=False, action='store_true', - help='Do post-release cleanups (i.e. you\'re done!)') - p.add_option('--skip-docs', - default=False, action='store_true', - help='Skip building the documentation during export') + p.add_option( + "-b", + "--bump", + default=False, + action="store_true", + help="bump the revision number in important files", + ) + p.add_option( + "-e", + "--export", + default=False, + action="store_true", + help="Export the git tag to a tarball and build docs", + ) + p.add_option( + "-u", + "--upload", + metavar="username", + help="Upload the tarballs and docs to dinsdale", + ) + p.add_option( + "-m", + "--branch", + default=False, + action="store_true", + help="Create a maintenance branch to go along with the release", + ) + p.add_option( + "-t", + "--tag", + default=False, + action="store_true", + help="Tag the release in Subversion", + ) + p.add_option( + "-d", + "--done", + default=False, + action="store_true", + help="Do post-release cleanups (i.e. you're done!)", + ) + p.add_option( + "--skip-docs", + default=False, + action="store_true", + help="Skip building the documentation during export", + ) return p -def constant_replace(fn, updated_constants, - comment_start='/*', comment_end='*/'): - """Inserts in between --start constant-- and --end constant-- in a file - """ - start_tag = comment_start + '--start constants--' + comment_end - end_tag = comment_start + '--end constants--' + comment_end - with open(fn, encoding="ascii") as infile, \ - open(fn + '.new', 'w', encoding="ascii") as outfile: +def constant_replace(fn, updated_constants, comment_start="/*", comment_end="*/"): + """Inserts in between --start constant-- and --end constant-- in a file""" + start_tag = comment_start + "--start constants--" + comment_end + end_tag = comment_start + "--end constants--" + comment_end + with open(fn, encoding="ascii") as infile, open( + fn + ".new", "w", encoding="ascii" + ) as outfile: found_constants = False waiting_for_end = False for line in infile: @@ -155,12 +186,12 @@ def constant_replace(fn, updated_constants, else: outfile.write(line) if not found_constants: - error('Constant section delimiters not found: %s' % fn) + error("Constant section delimiters not found: %s" % fn) os.rename(fn + ".new", fn) def tweak_patchlevel(tag, done=False): - print('Updating Include/patchlevel.h...', end=' ') + print("Updating Include/patchlevel.h...", end=" ") template = ''' #define PY_MAJOR_VERSION\t{tag.major} #define PY_MINOR_VERSION\t{tag.minor} @@ -171,52 +202,53 @@ def tweak_patchlevel(tag, done=False): /* Version as a string */ #define PY_VERSION \t\"{tag.text}{plus}"'''.strip() level_def = { - 'a': 'PY_RELEASE_LEVEL_ALPHA', - 'b': 'PY_RELEASE_LEVEL_BETA', - 'rc': 'PY_RELEASE_LEVEL_GAMMA', - 'f': 'PY_RELEASE_LEVEL_FINAL', - }[tag.level] - new_constants = template.format(tag=tag, level_def=level_def, - plus=done and '+' or '') - if tag.as_tuple() >= (3, 7, 0, 'a', 3): + "a": "PY_RELEASE_LEVEL_ALPHA", + "b": "PY_RELEASE_LEVEL_BETA", + "rc": "PY_RELEASE_LEVEL_GAMMA", + "f": "PY_RELEASE_LEVEL_FINAL", + }[tag.level] + new_constants = template.format( + tag=tag, level_def=level_def, plus=done and "+" or "" + ) + if tag.as_tuple() >= (3, 7, 0, "a", 3): new_constants = new_constants.expandtabs() - constant_replace('Include/patchlevel.h', new_constants) - print('done') + constant_replace("Include/patchlevel.h", new_constants) + print("done") def bump(tag): - print('Bumping version to %s' % tag) + print("Bumping version to %s" % tag) tweak_patchlevel(tag) extra_work = False # Older releases have a plain text README, # newer releases have README.rst. - other_files = ['README.rst', 'README', 'Misc/NEWS'] + other_files = ["README.rst", "README", "Misc/NEWS"] if tag.patch == 0 and tag.level == "a" and tag.serial == 0: extra_work = True other_files += [ - 'configure.ac', - 'Doc/tutorial/interpreter.rst', - 'Doc/tutorial/stdlib.rst', - 'Doc/tutorial/stdlib2.rst', - 'LICENSE', - 'Doc/license.rst', - 'PC/pyconfig.h.in', - 'PCbuild/rt.bat', - ] - print('\nManual editing time...') + "configure.ac", + "Doc/tutorial/interpreter.rst", + "Doc/tutorial/stdlib.rst", + "Doc/tutorial/stdlib2.rst", + "LICENSE", + "Doc/license.rst", + "PC/pyconfig.h.in", + "PCbuild/rt.bat", + ] + print("\nManual editing time...") for fn in other_files: if os.path.exists(fn): - print('Edit %s' % fn) + print("Edit %s" % fn) manual_edit(fn) else: - print('Skipping %s' % fn) + print("Skipping %s" % fn) - print('Bumped revision') + print("Bumped revision") if extra_work: - print('configure.ac has change; re-run autotools!') - print('Please commit and use --tag') + print("configure.ac has change; re-run autotools!") + print("Please commit and use --tag") def manual_edit(fn: str) -> None: @@ -226,7 +258,7 @@ def manual_edit(fn: str) -> None: @contextmanager def pushd(new): - print('chdir\'ing to %s' % new) + print("chdir'ing to %s" % new) old = os.getcwd() os.chdir(new) try: @@ -234,23 +266,25 @@ def pushd(new): finally: os.chdir(old) + def make_dist(name): try: os.mkdir(name) except OSError: if os.path.isdir(name): - print('WARNING: dist dir %s already exists' % name, file=sys.stderr) + print("WARNING: dist dir %s already exists" % name, file=sys.stderr) else: - error('%s/ is not a directory' % name) + error("%s/ is not a directory" % name) else: - print('created dist directory %s' % name) + print("created dist directory %s" % name) + def tarball(source, clamp_mtime): """Build tarballs for a directory.""" - print('Making .tgz') + print("Making .tgz") base = os.path.basename(source) - tgz = os.path.join('src', base + '.tgz') - xz = os.path.join('src', base + '.tar.xz') + tgz = os.path.join("src", base + ".tgz") + xz = os.path.join("src", base + ".tar.xz") # Recommended options for creating reproducible tarballs from: # https://www.gnu.org/software/tar/manual/html_node/Reproducibility.html#Reproducibility # and https://reproducible-builds.org/docs/archives/ @@ -258,45 +292,64 @@ def tarball(source, clamp_mtime): # Sorts the entries in the tarball by name. "--sort=name", # Sets a maximum 'modified time' of entries in tarball. - f"--mtime={clamp_mtime}", "--clamp-mtime", + f"--mtime={clamp_mtime}", + "--clamp-mtime", # Sets the owner uid and gid to 0. - "--owner=0", "--group=0", "--numeric-owner", + "--owner=0", + "--group=0", + "--numeric-owner", # Omits process ID, file access, and status change times. "--pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime", # Omit irrelevant info about file permissions. - "--mode=go+u,go-w" + "--mode=go+u,go-w", ] - run_cmd(["tar", "cf", tgz, *repro_options, "--use-compress-program", - "gzip --no-name -9", source]) + run_cmd( + [ + "tar", + "cf", + tgz, + *repro_options, + "--use-compress-program", + "gzip --no-name -9", + source, + ] + ) print("Making .tar.xz") run_cmd(["tar", "cJf", xz, *repro_options, source]) - print('Calculating md5 sums') + print("Calculating md5 sums") checksum_tgz = hashlib.md5() - with open(tgz, 'rb') as data: + with open(tgz, "rb") as data: checksum_tgz.update(data.read()) checksum_xz = hashlib.md5() - with open(xz, 'rb') as data: + with open(xz, "rb") as data: checksum_xz.update(data.read()) - print(' %s %8s %s' % ( - checksum_tgz.hexdigest(), int(os.path.getsize(tgz)), tgz)) - print(' %s %8s %s' % ( - checksum_xz.hexdigest(), int(os.path.getsize(xz)), xz)) + print(" %s %8s %s" % (checksum_tgz.hexdigest(), int(os.path.getsize(tgz)), tgz)) + print(" %s %8s %s" % (checksum_xz.hexdigest(), int(os.path.getsize(xz)), xz)) def export(tag, silent=False, skip_docs=False): make_dist(tag.text) - print('Exporting tag:', tag.text) - archivename = 'Python-%s' % tag.text + print("Exporting tag:", tag.text) + archivename = "Python-%s" % tag.text # I have not figured out how to get git to directly produce an # archive directory like hg can, so use git to produce a temporary # tarball then expand it with tar. - archivetempfile = '%s.tar' % archivename - run_cmd(['git', 'archive', '--format=tar', - '--prefix=%s/' % archivename, - '-o', archivetempfile, tag.gitname], silent=silent) + archivetempfile = "%s.tar" % archivename + run_cmd( + [ + "git", + "archive", + "--format=tar", + "--prefix=%s/" % archivename, + "-o", + archivetempfile, + tag.gitname, + ], + silent=silent, + ) with pushd(tag.text): - archivetempfile = '../%s' % archivetempfile - run_cmd(['tar', '-xf', archivetempfile], silent=silent) + archivetempfile = "../%s" % archivetempfile + run_cmd(["tar", "-xf", archivetempfile], silent=silent) os.unlink(archivetempfile) with pushd(archivename): # Touch a few files that get generated so they're up-to-date in @@ -305,14 +358,18 @@ def export(tag, silent=False, skip_docs=False): # Note, with the demise of "make touch" and the hg touch # extension, touches should not be needed anymore, # but keep it for now as a reminder. - maybe_touchables = ['Include/Python-ast.h', - 'Include/internal/pycore_ast.h', - 'Include/internal/pycore_ast_state.h', - 'Python/Python-ast.c', - 'Python/opcode_targets.h'] + maybe_touchables = [ + "Include/Python-ast.h", + "Include/internal/pycore_ast.h", + "Include/internal/pycore_ast_state.h", + "Python/Python-ast.c", + "Python/opcode_targets.h", + ] touchables = [file for file in maybe_touchables if os.path.exists(file)] - print('Touching:', COMMASPACE.join(name.rsplit('/', 1)[-1] - for name in touchables)) + print( + "Touching:", + COMMASPACE.join(name.rsplit("/", 1)[-1] for name in touchables), + ) for name in touchables: os.utime(name, None) @@ -321,18 +378,26 @@ def export(tag, silent=False, skip_docs=False): # and we remove Misc/NEWS.d as part of cleanup for export # # If --skip-docs is provided we don't build and docs. - if not skip_docs and (tag.is_final or tag.level == 'rc'): + if not skip_docs and (tag.is_final or tag.level == "rc"): docdist = build_docs() - print('Using blurb to build Misc/NEWS') + print("Using blurb to build Misc/NEWS") run_cmd(["blurb", "merge"], silent=silent) # Remove files we don't want to ship in tarballs. - print('Removing VCS .*ignore, .git*, Misc/NEWS.d, et al') - for name in ('.gitattributes', '.gitignore', - '.hgignore', '.hgeol', '.hgtags', '.hgtouch', - '.bzrignore', '.codecov.yml', - '.mention-bot', '.travis.yml', ): + print("Removing VCS .*ignore, .git*, Misc/NEWS.d, et al") + for name in ( + ".gitattributes", + ".gitignore", + ".hgignore", + ".hgeol", + ".hgtags", + ".hgtouch", + ".bzrignore", + ".codecov.yml", + ".mention-bot", + ".travis.yml", + ): try: os.unlink(name) except OSError: @@ -340,63 +405,84 @@ def export(tag, silent=False, skip_docs=False): # Remove directories we don't want to ship in tarballs. run_cmd(["blurb", "export"], silent=silent) - for name in ('.azure-pipelines', '.git', '.github', '.hg'): + for name in (".azure-pipelines", ".git", ".github", ".hg"): shutil.rmtree(name, ignore_errors=True) - if not skip_docs and (tag.is_final or tag.level == 'rc'): - shutil.copytree(docdist, 'docs') + if not skip_docs and (tag.is_final or tag.level == "rc"): + shutil.copytree(docdist, "docs") - with pushd(os.path.join(archivename, 'Doc')): - print('Removing doc build artifacts') - shutil.rmtree('venv', ignore_errors=True) - shutil.rmtree('build', ignore_errors=True) - shutil.rmtree('dist', ignore_errors=True) - shutil.rmtree('tools/docutils', ignore_errors=True) - shutil.rmtree('tools/jinja2', ignore_errors=True) - shutil.rmtree('tools/pygments', ignore_errors=True) - shutil.rmtree('tools/sphinx', ignore_errors=True) + with pushd(os.path.join(archivename, "Doc")): + print("Removing doc build artifacts") + shutil.rmtree("venv", ignore_errors=True) + shutil.rmtree("build", ignore_errors=True) + shutil.rmtree("dist", ignore_errors=True) + shutil.rmtree("tools/docutils", ignore_errors=True) + shutil.rmtree("tools/jinja2", ignore_errors=True) + shutil.rmtree("tools/pygments", ignore_errors=True) + shutil.rmtree("tools/sphinx", ignore_errors=True) with pushd(archivename): - print('Zapping pycs') - run_cmd(['find', '.', '-depth', '-name', '__pycache__', - '-exec', 'rm', '-rf', '{}', ';'], silent=silent) - run_cmd(['find', '.', '-name', '*.py[co]', - '-exec', 'rm', '-f', '{}', ';'], silent=silent) - - os.mkdir('src') + print("Zapping pycs") + run_cmd( + [ + "find", + ".", + "-depth", + "-name", + "__pycache__", + "-exec", + "rm", + "-rf", + "{}", + ";", + ], + silent=silent, + ) + run_cmd( + ["find", ".", "-name", "*.py[co]", "-exec", "rm", "-f", "{}", ";"], + silent=silent, + ) + + os.mkdir("src") tarball(archivename, tag.committed_at.strftime("%Y-%m-%d %H:%M:%SZ")) print() - print('**Now extract the archives in %s/src and run the tests**' % tag.text) - print('**You may also want to run make install and re-test**') + print("**Now extract the archives in %s/src and run the tests**" % tag.text) + print("**You may also want to run make install and re-test**") def build_docs(): """Build and tarball the documentation""" print("Building docs") with tempfile.TemporaryDirectory() as venv: - run_cmd(['python3', '-m', 'venv', venv]) - pip = os.path.join(venv, 'bin', 'pip') - run_cmd([pip, 'install', '-r', 'Doc/requirements.txt']) - sphinx_build = os.path.join(venv, 'bin', 'sphinx-build') - blurb = os.path.join(venv, 'bin', 'blurb') - with pushd('Doc'): - run_cmd(['make', 'dist', 'SPHINXBUILD=' + sphinx_build, - 'BLURB=' + blurb], - env={**os.environ, "SPHINXOPTS": "-j10"}) - return os.path.abspath('dist') + run_cmd(["python3", "-m", "venv", venv]) + pip = os.path.join(venv, "bin", "pip") + run_cmd([pip, "install", "-r", "Doc/requirements.txt"]) + sphinx_build = os.path.join(venv, "bin", "sphinx-build") + blurb = os.path.join(venv, "bin", "blurb") + with pushd("Doc"): + run_cmd( + ["make", "dist", "SPHINXBUILD=" + sphinx_build, "BLURB=" + blurb], + env={**os.environ, "SPHINXOPTS": "-j10"}, + ) + return os.path.abspath("dist") + def upload(tag, username): """scp everything to dinsdale""" - address ='"%s@dinsdale.python.org:' % username + address = '"%s@dinsdale.python.org:' % username + def scp(from_loc, to_loc): - run_cmd(['scp', from_loc, address + to_loc]) + run_cmd(["scp", from_loc, address + to_loc]) + with pushd(tag.text): print("Uploading source tarballs") - scp('src', '/data/python-releases/%s' % tag.nickname) + scp("src", "/data/python-releases/%s" % tag.nickname) print("Upload doc tarballs") - scp('docs', '/data/python-releases/doc/%s' % tag.nickname) - print("* Now change the permissions on the tarballs so they are " - "writable by the webmaster group. *") + scp("docs", "/data/python-releases/doc/%s" % tag.nickname) + print( + "* Now change the permissions on the tarballs so they are " + "writable by the webmaster group. *" + ) class Tag: @@ -409,7 +495,7 @@ def __init__(self, tag_name): tag_name = os.path.basename(os.getcwd()) result = tag_cre.match(tag_name) if result is None: - error('tag %s is not valid' % tag_name) + error("tag %s is not valid" % tag_name) data = list(result.groups()) if data[3] is None: # A final release. @@ -430,7 +516,7 @@ def __init__(self, tag_name): self.text = self.normalized() if self.level != "f": self.text += self.level + str(self.serial) - self.basic_version = f'{self.major}.{self.minor}' + self.basic_version = f"{self.major}.{self.minor}" def __str__(self): return self.text @@ -456,11 +542,11 @@ def is_feature_freeze_release(self): @property def nickname(self): - return self.text.replace('.', '') + return self.text.replace(".", "") @property def gitname(self): - return 'v' + self.text + return "v" + self.text def next_minor_release(self): return self.__class__(f"{self.major}.{int(self.minor)+1}.0a0") @@ -471,10 +557,14 @@ def as_tuple(self): @property def committed_at(self): # Fetch the epoch of the tagged commit for build reproducibility. - proc = subprocess.run(["git", "log", self.gitname, "-1", "--pretty=%ct"], stdout=subprocess.PIPE) + proc = subprocess.run( + ["git", "log", self.gitname, "-1", "--pretty=%ct"], stdout=subprocess.PIPE + ) if proc.returncode != 0: error(f"Couldn't fetch the epoch of tag {self.gitname}") - return datetime.datetime.fromtimestamp(int(proc.stdout.decode().strip()), tz=datetime.timezone.utc) + return datetime.datetime.fromtimestamp( + int(proc.stdout.decode().strip()), tz=datetime.timezone.utc + ) def make_tag(tag): @@ -485,30 +575,33 @@ def make_tag(tag): if bad_files or not good_files: print('It doesn\'t look like you ran "blurb release" yet.') if bad_files: - print('There are still reST files in NEWS.d/next/...') + print("There are still reST files in NEWS.d/next/...") if not good_files: - print(f'There is no Misc/NEWS.d/{tag}.rst file.') - if input('Are you sure you want to tag? (y/n) > ') not in ("y", "yes"): + print(f"There is no Misc/NEWS.d/{tag}.rst file.") + if input("Are you sure you want to tag? (y/n) > ") not in ("y", "yes"): print("Aborting.") return False # make sure we're on the correct branch if tag.patch > 0: - if get_output( - ['git', 'name-rev', '--name-only', 'HEAD'] - ).strip().decode() != tag.basic_version: - print('It doesn\'t look like you\'re on the correct branch.') - if input('Are you sure you want to tag? (y/n) > ').lower() not in ("y", "yes"): + if ( + get_output(["git", "name-rev", "--name-only", "HEAD"]).strip().decode() + != tag.basic_version + ): + print("It doesn't look like you're on the correct branch.") + if input("Are you sure you want to tag? (y/n) > ").lower() not in ( + "y", + "yes", + ): print("Aborting.") return False - print('Signing tag') + print("Signing tag") uid = os.environ.get("GPG_KEY_FOR_RELEASE") if not uid: - print('List of available private keys:') + print("List of available private keys:") run_cmd(['gpg -K | grep -A 1 "^sec"'], shell=True) - uid = input('Please enter key ID to use for signing: ') - run_cmd(['git', 'tag', '-s', '-u', uid, tag.gitname, '-m', - 'Python ' + str(tag)]) + uid = input("Please enter key ID to use for signing: ") + run_cmd(["git", "tag", "-s", "-u", uid, tag.gitname, "-m", "Python " + str(tag)]) return True @@ -523,10 +616,10 @@ def main(argv): if options.skip_docs and not options.export: error("--skip-docs option has no effect without --export") if len(args) != 2: - if 'RELEASE_TAG' not in os.environ: + if "RELEASE_TAG" not in os.environ: parser.print_usage() sys.exit(1) - tagname = os.environ['RELEASE_TAG'] + tagname = os.environ["RELEASE_TAG"] else: tagname = args[1] tag = Tag(tagname) @@ -544,5 +637,5 @@ def main(argv): done(tag) -if __name__ == '__main__': +if __name__ == "__main__": main(sys.argv) diff --git a/run_release.py b/run_release.py index cff01fa8..fa014902 100755 --- a/run_release.py +++ b/run_release.py @@ -414,13 +414,16 @@ def run_autoconf(db: DbfilenameShelf) -> None: regen_configure_sh = db["git_repo"] / "Tools/build/regen-configure.sh" if regen_configure_sh.exists(): subprocess.check_call( - [regen_configure_sh], cwd=db["git_repo"], + [regen_configure_sh], + cwd=db["git_repo"], ) # Python 3.11 and prior rely on autoconf built within a container # in order to maintain stability of autoconf generation. else: # Corresponds to the tag '269' and 'cp311' - cpython_autoconf_sha256 = "f370fee95eefa3d57b00488bce4911635411fa83e2d293ced8cf8a3674ead939" + cpython_autoconf_sha256 = ( + "f370fee95eefa3d57b00488bce4911635411fa83e2d293ced8cf8a3674ead939" + ) subprocess.check_call( [ "docker", @@ -483,25 +486,29 @@ def wait_for_source_and_docs_artifacts(db: DbfilenameShelf) -> None: # Build the list of filepaths we're expecting. wait_for_paths = [ release_path / "src" / f"Python-{release_tag}.tgz", - release_path / "src" / f"Python-{release_tag}.tar.xz" + release_path / "src" / f"Python-{release_tag}.tar.xz", ] if should_wait_for_docs: docs_path = release_path / "docs" - wait_for_paths.extend([ - docs_path / f"python-{release_tag}-docs.epub", - docs_path / f"python-{release_tag}-docs-html.tar.bz2", - docs_path / f"python-{release_tag}-docs-html.zip", - docs_path / f"python-{release_tag}-docs-pdf-a4.tar.bz2", - docs_path / f"python-{release_tag}-docs-pdf-a4.zip", - docs_path / f"python-{release_tag}-docs-pdf-letter.tar.bz2", - docs_path / f"python-{release_tag}-docs-pdf-letter.zip", - docs_path / f"python-{release_tag}-docs-texinfo.tar.bz2", - docs_path / f"python-{release_tag}-docs-texinfo.zip", - docs_path / f"python-{release_tag}-docs-text.tar.bz2", - docs_path / f"python-{release_tag}-docs-text.zip", - ]) - - print(f"Waiting for source{' and docs' if should_wait_for_docs else ''} artifacts to be built") + wait_for_paths.extend( + [ + docs_path / f"python-{release_tag}-docs.epub", + docs_path / f"python-{release_tag}-docs-html.tar.bz2", + docs_path / f"python-{release_tag}-docs-html.zip", + docs_path / f"python-{release_tag}-docs-pdf-a4.tar.bz2", + docs_path / f"python-{release_tag}-docs-pdf-a4.zip", + docs_path / f"python-{release_tag}-docs-pdf-letter.tar.bz2", + docs_path / f"python-{release_tag}-docs-pdf-letter.zip", + docs_path / f"python-{release_tag}-docs-texinfo.tar.bz2", + docs_path / f"python-{release_tag}-docs-texinfo.zip", + docs_path / f"python-{release_tag}-docs-text.tar.bz2", + docs_path / f"python-{release_tag}-docs-text.zip", + ] + ) + + print( + f"Waiting for source{' and docs' if should_wait_for_docs else ''} artifacts to be built" + ) print(f"Artifacts should be placed at '{release_path}':") for path in wait_for_paths: print(f"- '{os.path.relpath(path, release_path)}'") @@ -511,23 +518,32 @@ def wait_for_source_and_docs_artifacts(db: DbfilenameShelf) -> None: def sign_source_artifacts(db: DbfilenameShelf) -> None: - print('Signing tarballs with GPG') + print("Signing tarballs with GPG") uid = os.environ.get("GPG_KEY_FOR_RELEASE") if not uid: - print('List of available private keys:') + print("List of available private keys:") subprocess.check_call('gpg -K | grep -A 1 "^sec"', shell=True) - uid = input('Please enter key ID to use for signing: ') + uid = input("Please enter key ID to use for signing: ") tarballs_path = pathlib.Path(db["git_repo"] / str(db["release"]) / "src") tgz = str(tarballs_path / ("Python-%s.tgz" % db["release"])) xz = str(tarballs_path / ("Python-%s.tar.xz" % db["release"])) - subprocess.check_call(['gpg', '-bas', '-u', uid, tgz]) - subprocess.check_call(['gpg', '-bas', '-u', uid, xz]) + subprocess.check_call(["gpg", "-bas", "-u", uid, tgz]) + subprocess.check_call(["gpg", "-bas", "-u", uid, xz]) - print('Signing tarballs with Sigstore') - subprocess.check_call(['python3', '-m', 'sigstore', 'sign', - '--oidc-disable-ambient-providers', tgz, xz]) + print("Signing tarballs with Sigstore") + subprocess.check_call( + [ + "python3", + "-m", + "sigstore", + "sign", + "--oidc-disable-ambient-providers", + tgz, + xz, + ] + ) def build_sbom_artifacts(db): @@ -721,29 +737,43 @@ def execute_command(command): def start_build_of_source_and_docs(db: DbfilenameShelf) -> None: # Get the git commit SHA for the tag - commit_sha = subprocess.check_output( - ["git", "rev-list", "-n", "1", db["release"].gitname], - cwd=db["git_repo"] - ).decode().strip() + commit_sha = ( + subprocess.check_output( + ["git", "rev-list", "-n", "1", db["release"].gitname], cwd=db["git_repo"] + ) + .decode() + .strip() + ) # Get the owner of the GitHub repo (first path segment in a 'github.com' remote URL) # This works for both 'https' and 'ssh' style remote URLs. - origin_remote_url = subprocess.check_output( - ["git", "ls-remote", "--get-url", "origin"], - cwd=db["git_repo"] - ).decode().strip() + origin_remote_url = ( + subprocess.check_output( + ["git", "ls-remote", "--get-url", "origin"], cwd=db["git_repo"] + ) + .decode() + .strip() + ) if https_match := re.match(r"github\.com/([^/]+)/", origin_remote_url): origin_remote_github_owner = https_match.group(1) elif ssh_match := re.match(r"^git@github\.com:([^/]+)/", origin_remote_url): origin_remote_github_owner = ssh_match.group(1) else: - raise ReleaseException(f"Could not parse GitHub owner from 'origin' remote URL: {origin_remote_url}") + raise ReleaseException( + f"Could not parse GitHub owner from 'origin' remote URL: {origin_remote_url}" + ) # We ask for human verification at this point since this commit SHA is 'locked in' print() - print(f"Go to https://github.com/{origin_remote_github_owner}/cpython/commit/{commit_sha}") - print("- Ensure that there is no warning that the commit does not belong to this repository.") + print( + f"Go to https://github.com/{origin_remote_github_owner}/cpython/commit/{commit_sha}" + ) + print( + "- Ensure that there is no warning that the commit does not belong to this repository." + ) print("- Ensure that the commit diff does not contain any unexpected changes.") - print("- For the next step, ensure the commit SHA matches the one you verified on GitHub in this step.") + print( + "- For the next step, ensure the commit SHA matches the one you verified on GitHub in this step." + ) print() if not ask_question( "Have you verified the release commit hasn't been tampered with on GitHub?" @@ -753,16 +783,16 @@ def start_build_of_source_and_docs(db: DbfilenameShelf) -> None: # After visually confirming the release manager can start the build process # with the known good commit SHA. print() - print("Go to https://github.com/python/release-tools/actions/workflows/source-and-docs-release.yml") + print( + "Go to https://github.com/python/release-tools/actions/workflows/source-and-docs-release.yml" + ) print("Select 'Run workflow' and enter the following values:") print(f"- Git remote to checkout: {origin_remote_github_owner}") print(f"- Git commit to target for the release: {commit_sha}") print(f"- CPython release number: {db['release']}") print() - if not ask_question( - "Have you started the source and docs build?" - ): + if not ask_question("Have you started the source and docs build?"): raise ReleaseException("Source and docs build must be started") @@ -1012,7 +1042,8 @@ def _push_to_local(dry_run=False): git_command += ["HEAD", "--tags"] subprocess.check_call( - git_command, cwd=db["git_repo"], + git_command, + cwd=db["git_repo"], ) _push_to_local(dry_run=True) @@ -1146,12 +1177,18 @@ def _api_key(api_key): Task(check_cpython_repo_is_clean, "Checking git repository is clean"), Task(create_tag, "Create tag"), Task(push_to_local_fork, "Push new tags and branches to private fork"), - Task(start_build_of_source_and_docs, "Start the builds for source and docs artifacts"), + Task( + start_build_of_source_and_docs, + "Start the builds for source and docs artifacts", + ), Task( send_email_to_platform_release_managers, "Platform release managers have been notified of the commit SHA", ), - Task(wait_for_source_and_docs_artifacts, "Wait for source and docs artifacts to build"), + Task( + wait_for_source_and_docs_artifacts, + "Wait for source and docs artifacts to build", + ), Task(build_sbom_artifacts, "Building SBOM artifacts"), Task(sign_source_artifacts, "Sign source artifacts"), Task(upload_files_to_server, "Upload files to the PSF server"), diff --git a/sbom.py b/sbom.py index 432060a9..96c16714 100644 --- a/sbom.py +++ b/sbom.py @@ -107,10 +107,13 @@ def calculate_package_verification_codes(sbom) -> None: def get_release_tools_commit_sha() -> str: """Gets the git commit SHA of the release-tools repository""" git_prefix = os.path.abspath(os.path.dirname(__file__)) - stdout = subprocess.check_output( - ["git", "rev-parse", "--prefix", git_prefix, "HEAD"], - cwd=git_prefix - ).decode("ascii").strip() + stdout = ( + subprocess.check_output( + ["git", "rev-parse", "--prefix", git_prefix, "HEAD"], cwd=git_prefix + ) + .decode("ascii") + .strip() + ) assert re.fullmatch(r"^[a-f0-9]{40,}$", stdout) return stdout @@ -120,6 +123,7 @@ def normalize_sbom_data(sbom_data): Normalize SBOM data in-place by recursion and sorting lists by some repeatable key. """ + def recursive_sort_in_place(value): if isinstance(value, list): # We need to recurse first so bottom-most elements are sorted first. @@ -138,7 +142,9 @@ def recursive_sort_in_place(value): recursive_sort_in_place(sbom_data) -def fetch_package_metadata_from_pypi(project: str, version: str, filename: str | None = None) -> tuple[str, str] | None: +def fetch_package_metadata_from_pypi( + project: str, version: str, filename: str | None = None +) -> tuple[str, str] | None: """ Fetches the SHA256 checksum and download location from PyPI. If we're given a filename then we match with that, otherwise we use wheels. @@ -154,9 +160,8 @@ def fetch_package_metadata_from_pypi(project: str, version: str, filename: str | for url in release_metadata["urls"]: # pip can only use Python-only dependencies, so there's # no risk of picking the 'incorrect' wheel here. - if ( - (filename is None and url["packagetype"] == "bdist_wheel") - or (filename is not None and url["filename"] == filename) + if (filename is None and url["packagetype"] == "bdist_wheel") or ( + filename is not None and url["filename"] == filename ): break else: @@ -168,7 +173,9 @@ def fetch_package_metadata_from_pypi(project: str, version: str, filename: str | return download_url, checksum_sha256 except (OSError, ValueError) as e: - raise ValueError(f"Couldn't fetch metadata for project '{project}' from PyPI: {e}") + raise ValueError( + f"Couldn't fetch metadata for project '{project}' from PyPI: {e}" + ) def remove_pip_from_sbom(sbom_data: dict[str, typing.Any]) -> None: @@ -190,19 +197,19 @@ def remove_pip_from_sbom(sbom_data: dict[str, typing.Any]) -> None: # Remove all the packages and relationships. sbom_data["packages"] = [ - sbom_package for sbom_package in sbom_data["packages"] + sbom_package + for sbom_package in sbom_data["packages"] if sbom_package["SPDXID"] not in sbom_spdx_ids_to_remove ] sbom_data["relationships"] = [ - sbom_relationship for sbom_relationship in sbom_data["relationships"] + sbom_relationship + for sbom_relationship in sbom_data["relationships"] if sbom_relationship["relatedSpdxElement"] not in sbom_spdx_ids_to_remove ] def create_pip_sbom_from_wheel( - sbom_data: dict[str, typing.Any], - pip_wheel_filename: str, - pip_wheel_bytes: bytes + sbom_data: dict[str, typing.Any], pip_wheel_filename: str, pip_wheel_bytes: bytes ) -> None: """ pip is a part of a packaging ecosystem (Python, surprise!) so it's actually @@ -240,7 +247,9 @@ def create_pip_sbom_from_wheel( # Non-empty lines we must be able to match. match = version_pin_re.match(line) - assert match is not None, f"Unparseable line in vendor.txt: {line!r}" # Make mypy happy. + assert ( + match is not None + ), f"Unparseable line in vendor.txt: {line!r}" # Make mypy happy. # Parse out and normalize the project name. project_name, project_version = match.groups() @@ -254,24 +263,29 @@ def create_pip_sbom_from_wheel( # Update our SBOM data with what we received from PyPI. sbom_project_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{project_name}") sbom_pip_dependency_spdx_ids.add(sbom_project_spdx_id) - sbom_data["packages"].append({ - "SPDXID": sbom_project_spdx_id, - "name": project_name, - "versionInfo": project_version, - "downloadLocation": project_download_url, - "checksums": [ - {"algorithm": "SHA256", "checksumValue": project_checksum_sha256} - ], - "externalRefs": [ - { - "referenceCategory": "PACKAGE_MANAGER", - "referenceLocator": f"pkg:pypi/{project_name}@{project_version}", - "referenceType": "purl", - }, - ], - "primaryPackagePurpose": "SOURCE", - "licenseConcluded": "NOASSERTION", - }) + sbom_data["packages"].append( + { + "SPDXID": sbom_project_spdx_id, + "name": project_name, + "versionInfo": project_version, + "downloadLocation": project_download_url, + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": project_checksum_sha256, + } + ], + "externalRefs": [ + { + "referenceCategory": "PACKAGE_MANAGER", + "referenceLocator": f"pkg:pypi/{project_name}@{project_version}", + "referenceType": "purl", + }, + ], + "primaryPackagePurpose": "SOURCE", + "licenseConcluded": "NOASSERTION", + } + ) # Now we add pip to the SBOM and dependency relationships sbom_pip_spdx_id = spdx_id("SPDXRef-PACKAGE-pip") @@ -302,11 +316,13 @@ def create_pip_sbom_from_wheel( } ) for sbom_dep_spdx_id in sorted(sbom_pip_dependency_spdx_ids): - sbom_data["relationships"].append({ - "spdxElementId": sbom_pip_spdx_id, - "relatedSpdxElement": sbom_dep_spdx_id, - "relationshipType": "DEPENDS_ON" - }) + sbom_data["relationships"].append( + { + "spdxElementId": sbom_pip_spdx_id, + "relatedSpdxElement": sbom_dep_spdx_id, + "relationshipType": "DEPENDS_ON", + } + ) # Finally, CPython depends on pip. sbom_data["relationships"].append( @@ -333,28 +349,31 @@ def create_cpython_sbom( with open(artifact_path, mode="rb") as f: artifact_checksum_sha256 = hashlib.sha256(f.read()).hexdigest() - sbom_data.update({ - "SPDXID": "SPDXRef-DOCUMENT", - "spdxVersion": "SPDX-2.3", - "name": "CPython SBOM", - "dataLicense": "CC0-1.0", - # Naming done according to OpenSSF SBOM WG recommendations. - # See: https://github.com/ossf/sbom-everywhere/blob/main/reference/sbom_naming.md - "documentNamespace": f"{artifact_download_location}.spdx.json", - "creationInfo": { - "created": ( - datetime.datetime.now(tz=datetime.timezone.utc) - .strftime("%Y-%m-%dT%H:%M:%SZ") - ), - "creators": [ - "Person: Python Release Managers", - f"Tool: ReleaseTools-{get_release_tools_commit_sha()}", - ], - # Version of the SPDX License ID list. - # This shouldn't need to be updated often, if ever. - "licenseListVersion": "3.22", - }, - }) + sbom_data.update( + { + "SPDXID": "SPDXRef-DOCUMENT", + "spdxVersion": "SPDX-2.3", + "name": "CPython SBOM", + "dataLicense": "CC0-1.0", + # Naming done according to OpenSSF SBOM WG recommendations. + # See: https://github.com/ossf/sbom-everywhere/blob/main/reference/sbom_naming.md + "documentNamespace": f"{artifact_download_location}.spdx.json", + "creationInfo": { + "created": ( + datetime.datetime.now(tz=datetime.timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + ), + "creators": [ + "Person: Python Release Managers", + f"Tool: ReleaseTools-{get_release_tools_commit_sha()}", + ], + # Version of the SPDX License ID list. + # This shouldn't need to be updated often, if ever. + "licenseListVersion": "3.22", + }, + } + ) # Create the SBOM entry for the CPython package. We use # the SPDXID later on for creating relationships to files. @@ -375,16 +394,20 @@ def create_cpython_sbom( ], "primaryPackagePurpose": "SOURCE", "downloadLocation": artifact_download_location, - "checksums": [{"algorithm": "SHA256", "checksumValue": artifact_checksum_sha256}], + "checksums": [ + {"algorithm": "SHA256", "checksumValue": artifact_checksum_sha256} + ], } # The top-level CPython package depends on every vendored sub-package. for sbom_package in sbom_data["packages"]: - sbom_data["relationships"].append({ - "spdxElementId": sbom_cpython_package["SPDXID"], - "relatedSpdxElement": sbom_package["SPDXID"], - "relationshipType": "DEPENDS_ON", - }) + sbom_data["relationships"].append( + { + "spdxElementId": sbom_cpython_package["SPDXID"], + "relatedSpdxElement": sbom_package["SPDXID"], + "relationshipType": "DEPENDS_ON", + } + ) sbom_data["packages"].append(sbom_cpython_package) @@ -408,7 +431,9 @@ def create_sbom_for_source_tarball(tarball_path: str): # There should be an SBOM included in the tarball. # If there's not we can't create an SBOM. try: - sbom_tarball_member = tarball.getmember(f"Python-{cpython_version}/Misc/sbom.spdx.json") + sbom_tarball_member = tarball.getmember( + f"Python-{cpython_version}/Misc/sbom.spdx.json" + ) except KeyError: raise ValueError( "Tarball doesn't contain an SBOM at 'Misc/sbom.spdx.json'" @@ -416,12 +441,17 @@ def create_sbom_for_source_tarball(tarball_path: str): sbom_bytes = tarball.extractfile(sbom_tarball_member).read() sbom_data = json.loads(sbom_bytes) - create_cpython_sbom(sbom_data, cpython_version=cpython_version, artifact_path=tarball_path) + create_cpython_sbom( + sbom_data, cpython_version=cpython_version, artifact_path=tarball_path + ) sbom_cpython_package_spdx_id = spdx_id("SPDXRef-PACKAGE-cpython") # Find the pip wheel in ensurepip in the tarball for member in tarball.getmembers(): - match = re.match(rf"^Python-{cpython_version}/Lib/ensurepip/_bundled/(pip-.*\.whl)$", member.name) + match = re.match( + rf"^Python-{cpython_version}/Lib/ensurepip/_bundled/(pip-.*\.whl)$", + member.name, + ) if match is not None: pip_wheel_filename = match.group(1) pip_wheel_bytes = tarball.extractfile(member).read() @@ -434,7 +464,7 @@ def create_sbom_for_source_tarball(tarball_path: str): create_pip_sbom_from_wheel( sbom_data=sbom_data, pip_wheel_filename=pip_wheel_filename, - pip_wheel_bytes=pip_wheel_bytes + pip_wheel_bytes=pip_wheel_bytes, ) # Extract all currently known files from the SBOM with their checksums. @@ -445,9 +475,7 @@ def create_sbom_for_source_tarball(tarball_path: str): # Look for the expected SHA256 checksum. for sbom_file_checksum in sbom_file["checksums"]: if sbom_file_checksum["algorithm"] == "SHA256": - known_sbom_files[sbom_filename] = ( - sbom_file_checksum["checksumValue"] - ) + known_sbom_files[sbom_filename] = sbom_file_checksum["checksumValue"] break else: raise ValueError( @@ -472,14 +500,16 @@ def create_sbom_for_source_tarball(tarball_path: str): actual_file_checksum_sha256 = hashlib.sha256(file_bytes).hexdigest() # Remove the 'Python-{version}/...' prefix for the SPDXID and fileName. - member_name_no_prefix = member.name.split('/', 1)[1] + member_name_no_prefix = member.name.split("/", 1)[1] # We've already seen this file, so we check it hasn't been modified and continue on. if member_name_no_prefix in known_sbom_files: # If there's a hash mismatch we raise an error, something isn't right! expected_file_checksum_sha256 = known_sbom_files.pop(member_name_no_prefix) if expected_file_checksum_sha256 != actual_file_checksum_sha256: - raise ValueError(f"Mismatched checksum for file '{member_name_no_prefix}'") + raise ValueError( + f"Mismatched checksum for file '{member_name_no_prefix}'" + ) # If this is a new file, then it's a part of the 'CPython' SBOM package. else: @@ -539,7 +569,9 @@ def create_sbom_for_source_tarball(tarball_path: str): def create_sbom_for_windows_artifact(artifact_path, cpython_source_dir: str): artifact_name = os.path.basename(artifact_path) - cpython_version = re.match(r"^python-([0-9abrc.]+)(?:-|\.exe|\.zip)", artifact_name).group(1) + cpython_version = re.match( + r"^python-([0-9abrc.]+)(?:-|\.exe|\.zip)", artifact_name + ).group(1) if not cpython_source_dir: raise ValueError("Must specify --cpython-source-dir for Windows artifacts") @@ -561,9 +593,7 @@ def create_sbom_for_windows_artifact(artifact_path, cpython_source_dir: str): sbom_data["packages"].append(sbom_package) create_cpython_sbom( - sbom_data, - cpython_version=cpython_version, - artifact_path=artifact_path + sbom_data, cpython_version=cpython_version, artifact_path=artifact_path ) sbom_cpython_package_spdx_id = spdx_id("SPDXRef-PACKAGE-cpython") @@ -575,7 +605,9 @@ def create_sbom_for_windows_artifact(artifact_path, cpython_source_dir: str): for pathname in os.listdir(cpython_source_dir / "Lib/ensurepip/_bundled"): if pathname.startswith("pip-") and pathname.endswith(".whl"): pip_wheel_filename = pathname - pip_wheel_bytes = (cpython_source_dir / f"Lib/ensurepip/_bundled/{pathname}").read_bytes() + pip_wheel_bytes = ( + cpython_source_dir / f"Lib/ensurepip/_bundled/{pathname}" + ).read_bytes() break else: raise ValueError("Could not find pip wheel in 'Lib/ensurepip/_bundled/...'") @@ -619,8 +651,7 @@ def main() -> None: # Windows MSI and Embed artifacts if artifact_path.endswith(".exe") or artifact_path.endswith(".zip"): sbom_data = create_sbom_for_windows_artifact( - artifact_path, - cpython_source_dir=cpython_source_dir + artifact_path, cpython_source_dir=cpython_source_dir ) # Source artifacts else: diff --git a/size.py b/size.py index f0a608e7..1745504b 100755 --- a/size.py +++ b/size.py @@ -6,12 +6,16 @@ import os import sys -DOT = '.' +DOT = "." # For consistency with historical use. -sort_order = {ext: i for i, ext in enumerate( - ('tgz', 'tar.bz2', 'tar.xz', 'pdb.zip', 'amd64.msi', 'msi', 'chm', 'dmg'))} +sort_order = { + ext: i + for i, ext in enumerate( + ("tgz", "tar.bz2", "tar.xz", "pdb.zip", "amd64.msi", "msi", "chm", "dmg") + ) +} def ignore(filename: str) -> bool: @@ -33,11 +37,11 @@ def main() -> None: if ignore(filename): continue md5 = hashlib.md5() - with open(filename, 'rb') as fp: + with open(filename, "rb") as fp: md5.update(fp.read()) size = os.stat(filename).st_size - print(f' {md5.hexdigest()} {size:8} {filename}') + print(f" {md5.hexdigest()} {size:8} {filename}") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tests/test_sbom.py b/tests/test_sbom.py index 1b621e19..da14ef54 100644 --- a/tests/test_sbom.py +++ b/tests/test_sbom.py @@ -18,8 +18,11 @@ # One file -> SHA1(SHA1(file)) (["F" * 40], hashlib.sha1(b"f" * 40).hexdigest()), # Tests ordering and lowercasing of SHA1s - (["0" * 40, "e" * 40, "F" * 40], hashlib.sha1((b"0" * 40) + (b"e" * 40) + (b"f" * 40)).hexdigest()) - ] + ( + ["0" * 40, "e" * 40, "F" * 40], + hashlib.sha1((b"0" * 40) + (b"e" * 40) + (b"f" * 40)).hexdigest(), + ), + ], ) def test_calculate_package_verification_code(package_sha1s, package_verification_code): # Randomize because PackageVerificationCode is deterministic. @@ -29,18 +32,19 @@ def test_calculate_package_verification_code(package_sha1s, package_verification "files": [ { "SPDXID": f"SPDXRef-FILE-{package_sha1}", - "checksums": [{"algorithm": "SHA1", "checksumValue": package_sha1}] - } for package_sha1 in package_sha1s + "checksums": [{"algorithm": "SHA1", "checksumValue": package_sha1}], + } + for package_sha1 in package_sha1s ], "packages": [{"SPDXID": "SPDXRef-PACKAGE", "filesAnalyzed": True}], "relationships": [ { "spdxElementId": "SPDXRef-PACKAGE", "relatedSpdxElement": f"SPDXRef-FILE-{package_sha1}", - "relationshipType": "CONTAINS" + "relationshipType": "CONTAINS", } for package_sha1 in package_sha1s - ] + ], } sbom.calculate_package_verification_codes(input_sbom) @@ -57,12 +61,12 @@ def test_normalization(): data = { "a": [1, 2, 3, {"b": [4, "c", [7, True, "2", {}]]}], # This line tests that inner structures are sorted first. - "b": [[1, 2, "b"], [2, 1, "a"]] + "b": [[1, 2, "b"], [2, 1, "a"]], } sbom.normalize_sbom_data(data) assert data == { "a": [1, 2, 3, {"b": ["c", 4, ["2", 7, True, {}]]}], - "b": [["a", 1, 2], ["b", 1, 2]] + "b": [["a", 1, 2], ["b", 1, 2]], } @@ -72,30 +76,32 @@ def test_fetch_project_metadata_from_pypi(mocker): # This is only a partial response using the information # that this function uses. - mock_urlopen.return_value.read.return_value = json.dumps({ - "urls": [ - { - "digests": { - "blake2b_256": "94596638090c25e9bc4ce0c42817b5a234e183872a1129735a9330c472cc2056", - "md5": "1331aabb4d1a2677f493effeebda3605", - "sha256": "ea9bd1a847e8c5774a5777bb398c19e80bcd4e2aa16a4b301b718fe6f593aba2" + mock_urlopen.return_value.read.return_value = json.dumps( + { + "urls": [ + { + "digests": { + "blake2b_256": "94596638090c25e9bc4ce0c42817b5a234e183872a1129735a9330c472cc2056", + "md5": "1331aabb4d1a2677f493effeebda3605", + "sha256": "ea9bd1a847e8c5774a5777bb398c19e80bcd4e2aa16a4b301b718fe6f593aba2", + }, + "filename": "pip-24.0.tar.gz", + "packagetype": "sdist", + "url": "https://files.pythonhosted.org/packages/.../pip-24.0.tar.gz", }, - "filename": "pip-24.0.tar.gz", - "packagetype": "sdist", - "url": "https://files.pythonhosted.org/packages/.../pip-24.0.tar.gz", - }, - { - "digests": { - "blake2b_256": "8a6a19e9fe04fca059ccf770861c7d5721ab4c2aebc539889e97c7977528a53b", - "md5": "74e3c5e4082113b1239ca0e9abfd1e82", - "sha256": "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc" + { + "digests": { + "blake2b_256": "8a6a19e9fe04fca059ccf770861c7d5721ab4c2aebc539889e97c7977528a53b", + "md5": "74e3c5e4082113b1239ca0e9abfd1e82", + "sha256": "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc", + }, + "filename": "pip-24.0-py3-none-any.whl", + "packagetype": "bdist_wheel", + "url": "https://files.pythonhosted.org/packages/.../pip-24.0-py3-none-any.whl", }, - "filename": "pip-24.0-py3-none-any.whl", - "packagetype": "bdist_wheel", - "url": "https://files.pythonhosted.org/packages/.../pip-24.0-py3-none-any.whl", - } - ] - }).encode() + ] + } + ).encode() # Default filename is the wheel download_url, checksum_sha256 = sbom.fetch_package_metadata_from_pypi( @@ -104,19 +110,26 @@ def test_fetch_project_metadata_from_pypi(mocker): ) mock_urlopen.assert_called_once_with("https://pypi.org/pypi/pip/24.0/json") - assert download_url == "https://files.pythonhosted.org/packages/.../pip-24.0-py3-none-any.whl" - assert checksum_sha256 == "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc" + assert ( + download_url + == "https://files.pythonhosted.org/packages/.../pip-24.0-py3-none-any.whl" + ) + assert ( + checksum_sha256 + == "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc" + ) # If we ask for the sdist (which we don't do normally) # then it'll be returned instead. download_url, checksum_sha256 = sbom.fetch_package_metadata_from_pypi( - project="pip", - version="24.0", - filename="pip-24.0.tar.gz" + project="pip", version="24.0", filename="pip-24.0.tar.gz" ) assert download_url == "https://files.pythonhosted.org/packages/.../pip-24.0.tar.gz" - assert checksum_sha256 == "ea9bd1a847e8c5774a5777bb398c19e80bcd4e2aa16a4b301b718fe6f593aba2" + assert ( + checksum_sha256 + == "ea9bd1a847e8c5774a5777bb398c19e80bcd4e2aa16a4b301b718fe6f593aba2" + ) def test_create_cpython_sbom(): @@ -129,11 +142,10 @@ def test_create_cpython_sbom(): assert re.fullmatch( r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - sbom_data["creationInfo"].pop("created") + sbom_data["creationInfo"].pop("created"), ) assert re.fullmatch( - r"^Tool: ReleaseTools-[a-f0-9]+$", - sbom_data["creationInfo"]["creators"].pop(1) + r"^Tool: ReleaseTools-[a-f0-9]+$", sbom_data["creationInfo"]["creators"].pop(1) ) assert sbom_data == { @@ -184,9 +196,11 @@ def test_create_cpython_sbom(): ("3.11.0a1", "https://www.python.org/ftp/python/3.11.0/fake-artifact.txt"), ("3.12.0b2", "https://www.python.org/ftp/python/3.12.0/fake-artifact.txt"), ("3.13.0rc3", "https://www.python.org/ftp/python/3.13.0/fake-artifact.txt"), - ] + ], ) -def test_create_cpython_sbom_pre_release_download_location(cpython_version, download_location): +def test_create_cpython_sbom_pre_release_download_location( + cpython_version, download_location +): sbom_data = {"packages": []} artifact_path = str(pathlib.Path(__file__).parent / "fake-artifact.txt") diff --git a/windows-release/purge.py b/windows-release/purge.py index 18d214fa..f5296edf 100644 --- a/windows-release/purge.py +++ b/windows-release/purge.py @@ -4,19 +4,19 @@ # py -3 purge.py 3.5.1rc1 # -__author__ = 'Steve Dower ' -__version__ = '1.0.0' +__author__ = "Steve Dower " +__version__ = "1.0.0" import re import sys from urllib.request import Request, urlopen -VERSION_RE = re.compile(r'(\d+\.\d+\.\d+)([A-Za-z_]+\d+)?$') +VERSION_RE = re.compile(r"(\d+\.\d+\.\d+)([A-Za-z_]+\d+)?$") try: m = VERSION_RE.match(sys.argv[1]) if not m: - print('Invalid version:', sys.argv[1]) + print("Invalid version:", sys.argv[1]) print('Expected something like "3.5.1rc1"') sys.exit(1) except LookupError: @@ -24,7 +24,7 @@ sys.exit(1) URL = f"https://www.python.org/ftp/python/{m.group(1)}/" -REL = m.group(2) or '' +REL = m.group(2) or "" FILES = [ "core.msi", @@ -78,9 +78,9 @@ ] PATHS = PATHS + [p + ".asc" for p in PATHS] -print('Purged:') +print("Purged:") for n in PATHS: u = URL + n - with urlopen(Request(u, method='PURGE', headers={'Fastly-Soft-Purge': 1})) as r: + with urlopen(Request(u, method="PURGE", headers={"Fastly-Soft-Purge": 1})) as r: r.read() - print(' ', u) + print(" ", u) From f5cdc882568938547ce33e612ec01bef6613dc6a Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sat, 8 Jun 2024 18:32:52 +0300 Subject: [PATCH 3/3] Use f-strings --- add-to-pydotorg.py | 24 ++++++++++++------------ release.py | 38 +++++++++++++++++++------------------- run_release.py | 4 ++-- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/add-to-pydotorg.py b/add-to-pydotorg.py index a33fe330..a383e26e 100755 --- a/add-to-pydotorg.py +++ b/add-to-pydotorg.py @@ -47,14 +47,14 @@ def run_cmd(cmd, silent=False, shell=True, **kwargs): if shell: cmd = " ".join(cmd) if not silent: - print("Executing %s" % cmd) + print(f"Executing {cmd}") try: if silent: subprocess.check_call(cmd, shell=shell, stdout=subprocess.PIPE, **kwargs) else: subprocess.check_call(cmd, shell=shell, **kwargs) except subprocess.CalledProcessError: - error("%s failed" % cmd) + error(f"{cmd} failed") try: @@ -72,7 +72,7 @@ def run_cmd(cmd, silent=False, shell=True, **kwargs): tag_cre = re.compile(r"(\d+)(?:\.(\d+)(?:\.(\d+))?)?(?:([ab]|rc)(\d+))?$") -headers = {"Authorization": "ApiKey %s" % auth_info, "Content-Type": "application/json"} +headers = {"Authorization": f"ApiKey {auth_info}", "Content-Type": "application/json"} github_oidc_provider = "https://github.com/login/oauth" google_oidc_provider = "https://accounts.google.com" @@ -153,9 +153,9 @@ def get_file_descriptions(release): def changelog_for(release): - new_url = "http://docs.python.org/release/%s/whatsnew/changelog.html" % release + new_url = f"http://docs.python.org/release/{release}/whatsnew/changelog.html" if requests.head(new_url).status_code != 200: - return "http://hg.python.org/cpython/file/v%s/Misc/NEWS" % release + return f"http://hg.python.org/cpython/file/v{release}/Misc/NEWS" def slug_for(release): @@ -204,8 +204,8 @@ def build_file_dict(release, rfile, rel_pk, file_desc, os_pk, add_download, add_ d = { "name": file_desc, "slug": slug_for(release) + "-" + make_slug(file_desc)[:40], - "os": "/api/v1/downloads/os/%s/" % os_pk, - "release": "/api/v1/downloads/release/%s/" % rel_pk, + "os": f"/api/v1/downloads/os/{os_pk}/", + "release": f"/api/v1/downloads/release/{rel_pk}/", "description": add_desc, "is_source": os_pk == 3, "url": download_root + f"{base_version(release)}/{rfile}", @@ -276,7 +276,7 @@ def list_files(release): def query_object(objtype, **params): """Find an API object by query parameters.""" - uri = base_url + "downloads/%s/" % objtype + uri = base_url + f"downloads/{objtype}/" uri += "?" + "&".join(f"{k}={v}" for k, v in params.items()) resp = requests.get(uri, headers=headers) if resp.status_code != 200 or not json.loads(resp.text)["objects"]: @@ -391,20 +391,20 @@ def main(): continue print("Creating ReleaseFile object for", rfile, key) if key in file_dicts: - raise RuntimeError("duplicate slug generated: %s" % key) + raise RuntimeError(f"duplicate slug generated: {key}") file_dicts[key] = file_dict print("Deleting previous release files") resp = requests.delete( - base_url + "downloads/release_file/?release=%s" % rel_pk, headers=headers + base_url + f"downloads/release_file/?release={rel_pk}", headers=headers ) if resp.status_code != 204: - raise RuntimeError("deleting previous releases failed: %s" % resp.status_code) + raise RuntimeError(f"deleting previous releases failed: {resp.status_code}") for file_dict in file_dicts.values(): file_pk = post_object("release_file", file_dict) if file_pk >= 0: print("Created as id =", file_pk) n += 1 - print("Done - %d files added" % n) + print(f"Done - {n} files added") if not sys.flags.interactive: diff --git a/release.py b/release.py index e6f02ea5..180859dd 100755 --- a/release.py +++ b/release.py @@ -38,14 +38,14 @@ def run_cmd(cmd, silent=False, shell=False, **kwargs): if shell: cmd = SPACE.join(cmd) if not silent: - print("Executing %s" % cmd) + print(f"Executing {cmd}") try: if silent: subprocess.check_call(cmd, shell=shell, stdout=subprocess.PIPE, **kwargs) else: subprocess.check_call(cmd, shell=shell, **kwargs) except subprocess.CalledProcessError: - error("%s failed" % cmd) + error(f"{cmd} failed") readme_re = re.compile(r"This is Python version [23]\.\d").match @@ -186,7 +186,7 @@ def constant_replace(fn, updated_constants, comment_start="/*", comment_end="*/" else: outfile.write(line) if not found_constants: - error("Constant section delimiters not found: %s" % fn) + error(f"Constant section delimiters not found: {fn}") os.rename(fn + ".new", fn) @@ -217,7 +217,7 @@ def tweak_patchlevel(tag, done=False): def bump(tag): - print("Bumping version to %s" % tag) + print(f"Bumping version to {tag}") tweak_patchlevel(tag) @@ -240,10 +240,10 @@ def bump(tag): print("\nManual editing time...") for fn in other_files: if os.path.exists(fn): - print("Edit %s" % fn) + print(f"Edit {fn}") manual_edit(fn) else: - print("Skipping %s" % fn) + print(f"Skipping {fn}") print("Bumped revision") if extra_work: @@ -258,7 +258,7 @@ def manual_edit(fn: str) -> None: @contextmanager def pushd(new): - print("chdir'ing to %s" % new) + print(f"chdir'ing to {new}") old = os.getcwd() os.chdir(new) try: @@ -272,11 +272,11 @@ def make_dist(name): os.mkdir(name) except OSError: if os.path.isdir(name): - print("WARNING: dist dir %s already exists" % name, file=sys.stderr) + print(f"WARNING: dist dir {name} already exists", file=sys.stderr) else: - error("%s/ is not a directory" % name) + error(f"{name}/ is not a directory") else: - print("created dist directory %s" % name) + print(f"created dist directory {name}") def tarball(source, clamp_mtime): @@ -330,17 +330,17 @@ def tarball(source, clamp_mtime): def export(tag, silent=False, skip_docs=False): make_dist(tag.text) print("Exporting tag:", tag.text) - archivename = "Python-%s" % tag.text + archivename = f"Python-{tag.text}" # I have not figured out how to get git to directly produce an # archive directory like hg can, so use git to produce a temporary # tarball then expand it with tar. - archivetempfile = "%s.tar" % archivename + archivetempfile = f"{archivename}.tar" run_cmd( [ "git", "archive", "--format=tar", - "--prefix=%s/" % archivename, + f"--prefix={archivename}/", "-o", archivetempfile, tag.gitname, @@ -348,7 +348,7 @@ def export(tag, silent=False, skip_docs=False): silent=silent, ) with pushd(tag.text): - archivetempfile = "../%s" % archivetempfile + archivetempfile = f"../{archivetempfile}" run_cmd(["tar", "-xf", archivetempfile], silent=silent) os.unlink(archivetempfile) with pushd(archivename): @@ -446,7 +446,7 @@ def export(tag, silent=False, skip_docs=False): os.mkdir("src") tarball(archivename, tag.committed_at.strftime("%Y-%m-%d %H:%M:%SZ")) print() - print("**Now extract the archives in %s/src and run the tests**" % tag.text) + print(f"**Now extract the archives in {tag.text}/src and run the tests**") print("**You may also want to run make install and re-test**") @@ -469,16 +469,16 @@ def build_docs(): def upload(tag, username): """scp everything to dinsdale""" - address = '"%s@dinsdale.python.org:' % username + address = f'"{username}@dinsdale.python.org:' def scp(from_loc, to_loc): run_cmd(["scp", from_loc, address + to_loc]) with pushd(tag.text): print("Uploading source tarballs") - scp("src", "/data/python-releases/%s" % tag.nickname) + scp("src", f"/data/python-releases/{tag.nickname}") print("Upload doc tarballs") - scp("docs", "/data/python-releases/doc/%s" % tag.nickname) + scp("docs", f"/data/python-releases/doc/{tag.nickname}") print( "* Now change the permissions on the tarballs so they are " "writable by the webmaster group. *" @@ -495,7 +495,7 @@ def __init__(self, tag_name): tag_name = os.path.basename(os.getcwd()) result = tag_cre.match(tag_name) if result is None: - error("tag %s is not valid" % tag_name) + error(f"tag {tag_name} is not valid") data = list(result.groups()) if data[3] is None: # A final release. diff --git a/run_release.py b/run_release.py index fa014902..b97ca181 100755 --- a/run_release.py +++ b/run_release.py @@ -526,8 +526,8 @@ def sign_source_artifacts(db: DbfilenameShelf) -> None: uid = input("Please enter key ID to use for signing: ") tarballs_path = pathlib.Path(db["git_repo"] / str(db["release"]) / "src") - tgz = str(tarballs_path / ("Python-%s.tgz" % db["release"])) - xz = str(tarballs_path / ("Python-%s.tar.xz" % db["release"])) + tgz = str(tarballs_path / (f"Python-{db['release']}.tgz")) + xz = str(tarballs_path / (f"Python-{db['release']}.tar.xz")) subprocess.check_call(["gpg", "-bas", "-u", uid, tgz]) subprocess.check_call(["gpg", "-bas", "-u", uid, xz])