diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/__pycache__/__init__.cpython-312.pyc b/scripts/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..9d7d71e Binary files /dev/null and b/scripts/__pycache__/__init__.cpython-312.pyc differ diff --git a/scripts/__pycache__/fetch_updates.cpython-312.pyc b/scripts/__pycache__/fetch_updates.cpython-312.pyc new file mode 100644 index 0000000..c91974b Binary files /dev/null and b/scripts/__pycache__/fetch_updates.cpython-312.pyc differ diff --git a/scripts/__pycache__/parse_and_validate_properties_txt.cpython-312.pyc b/scripts/__pycache__/parse_and_validate_properties_txt.cpython-312.pyc new file mode 100644 index 0000000..691f58f Binary files /dev/null and b/scripts/__pycache__/parse_and_validate_properties_txt.cpython-312.pyc differ diff --git a/scripts/fetch_updates.py b/scripts/fetch_updates.py index 971c346..57e4e3c 100644 --- a/scripts/fetch_updates.py +++ b/scripts/fetch_updates.py @@ -1,121 +1,120 @@ """ Reads in the contributions.yaml file, and updates the entries by hitting the 'source' url. """ + import argparse from datetime import datetime, UTC import pathlib from ruamel.yaml import YAML from multiprocessing import Pool -from parse_and_validate_properties_txt import read_properties_txt, parse_text, validate_existing +from scripts.parse_and_validate_properties_txt import ( + read_properties_txt, + parse_text, + validate_existing +) def update_contribution(contribution, props): - datetime_today = datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%S%z') - contribution['lastUpdated'] = datetime_today - if 'previousVersions' not in contribution: - contribution['previousVersions'] = [] - contribution['previousVersions'].append(contribution['prettyVersion']) - - # update from online - for field in props.keys(): - # process category list - if field == 'categories': - if props[field]: - contribution[field] = sorted(props[field].strip('"').split(',')) - else: - contribution[field] = [] - else: - contribution[field] = props[field] + datetime_today = datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%S%z') + contribution['lastUpdated'] = datetime_today + + if 'previousVersions' not in contribution: + contribution['previousVersions'] = [] + + contribution['previousVersions'].append(contribution['prettyVersion']) + + for field in props.keys(): + if field == 'categories': + if props[field]: + contribution[field] = sorted(props[field].strip('"').split(',')) + else: + contribution[field] = [] + else: + contribution[field] = props[field] + + if 'download' not in contribution: + contribution['download'] = contribution['source'][:contribution['source'].rfind('.')] + '.zip' - if 'download' not in contribution: - contribution['download'] = contribution['source'][:contribution['source'].rfind('.')] + '.zip' - def log_broken(contribution, msg): - if contribution['status'] == 'VALID': - contribution['status'] = 'BROKEN' + if contribution['status'] == 'VALID': + contribution['status'] = 'BROKEN' + if 'log' not in contribution: - contribution['log'] = [] + contribution['log'] = [] + contribution['log'].append(msg) + def process_contribution(item): - index, contribution = item + index, contribution = item - date_today = datetime.now(UTC).strftime('%Y-%m-%d') - this_version = '0' + date_today = datetime.now(UTC).strftime('%Y-%m-%d') + this_version = '0' - if contribution['status'] != 'DEPRECATED': - # compare version to what is at url. If has changed, update contribution to - # what is online - if 'version' in contribution: - this_version = contribution['version'] + if contribution['status'] != 'DEPRECATED': + if 'version' in contribution: + this_version = contribution['version'] - try: - properties_raw = read_properties_txt(contribution['source']) - except FileNotFoundError as e: - log_broken(contribution, f'file not found, {e}, {date_today}') - return index, contribution - except Exception: - log_broken(contribution, f'url timeout, {date_today}') - return index, contribution + try: + properties_raw = read_properties_txt(contribution['source']) + except FileNotFoundError as e: + log_broken(contribution, f'file not found, {e}, {date_today}') + return index, contribution + except Exception: + log_broken(contribution, f'url timeout, {date_today}') + return index, contribution - try: - props = validate_existing(parse_text(properties_raw)) - except Exception: - log_broken(contribution, f'invalid file, {date_today}') - return index, contribution + try: + props = validate_existing(parse_text(properties_raw)) + except Exception: + log_broken(contribution, f'invalid file, {date_today}') + return index, contribution - # some library files have field lastUpdated. This also exists in the database, but is defined - # by our scripts, so remove this field. - contribution.pop('lastUpdated', None) + contribution.pop('lastUpdated', None) + contribution['status'] = 'VALID' - contribution['status'] = 'VALID' + if props['version'] != this_version: + update_contribution(contribution, props) - if props['version'] != this_version: - # update from online - update_contribution(contribution, props) - return index, contribution + return index, contribution if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('--index') - args = parser.parse_args() - - index = 'all' - if args.index: - index = args.index - - database_file = pathlib.Path(__file__).parent.parent / 'contributions.yaml' - - # read in database yaml file - yaml = YAML() - with open(database_file, 'r') as db: - data = yaml.load(db) - - contributions_list = data['contributions'] - - if index == 'all': - total = len(contributions_list) - completed = 0 - print(f"Starting processing of {total} contributions...") - - with Pool(processes=256) as pool: - for index, contribution in pool.imap_unordered(process_contribution, enumerate(contributions_list)): - contributions_list[index] = contribution - completed += 1 - print(f"Progress: {completed}/{total} ({(completed/total*100):.1f}%)") - - print("All processing complete") - else: - # update only contribution with id==index - contribution = next((x for x in contributions_list if x['id'] == int(index)), None) - print(contribution) - process_contribution((index, contribution)) - print(contribution) - - # write all contributions to database file - yaml = YAML() - with open(database_file, 'w') as outfile: - yaml.dump({"contributions": contributions_list}, outfile) + parser = argparse.ArgumentParser() + parser.add_argument('--index') + args = parser.parse_args() + + index = 'all' + if args.index: + index = args.index + + database_file = pathlib.Path(__file__).parent.parent / 'contributions.yaml' + yaml = YAML() + + with open(database_file, 'r') as db: + data = yaml.load(db) + + contributions_list = data['contributions'] + + if index == 'all': + total = len(contributions_list) + completed = 0 + print(f"Starting processing of {total} contributions...") + + with Pool(processes=256) as pool: + for index, contribution in pool.imap_unordered(process_contribution, enumerate(contributions_list)): + contributions_list[index] = contribution + completed += 1 + print(f"Progress: {completed}/{total} ({(completed/total*100):.1f}%)") + + print("All processing complete") + else: + contribution = next((x for x in contributions_list if x['id'] == int(index)), None) + print(contribution) + process_contribution((index, contribution)) + print(contribution) + + with open(database_file, 'w') as outfile: + yaml.dump({"contributions": contributions_list}, outfile) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/__pycache__/__init__.cpython-312.pyc b/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..0a4e571 Binary files /dev/null and b/tests/__pycache__/__init__.cpython-312.pyc differ diff --git a/tests/__pycache__/test_fetch_updates.cpython-312-pytest-8.4.1.pyc b/tests/__pycache__/test_fetch_updates.cpython-312-pytest-8.4.1.pyc new file mode 100644 index 0000000..c8b0e68 Binary files /dev/null and b/tests/__pycache__/test_fetch_updates.cpython-312-pytest-8.4.1.pyc differ diff --git a/tests/test_fetch_updates.py b/tests/test_fetch_updates.py new file mode 100644 index 0000000..dd642c8 --- /dev/null +++ b/tests/test_fetch_updates.py @@ -0,0 +1,42 @@ +import os +import sys + +# Allow imports from project root +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from scripts import fetch_updates + + +def test_log_broken(): + contribution = { + "status": "VALID" + } + + fetch_updates.log_broken(contribution, "file not found") + + assert contribution["status"] == "BROKEN" + assert "log" in contribution + assert contribution["log"] == ["file not found"] + + +def test_update_contribution(): + contribution = { + "id": "test-id", + "prettyVersion": "1.0", + "source": "http://example.com/library.properties" + } + + props = { + "version": "2.0", + "categories": '"Graphics,Animation"', + "author": "Test Author" + } + + fetch_updates.update_contribution(contribution, props) + + assert "lastUpdated" in contribution + assert contribution["previousVersions"] == ["1.0"] + assert contribution["version"] == "2.0" + assert contribution["categories"] == ["Animation", "Graphics"] + assert contribution["author"] == "Test Author" + assert contribution["download"].endswith(".zip")