From f21311da9f636839c387559e6600ba8892fd520f Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Wed, 8 Jan 2025 14:00:22 -0800 Subject: [PATCH 1/9] updated cli to use updated download functionality --- coderdata/cli.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/coderdata/cli.py b/coderdata/cli.py index 1f7cc0b3..05ebf88d 100644 --- a/coderdata/cli.py +++ b/coderdata/cli.py @@ -1,5 +1,6 @@ import argparse -from .download.downloader import download +import sys +from .download.downloader import download as download_datasets def main(): parser = argparse.ArgumentParser(prog='coderdata') @@ -7,20 +8,25 @@ def main(): # Subcommand 'download' parser_download = subparsers.add_parser('download', help='Download datasets') - parser_download.add_argument('--prefix', type=str, default=None, - help='Prefix of the dataset to download (e.g., "hcmi"), "all", or leave empty for all files.') + parser_download.add_argument( + '-n', '--name', + dest='DATASET_NAME', + type=str, + default='all', + help='Name of the dataset to download (e.g., "beataml"). ' + 'Alternatively "all" will download the full repository of ' + 'coderdata datasets. See "coderdata --list" for a complete list ' + 'of available datasets. Defaults to "all"' + ) parser_download.set_defaults(func=download) - + if len(sys.argv) == 1: + parser.print_help(sys.stderr) + sys.exit(0) args = parser.parse_args() - if hasattr(args, 'func'): - # Check if 'prefix' argument is present and pass it directly to the function - if 'prefix' in args: - args.func(args.prefix) - else: - args.func() - else: - parser.print_help() + args.func(args) +def download(args): + download_datasets(name=args.DATASET_NAME) if __name__ == '__main__': main() From 3f6a6f5c02253aaead59a2392696c58640351de7 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Wed, 8 Jan 2025 14:58:30 -0800 Subject: [PATCH 2/9] added additional arguments to download routine to be able to specify storage folder and allow for files to be overwritten --- coderdata/cli.py | 50 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/coderdata/cli.py b/coderdata/cli.py index 05ebf88d..1e3cfe65 100644 --- a/coderdata/cli.py +++ b/coderdata/cli.py @@ -1,4 +1,7 @@ import argparse +from os import PathLike +from pathlib import Path +from typing import Union import sys from .download.downloader import download as download_datasets @@ -17,6 +20,21 @@ def main(): 'Alternatively "all" will download the full repository of ' 'coderdata datasets. See "coderdata --list" for a complete list ' 'of available datasets. Defaults to "all"' + ) + parser_download.add_argument( + '-p', '--local_path', + dest="LOCAL_PATH", + type=check_folder, + default=Path.cwd(), + help='Defines the folder the datasets should be stored in. Defaults ' + 'to the current working directory if omitted.' + ) + parser_download.add_argument( + '-o', '--overwrite', + dest="OVERWRITE", + default=False, + action='store_true', + help='Allow dataset files to be overwritten if they already exist.' ) parser_download.set_defaults(func=download) if len(sys.argv) == 1: @@ -26,7 +44,37 @@ def main(): args.func(args) def download(args): - download_datasets(name=args.DATASET_NAME) + download_datasets( + name=args.DATASET_NAME, + local_path=args.LOCAL_PATH, + exist_ok=args.OVERWRITE, + ) + + +def check_folder(path: Union[str, PathLike, Path]) -> Path: + """ + Helper function to check if a defined folder exists + """ + + if not isinstance(path, (str, PathLike, Path)): + raise TypeError( + f"'path' must be of type str, PathLike or Path. Supplied argument " + f"is of type {type(path)}." + ) + if not isinstance(path, Path): + abs_path = Path(path).absolute() + else: + abs_path = path.absolute() + + if not abs_path.is_dir(): + raise OSError( + f"The defined folder path '{path}' does not exist or is not a " + f"folder." + ) + + return abs_path + + if __name__ == '__main__': main() From 0aa5314767a4d5bf78877fc6cc13d5e17a33630c Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Wed, 8 Jan 2025 14:59:53 -0800 Subject: [PATCH 3/9] added check to exit quitely if CLI is interrupted --- coderdata/cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/coderdata/cli.py b/coderdata/cli.py index 1e3cfe65..63f07672 100644 --- a/coderdata/cli.py +++ b/coderdata/cli.py @@ -76,6 +76,9 @@ def check_folder(path: Union[str, PathLike, Path]) -> Path: if __name__ == '__main__': - main() + try: + main() + except KeyboardInterrupt: + pass From a6e751bf9f32b79bad092b81c8b619a20ce193fe Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Wed, 8 Jan 2025 15:00:38 -0800 Subject: [PATCH 4/9] added flags to CLI to print version numbers and list available datasets --- coderdata/cli.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/coderdata/cli.py b/coderdata/cli.py index 63f07672..7414ba28 100644 --- a/coderdata/cli.py +++ b/coderdata/cli.py @@ -4,6 +4,8 @@ from typing import Union import sys from .download.downloader import download as download_datasets +from .utils import version +from .utils import list_datasets def main(): parser = argparse.ArgumentParser(prog='coderdata') @@ -37,12 +39,35 @@ def main(): help='Allow dataset files to be overwritten if they already exist.' ) parser_download.set_defaults(func=download) + + grp = parser.add_mutually_exclusive_group() + grp.add_argument( + '-l', '--list', + dest="LIST", + action='store_true', + help="Prints list of available datasets and exits program." + ) + grp.add_argument( + '-v', '--version', + dest="VERSION", + action='store_true', + help="Prints the versions of the coderdata API and dataset and exits the program" + ) + parser.set_defaults(func=info) + if len(sys.argv) == 1: parser.print_help(sys.stderr) sys.exit(0) args = parser.parse_args() args.func(args) +def info(args): + if args.LIST: + print(list_datasets()) + elif args.VERSION: + print(version()) + + def download(args): download_datasets( name=args.DATASET_NAME, From 9fcd41a127dfdd00d5c227b6515f69a9f8b1c327 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Wed, 8 Jan 2025 15:40:25 -0800 Subject: [PATCH 5/9] made CLI arguments --version & --list more verbose --- coderdata/cli.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/coderdata/cli.py b/coderdata/cli.py index 7414ba28..278cc0b4 100644 --- a/coderdata/cli.py +++ b/coderdata/cli.py @@ -63,9 +63,28 @@ def main(): def info(args): if args.LIST: - print(list_datasets()) + print( + '\n' + 'Available datasets\n' + '------------------\n' + ) + list_datasets() + print( + '\n' + '------------------\n\n' + 'To download individual datasets run "coderdata download -name ' + 'DATASET_NAME" where "DATASET_NAME" is for example "beataml".' + ) elif args.VERSION: - print(version()) + version_numbers = version() + print( + *( + f"package version: {version_numbers['package']}", + f"dataset version: {version_numbers['dataset']}" + ), + sep='\n', + file=sys.stdout, + ) def download(args): From 8a998e93c7bd7e21093f1b95b46a7a435c505566 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Wed, 8 Jan 2025 16:38:49 -0800 Subject: [PATCH 6/9] added more comprehensive code documentation to cli --- coderdata/cli.py | 93 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 81 insertions(+), 12 deletions(-) diff --git a/coderdata/cli.py b/coderdata/cli.py index 278cc0b4..228b2a75 100644 --- a/coderdata/cli.py +++ b/coderdata/cli.py @@ -1,25 +1,42 @@ +""" +Command Line Interface to retrieve coderdata datasets. +""" + import argparse from os import PathLike from pathlib import Path from typing import Union import sys + from .download.downloader import download as download_datasets from .utils import version from .utils import list_datasets def main(): - parser = argparse.ArgumentParser(prog='coderdata') - subparsers = parser.add_subparsers(dest='command') + """ + Main method containing the argument parsing and execution of + individual subroutines. + """ + parser = argparse.ArgumentParser(prog='coderdata', add_help=True) + parser.set_defaults(func=info) - # Subcommand 'download' - parser_download = subparsers.add_parser('download', help='Download datasets') + # subparser for the 'download' subroutine + subparsers = parser.add_subparsers( + title="commands", + dest='command' + ) + parser_download = subparsers.add_parser( + 'download', + help='subroutine to download datasets. See "coderdata download -h" ' + 'for more options.' + ) parser_download.add_argument( '-n', '--name', dest='DATASET_NAME', type=str, default='all', - help='Name of the dataset to download (e.g., "beataml"). ' - 'Alternatively "all" will download the full repository of ' + help='name of the dataset to download (e.g., "beataml"). ' + 'Alternatively, "all" will download the full repository of ' 'coderdata datasets. See "coderdata --list" for a complete list ' 'of available datasets. Defaults to "all"' ) @@ -28,7 +45,7 @@ def main(): dest="LOCAL_PATH", type=check_folder, default=Path.cwd(), - help='Defines the folder the datasets should be stored in. Defaults ' + help='defines the folder the datasets should be stored in. Defaults ' 'to the current working directory if omitted.' ) parser_download.add_argument( @@ -36,32 +53,55 @@ def main(): dest="OVERWRITE", default=False, action='store_true', - help='Allow dataset files to be overwritten if they already exist.' + help='allow dataset files to be overwritten if they already exist.' ) parser_download.set_defaults(func=download) + # argument group that contains flags for additional information grp = parser.add_mutually_exclusive_group() grp.add_argument( '-l', '--list', dest="LIST", action='store_true', - help="Prints list of available datasets and exits program." + help="prints list of available datasets and exits program." ) grp.add_argument( '-v', '--version', dest="VERSION", action='store_true', - help="Prints the versions of the coderdata API and dataset and exits the program" + help='prints the versions of the coderdata API and dataset and exits ' + 'the program' ) - parser.set_defaults(func=info) + # checks if 'coderdata' was executed without additional arguments + # and if so prints help message and exits if len(sys.argv) == 1: parser.print_help(sys.stderr) sys.exit(0) + + # parse arguments and execute defined functions by `set_default()` + # according to which subcommands / arguments where passed on the + # command line args = parser.parse_args() args.func(args) + def info(args): + """ + Helper function that takes the parsed command line arguments and + prints either verison information or information on the available + datasets depending on the arguments in ``args``. + + Parameters + ---------- + args : Namespace + A Namespace object that contains commandline arguments parsed by + ``ArgumentParser.parse_args()``. + """ + + # retrieve the dataset information stored in dataset.yml via + # coderdata.utils.list_dataset() and print the information to + # sys.stdout if args.LIST: print( '\n' @@ -75,6 +115,9 @@ def info(args): 'To download individual datasets run "coderdata download -name ' 'DATASET_NAME" where "DATASET_NAME" is for example "beataml".' ) + + # retrieve version number information stored in dataset.yml via + # coderdata.utils.version() and print the information to sys.stdout elif args.VERSION: version_numbers = version() print( @@ -88,6 +131,17 @@ def info(args): def download(args): + """ + Wrapper function to download datasets via ``coderdata.download()``. + Function passes commandline arguments to the internal download + function. + + Parameters + ---------- + args : Namespace + A Namespace object that contains commandline arguments parsed by + ``ArgumentParser.parse_args()``. + """ download_datasets( name=args.DATASET_NAME, local_path=args.LOCAL_PATH, @@ -97,7 +151,21 @@ def download(args): def check_folder(path: Union[str, PathLike, Path]) -> Path: """ - Helper function to check if a defined folder exists + Helper function to check if a defined folder exists. + + Returns + ------- + Path + Cleaned path object with the absolute path to the folder passed + to the function. + + Raises + ------ + TypeError + If passed path argument is not of the requested type. + OSError + If the passed path argument does not link to a valid existing + folder. """ if not isinstance(path, (str, PathLike, Path)): @@ -119,6 +187,7 @@ def check_folder(path: Union[str, PathLike, Path]) -> Path: return abs_path +# Routine to execute the main function. if __name__ == '__main__': try: main() From 9a921209e72b5b8a4dbd478eac8c260d69e9ece1 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Tue, 14 Jan 2025 10:37:13 -0800 Subject: [PATCH 7/9] fixed typo --- coderdata/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderdata/cli.py b/coderdata/cli.py index 228b2a75..326697c5 100644 --- a/coderdata/cli.py +++ b/coderdata/cli.py @@ -112,7 +112,7 @@ def info(args): print( '\n' '------------------\n\n' - 'To download individual datasets run "coderdata download -name ' + 'To download individual datasets run "coderdata download --name ' 'DATASET_NAME" where "DATASET_NAME" is for example "beataml".' ) From bc4b669564723e9f65eab2f34f86e63e378fd9df Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Tue, 14 Jan 2025 10:37:36 -0800 Subject: [PATCH 8/9] fixed dataset version number --- coderdata/dataset.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderdata/dataset.yml b/coderdata/dataset.yml index efd6a666..6ca27429 100644 --- a/coderdata/dataset.yml +++ b/coderdata/dataset.yml @@ -1,5 +1,5 @@ figshare: "https://api.figshare.com/v2/articles/26409316" -version: "v0.1.4" +version: "0.1.4" datasets: beataml: description: "Beat acute myeloid leukemia (BeatAML) focuses on acute myeloid leukemia tumor data. Data includes drug response, proteomics, and transcriptomics datasets." From 5659516e7b916df8321ed8b18717377d73066ed5 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Tue, 14 Jan 2025 10:38:11 -0800 Subject: [PATCH 9/9] set default of `name` in `cd.download()` to 'all' --- coderdata/download/downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderdata/download/downloader.py b/coderdata/download/downloader.py index fb2d662e..a513963a 100644 --- a/coderdata/download/downloader.py +++ b/coderdata/download/downloader.py @@ -10,7 +10,7 @@ import yaml def download( - name: str=None, + name: str='all', local_path: PathLike=Path.cwd(), exist_ok: bool=False ):