From b14d1c39ada40a35428a4fe8ed53da4673a6a9f5 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Mon, 18 Nov 2024 15:12:45 -0800 Subject: [PATCH 01/24] first draft of `Dataset` / `DataLoader` rewrite --- coderdata/__init__.py | 4 + coderdata/dataset/__init__.py | 2 + coderdata/dataset/dataset.py | 273 ++++++++++++++++++++++++++++++++++ 3 files changed, 279 insertions(+) create mode 100644 coderdata/dataset/__init__.py create mode 100644 coderdata/dataset/dataset.py diff --git a/coderdata/__init__.py b/coderdata/__init__.py index d3883db3..db739e6a 100644 --- a/coderdata/__init__.py +++ b/coderdata/__init__.py @@ -1,3 +1,7 @@ from .download.downloader import download_data_by_prefix from .load.loader import DatasetLoader, join_datasets from .split.splitter import train_test_validate +from .dataset.dataset import ( + Dataset, + load, +) diff --git a/coderdata/dataset/__init__.py b/coderdata/dataset/__init__.py new file mode 100644 index 00000000..3399f0f6 --- /dev/null +++ b/coderdata/dataset/__init__.py @@ -0,0 +1,2 @@ +from .dataset import Dataset +from .dataset import load \ No newline at end of file diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py new file mode 100644 index 00000000..aecb3e58 --- /dev/null +++ b/coderdata/dataset/dataset.py @@ -0,0 +1,273 @@ +from pathlib import Path +import sys + +import pandas as pd + +class Dataset: + + def __init__( + self, + name: str, + transcriptomics: pd.DataFrame=None, + proteomics: pd.DataFrame=None, + mutations: pd.DataFrame=None, + copy_number: pd.DataFrame=None, + samples: pd.DataFrame=None, + drugs: pd.DataFrame=None, + mirna: pd.DataFrame=None, + experiments: pd.DataFrame=None, + methylation: pd.DataFrame=None, + metabolomics: pd.DataFrame=None, + genes: pd.DataFrame=None, + full: pd.DataFrame=None, + ): + + self.name = name + self.transcriptomics = transcriptomics + self.proteomics = proteomics + self.mutations = mutations + self.copy_number = copy_number + self.samples = samples + self.drugs = drugs + self.mirna = mirna + self.experiments = experiments + self.methylation = methylation + self.metabolomics = metabolomics + self.genes = genes + self.full = full + + + #----------------------------- + # getters / setters & deleters + # ---------------------------- + + + @property + def name(self): + return self._name + + @name.setter + def name(self, value): + self._name = value + + @name.deleter + def name(self): + del self._name + + + @property + def transcriptomics(self): + return self._transcriptomics + + @transcriptomics.setter + def transcriptomics(self, value): + self._transcriptomics = value + + @transcriptomics.deleter + def transcriptomics(self): + del self._transcriptomics + + + @property + def proteomics(self): + return self._proteomics + + @proteomics.setter + def proteomics(self, value): + self._proteomics = value + + @proteomics.deleter + def proteomics(self): + del self._proteomics + + + @property + def mutations(self): + return self._mutations + + @mutations.setter + def mutations(self, value): + self._mutations = value + + @mutations.deleter + def mutations(self): + del self._mutations + + + @property + def copy_number(self): + return self._copy_number + + @copy_number.setter + def copy_number(self, value): + self._copy_number = value + + @copy_number.deleter + def copy_number(self): + del self._copy_number + + + @property + def samples(self): + return self._samples + + @samples.setter + def samples(self, value): + self._samples = value + + @samples.deleter + def samples(self): + del self._samples + + + @property + def drugs(self): + return self._drugs + + @drugs.setter + def drugs(self, value): + self._drugs = value + + @drugs.deleter + def drugs(self): + del self._drugs + + + @property + def mirna(self): + return self._mirna + + @mirna.setter + def mirna(self, value): + self._mirna = value + + @mirna.deleter + def mirna(self): + del self._mirna + + + @property + def experiments(self): + return self._experiments + + @experiments.setter + def experiments(self, value): + self._experiments = value + + @experiments.deleter + def experiments(self): + del self._experiments + + + @property + def methylation(self): + return self._methylation + + @methylation.setter + def methylation(self, value): + self._methylation = value + + @methylation.deleter + def methylation(self): + del self._methylation + + + @property + def metabolomics(self): + return self._metabolomics + + @metabolomics.setter + def metabolomics(self, value): + self._metabolomics = value + + @metabolomics.deleter + def metabolomics(self): + del self._metabolomics + + + @property + def genes(self): + return self._genes + + @genes.setter + def genes(self, value): + self._genes = value + + @genes.deleter + def genes(self): + del self._genes + + + @property + def full(self): + return self._full + + @full.setter + def full(self, value): + self._full = value + + @full.deleter + def full(self): + del self._full + + +# --------------------------------------------------------------------- +# Functions that are Dataset related but not Class / Instance functions +# --------------------------------------------------------------------- + +def load(name: str, directory: str|Path=Path.cwd()) -> Dataset: + print("Processing Data...", file=sys.stderr) + + if type(directory) is not Path: + try: + directory = Path(directory) + if not directory.exists(): + raise OSError( + f"Given path / directory does not exist: '{directory}'" + ) + except TypeError: + raise TypeError( + f"Invalid path / directory defined: '{directory}'" + ) + + accepted_file_endings = ('.csv', '.tsv', '.csv.gz', '.tsv.gz') + dataset = Dataset(name) + + for child in directory.iterdir(): + if child.name in ["genes.csv", "genes.csv.gz"]: + dataset.genes = _load_file(child) + print("Loaded genes dataset.", file=sys.stderr) + + if ( + child.name.startswith(name) + and child.name.endswith(accepted_file_endings) + ): + + dataset_type = child.name[len(name)+1:].split('.')[0] + print(dataset_type) + if hasattr(dataset, dataset_type): + setattr(dataset, dataset_type, _load_file(child)) + + return dataset + + +def _load_file(file_path: Path) -> pd.DataFrame: + if file_path.suffix == '.gz': + return pd.read_csv( + file_path, + compression='gzip', + delimiter=_determine_delimiter(file_path), + ) + else: + return pd.read_csv( + file_path, + delimiter=_determine_delimiter(file_path), + ) + + +def _determine_delimiter(file_path): + print(file_path.suffixes) + if '.tsv' in file_path.suffixes: + return '\t' + else: + return ',' From 27813097f08ae1c6910bf96bac689146753e4a23 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Mon, 18 Nov 2024 15:13:15 -0800 Subject: [PATCH 02/24] added `Dataset.types()` function --- coderdata/dataset/dataset.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index aecb3e58..ee38b5a2 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -210,6 +210,32 @@ def full(self, value): def full(self): del self._full + + # ---------------------------- + # Class and Instance functions + # ---------------------------- + + def types(self) -> list: + data_types = [ + 'transcriptomics', + 'proteomics', + 'mutations', + 'copy_number', + 'samples', + 'drugs', + 'mirna', + 'experiments', + 'methylation', + 'metabolomics', + 'genes', + ] + data_types_present = [] + for data_type in data_types: + if getattr(self, data_type) is not None: + data_types_present.append(data_type) + + return data_types_present + # --------------------------------------------------------------------- # Functions that are Dataset related but not Class / Instance functions From 748e3833ce1d253e6c9909f0ab1b3b5949c4d3fd Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Mon, 18 Nov 2024 15:44:03 -0800 Subject: [PATCH 03/24] addded train_test_validate as instance method --- coderdata/dataset/dataset.py | 526 +++++++++++++++++++++++++++++++++++ 1 file changed, 526 insertions(+) diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index ee38b5a2..b0c4b4a7 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -1,8 +1,29 @@ + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import dataclass from pathlib import Path import sys +from typing import Literal +import numpy as np +from numpy.random import RandomState import pandas as pd +from sklearn.model_selection import GroupShuffleSplit +from sklearn.model_selection import ShuffleSplit +from sklearn.model_selection import StratifiedGroupKFold +from sklearn.model_selection import StratifiedShuffleSplit + + +@dataclass +class Split: + train: Dataset + test: Dataset + validate: Dataset + + class Dataset: def __init__( @@ -235,6 +256,342 @@ def types(self) -> list: data_types_present.append(data_type) return data_types_present + + + + def train_test_validate( + data: Dataset, + split_type: Literal[ + 'mixed-set', 'drug-blind', 'cancer-blind' + ]='mixed-set', + ratio: tuple[int, int, int]=(8,1,1), + stratify_by: (str | None)=None, + random_state: (int | RandomState | None)=None, + **kwargs: dict, + ) -> Split: + """ + Splits a `CoderData` object (see also + `coderdata.load.loader.DatasetLoader`) into three subsets for + training, testing and validating machine learning algorithms. + + The size of the splits can be adjusted to be different from 80:10:10 + (the default)for train:test:validate. The function also allows for + additional optional arguments, that define the type of split that is + performed ('mixed-set', 'drug-blind', 'cancer-blind'), if the splits + should be stratified (and which drug response metric to use), as + well as a random seed to enable the creation of reproducable splits. + Furhermore, a list of keyword arguments can be defined that will be + passed to the stratification function if so desired. + + Parameters + ---------- + data : DatasetLoader + CoderData object containing a full dataset either downloaded + from the CoderData repository (see also + `coderdata.download.downloader.download_data_by_prefix`) or + built locally via the `build_all` process. The object must first + be loaded via `coderdata.load.loader.DatasetLoader`. + split_type : {'mixed-set', 'drug-blind', 'cancer-blind'}, \ + default='mixed-set' + + Defines the type of split that should be generated: + + - *mixed-set*: Splits randomly independent of drug / cancer + association of the samples. Individual drugs or cancer types + can appear in all three splits + - *drug-blind*: Splits according to drug association. Any sample + associated with a drug will be unique to one of the splits. + For example samples with association to drug A will only be + present in the train split, but never in test or validate. + - *cancer-blind*: Splits according to cancer association. + Equivalent to drug-blind, except cancer types will be unique + to splits. + ratio : tuple[int, int, int], default=(8,1,1) + Defines the size ratio of the resulting test, train and + validation sets. + stratify_by : str | None, default=None + Defines if the training, testing and validation sets should be + stratified. Any value other than None indicates stratification + and defines which drug response value should be used as basis + for the stratification. _None_ indicates that no stratfication + should be performed. + random_state : int | RandomState | None, defaul=None + Defines a seed value for the randomization of the splits. Will + get passed to internal functions. Providing the seed will enable + reproducability of the generated splits. + **kwargs + Additional keyword arguments that will be passed to the function + that generates classes for the stratification + (see also ``_create_classes``). + + Returns + ------- + Splits : Split + A ``Split`` object that contains three Dataset objects as + attributes (``Split.train``, ``Split.test``, + ``Split.validate``) + + Raises + ------- + ValueError : + If supplied `split_type` is not in the list of accepted values. + + """ + + # reading in the potential keyword arguments that will be passed to + # _create_classes(). + thresh = kwargs.get('thresh', None) + num_classes = kwargs.get('num_classes', 2) + quantiles = kwargs.get('quantiles', True) + + # Type checking split_type + if split_type not in [ + 'mixed-set', 'drug-blind', 'cancer-blind' + ]: + raise ValueError( + f"{split_type} not an excepted input for 'split_type'" + ) + + # A wide (pivoted) table is more easy to work with in this instance. + # The pivot is done using all columns but the 'dose_respones_value' + # and 'dose_respones_metric' as index. df.pivot will generate a + # MultiIndex which complicates things further down the line. To that + # end 'reset_index()' is used to remove the MultiIndex + df_full = data.experiments.copy() + df_full = df_full.pivot( + index = [ + 'source', + 'improve_sample_id', + 'improve_drug_id', + 'study', + 'time', + 'time_unit' + ], + columns = 'dose_response_metric', + values = 'dose_response_value' + ).reset_index() + + # Defining the split sizes. + train_size = float(ratio[0]) / sum(ratio) + test_val_size = float(ratio[1] + ratio[2]) / sum(ratio) + test_size = float(ratio[1]) / (ratio[1] + ratio[2]) + validate_size = 1 - test_size + + # ShuffleSplit is a method/class implemented by scikit-learn that + # enables creating splits where the data is shuffled and then + # randomly distributed into train and test sets according to the + # defined ratio. + # + # n_splits defines how often a train/test split is generated. + # Individual splits (if more than 1 is generated) are not guaranteed + # to be disjoint i.e. test sets from individual splits can overlap. + # + # ShuffleSplit will be used for non stratified mixed-set splitting + # since there is no requirement for disjoint groups (i.e. drug / + # sample ids). + shs_1 = ShuffleSplit( + n_splits=1, + train_size=train_size, + test_size=test_val_size, + random_state=random_state + ) + shs_2 = ShuffleSplit( + n_splits=1, + train_size=test_size, + test_size=validate_size, + random_state=random_state + ) + + # GroupShuffleSplit is an extension to ShuffleSplit that also + # factors in a group that is used to generate disjoint train and + # test sets, e.g. in this particular case the drug or sample id to + # generate drug-blind or sample-blind train and test sets. + # + # GroupShuffleSplit will be used for non stratified drug-/sample- + # blind splitting, i.e. there is a requirement that instances from + # one group (e.g. a specific drug) are only present in the training + # set but not in the test set. + gss_1 = GroupShuffleSplit( + n_splits=1, + train_size=train_size, + test_size=test_val_size, + random_state=random_state + ) + gss_2 = GroupShuffleSplit( + n_splits=1, + train_size=test_size, + test_size=validate_size, + random_state=random_state + ) + + # StratifiedShuffleSplit is similar to ShuffleSplit with the added + # functionality to also stratify the splits according to defined + # class labels. + # + # StratifiedShuffleSplit will be used for stratified mixed-set + # train/test/validate sets. + + sss_1 = StratifiedShuffleSplit( + n_splits=1, + train_size=train_size, + test_size=test_val_size, + random_state=random_state + ) + sss_2 = StratifiedShuffleSplit( + n_splits=1, + train_size=test_size, + test_size=validate_size, + random_state=random_state + ) + + # StratifiedGroupKFold generates K folds that take the group into + # account when generating folds, i.e. a group will only be present + # in one fold. It further tries to stratify the folds based on the + # defined classes. + # + # StratifiedGroupKFold will be used for stratified drug-/sample- + # blind splitting. + # + # The way the K folds are utilized is to combine i, j, & k folds + # (according to the defined ratio) into training, testing and + # validation sets. + sgk = StratifiedGroupKFold( + n_splits=sum(ratio), + shuffle=True, + random_state=random_state + ) + + # The "actual" splitting logic using the defined Splitters as above + # follows here starting with the non-stratified splitting: + if stratify_by is None: + if split_type == 'mixed-set': + # Using ShuffleSplit to generate randomized train and + # 'other' set, since there is no need for grouping. + idx1, idx2 = next( + shs_1.split(df_full) + ) + elif split_type == 'drug-blind': + # Using GroupShuffleSplit to created disjoint train and + # 'other' sets by drug id + idx1, idx2 = next( + gss_1.split(df_full, groups=df_full.improve_drug_id) + ) + elif split_type == 'cancer-blind': + # same as above we just group over the sample id + idx1, idx2 = next( + gss_1.split(df_full, groups=df_full.improve_sample_id) + ) + else: + raise Exception(f"Should be unreachable") + + # generate new DFs containing the subset of items extracted for + # train and other + df_train = df_full.iloc[idx1] + df_other = df_full.iloc[idx2] + + # follows same logic as previous splitting with the difference + # that only "other" is sampled and split + if split_type == 'mixed-set': + idx1, idx2 = next( + shs_2.split(df_other, groups=None) + ) + elif split_type == 'drug-blind': + idx1, idx2 = next( + gss_2.split(df_other, groups=df_other.improve_drug_id) + ) + elif split_type == 'cancer-blind': + idx1, idx2 = next( + gss_2.split(df_other, groups=df_other.improve_sample_id) + ) + else: + raise Exception(f"Should be unreachable") + + # extract itmes for test and validate from other based on the + # sampled indices + df_test = df_other.iloc[idx1] + df_val = df_other.iloc[idx2] + + # The following block contains the stratified splitting logic + else: + # First the classes that are needed for the stratification are + # generated. `num_classes`, `thresh` and `quantiles` were + # previously defined as possible keyword arguments. + df_full = _create_classes( + data=df_full, + metric=stratify_by, + num_classes=num_classes, + thresh=thresh, + quantiles=quantiles, + ) + if split_type == 'mixed-set': + # Using StratifiedShuffleSplit to generate randomized train + # and 'other' set, since there is no need for grouping. + idx_train, idx_other = next( + sss_1.split(X=df_full, y=df_full['split_class']) + ) + df_train = df_full.iloc[idx_train] + df_other = df_full.iloc[idx_other] + # Splitting 'other' further into test and validate + idx_test, idx_val = next( + sss_2.split(X=df_other, y=df_other['split_class']) + ) + df_test = df_other.iloc[idx_test] + df_val = df_other.iloc[idx_val] + + # using StratifiedGroupKSplit for the stratified drug-/sample- + # blind splits. + elif split_type == 'drug-blind' or split_type == 'cancer-blind': + if split_type == 'drug-blind': + splitter = enumerate( + sgk.split( + X=df_full, + y=df_full['split_class'], + groups=df_full.improve_drug_id + ) + ) + elif split_type == 'cancer-blind': + splitter = enumerate( + sgk.split( + X=df_full, + y=df_full['split_class'], + groups=df_full.improve_sample_id + ) + ) + + # StratifiedGroupKSplit is setup to generate K splits where + # K=sum(ratios) (e.g. 10 if ratio=8:1:1). To obtain three + # sets (train/test/validate) the individual splits need to + # be combined (e.g. k=[1:8] -> train, k=9 -> test, k=10 -> + # validate). The code block below does that by combining + # all indices (row numbers) that go into individual sets and + # then extracting and adding those rows into the individual + # sets. + idx_train = [] + idx_test = [] + idx_val = [] + for i, (idx1, idx2) in splitter: + if i < ratio[0]: + idx_train.extend(idx2) + elif i >= ratio[0] and i < (ratio[0] + ratio[1]): + idx_test.extend(idx2) + elif i >= (ratio[0] + ratio[1]) and i < (ratio[0] + ratio[1] + ratio[2]): + idx_val.extend(idx2) + df_full.drop(labels=['split_class'], axis=1, inplace=True) + df_train = df_full.iloc[idx_train] + df_test = df_full.iloc[idx_test] + df_val = df_full.iloc[idx_val] + else: + raise Exception(f"Should be unreachable") + + + # generating filtered CoderData objects that contain only the + # respective data for each split + data_train = _filter(data, df_train) + data_test = _filter(data, df_test) + data_val = _filter(data, df_val) + + return Split(data_train, data_test, data_val) + # --------------------------------------------------------------------- @@ -297,3 +654,172 @@ def _determine_delimiter(file_path): return '\t' else: return ',' + + +def _filter(data: Dataset, split: pd.DataFrame) -> Dataset: + """ + Helper function to filter down the CoderData object(s) to create + indipendent more concise CoderData objects for training, testing + and validation splits. + + Parameters + ---------- + data : DatasetLoader + CoderData object containing the "full" dataset, e.g. the dataset + that splits are based on. + split : pandas.DataFrame + Contains a subset of rows from the data.experiments DataFrame + that correspond to the generated split. + + Returns + ------- + DatasetLoader + A CoderData object that is a subset of ``data`` containing only + the data points that pertain to the information in ``split``. + """ + + # extracting improve sample and drug ids from the provided split + sample_ids = np.unique(split['improve_sample_id'].values) + drug_ids = np.unique(split['improve_drug_id'].values) + + # melt the wide table to store a long table in the experiments + # object of the returned CoderData object (to be consistent with the + # initial orginial experiments table) + split_long = pd.melt( + split, + id_vars=[ + 'source', + 'improve_sample_id', + 'improve_drug_id', + 'study', + 'time', + 'time_unit', + ], + var_name='dose_response_metric', + value_name='dose_response_value' + ) + + # creating a deep copy of the CoderData object such that any + # further operations on the object are not changing the original + # object / data + data_ret = deepcopy(data) + + # filtering each individual data type down by only the improve + # sample / drug ids that are present in the split (extracted above) + # + # Datapoints in the individual datasets in the CoderData object are + # filtered by the following "rules": + # + # cd.drugs -> reduce based on improve_drug_id + # cd.mutations -> reduce based on improve_sample_id + # cd.proteomics -> reduce based on improve_sample_id + # cd.samples -> reduce based on improve_sample_id + # cd.transcriptomics -> reduce based on improve_sample_id + + data_ret.drugs = data_ret.drugs[ + data_ret.drugs['improve_drug_id'].isin(drug_ids) + ] + data_ret.mutations = data_ret.mutations[ + data_ret.mutations['improve_sample_id'].isin(sample_ids) + ] + data_ret.proteomics = data_ret.proteomics[ + data_ret.proteomics['improve_sample_id'].isin(sample_ids) + ] + data_ret.samples = data_ret.samples[ + data_ret.samples['improve_sample_id'].isin(sample_ids) + ] + data_ret.transcriptomics = data_ret.transcriptomics[ + data_ret.transcriptomics['improve_sample_id'].isin(sample_ids) + ] + data_ret.experiments = split_long + + return data_ret + + +def _create_classes( + data: pd.DataFrame, + metric: str, + num_classes: int=2, + quantiles: bool=True, + thresh: float=None, + ) -> pd.DataFrame: + """ + Helper function that bins experiment data into a number of defined + classes for use with Stratified Splits. + + Parameters + ---------- + data : pandas.DataFrame + The DataFrame containing drug response data (experiment) which + is subject to binning / creating classes based on the + drug response metric + metric : str + The drug response metric upon which the class generation should + be based on. Needs to be in data['drug_response_metric']. + num_classes : int, default=2 + Number of classes that should be generated. Defaults to 2. + quantiles : bool, default=True + Defines whether the individual bins should be based on quantiles + (or percentiles) instead of "evenly" spaced. If true then the + "bin" size will be chosen such that roughly the same number of + data points fall into each class + thresh : float, default=None + Optional argument that defines a threshold other than the mean + of the drug response metric if ``num_classes=2``. Can be used to + generate "uneven" classes. + + Returns + ------- + pandas.DataFrame + DataFrame that is the same as the input with additional column + that defines the established class association of each data + point. + + Raises + ------ + ValueError + If the chosen ``metric`` is not present in the + `drug_response_metric` column of ``data``. + ValueError + If ``num_classes`` < 2. + ValueError + If ``thresh`` is defined but ``num_classes`` > 2. + """ + + if metric not in data.columns: + raise ValueError( + f"Defined metric '{metric}' not present in data." + ) + + if num_classes < 2: + raise ValueError( + f"'num_classes must be >=2. 'num_classes' given: '{num_classes}'" + ) + + if thresh is None: + if quantiles: + data['split_class'] = pd.qcut( + data[metric], + q=num_classes, + labels=False, + ) + else: + data['split_class'] = pd.cut( + data[metric], + bins=num_classes, + labels=False + ) + elif num_classes == 2: + data['split_class'] = pd.cut( + data[metric], + bins=[float(min(data[metric])), thresh, float(max(data[metric]))], + labels=False, + include_lowest=True, + ) + else: + raise ValueError( + f"'thresh' can only be defined if num_classes == 2. num_classes " + f"passed to function: '{num_classes}'" + ) + + return data From 8995b10330505a1ae5852bf80f91e8aec6eaa84a Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Mon, 18 Nov 2024 16:14:08 -0800 Subject: [PATCH 04/24] moved `train_test_validate` out of `Dataset`. Is now called from within `cd.Dataset.train_test_validate()` but can also be called as standalone function (`cd.train_test_validate()`) --- coderdata/dataset/dataset.py | 698 ++++++++++++++++++----------------- 1 file changed, 363 insertions(+), 335 deletions(-) diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index b0c4b4a7..7df58ad2 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -236,6 +236,29 @@ def full(self): # Class and Instance functions # ---------------------------- + def train_test_validate( + self, + split_type: Literal[ + 'mixed-set', 'drug-blind', 'cancer-blind' + ]='mixed-set', + ratio: tuple[int, int, int]=(8,1,1), + stratify_by: (str | None)=None, + random_state: (int | RandomState | None)=None, + **kwargs: dict, + ) -> Split: + + split = train_test_validate( + data=self, + split_type=split_type, + ratio=ratio, + stratify_by=stratify_by, + random_state=random_state, + **kwargs + ) + + return split + + def types(self) -> list: data_types = [ 'transcriptomics', @@ -259,346 +282,12 @@ def types(self) -> list: - def train_test_validate( - data: Dataset, - split_type: Literal[ - 'mixed-set', 'drug-blind', 'cancer-blind' - ]='mixed-set', - ratio: tuple[int, int, int]=(8,1,1), - stratify_by: (str | None)=None, - random_state: (int | RandomState | None)=None, - **kwargs: dict, - ) -> Split: - """ - Splits a `CoderData` object (see also - `coderdata.load.loader.DatasetLoader`) into three subsets for - training, testing and validating machine learning algorithms. - - The size of the splits can be adjusted to be different from 80:10:10 - (the default)for train:test:validate. The function also allows for - additional optional arguments, that define the type of split that is - performed ('mixed-set', 'drug-blind', 'cancer-blind'), if the splits - should be stratified (and which drug response metric to use), as - well as a random seed to enable the creation of reproducable splits. - Furhermore, a list of keyword arguments can be defined that will be - passed to the stratification function if so desired. - - Parameters - ---------- - data : DatasetLoader - CoderData object containing a full dataset either downloaded - from the CoderData repository (see also - `coderdata.download.downloader.download_data_by_prefix`) or - built locally via the `build_all` process. The object must first - be loaded via `coderdata.load.loader.DatasetLoader`. - split_type : {'mixed-set', 'drug-blind', 'cancer-blind'}, \ - default='mixed-set' - - Defines the type of split that should be generated: - - - *mixed-set*: Splits randomly independent of drug / cancer - association of the samples. Individual drugs or cancer types - can appear in all three splits - - *drug-blind*: Splits according to drug association. Any sample - associated with a drug will be unique to one of the splits. - For example samples with association to drug A will only be - present in the train split, but never in test or validate. - - *cancer-blind*: Splits according to cancer association. - Equivalent to drug-blind, except cancer types will be unique - to splits. - ratio : tuple[int, int, int], default=(8,1,1) - Defines the size ratio of the resulting test, train and - validation sets. - stratify_by : str | None, default=None - Defines if the training, testing and validation sets should be - stratified. Any value other than None indicates stratification - and defines which drug response value should be used as basis - for the stratification. _None_ indicates that no stratfication - should be performed. - random_state : int | RandomState | None, defaul=None - Defines a seed value for the randomization of the splits. Will - get passed to internal functions. Providing the seed will enable - reproducability of the generated splits. - **kwargs - Additional keyword arguments that will be passed to the function - that generates classes for the stratification - (see also ``_create_classes``). - - Returns - ------- - Splits : Split - A ``Split`` object that contains three Dataset objects as - attributes (``Split.train``, ``Split.test``, - ``Split.validate``) - - Raises - ------- - ValueError : - If supplied `split_type` is not in the list of accepted values. - - """ - - # reading in the potential keyword arguments that will be passed to - # _create_classes(). - thresh = kwargs.get('thresh', None) - num_classes = kwargs.get('num_classes', 2) - quantiles = kwargs.get('quantiles', True) - - # Type checking split_type - if split_type not in [ - 'mixed-set', 'drug-blind', 'cancer-blind' - ]: - raise ValueError( - f"{split_type} not an excepted input for 'split_type'" - ) - - # A wide (pivoted) table is more easy to work with in this instance. - # The pivot is done using all columns but the 'dose_respones_value' - # and 'dose_respones_metric' as index. df.pivot will generate a - # MultiIndex which complicates things further down the line. To that - # end 'reset_index()' is used to remove the MultiIndex - df_full = data.experiments.copy() - df_full = df_full.pivot( - index = [ - 'source', - 'improve_sample_id', - 'improve_drug_id', - 'study', - 'time', - 'time_unit' - ], - columns = 'dose_response_metric', - values = 'dose_response_value' - ).reset_index() - - # Defining the split sizes. - train_size = float(ratio[0]) / sum(ratio) - test_val_size = float(ratio[1] + ratio[2]) / sum(ratio) - test_size = float(ratio[1]) / (ratio[1] + ratio[2]) - validate_size = 1 - test_size - - # ShuffleSplit is a method/class implemented by scikit-learn that - # enables creating splits where the data is shuffled and then - # randomly distributed into train and test sets according to the - # defined ratio. - # - # n_splits defines how often a train/test split is generated. - # Individual splits (if more than 1 is generated) are not guaranteed - # to be disjoint i.e. test sets from individual splits can overlap. - # - # ShuffleSplit will be used for non stratified mixed-set splitting - # since there is no requirement for disjoint groups (i.e. drug / - # sample ids). - shs_1 = ShuffleSplit( - n_splits=1, - train_size=train_size, - test_size=test_val_size, - random_state=random_state - ) - shs_2 = ShuffleSplit( - n_splits=1, - train_size=test_size, - test_size=validate_size, - random_state=random_state - ) - - # GroupShuffleSplit is an extension to ShuffleSplit that also - # factors in a group that is used to generate disjoint train and - # test sets, e.g. in this particular case the drug or sample id to - # generate drug-blind or sample-blind train and test sets. - # - # GroupShuffleSplit will be used for non stratified drug-/sample- - # blind splitting, i.e. there is a requirement that instances from - # one group (e.g. a specific drug) are only present in the training - # set but not in the test set. - gss_1 = GroupShuffleSplit( - n_splits=1, - train_size=train_size, - test_size=test_val_size, - random_state=random_state - ) - gss_2 = GroupShuffleSplit( - n_splits=1, - train_size=test_size, - test_size=validate_size, - random_state=random_state - ) - - # StratifiedShuffleSplit is similar to ShuffleSplit with the added - # functionality to also stratify the splits according to defined - # class labels. - # - # StratifiedShuffleSplit will be used for stratified mixed-set - # train/test/validate sets. - - sss_1 = StratifiedShuffleSplit( - n_splits=1, - train_size=train_size, - test_size=test_val_size, - random_state=random_state - ) - sss_2 = StratifiedShuffleSplit( - n_splits=1, - train_size=test_size, - test_size=validate_size, - random_state=random_state - ) - - # StratifiedGroupKFold generates K folds that take the group into - # account when generating folds, i.e. a group will only be present - # in one fold. It further tries to stratify the folds based on the - # defined classes. - # - # StratifiedGroupKFold will be used for stratified drug-/sample- - # blind splitting. - # - # The way the K folds are utilized is to combine i, j, & k folds - # (according to the defined ratio) into training, testing and - # validation sets. - sgk = StratifiedGroupKFold( - n_splits=sum(ratio), - shuffle=True, - random_state=random_state - ) - - # The "actual" splitting logic using the defined Splitters as above - # follows here starting with the non-stratified splitting: - if stratify_by is None: - if split_type == 'mixed-set': - # Using ShuffleSplit to generate randomized train and - # 'other' set, since there is no need for grouping. - idx1, idx2 = next( - shs_1.split(df_full) - ) - elif split_type == 'drug-blind': - # Using GroupShuffleSplit to created disjoint train and - # 'other' sets by drug id - idx1, idx2 = next( - gss_1.split(df_full, groups=df_full.improve_drug_id) - ) - elif split_type == 'cancer-blind': - # same as above we just group over the sample id - idx1, idx2 = next( - gss_1.split(df_full, groups=df_full.improve_sample_id) - ) - else: - raise Exception(f"Should be unreachable") - - # generate new DFs containing the subset of items extracted for - # train and other - df_train = df_full.iloc[idx1] - df_other = df_full.iloc[idx2] - - # follows same logic as previous splitting with the difference - # that only "other" is sampled and split - if split_type == 'mixed-set': - idx1, idx2 = next( - shs_2.split(df_other, groups=None) - ) - elif split_type == 'drug-blind': - idx1, idx2 = next( - gss_2.split(df_other, groups=df_other.improve_drug_id) - ) - elif split_type == 'cancer-blind': - idx1, idx2 = next( - gss_2.split(df_other, groups=df_other.improve_sample_id) - ) - else: - raise Exception(f"Should be unreachable") - - # extract itmes for test and validate from other based on the - # sampled indices - df_test = df_other.iloc[idx1] - df_val = df_other.iloc[idx2] - - # The following block contains the stratified splitting logic - else: - # First the classes that are needed for the stratification are - # generated. `num_classes`, `thresh` and `quantiles` were - # previously defined as possible keyword arguments. - df_full = _create_classes( - data=df_full, - metric=stratify_by, - num_classes=num_classes, - thresh=thresh, - quantiles=quantiles, - ) - if split_type == 'mixed-set': - # Using StratifiedShuffleSplit to generate randomized train - # and 'other' set, since there is no need for grouping. - idx_train, idx_other = next( - sss_1.split(X=df_full, y=df_full['split_class']) - ) - df_train = df_full.iloc[idx_train] - df_other = df_full.iloc[idx_other] - # Splitting 'other' further into test and validate - idx_test, idx_val = next( - sss_2.split(X=df_other, y=df_other['split_class']) - ) - df_test = df_other.iloc[idx_test] - df_val = df_other.iloc[idx_val] - - # using StratifiedGroupKSplit for the stratified drug-/sample- - # blind splits. - elif split_type == 'drug-blind' or split_type == 'cancer-blind': - if split_type == 'drug-blind': - splitter = enumerate( - sgk.split( - X=df_full, - y=df_full['split_class'], - groups=df_full.improve_drug_id - ) - ) - elif split_type == 'cancer-blind': - splitter = enumerate( - sgk.split( - X=df_full, - y=df_full['split_class'], - groups=df_full.improve_sample_id - ) - ) - - # StratifiedGroupKSplit is setup to generate K splits where - # K=sum(ratios) (e.g. 10 if ratio=8:1:1). To obtain three - # sets (train/test/validate) the individual splits need to - # be combined (e.g. k=[1:8] -> train, k=9 -> test, k=10 -> - # validate). The code block below does that by combining - # all indices (row numbers) that go into individual sets and - # then extracting and adding those rows into the individual - # sets. - idx_train = [] - idx_test = [] - idx_val = [] - for i, (idx1, idx2) in splitter: - if i < ratio[0]: - idx_train.extend(idx2) - elif i >= ratio[0] and i < (ratio[0] + ratio[1]): - idx_test.extend(idx2) - elif i >= (ratio[0] + ratio[1]) and i < (ratio[0] + ratio[1] + ratio[2]): - idx_val.extend(idx2) - df_full.drop(labels=['split_class'], axis=1, inplace=True) - df_train = df_full.iloc[idx_train] - df_test = df_full.iloc[idx_test] - df_val = df_full.iloc[idx_val] - else: - raise Exception(f"Should be unreachable") - - - # generating filtered CoderData objects that contain only the - # respective data for each split - data_train = _filter(data, df_train) - data_test = _filter(data, df_test) - data_val = _filter(data, df_val) - - return Split(data_train, data_test, data_val) - - - # --------------------------------------------------------------------- # Functions that are Dataset related but not Class / Instance functions # --------------------------------------------------------------------- def load(name: str, directory: str|Path=Path.cwd()) -> Dataset: + print("Processing Data...", file=sys.stderr) if type(directory) is not Path: @@ -634,6 +323,345 @@ def load(name: str, directory: str|Path=Path.cwd()) -> Dataset: return dataset + + +def train_test_validate( + data: Dataset, + split_type: Literal[ + 'mixed-set', 'drug-blind', 'cancer-blind' + ]='mixed-set', + ratio: tuple[int, int, int]=(8,1,1), + stratify_by: (str | None)=None, + random_state: (int | RandomState | None)=None, + **kwargs: dict, + ) -> Split: + """ + Splits a `CoderData` object (see also + `coderdata.load.loader.DatasetLoader`) into three subsets for + training, testing and validating machine learning algorithms. + + The size of the splits can be adjusted to be different from 80:10:10 + (the default)for train:test:validate. The function also allows for + additional optional arguments, that define the type of split that is + performed ('mixed-set', 'drug-blind', 'cancer-blind'), if the splits + should be stratified (and which drug response metric to use), as + well as a random seed to enable the creation of reproducable splits. + Furhermore, a list of keyword arguments can be defined that will be + passed to the stratification function if so desired. + + Parameters + ---------- + data : DatasetLoader + CoderData object containing a full dataset either downloaded + from the CoderData repository (see also + `coderdata.download.downloader.download_data_by_prefix`) or + built locally via the `build_all` process. The object must first + be loaded via `coderdata.load.loader.DatasetLoader`. + split_type : {'mixed-set', 'drug-blind', 'cancer-blind'}, \ + default='mixed-set' + + Defines the type of split that should be generated: + + - *mixed-set*: Splits randomly independent of drug / cancer + association of the samples. Individual drugs or cancer types + can appear in all three splits + - *drug-blind*: Splits according to drug association. Any sample + associated with a drug will be unique to one of the splits. + For example samples with association to drug A will only be + present in the train split, but never in test or validate. + - *cancer-blind*: Splits according to cancer association. + Equivalent to drug-blind, except cancer types will be unique + to splits. + ratio : tuple[int, int, int], default=(8,1,1) + Defines the size ratio of the resulting test, train and + validation sets. + stratify_by : str | None, default=None + Defines if the training, testing and validation sets should be + stratified. Any value other than None indicates stratification + and defines which drug response value should be used as basis + for the stratification. _None_ indicates that no stratfication + should be performed. + random_state : int | RandomState | None, defaul=None + Defines a seed value for the randomization of the splits. Will + get passed to internal functions. Providing the seed will enable + reproducability of the generated splits. + **kwargs + Additional keyword arguments that will be passed to the function + that generates classes for the stratification + (see also ``_create_classes``). + + Returns + ------- + Splits : Split + A ``Split`` object that contains three Dataset objects as + attributes (``Split.train``, ``Split.test``, + ``Split.validate``) + + Raises + ------- + ValueError : + If supplied `split_type` is not in the list of accepted values. + + """ + + # reading in the potential keyword arguments that will be passed to + # _create_classes(). + thresh = kwargs.get('thresh', None) + num_classes = kwargs.get('num_classes', 2) + quantiles = kwargs.get('quantiles', True) + + # Type checking split_type + if split_type not in [ + 'mixed-set', 'drug-blind', 'cancer-blind' + ]: + raise ValueError( + f"{split_type} not an excepted input for 'split_type'" + ) + + # A wide (pivoted) table is more easy to work with in this instance. + # The pivot is done using all columns but the 'dose_respones_value' + # and 'dose_respones_metric' as index. df.pivot will generate a + # MultiIndex which complicates things further down the line. To that + # end 'reset_index()' is used to remove the MultiIndex + df_full = data.experiments.copy() + df_full = df_full.pivot( + index = [ + 'source', + 'improve_sample_id', + 'improve_drug_id', + 'study', + 'time', + 'time_unit' + ], + columns = 'dose_response_metric', + values = 'dose_response_value' + ).reset_index() + + # Defining the split sizes. + train_size = float(ratio[0]) / sum(ratio) + test_val_size = float(ratio[1] + ratio[2]) / sum(ratio) + test_size = float(ratio[1]) / (ratio[1] + ratio[2]) + validate_size = 1 - test_size + + # ShuffleSplit is a method/class implemented by scikit-learn that + # enables creating splits where the data is shuffled and then + # randomly distributed into train and test sets according to the + # defined ratio. + # + # n_splits defines how often a train/test split is generated. + # Individual splits (if more than 1 is generated) are not guaranteed + # to be disjoint i.e. test sets from individual splits can overlap. + # + # ShuffleSplit will be used for non stratified mixed-set splitting + # since there is no requirement for disjoint groups (i.e. drug / + # sample ids). + shs_1 = ShuffleSplit( + n_splits=1, + train_size=train_size, + test_size=test_val_size, + random_state=random_state + ) + shs_2 = ShuffleSplit( + n_splits=1, + train_size=test_size, + test_size=validate_size, + random_state=random_state + ) + + # GroupShuffleSplit is an extension to ShuffleSplit that also + # factors in a group that is used to generate disjoint train and + # test sets, e.g. in this particular case the drug or sample id to + # generate drug-blind or sample-blind train and test sets. + # + # GroupShuffleSplit will be used for non stratified drug-/sample- + # blind splitting, i.e. there is a requirement that instances from + # one group (e.g. a specific drug) are only present in the training + # set but not in the test set. + gss_1 = GroupShuffleSplit( + n_splits=1, + train_size=train_size, + test_size=test_val_size, + random_state=random_state + ) + gss_2 = GroupShuffleSplit( + n_splits=1, + train_size=test_size, + test_size=validate_size, + random_state=random_state + ) + + # StratifiedShuffleSplit is similar to ShuffleSplit with the added + # functionality to also stratify the splits according to defined + # class labels. + # + # StratifiedShuffleSplit will be used for stratified mixed-set + # train/test/validate sets. + + sss_1 = StratifiedShuffleSplit( + n_splits=1, + train_size=train_size, + test_size=test_val_size, + random_state=random_state + ) + sss_2 = StratifiedShuffleSplit( + n_splits=1, + train_size=test_size, + test_size=validate_size, + random_state=random_state + ) + + # StratifiedGroupKFold generates K folds that take the group into + # account when generating folds, i.e. a group will only be present + # in one fold. It further tries to stratify the folds based on the + # defined classes. + # + # StratifiedGroupKFold will be used for stratified drug-/sample- + # blind splitting. + # + # The way the K folds are utilized is to combine i, j, & k folds + # (according to the defined ratio) into training, testing and + # validation sets. + sgk = StratifiedGroupKFold( + n_splits=sum(ratio), + shuffle=True, + random_state=random_state + ) + + # The "actual" splitting logic using the defined Splitters as above + # follows here starting with the non-stratified splitting: + if stratify_by is None: + if split_type == 'mixed-set': + # Using ShuffleSplit to generate randomized train and + # 'other' set, since there is no need for grouping. + idx1, idx2 = next( + shs_1.split(df_full) + ) + elif split_type == 'drug-blind': + # Using GroupShuffleSplit to created disjoint train and + # 'other' sets by drug id + idx1, idx2 = next( + gss_1.split(df_full, groups=df_full.improve_drug_id) + ) + elif split_type == 'cancer-blind': + # same as above we just group over the sample id + idx1, idx2 = next( + gss_1.split(df_full, groups=df_full.improve_sample_id) + ) + else: + raise Exception(f"Should be unreachable") + + # generate new DFs containing the subset of items extracted for + # train and other + df_train = df_full.iloc[idx1] + df_other = df_full.iloc[idx2] + + # follows same logic as previous splitting with the difference + # that only "other" is sampled and split + if split_type == 'mixed-set': + idx1, idx2 = next( + shs_2.split(df_other, groups=None) + ) + elif split_type == 'drug-blind': + idx1, idx2 = next( + gss_2.split(df_other, groups=df_other.improve_drug_id) + ) + elif split_type == 'cancer-blind': + idx1, idx2 = next( + gss_2.split(df_other, groups=df_other.improve_sample_id) + ) + else: + raise Exception(f"Should be unreachable") + + # extract itmes for test and validate from other based on the + # sampled indices + df_test = df_other.iloc[idx1] + df_val = df_other.iloc[idx2] + + # The following block contains the stratified splitting logic + else: + # First the classes that are needed for the stratification are + # generated. `num_classes`, `thresh` and `quantiles` were + # previously defined as possible keyword arguments. + df_full = _create_classes( + data=df_full, + metric=stratify_by, + num_classes=num_classes, + thresh=thresh, + quantiles=quantiles, + ) + if split_type == 'mixed-set': + # Using StratifiedShuffleSplit to generate randomized train + # and 'other' set, since there is no need for grouping. + idx_train, idx_other = next( + sss_1.split(X=df_full, y=df_full['split_class']) + ) + df_train = df_full.iloc[idx_train] + df_other = df_full.iloc[idx_other] + # Splitting 'other' further into test and validate + idx_test, idx_val = next( + sss_2.split(X=df_other, y=df_other['split_class']) + ) + df_test = df_other.iloc[idx_test] + df_val = df_other.iloc[idx_val] + + # using StratifiedGroupKSplit for the stratified drug-/sample- + # blind splits. + elif split_type == 'drug-blind' or split_type == 'cancer-blind': + if split_type == 'drug-blind': + splitter = enumerate( + sgk.split( + X=df_full, + y=df_full['split_class'], + groups=df_full.improve_drug_id + ) + ) + elif split_type == 'cancer-blind': + splitter = enumerate( + sgk.split( + X=df_full, + y=df_full['split_class'], + groups=df_full.improve_sample_id + ) + ) + + # StratifiedGroupKSplit is setup to generate K splits where + # K=sum(ratios) (e.g. 10 if ratio=8:1:1). To obtain three + # sets (train/test/validate) the individual splits need to + # be combined (e.g. k=[1:8] -> train, k=9 -> test, k=10 -> + # validate). The code block below does that by combining + # all indices (row numbers) that go into individual sets and + # then extracting and adding those rows into the individual + # sets. + idx_train = [] + idx_test = [] + idx_val = [] + for i, (idx1, idx2) in splitter: + if i < ratio[0]: + idx_train.extend(idx2) + elif i >= ratio[0] and i < (ratio[0] + ratio[1]): + idx_test.extend(idx2) + elif ( + i >= (ratio[0] + ratio[1]) + and i < (ratio[0] + ratio[1] + ratio[2]) + ): + idx_val.extend(idx2) + df_full.drop(labels=['split_class'], axis=1, inplace=True) + df_train = df_full.iloc[idx_train] + df_test = df_full.iloc[idx_test] + df_val = df_full.iloc[idx_val] + else: + raise Exception(f"Should be unreachable") + + + # generating filtered CoderData objects that contain only the + # respective data for each split + data_train = _filter(data, df_train) + data_test = _filter(data, df_test) + data_val = _filter(data, df_val) + + return Split(data_train, data_test, data_val) + + def _load_file(file_path: Path) -> pd.DataFrame: if file_path.suffix == '.gz': return pd.read_csv( From f689d005f6dcac32c9c33a4b135e150120b9b21b Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Tue, 3 Dec 2024 10:55:52 -0800 Subject: [PATCH 05/24] added `dataset.save()` function --- coderdata/dataset/dataset.py | 49 ++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index 7df58ad2..49dd0774 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -4,6 +4,7 @@ from copy import deepcopy from dataclasses import dataclass from pathlib import Path +import pickle import sys from typing import Literal @@ -28,20 +29,27 @@ class Dataset: def __init__( self, - name: str, + name: str=None, transcriptomics: pd.DataFrame=None, proteomics: pd.DataFrame=None, mutations: pd.DataFrame=None, copy_number: pd.DataFrame=None, samples: pd.DataFrame=None, drugs: pd.DataFrame=None, + drug_descriptors: pd.DataFrame=None, mirna: pd.DataFrame=None, experiments: pd.DataFrame=None, methylation: pd.DataFrame=None, metabolomics: pd.DataFrame=None, genes: pd.DataFrame=None, - full: pd.DataFrame=None, + combinations: pd.DataFrame=None, ): + """ + Load datasets of a specific type into predefined attributes of this class instance. + + Each attribute will be a Pandas DataFrame corresponding to a file with the dataset prefix. + Attributes include transcriptomics, proteomics, mutations, etc. + """ self.name = name self.transcriptomics = transcriptomics @@ -50,12 +58,13 @@ def __init__( self.copy_number = copy_number self.samples = samples self.drugs = drugs + self.drug_descriptors = drug_descriptors self.mirna = mirna self.experiments = experiments self.methylation = methylation self.metabolomics = metabolomics self.genes = genes - self.full = full + self.combinations = combinations #----------------------------- @@ -154,6 +163,19 @@ def drugs(self): del self._drugs + @property + def drug_descriptors(self): + return self._drug_descriptors + + @drug_descriptors.setter + def drug_descriptors(self, value): + self._drug_descriptors = value + + @drug_descriptors.deleter + def drug_descriptors(self): + del self._drug_descriptors + + @property def mirna(self): return self._mirna @@ -220,16 +242,16 @@ def genes(self): @property - def full(self): - return self._full + def combinations(self): + return self._combinations - @full.setter - def full(self, value): - self._full = value + @combinations.setter + def combinations(self, value): + self._combinations = value - @full.deleter - def full(self): - del self._full + @combinations.deleter + def combinations(self): + del self._combinations # ---------------------------- @@ -280,6 +302,11 @@ def types(self) -> list: return data_types_present + def save(self, path: Path) -> None: + + with open(path, 'wb') as f_path: + pickle.dump(self, file=f_path) + # --------------------------------------------------------------------- From b822acdc378a412acea509375040973d700e8574 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Tue, 3 Dec 2024 10:56:30 -0800 Subject: [PATCH 06/24] added option to load from pickled object file to `dataset.load()` function --- coderdata/dataset/dataset.py | 71 +++++++++++++++++++++++++++--------- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index 49dd0774..1ca69d42 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -313,8 +313,33 @@ def save(self, path: Path) -> None: # Functions that are Dataset related but not Class / Instance functions # --------------------------------------------------------------------- -def load(name: str, directory: str|Path=Path.cwd()) -> Dataset: - +def load( + name: str, + directory: str|Path=Path.cwd(), + from_pickle:bool=False + ) -> Dataset: + """ + _summary_ + + Parameters + ---------- + name : str + _description_ + directory : str | Path, optional + _description_, by default Path.cwd() + + Returns + ------- + Dataset + _description_ + + Raises + ------ + OSError + _description_ + TypeError + _description_ + """ print("Processing Data...", file=sys.stderr) if type(directory) is not Path: @@ -329,27 +354,37 @@ def load(name: str, directory: str|Path=Path.cwd()) -> Dataset: f"Invalid path / directory defined: '{directory}'" ) - accepted_file_endings = ('.csv', '.tsv', '.csv.gz', '.tsv.gz') - dataset = Dataset(name) - for child in directory.iterdir(): - if child.name in ["genes.csv", "genes.csv.gz"]: - dataset.genes = _load_file(child) - print("Loaded genes dataset.", file=sys.stderr) + if not from_pickle: + dataset = Dataset(name) + accepted_file_endings = ('.csv', '.tsv', '.csv.gz', '.tsv.gz') + for child in directory.iterdir(): + if child.name in ["genes.csv", "genes.csv.gz"]: + dataset.genes = _load_file(child) + print("Loaded genes dataset.", file=sys.stderr) - if ( - child.name.startswith(name) - and child.name.endswith(accepted_file_endings) - ): - - dataset_type = child.name[len(name)+1:].split('.')[0] - print(dataset_type) - if hasattr(dataset, dataset_type): - setattr(dataset, dataset_type, _load_file(child)) + if ( + child.name.startswith(name) + and child.name.endswith(accepted_file_endings) + ): - return dataset + dataset_type = child.name[len(name)+1:].split('.')[0] + print(dataset_type) + if hasattr(dataset, dataset_type): + setattr(dataset, dataset_type, _load_file(child)) + return dataset + else: + accepted_file_endings = ('.pkl', '.pickle') + for child in directory.iterdir(): + if ( + child.name.startswith(name) + and child.name.endswith(accepted_file_endings) + ): + with open(child, 'rb') as file: + dataset = pickle.load(file=file) + return dataset def train_test_validate( From 37d5f90f74630b1fdebcae3ca92a5904b4d3b42e Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Tue, 3 Dec 2024 11:03:08 -0800 Subject: [PATCH 07/24] added skeleton for `dataset.format()` --- coderdata/dataset/dataset.py | 49 ++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index 1ca69d42..0b74ec98 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -258,6 +258,19 @@ def combinations(self): # Class and Instance functions # ---------------------------- + def format( + self, + data_type: Literal[ + 'transcriptomics', 'mutations', 'copy_number', 'proteomics', + 'experiments', 'combinations', 'drug_descriptor', 'drugs', + 'genes', 'samples', + ], + use_polars: bool=False, + **kwargs: dict, + ): + return format(self, data_type=data_type, use_polars=use_polars, **kwargs) + + def train_test_validate( self, split_type: Literal[ @@ -387,6 +400,42 @@ def load( return dataset + +def format( + data: Dataset, + data_type: Literal[ + 'transcriptomics', 'mutations', 'copy_number', 'proteomics', + 'experiments', 'combinations', 'drug_descriptor', 'drugs', + 'genes', 'samples', + ], + use_polars: bool=False, + **kwargs: dict, + ): + + if data_type == "transcriptomics": + pass + + elif data_type == "mutations": + pass + elif data_type == "copy_number": + pass + elif data_type == "proteomics": + pass + elif data_type == "experiments": + pass + elif data_type == "combinations": + pass + elif data_type == "drugs": + pass + elif data_type == "genes": + pass + elif data_type == "samples": + pass + else: + pass + + return ret + def train_test_validate( data: Dataset, split_type: Literal[ From e899ec9ebfd692904c438b9e0359cb1a0243d0f0 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Tue, 3 Dec 2024 11:04:02 -0800 Subject: [PATCH 08/24] added "mutations" data_type to `dataset.format()` --- coderdata/dataset/dataset.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index 0b74ec98..658c7192 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -416,7 +416,23 @@ def format( pass elif data_type == "mutations": - pass + if data.mutations is None: + raise ValueError( + "'mutations' attribute of Dataset cannot be 'None'" + ) + mutation_type = kwargs.get('mutation_type', None) + if mutation_type is None: + raise ValueError( + "'mutation_type' must be defined if 'data_type'=='mutations'" + ) + tmp = data.mutations[ + data.mutations['variant_classification'] == mutation_type + ] + ret = pd.crosstab( + index=tmp['entrez_id'], + columns=tmp['improve_sample_id'] + ) + elif data_type == "copy_number": pass elif data_type == "proteomics": From 68a5719342db722c685601323fe4383b132a92e8 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Tue, 3 Dec 2024 11:31:26 -0800 Subject: [PATCH 09/24] added handeling of 'combinations', 'drugs', 'genes' & 'samples' in `dataset.format()` --- coderdata/dataset/dataset.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index 658c7192..8e7ae504 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -439,16 +439,37 @@ def format( pass elif data_type == "experiments": pass + elif data_type == "combinations": - pass + raise NotImplementedError( + f"'data_type' {data_type} is currently not implemented" + ) + elif data_type == "drugs": - pass + if data.drugs is None: + raise ValueError( + "'drugs' attribute of Dataset cannot be 'None'" + ) + ret = data.drugs + elif data_type == "genes": - pass + if data.genes is None: + raise ValueError( + "'genes' attribute of Dataset cannot be 'None'" + ) + ret = data.genes + elif data_type == "samples": - pass + if data.samples is None: + raise ValueError( + "'samples' attribute of Dataset cannot be 'None'" + ) + ret = data.samples + else: - pass + raise ValueError( + f"'data_type' contains unsupported value: '{data_type}'" + ) return ret From 50e4ee28a98dc0394c4876621dd672276788b351 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Tue, 3 Dec 2024 11:32:02 -0800 Subject: [PATCH 10/24] added basic handling of 'proteomics' to `dataset.format()` --- coderdata/dataset/dataset.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index 8e7ae504..70c24bbf 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -436,7 +436,19 @@ def format( elif data_type == "copy_number": pass elif data_type == "proteomics": - pass + if data.proteomics is None: + raise ValueError( + "'proteomics' attribute of Dataset cannot be 'None'" + ) + + # TODO: currently assumes that the proteomics table is properly formatted + ret = pd.pivot_table( + data=data.proteomics, + values='proteomics', + index='entrez_id', + columns='improve_sample_id' + ) + elif data_type == "experiments": pass From 8ebb43220f9b9c4103426f414da69ee7a7644388 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Tue, 3 Dec 2024 13:31:14 -0800 Subject: [PATCH 11/24] added handeling of 'transcriptomics' in `dataset.format()` --- coderdata/dataset/dataset.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index 70c24bbf..98dcef28 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -413,7 +413,16 @@ def format( ): if data_type == "transcriptomics": - pass + if data.transcriptomics is None: + raise ValueError( + f"'{data_type}' attribute of Dataset cannot be 'None'" + ) + ret = pd.pivot_table( + data=data.transcriptomics, + values='transcriptomics', + index='entrez_id', + columns='improve_sample_id' + ) elif data_type == "mutations": if data.mutations is None: From 64ce81aa4014f69079583ea4bfc6ed654f34e78a Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Tue, 3 Dec 2024 13:32:31 -0800 Subject: [PATCH 12/24] generalized error handling --- coderdata/dataset/dataset.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index 98dcef28..b9e544c4 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -427,13 +427,13 @@ def format( elif data_type == "mutations": if data.mutations is None: raise ValueError( - "'mutations' attribute of Dataset cannot be 'None'" + f"'{data_type}' attribute of Dataset cannot be 'None'" ) mutation_type = kwargs.get('mutation_type', None) if mutation_type is None: raise ValueError( "'mutation_type' must be defined if 'data_type'=='mutations'" - ) + ) tmp = data.mutations[ data.mutations['variant_classification'] == mutation_type ] @@ -447,7 +447,7 @@ def format( elif data_type == "proteomics": if data.proteomics is None: raise ValueError( - "'proteomics' attribute of Dataset cannot be 'None'" + f"'{data_type}' attribute of Dataset cannot be 'None'" ) # TODO: currently assumes that the proteomics table is properly formatted @@ -469,22 +469,22 @@ def format( elif data_type == "drugs": if data.drugs is None: raise ValueError( - "'drugs' attribute of Dataset cannot be 'None'" - ) + f"'{data_type}' attribute of Dataset cannot be 'None'" + ) ret = data.drugs elif data_type == "genes": if data.genes is None: raise ValueError( - "'genes' attribute of Dataset cannot be 'None'" - ) + f"'{data_type}' attribute of Dataset cannot be 'None'" + ) ret = data.genes elif data_type == "samples": if data.samples is None: raise ValueError( - "'samples' attribute of Dataset cannot be 'None'" - ) + f"'{data_type}' attribute of Dataset cannot be 'None'" + ) ret = data.samples else: From 101ad746108a288e111d065c509f27366bd05a2c Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Tue, 3 Dec 2024 13:33:00 -0800 Subject: [PATCH 13/24] added handling of 'experiments' in `dataset.format()` --- coderdata/dataset/dataset.py | 47 ++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index b9e544c4..6783de81 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -459,8 +459,51 @@ def format( ) elif data_type == "experiments": - pass - + if data.experiments is None: + raise ValueError( + f"'{data_type}' attribute of Dataset cannot be 'None'" + ) + shape = kwargs.get('shape', 'long') + legal_shapes = ['long', 'wide', 'matrix'] + metrics = kwargs.get('metrics', ['fit_auc']) + if type(metrics) is not list: + metrics = [metrics] + if shape not in legal_shapes: + raise ValueError( + f"'shape' has to be one of '{legal_shapes}'" + ) + tmp = data.experiments[ + data.experiments['dose_response_metric'].isin(metrics) + ] + if shape == 'long': + ret = tmp + elif shape == 'wide': + ret = tmp.pivot( + index = [ + 'source', + 'improve_sample_id', + 'improve_drug_id', + 'study', + 'time', + 'time_unit' + ], + columns = 'dose_response_metric', + values = 'dose_response_value' + ).reset_index().rename_axis(None, axis=1) + elif shape == 'matrix': + if len(metrics) > 1: + raise ValueError( + f"'metrics' cannot contain more than 1 value if " + f"shape=='{shape}'" + ) + ret = pd.pivot_table( + data=tmp, + values='dose_response_value', + index='improve_drug_id', + columns='improve_sample_id' + ) + return ret + elif data_type == "combinations": raise NotImplementedError( f"'data_type' {data_type} is currently not implemented" From a3972818934a7b5684c8fbfc4d11979734301c59 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Mon, 9 Dec 2024 12:03:26 -0800 Subject: [PATCH 14/24] added handling of `copy_number` --- coderdata/dataset/dataset.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index 6783de81..52441d7f 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -443,7 +443,20 @@ def format( ) elif data_type == "copy_number": - pass + if data.copy_number is None: + raise ValueError( + f"'{data_type}' attribute of Dataset cannot be 'None'" + ) + + # TODO: add way to extract copy_call + ret = pd.pivot_table( + data=data.copy_number, + index='entrez_id', + columns='improve_sample_id', + values='copy_number', + aggfunc='mean', + ) + elif data_type == "proteomics": if data.proteomics is None: raise ValueError( From 6bd844a3c4ebd6e95e08b8535c91e3fc891ac851 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Mon, 9 Dec 2024 12:03:50 -0800 Subject: [PATCH 15/24] added handling of `drug_descriptor` --- coderdata/dataset/dataset.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index 52441d7f..269473e1 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -522,6 +522,36 @@ def format( f"'data_type' {data_type} is currently not implemented" ) + elif data_type == "drug_descriptor": + if data.drug_descriptors is None: + raise ValueError( + f"'{data_type}' attribute of Dataset cannot be 'None'" + ) + shape = kwargs.get('shape', 'long') + legal_shapes = ['long', 'wide'] + if shape not in legal_shapes: + raise ValueError( + f"'shape' has to be one of '{legal_shapes}'" + ) + type = kwargs.get('type', None) + + if type is None: + tmp = data.drug_descriptors + else: + # TODO: potentially allow for list of columns to retain + tmp = data.drug_descriptors[ + data.drug_descriptors['structural_descriptor'] == type + ] + if shape == 'long': + ret = tmp + else: + ret = tmp.pivot( + index = 'improve_drug_id', + columns = 'structural_descriptor', + values = 'descriptor_value' + ) + + elif data_type == "drugs": if data.drugs is None: raise ValueError( From 7c7342cfa4409c703592dcd7a51df6ee78d6f6e4 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Mon, 9 Dec 2024 12:05:02 -0800 Subject: [PATCH 16/24] changed `format('mutations')` to use `pd.pivot_table` instead of `pd.crosstab` --- coderdata/dataset/dataset.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index 269473e1..c0a25e21 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -434,12 +434,16 @@ def format( raise ValueError( "'mutation_type' must be defined if 'data_type'=='mutations'" ) - tmp = data.mutations[ + tmp = deepcopy(data.mutations[ data.mutations['variant_classification'] == mutation_type - ] - ret = pd.crosstab( - index=tmp['entrez_id'], - columns=tmp['improve_sample_id'] + ]) + tmp['exists'] = 1 + ret = pd.pivot_table( + data=tmp, + index='entrez_id', + columns='improve_sample_id', + values='exists', + fill_value=0, ) elif data_type == "copy_number": @@ -463,7 +467,6 @@ def format( f"'{data_type}' attribute of Dataset cannot be 'None'" ) - # TODO: currently assumes that the proteomics table is properly formatted ret = pd.pivot_table( data=data.proteomics, values='proteomics', From 8ab2d779c6e16c2edd0de5574068b4efc2090c39 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Tue, 10 Dec 2024 10:22:27 -0800 Subject: [PATCH 17/24] renamed `download_by_prefix` to `download` --- coderdata/__init__.py | 2 +- coderdata/cli.py | 4 ++-- coderdata/download/__init__.py | 2 +- coderdata/download/downloader.py | 9 ++++++++- tests/test_download_and_load_beataml.py | 4 ++-- tests/test_download_and_load_cell_line.py | 4 ++-- tests/test_download_and_load_cptac.py | 4 ++-- tests/test_download_and_load_hcmi.py | 4 ++-- tests/test_reload_all.py | 4 ++-- tests/test_reload_one.py | 4 ++-- 10 files changed, 24 insertions(+), 17 deletions(-) diff --git a/coderdata/__init__.py b/coderdata/__init__.py index db739e6a..b50f95da 100644 --- a/coderdata/__init__.py +++ b/coderdata/__init__.py @@ -1,4 +1,4 @@ -from .download.downloader import download_data_by_prefix +from .download.downloader import download from .load.loader import DatasetLoader, join_datasets from .split.splitter import train_test_validate from .dataset.dataset import ( diff --git a/coderdata/cli.py b/coderdata/cli.py index eb666752..1f7cc0b3 100644 --- a/coderdata/cli.py +++ b/coderdata/cli.py @@ -1,5 +1,5 @@ import argparse -from .download.downloader import download_data_by_prefix +from .download.downloader import download def main(): parser = argparse.ArgumentParser(prog='coderdata') @@ -9,7 +9,7 @@ def main(): parser_download = subparsers.add_parser('download', help='Download datasets') parser_download.add_argument('--prefix', type=str, default=None, help='Prefix of the dataset to download (e.g., "hcmi"), "all", or leave empty for all files.') - parser_download.set_defaults(func=download_data_by_prefix) + parser_download.set_defaults(func=download) args = parser.parse_args() if hasattr(args, 'func'): diff --git a/coderdata/download/__init__.py b/coderdata/download/__init__.py index f39a5195..40efb436 100644 --- a/coderdata/download/__init__.py +++ b/coderdata/download/__init__.py @@ -1,2 +1,2 @@ -from .downloader import download_data_by_prefix +from .downloader import download diff --git a/coderdata/download/downloader.py b/coderdata/download/downloader.py index 21bb4203..1bf3011d 100644 --- a/coderdata/download/downloader.py +++ b/coderdata/download/downloader.py @@ -1,8 +1,15 @@ # coderdata/download/downloader.py +from pathlib import Path +from os import PathLike +import os import requests -def download_data_by_prefix(dataset_prefix=None): +def download( + name: str=None, + local_path: PathLike=Path.cwd(), + exist_ok: bool=False + ): """ Download the most recent version of files from a Figshare dataset, filtered by a specific prefix or all files. diff --git a/tests/test_download_and_load_beataml.py b/tests/test_download_and_load_beataml.py index f07c5c60..0a39773e 100644 --- a/tests/test_download_and_load_beataml.py +++ b/tests/test_download_and_load_beataml.py @@ -1,6 +1,6 @@ # tests/test_download_beataml.py -from coderdata.download.downloader import download_data_by_prefix +from coderdata.download.downloader import download from coderdata.load.loader import DatasetLoader import os import glob @@ -9,7 +9,7 @@ def test_download_data_beataml(): #BeatAML - download_data_by_prefix('beataml') + download('beataml') beataml_drugs = glob.glob('beataml_drugs*') assert len(beataml_drugs) > 0, "File beataml_drugs does not exist." diff --git a/tests/test_download_and_load_cell_line.py b/tests/test_download_and_load_cell_line.py index 2f7f7611..e9526fab 100644 --- a/tests/test_download_and_load_cell_line.py +++ b/tests/test_download_and_load_cell_line.py @@ -1,6 +1,6 @@ # tests/test_download_depmap.py -from coderdata.download.downloader import download_data_by_prefix +from coderdata.download.downloader import download from coderdata.load.loader import DatasetLoader import os import glob @@ -9,7 +9,7 @@ def test_download_data_depmap(): #depmap - download_data_by_prefix('depmap') + download('depmap') depmap_samples = glob.glob('depmap_samples*') assert len(depmap_samples) > 0, "File depmap_samples does not exist." diff --git a/tests/test_download_and_load_cptac.py b/tests/test_download_and_load_cptac.py index 742ad8a5..de9dc92a 100644 --- a/tests/test_download_and_load_cptac.py +++ b/tests/test_download_and_load_cptac.py @@ -1,7 +1,7 @@ # tests/test_download_cptac.py -from coderdata.download.downloader import download_data_by_prefix +from coderdata.download.downloader import download from coderdata.load.loader import DatasetLoader import os import glob @@ -10,7 +10,7 @@ def test_download_data_cptac(): #CPTAC - download_data_by_prefix('cptac') + download('cptac') cptac_copy_number = glob.glob('cptac_copy_number*') assert len(cptac_copy_number) > 0, "File cptac_copy_number does not exist." diff --git a/tests/test_download_and_load_hcmi.py b/tests/test_download_and_load_hcmi.py index ab73e76e..86de23c5 100644 --- a/tests/test_download_and_load_hcmi.py +++ b/tests/test_download_and_load_hcmi.py @@ -1,6 +1,6 @@ # tests/test_download_hcmi.py -from coderdata.download.downloader import download_data_by_prefix +from coderdata.download.downloader import download from coderdata.load.loader import DatasetLoader import os import glob @@ -9,7 +9,7 @@ def test_download_data_hcmi(): #HCMI - download_data_by_prefix('hcmi') + download('hcmi') hcmi_mutations = glob.glob('hcmi_mutations*') assert len(hcmi_mutations) > 0, "File hcmi_mutations does not exist." diff --git a/tests/test_reload_all.py b/tests/test_reload_all.py index a6c13ebe..b320b432 100644 --- a/tests/test_reload_all.py +++ b/tests/test_reload_all.py @@ -4,11 +4,11 @@ import pandas as pd import os from coderdata.load.loader import DatasetLoader -from coderdata.download import download_data_by_prefix +from coderdata.download import download def test_reload_all_datasets(): - download_data_by_prefix("hcmi") + download("hcmi") loader = DatasetLoader("hcmi") # Ensure the datasets are initially empty diff --git a/tests/test_reload_one.py b/tests/test_reload_one.py index e0ef482f..bca44890 100644 --- a/tests/test_reload_one.py +++ b/tests/test_reload_one.py @@ -4,11 +4,11 @@ import pandas as pd import os from coderdata.load.loader import DatasetLoader -from coderdata.download import download_data_by_prefix +from coderdata.download import download def test_reload_specific_dataset(): - download_data_by_prefix("hcmi") + download("hcmi") loader = DatasetLoader("hcmi") # Ensure the dataset is initially empty From 8cba605fb496dd49881d531a419ab8a92b10a439 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Tue, 10 Dec 2024 10:22:54 -0800 Subject: [PATCH 18/24] added option to download to specified folder in `download()` --- coderdata/download/downloader.py | 65 +++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 17 deletions(-) diff --git a/coderdata/download/downloader.py b/coderdata/download/downloader.py index 1bf3011d..18cf3ade 100644 --- a/coderdata/download/downloader.py +++ b/coderdata/download/downloader.py @@ -11,46 +11,71 @@ def download( exist_ok: bool=False ): """ - Download the most recent version of files from a Figshare dataset, filtered by a specific prefix or all files. + Download the most recent version of files from a Figshare dataset, + filtered by a specific prefix or all files. - This function queries the Figshare API to retrieve details of a dataset and then downloads files from it. - Files can be filtered by a specified prefix such as hcmi, beataml, etc. If 'all', an empty string, or None is passed as the prefix, - all files in the dataset are downloaded. The function identifies the most recent version of a file - by selecting the one with the highest ID among duplicates with the same name. + This function queries the Figshare API to retrieve details of a + dataset and then downloads files from it. Files can be filtered by a + specified prefix such as hcmi, beataml, etc. If 'all', an empty + string, or None is passed as the prefix, all files in the dataset + are downloaded. The function identifies the most recent version of a + file by selecting the one with the highest ID among duplicates with + the same name. Parameters ---------- dataset_prefix : str, optional - The prefix of the dataset to download (e.g., 'hcmi'). If 'all', an empty string, or None, - all files in the dataset are downloaded. Default is None. + The prefix of the dataset to download (e.g., 'hcmi'). If 'all', + an empty string, or None, all files in the dataset are + downloaded. Default is None. Returns ------- None - The function downloads files to the local repository and does not return any value. + The function downloads files to the local repository and does + not return any value. """ + # Create Path object from `local_path` + if type(local_path) != Path: + local_path = Path(local_path) + + if not local_path.exists(): + Path.mkdir(local_path) # Get the dataset details url = "https://api.figshare.com/v2/articles/25033697" response = requests.get(url) if response.status_code != 200: - raise Exception(f"Failed to get dataset details from Figshare: {response.text}") + raise Exception( + f"Failed to get dataset details from Figshare: {response.text}" + ) data = response.json() + # making sure that we are case insensitive + name = name.casefold() + # Filter files by the specified prefix - if dataset_prefix and dataset_prefix.lower() != "all": - filtered_files = [file for file in data['files'] if file['name'].startswith(dataset_prefix)] + if name != "all": + filtered_files = [ + file + for file + in data['files'] + if file['name'].startswith(name) + ] else: filtered_files = data['files'] # Group files by name and select the one with the highest ID unique_files = {} for file in filtered_files: - file_name = file['name'] + file_name = local_path.joinpath(file['name']) file_id = file['id'] - if file_name not in unique_files or file_id > unique_files[file_name]['id']: + if ( + file_name not in unique_files + or file_id > unique_files[file_name]['id'] + ): unique_files[file_name] = {'file_info': file, 'id': file_id} for file_name, file_data in unique_files.items(): @@ -60,11 +85,17 @@ def download( # Download the file with requests.get(file_url, stream=True) as r: r.raise_for_status() - with open(file_name, 'wb') as f: - for chunk in r.iter_content(chunk_size=8192): - f.write(chunk) + if file_name.exists() and not exist_ok: + os.warn( + f"{file_name} already exists. Use argument 'exist_ok=True'" + "to overwrite existing file." + ) + else: + with open(file_name, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) - print(f"Downloaded {file_name} to local repository.") + print(f"Downloaded '{file_url}' to '{file_name}'") return From 7c0d4b2830db82b4232dc72f32f219c90daf1bb3 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Thu, 12 Dec 2024 10:29:20 -0800 Subject: [PATCH 19/24] fixed import --- coderdata/download/downloader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/coderdata/download/downloader.py b/coderdata/download/downloader.py index 18cf3ade..b4b1701b 100644 --- a/coderdata/download/downloader.py +++ b/coderdata/download/downloader.py @@ -4,6 +4,7 @@ from os import PathLike import os import requests +import warnings def download( name: str=None, @@ -86,7 +87,7 @@ def download( with requests.get(file_url, stream=True) as r: r.raise_for_status() if file_name.exists() and not exist_ok: - os.warn( + warnings.warn( f"{file_name} already exists. Use argument 'exist_ok=True'" "to overwrite existing file." ) From 918436341cc86fb9e560ae4d35063a7e1033f58b Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Thu, 12 Dec 2024 10:30:07 -0800 Subject: [PATCH 20/24] added `copy_number` -> `copy_call` conversion --- coderdata/dataset/dataset.py | 69 +++++++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/coderdata/dataset/dataset.py b/coderdata/dataset/dataset.py index c0a25e21..aae224ee 100644 --- a/coderdata/dataset/dataset.py +++ b/coderdata/dataset/dataset.py @@ -1,3 +1,7 @@ +""" +_summary_ + +""" from __future__ import annotations @@ -11,6 +15,7 @@ import numpy as np from numpy.random import RandomState import pandas as pd +# import polars as pl from sklearn.model_selection import GroupShuffleSplit from sklearn.model_selection import ShuffleSplit @@ -27,6 +32,25 @@ class Split: class Dataset: + data_format_params = { + "samples": ( + "improve_sample_id", "cancer_type", "model_type", "common_name", + "other_id", "other_names", "id_source", "species" + ), + "transcriptomics": ( + "improve_sample_id", "entrez_id", "transcriptomics" + ), + "proteomics": ("improve_sample_id", "entrez_id", "proteomics"), + "mutations": ("improve_sample_id", "entrez_id", "mutation"), + "copy_number": ("improve_sample_id", "entrez_id", "copy_number"), + "methylation": ("improve_sample_id", "entrez_id", "methylation"), + "experiments": ( + "improve_sample_id", "improve_drug_id", "dose_response_value" + ), + "drugs": ("improve_drug_id", "chem_name", "isoSMILES"), + "genes": ("entrez_id", "gene_symbol", "other_id") + } + def __init__( self, name: str=None, @@ -49,6 +73,35 @@ def __init__( Each attribute will be a Pandas DataFrame corresponding to a file with the dataset prefix. Attributes include transcriptomics, proteomics, mutations, etc. + + Parameters + ---------- + name : str + The name of the dataset that is stored in the object + transcriptomics : pd.DataFrame, optional + _description_, by default None + proteomics : pd.DataFrame, optional + _description_, by default None + mutations : pd.DataFrame, optional + _description_, by default None + copy_number : pd.DataFrame, optional + _description_, by default None + samples : pd.DataFrame, optional + _description_, by default None + drugs : pd.DataFrame, optional + _description_, by default None + mirna : pd.DataFrame, optional + _description_, by default None + experiments : pd.DataFrame, optional + _description_, by default None + methylation : pd.DataFrame, optional + _description_, by default None + metabolomics : pd.DataFrame, optional + _description_, by default None + genes : pd.DataFrame, optional + _description_, by default None + full : pd.DataFrame, optional + _description_, by default None """ self.name = name @@ -72,6 +125,11 @@ def __init__( # ---------------------------- + @property + def data_format_params(self): + return self._data_format_params + + @property def name(self): return self._name @@ -451,8 +509,8 @@ def format( raise ValueError( f"'{data_type}' attribute of Dataset cannot be 'None'" ) + copy_call = kwargs.get('copy_call', False) - # TODO: add way to extract copy_call ret = pd.pivot_table( data=data.copy_number, index='entrez_id', @@ -460,6 +518,14 @@ def format( values='copy_number', aggfunc='mean', ) + if copy_call: + ret = ret.apply( + pd.cut, + bins = [0, 0.5210507, 0.7311832, 1.214125, 1.422233, 2], + labels = ['deep del', 'het loss', 'diploid', 'gain', 'amp'], + include_lowest=True + ) + elif data_type == "proteomics": if data.proteomics is None: @@ -920,6 +986,7 @@ def train_test_validate( return Split(data_train, data_test, data_val) + def _load_file(file_path: Path) -> pd.DataFrame: if file_path.suffix == '.gz': return pd.read_csv( From 7804cd68ee40d497a4ffe9fe74c7e8d022837e0c Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Mon, 16 Dec 2024 10:32:44 -0800 Subject: [PATCH 21/24] added utilization of __version__ --- coderdata/__init__.py | 13 +++++++++++++ coderdata/utils/utils.py | 13 +++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 coderdata/utils/utils.py diff --git a/coderdata/__init__.py b/coderdata/__init__.py index b50f95da..89b9476f 100644 --- a/coderdata/__init__.py +++ b/coderdata/__init__.py @@ -5,3 +5,16 @@ Dataset, load, ) + +# '_version.py' will be generated by hatchling once the switch away from +# setuptools.py is finished +try: + from ._version import __version__ +except ImportError: + __version__ = '0.1.40' +try: + from ._version import __version_tuple__ +except ImportError: + __version_tuple__ = (0, 1, 40) + +from .utils.utils import version \ No newline at end of file diff --git a/coderdata/utils/utils.py b/coderdata/utils/utils.py new file mode 100644 index 00000000..e4550815 --- /dev/null +++ b/coderdata/utils/utils.py @@ -0,0 +1,13 @@ +""" +Collection of small utility and helper functions. +""" +from .. import __version__ +from .. import __version_tuple__ + +def version(): + return { + 'package' : __version__, + 'dataset' : f"{__version_tuple__[0]}.{__version_tuple__[1]}" + } + + From 9862154c021740569fbd93952f1419de6e7db5f9 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Mon, 16 Dec 2024 10:49:22 -0800 Subject: [PATCH 22/24] added missing __init__ file --- coderdata/utils/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 coderdata/utils/__init__.py diff --git a/coderdata/utils/__init__.py b/coderdata/utils/__init__.py new file mode 100644 index 00000000..14679149 --- /dev/null +++ b/coderdata/utils/__init__.py @@ -0,0 +1 @@ +from .utils import version \ No newline at end of file From 1b16b8263d00f626112c18cfb4b714b3db6d951b Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Mon, 16 Dec 2024 11:36:32 -0800 Subject: [PATCH 23/24] added helper function to list all available datasets --- coderdata/__init__.py | 3 ++- coderdata/datasets.yml | 10 +++++++++ coderdata/utils/__init__.py | 3 ++- coderdata/utils/utils.py | 42 ++++++++++++++++++++++++++++++++++++- 4 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 coderdata/datasets.yml diff --git a/coderdata/__init__.py b/coderdata/__init__.py index 89b9476f..41b59c08 100644 --- a/coderdata/__init__.py +++ b/coderdata/__init__.py @@ -17,4 +17,5 @@ except ImportError: __version_tuple__ = (0, 1, 40) -from .utils.utils import version \ No newline at end of file +from .utils.utils import version +from .utils.utils import list_datasets \ No newline at end of file diff --git a/coderdata/datasets.yml b/coderdata/datasets.yml new file mode 100644 index 00000000..dbc09557 --- /dev/null +++ b/coderdata/datasets.yml @@ -0,0 +1,10 @@ +datasets: + beataml: + description: "Beat acute myeloid leukemia (BeatAML) focuses on acute myeloid leukemia tumor data. Data includes drug response, proteomics, and transcriptomics datasets." + doi: "https://doi.org/10.1016/j.ccell.2022.07.002" + cptac: + description: "The Clinical Proteomic Tumor Analysis Consortium (CPTAC) project is a collaborative network funded by the National Cancer Institute (NCI) focused on improving our understanding of cancer biology through the integration of transcriptomic, proteomic, and genomic data." + hcmi: + description: "Human Cancer Models Initiative (HCMI) encompasses numerous cancer types and includes cell line, organoid, and tumor data. Data includes the transcriptomics, somatic mutation, and copy number datasets." + mpnst: + description: "Malignant Peripheral Nerve Sheath Tumor is a rare, agressive sarcoma that affects peripheral nerves throughout the body." \ No newline at end of file diff --git a/coderdata/utils/__init__.py b/coderdata/utils/__init__.py index 14679149..3380dde2 100644 --- a/coderdata/utils/__init__.py +++ b/coderdata/utils/__init__.py @@ -1 +1,2 @@ -from .utils import version \ No newline at end of file +from .utils import version +from .utils import list_datasets \ No newline at end of file diff --git a/coderdata/utils/utils.py b/coderdata/utils/utils.py index e4550815..c6688ef9 100644 --- a/coderdata/utils/utils.py +++ b/coderdata/utils/utils.py @@ -1,13 +1,53 @@ """ Collection of small utility and helper functions. """ + +from importlib import resources +import yaml + from .. import __version__ from .. import __version_tuple__ -def version(): + +def version() -> dict: + """ + Helper function that returns the version strings for the package and + the dataset build. + + Returns + ------- + dict + Contains package and dataset build version. + """ return { 'package' : __version__, 'dataset' : f"{__version_tuple__[0]}.{__version_tuple__[1]}" } +def list_datasets(raw: bool=False) -> dict | None: + """ + Hepler function that returns a list of available datasets including + a short description and additional information available. + + Parameters + ---------- + raw : bool, default=False + If set to True returns a yaml dictionary containing all + available datasets including additional information. If set to + false prints information to stdout and returns None. + + Returns + ------- + dict | None + Returns a dict containing the information if ``raw==True``, + otherwise prints information to stdout and returns `None`. + """ + with resources.open_text('coderdata', 'datasets.yml') as f: + datasets = yaml.load(f, Loader=yaml.FullLoader) + if raw: + return datasets + else: + datasets = datasets['datasets'] + for dataset in datasets: + print(f'{dataset}: "{datasets[dataset]['description']}"') From 28ea7e3b6d9e2f83ce6e6556e6dfa712e7873d74 Mon Sep 17 00:00:00 2001 From: Yannick Mahlich Date: Mon, 16 Dec 2024 11:42:50 -0800 Subject: [PATCH 24/24] removed coderdata.DatasetLoader / coderdata.loader.* --- coderdata/__init__.py | 1 - coderdata/load/__init__.py | 2 - coderdata/load/loader.py | 376 ------------------------------------- 3 files changed, 379 deletions(-) delete mode 100644 coderdata/load/__init__.py delete mode 100644 coderdata/load/loader.py diff --git a/coderdata/__init__.py b/coderdata/__init__.py index 41b59c08..164ced89 100644 --- a/coderdata/__init__.py +++ b/coderdata/__init__.py @@ -1,5 +1,4 @@ from .download.downloader import download -from .load.loader import DatasetLoader, join_datasets from .split.splitter import train_test_validate from .dataset.dataset import ( Dataset, diff --git a/coderdata/load/__init__.py b/coderdata/load/__init__.py deleted file mode 100644 index 2eb603d9..00000000 --- a/coderdata/load/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .loader import DatasetLoader, join_datasets - diff --git a/coderdata/load/loader.py b/coderdata/load/loader.py deleted file mode 100644 index 452c51a3..00000000 --- a/coderdata/load/loader.py +++ /dev/null @@ -1,376 +0,0 @@ -# coderdata/load/loader.py - -import pandas as pd -import gzip -import os - -class DatasetLoader: - def __init__(self, dataset_type, data_directory="."): - """ - Load datasets of a specific type into predefined attributes of this class instance. - - Each attribute will be a Pandas DataFrame corresponding to a file with the dataset prefix. - Attributes include transcriptomics, proteomics, mutations, etc. - - Parameters - ---------- - dataset_type : str - The type of dataset to load (e.g., 'hcmi', 'beataml', 'cptac', 'depmap', "mpnst). - data_directory : str - The directory where dataset files are stored. - """ - # Initialize attributes as empty DataFrames - self.transcriptomics = pd.DataFrame() - self.proteomics = pd.DataFrame() - self.mutations = pd.DataFrame() - self.copy_number = pd.DataFrame() - self.samples = pd.DataFrame() - self.drugs = pd.DataFrame() - self.mirna = pd.DataFrame() - self.experiments = pd.DataFrame() - self.methylation = pd.DataFrame() - self.metabolomics = pd.DataFrame() - self.genes = pd.DataFrame() - # self.perturbations = pd.DataFrame() - self.full = pd.DataFrame() - self.dataset_type = dataset_type - self.data_directory = data_directory - self.load_datasets(dataset_type, data_directory) - self.data_format_params = { - 'samples': ('improve_sample_id', 'cancer_type', 'model_type',"common_name","other_id", "other_names", "id_source", "species"), - 'transcriptomics': ('improve_sample_id', 'entrez_id', 'transcriptomics'), - 'proteomics': ('improve_sample_id', 'entrez_id', 'proteomics'), - 'mutations': ('improve_sample_id', 'entrez_id', 'mutation'), - 'copy_number': ('improve_sample_id', 'entrez_id', 'copy_number'), - 'methylation': ('improve_sample_id', 'entrez_id', 'methylation'), - 'experiments': ('improve_sample_id', 'improve_drug_id', 'dose_response_value'), - 'drugs': ('improve_drug_id', 'chem_name', 'isoSMILES'), - 'genes': ('entrez_id', 'gene_symbol', 'other_id') - } - - def load_datasets(self, dataset_type, data_directory): - print("Processing Data...") - # Load genes.csv or genes.csv.gz if present - genes_file_path = None - for file_name in os.listdir(data_directory): - if file_name == 'genes.csv' or file_name == 'genes.csv.gz': - genes_file_path = os.path.join(data_directory, file_name) - self.genes = self.load_file(genes_file_path) - print("Loaded genes dataset.") - - if file_name.startswith(dataset_type) and (file_name.endswith(('.csv', '.tsv', '.csv.gz', '.tsv.gz'))): - file_path = os.path.join(data_directory, file_name) - dataset_name = file_name[len(dataset_type):].split('.')[0].strip('_') - if hasattr(self, dataset_name): - dataframe = self.load_file(file_path) - setattr(self, dataset_name, dataframe) - - @staticmethod - def load_file(file_path): - if file_path.endswith('.gz'): - with gzip.open(file_path, 'rt') as file: - return pd.read_csv(file, delimiter=DatasetLoader.determine_delimiter(file_path)) - else: - return pd.read_csv(file_path, delimiter=DatasetLoader.determine_delimiter(file_path)) - - @staticmethod - def determine_delimiter(file_path): - return '\t' if file_path.endswith('.tsv') or file_path.endswith('.tsv.gz') else ',' - - def merge_datasets(self, drop_na=False): - """ - Merge all non-empty available datasets into a single DataFrame. - The 'samples' attribute is merged last, after removing duplicates based on 'improve_sample_id'. - Merges on common keys ('improve_sample_id', 'entrez_id', 'improve_drug_id') where available. - """ - # Gather all non-empty DataFrame attributes - dataframes = {attr: getattr(self, attr) for attr in dir(self) - if isinstance(getattr(self, attr), pd.DataFrame) - and not getattr(self, attr).empty - and not attr.startswith('__') - and attr != 'full'} - - # Save the 'samples' DataFrame and remove it from the initial merging process - samples_df = dataframes.pop('samples', pd.DataFrame()) - - # List of potential merge keys - merge_keys = ['improve_sample_id', 'entrez_id', 'improve_drug_id'] - print("Merging Datasets into full.") - # Merge non-samples dataframes - for attr, df in dataframes.items(): - if self.full.empty: - self.full = df - else: - common_keys = [key for key in merge_keys if key in df.columns and key in self.full.columns] - if common_keys: - self.full = pd.merge(self.full, df, on=common_keys, how='outer') - - # Merge 'samples' DataFrame last - if not samples_df.empty: - samples_df = samples_df.drop_duplicates(subset=['improve_sample_id']) - common_keys = [key for key in merge_keys if key in samples_df.columns and key in self.full.columns] - if common_keys: - self.full = pd.merge(self.full, samples_df, on=common_keys, how='outer') - - merge_suffixes = ['_x', '_y', '_z'] # Add more if needed - for base_col in set(col.rsplit('_', 1)[0] for col in self.full.columns if any(col.endswith(suffix) for suffix in merge_suffixes)): - # Identify all suffix columns for the base column - suffix_cols = [col for col in self.full.columns if col.startswith(base_col) and any(col.endswith(suffix) for suffix in merge_suffixes)] - - # Start with the first suffix column - merged_col = self.full[suffix_cols[0]].copy() - - # Merge remaining suffix columns - for col in suffix_cols[1:]: - merged_col = merged_col.fillna(self.full[col]) - - # Replace the original column (if it exists) with the merged column, else create a new column - if base_col in self.full.columns: - self.full[base_col] = self.full[base_col].fillna(merged_col) - else: - self.full[base_col] = merged_col - - # Drop the suffix columns - self.full.drop(suffix_cols, axis=1, inplace=True) - - def reformat_dataset(self, *args): - """ - Reformat a given dataset or all datasets to either 'long' or 'wide' format. - - Parameters - ---------- - *args : variable length argument list - Can be either one argument (format) to reformat all datasets, or two arguments - (dataset_name, format) to reformat a specific dataset. - """ - # Validate and parse arguments - if len(args) == 1: - format = args[0] - datasets = self.data_format_params.keys() - elif len(args) == 2: - dataset_name, format = args - datasets = [dataset_name] - else: - raise ValueError("Invalid number of arguments. Expecting one or two arguments.") - - if format not in ['long', 'wide']: - raise ValueError("Format must be 'long' or 'wide'.") - - # Reformat datasets - for dataset_name in datasets: - if hasattr(self, dataset_name): - dataset = getattr(self, dataset_name) - if not dataset.empty and dataset_name in self.data_format_params: - id_vars, var_name, value_name = self.data_format_params[dataset_name] - - # Check if the dataset is already in the desired format - if format == 'wide' and self.is_wide_format(dataset, id_vars, var_name): - print(f"Dataset '{dataset_name}' is already in wide format.") - continue - elif format == 'long' and self.is_long_format(dataset, id_vars, var_name, value_name): - print(f"Dataset '{dataset_name}' is already in long format.") - continue - - # Convert to the desired format - if format == 'wide': - setattr(self, dataset_name, self.to_wide_format(dataset, id_vars, var_name, value_name)) - print(f"{dataset_name} successfully converted to wide format") - elif format == 'long': - setattr(self, dataset_name, self.to_long_format(dataset, id_vars, var_name, value_name)) - print(f"{dataset_name} successfully converted to long format") - else: - print(f"Dataset '{dataset_name}' is empty or already in desired format.") - - def to_wide_format(self, dataset, index, columns, values): - # Reset index if needed - if dataset.index.name == columns: - dataset = dataset.reset_index() - - # Check if the values column is numeric and apply the appropriate aggregation method. - if pd.api.types.is_numeric_dtype(dataset[values]): - dataset_agg = dataset.groupby([index, columns])[values].mean().reset_index() - else: - dataset_agg = dataset.groupby([index, columns])[values].first().reset_index() - - # Pivot the aggregated dataset - wide_format = dataset_agg.pivot(index=index, columns=columns, values=values) - return wide_format.reset_index() - - def to_long_format(self, dataset, id_vars, var_name, value_name): - long_format = pd.melt(dataset, id_vars=id_vars, var_name=var_name, value_name=value_name) - return long_format - - def is_wide_format(self, dataset, id_var, column_var): - """ - Check if the dataset is in wide format. - """ - return (dataset.columns.name == column_var and dataset.columns[0] == id_var) - - def is_long_format(self, dataset, id_vars, var_name, value_name): - """ - Check if the dataset is in long format. - """ - required_columns = set([id_vars, var_name, value_name]) - return required_columns.issubset(set(dataset.columns)) - - def reload_datasets(self, dataset_name=None): - """ - Reload a specific dataset or all datasets. - - Parameters - ---------- - dataset_name : str, optional - The name of the dataset to reload. If None, all datasets are reloaded. - """ - if dataset_name: - # Reload a specific dataset - if hasattr(self, dataset_name): - file_path = self._get_file_path(dataset_name) - if file_path: - dataframe = self.load_file(file_path) - setattr(self, dataset_name, dataframe) - print(f"Dataset '{dataset_name}' reloaded.") - else: - print(f"File for dataset '{dataset_name}' not found.") - else: - print(f"Dataset '{dataset_name}' does not exist.") - else: - # Reload all datasets - for attr in self.data_format_params.keys(): - if hasattr(self, attr): - file_path = self._get_file_path(attr) - if file_path: - dataframe = self.load_file(file_path) - setattr(self, attr, dataframe) - print(f"Dataset '{attr}' reloaded.") - else: - print(f"File for dataset '{attr}' not available.") - - def _get_file_path(self, dataset_name): - """ - Utility method to get the file path for a given dataset name. - This method assumes a specific naming convention and directory structure. - """ - for file_name in os.listdir(self.data_directory): - if file_name.startswith(self.dataset_type) and file_name.endswith(('.csv', '.tsv', '.csv.gz', '.tsv.gz')): - if dataset_name in file_name: - return os.path.join(self.data_directory, file_name) - return None - - def info(self): - """ - Display information about the datasets and datatypes. - """ - # Dataset descriptions - descriptions = { - 'depmap': 'The cell line datasets were collected from numerous resources such as the LINCS project, DepMap, and the Sanger Institute.', - 'cptac': 'The Clinical Proteomic Tumor Analysis Consortium (CPTAC) project is a collaborative network funded by the National Cancer Institute (NCI).', - 'hcmi': 'Human Cancer Models Initiative (HCMI) data was collected though the National Cancer Institute (NCI) Genomic Data Commons (GDC) Data Portal.', - 'beataml': 'Beat acute myeloid leukemia (BeatAML) data was collected though GitHub and Synapse.', - 'mpnst': 'A collection of NF1-MPNST patient-derived xenografts, organoids, and tumors. Data hosted on synapse.' - } - - # Check if this is a joined dataset - if hasattr(self, 'source_of_datatype'): - print("This is a joined dataset comprising of:") - for source in set(sum(self.source_of_datatype.values(), [])): - if source in descriptions: - print(f"- {source}: {descriptions[source]}") - else: - print(f"Dataset Type: {self.dataset_type}") - if self.dataset_type in descriptions: - print(descriptions[self.dataset_type]) - - # Print information about available datatypes and their formats - print("\nAvailable Datatypes and Their Formats:") - for attr in dir(self): - if isinstance(getattr(self, attr), pd.DataFrame) and not getattr(self, attr).empty: - format_params = self.data_format_params.get(attr, []) - if len(format_params) == 3: # Assuming format_params has three elements: id_vars, var_name, value_name - id_vars, var_name, value_name = format_params - format_type = 'long' if self.is_long_format(getattr(self, attr), id_vars, var_name, value_name) else 'wide' - print(f"- {attr}: {format_type} format") - else: - print(f"- {attr}: Format not specified") - -def join_datasets(*args): - """ - Joins datasets from multiple DatasetLoader instances or dataset type strings. - Includes datasets even if they exist in only one instance. - - Parameters - ---------- - *args : variable number of DatasetLoader instances or dataset type strings - - Returns - ------- - DatasetLoader object - A new DatasetLoader instance with the joined datasets. - """ - if not args: - raise ValueError("At least one dataset or loader must be provided.") - - loaders = [] - - for arg in args: - if isinstance(arg, str): - loaders.append(DatasetLoader(arg)) - elif isinstance(arg, DatasetLoader): - loaders.append(arg) - else: - raise ValueError("Arguments must be either dataset type strings or DatasetLoader instances.") - - joined_loader = DatasetLoader(loaders[0].dataset_type) - joined_loader.source_of_datatype = {} - - all_data_types = set() - for loader in loaders: - all_data_types.update(set(loader.data_format_params.keys())) - - for attr in all_data_types: - merged_df = pd.DataFrame() - sources = set() - - for loader in loaders: - df = getattr(loader, attr, pd.DataFrame()) - if not df.empty: - format_params = loader.data_format_params.get(attr, []) - if len(format_params) >= 3: # Assuming format_params has three elements: id_vars, var_name, value_name - id_vars, var_name, value_name = format_params[:3] - if not loader.is_long_format(df, id_vars, var_name, value_name): - print(f"Cannot merge {attr} as it is not in long format in {loader.dataset_type}.") - return None - - merge_keys = loader.data_format_params.get(attr, []) - - for key in merge_keys: - if key in df.columns: - if merged_df.empty or key not in merged_df.columns: - continue - - # Check if data types are different - if df[key].dtype != merged_df[key].dtype: - try: - # Try converting both columns to float - df[key] = df[key].astype(float) - merged_df[key] = merged_df[key].astype(float) - except ValueError: - # If conversion to float fails, convert to string - df[key] = df[key].astype(str) - merged_df[key] = merged_df[key].astype(str) - - if merged_df.empty: - merged_df = df - else: - merged_df = pd.concat((merged_df, df)) - - if hasattr(loader, 'source_of_datatype') and attr in loader.source_of_datatype: - sources.update(loader.source_of_datatype[attr]) - else: - sources.add(loader.dataset_type) - - if not merged_df.empty: - setattr(joined_loader, attr, merged_df) - joined_loader.source_of_datatype[attr] = list(sources) - - return joined_loader