diff --git a/Snakefile b/Snakefile index 5ad7aa185..33b0b848d 100644 --- a/Snakefile +++ b/Snakefile @@ -430,13 +430,17 @@ def get_gold_standard_pickle_file(wildcards): gs = parts[1] return SEP.join([out_dir, f'gs-{gs}-merged.pickle']) +# Return the dataset pickle file for a specific dataset +def get_dataset_pickle_file(wildcards): + dataset_label = get_dataset_label(wildcards) + return SEP.join([out_dir, f'dataset-{dataset_label}-merged.pickle']) + # Returns the dataset corresponding to the gold standard pair def get_dataset_label(wildcards): parts = wildcards.dataset_gold_standard_pair.split('-') dataset = parts[0] return dataset - # Returns all pathways for a specific dataset def collect_pathways_per_dataset(wildcards): dataset_label = get_dataset_label(wildcards) @@ -446,14 +450,16 @@ def collect_pathways_per_dataset(wildcards): rule evaluation_pr_per_pathways: input: node_gold_standard_file = get_gold_standard_pickle_file, + dataset_file = get_dataset_pickle_file, pathways = collect_pathways_per_dataset output: node_pr_file = SEP.join([out_dir, '{dataset_gold_standard_pair}-eval', "pr-per-pathway-nodes.txt"]), node_pr_png = SEP.join([out_dir, '{dataset_gold_standard_pair}-eval', 'pr-per-pathway-nodes.png']), run: node_table = Evaluation.from_file(input.node_gold_standard_file).node_table + input_nodes = Dataset.from_file(input.dataset_file).get_input_nodes() pr_df = Evaluation.node_precision_and_recall(input.pathways, node_table) - Evaluation.precision_and_recall_per_pathway(pr_df, output.node_pr_file, output.node_pr_png) + Evaluation.precision_and_recall_per_pathway(pr_df, input_nodes, node_table, output.node_pr_file, output.node_pr_png) # Returns all pathways for a specific algorithm and dataset def collect_pathways_per_algo_per_dataset(wildcards): @@ -465,14 +471,16 @@ def collect_pathways_per_algo_per_dataset(wildcards): rule evaluation_per_algo_pr_per_pathways: input: node_gold_standard_file = get_gold_standard_pickle_file, + dataset_file = get_dataset_pickle_file, pathways = collect_pathways_per_algo_per_dataset, output: node_pr_file = SEP.join([out_dir, '{dataset_gold_standard_pair}-eval', "pr-per-pathway-for-{algorithm}-nodes.txt"]), node_pr_png = SEP.join([out_dir, '{dataset_gold_standard_pair}-eval', 'pr-per-pathway-for-{algorithm}-nodes.png']), run: node_table = Evaluation.from_file(input.node_gold_standard_file).node_table + input_nodes = Dataset.from_file(input.dataset_file).get_input_nodes() pr_df = Evaluation.node_precision_and_recall(input.pathways, node_table) - Evaluation.precision_and_recall_per_pathway(pr_df, output.node_pr_file, output.node_pr_png, include_aggregate_algo_eval) + Evaluation.precision_and_recall_per_pathway(pr_df, input_nodes, node_table, output.node_pr_file, output.node_pr_png, include_aggregate_algo_eval) # Return pathway summary file per dataset def collect_summary_statistics_per_dataset(wildcards): @@ -490,6 +498,7 @@ def collect_pca_coordinates_per_dataset(wildcards): rule evaluation_pca_chosen: input: node_gold_standard_file = get_gold_standard_pickle_file, + dataset_file = get_dataset_pickle_file, pca_coordinates_file = collect_pca_coordinates_per_dataset, pathway_summary_file = collect_summary_statistics_per_dataset output: @@ -497,9 +506,10 @@ rule evaluation_pca_chosen: node_pca_chosen_pr_png = SEP.join([out_dir, '{dataset_gold_standard_pair}-eval', 'pr-pca-chosen-pathway-nodes.png']), run: node_table = Evaluation.from_file(input.node_gold_standard_file).node_table + input_nodes = Dataset.from_file(input.dataset_file).get_input_nodes() pca_chosen_pathway = Evaluation.pca_chosen_pathway(input.pca_coordinates_file, input.pathway_summary_file, out_dir) pr_df = Evaluation.node_precision_and_recall(pca_chosen_pathway, node_table) - Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, output.node_pca_chosen_pr_file, output.node_pca_chosen_pr_png) + Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, input_nodes, node_table, output.node_pca_chosen_pr_file, output.node_pca_chosen_pr_png) # Returns pca coordinates for a specific algorithm and dataset def collect_pca_coordinates_per_algo_per_dataset(wildcards): @@ -511,6 +521,7 @@ def collect_pca_coordinates_per_algo_per_dataset(wildcards): rule evaluation_per_algo_pca_chosen: input: node_gold_standard_file = get_gold_standard_pickle_file, + dataset_file = get_dataset_pickle_file, pca_coordinates_file = collect_pca_coordinates_per_algo_per_dataset, pathway_summary_file = collect_summary_statistics_per_dataset output: @@ -518,14 +529,10 @@ rule evaluation_per_algo_pca_chosen: node_pca_chosen_pr_png = SEP.join([out_dir, '{dataset_gold_standard_pair}-eval', 'pr-pca-chosen-pathway-per-algorithm-nodes.png']), run: node_table = Evaluation.from_file(input.node_gold_standard_file).node_table + input_nodes = Dataset.from_file(input.dataset_file).get_input_nodes() pca_chosen_pathways = Evaluation.pca_chosen_pathway(input.pca_coordinates_file, input.pathway_summary_file, out_dir) pr_df = Evaluation.node_precision_and_recall(pca_chosen_pathways, node_table) - Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, output.node_pca_chosen_pr_file, output.node_pca_chosen_pr_png, include_aggregate_algo_eval) - -# Return the dataset pickle file for a specific dataset -def get_dataset_pickle_file(wildcards): - dataset_label = get_dataset_label(wildcards) - return SEP.join([out_dir, f'dataset-{dataset_label}-merged.pickle']) + Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, input_nodes, node_table, output.node_pca_chosen_pr_file, output.node_pca_chosen_pr_png, include_aggregate_algo_eval) # Returns ensemble file for each dataset def collect_ensemble_per_dataset(wildcards): @@ -543,8 +550,9 @@ rule evaluation_ensemble_pr_curve: node_pr_curve_file = SEP.join([out_dir, '{dataset_gold_standard_pair}-eval', 'pr-curve-ensemble-nodes.txt']), run: node_table = Evaluation.from_file(input.node_gold_standard_file).node_table + input_nodes = Dataset.from_file(input.dataset_file).get_input_nodes() node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_file, input.dataset_file) - Evaluation.precision_recall_curve_node_ensemble(node_ensemble_dict, node_table, output.node_pr_curve_png, output.node_pr_curve_file) + Evaluation.precision_recall_curve_node_ensemble(node_ensemble_dict, node_table, input_nodes, output.node_pr_curve_png, output.node_pr_curve_file) # Returns list of algorithm specific ensemble files per dataset def collect_ensemble_per_algo_per_dataset(wildcards): @@ -562,8 +570,9 @@ rule evaluation_per_algo_ensemble_pr_curve: node_pr_curve_file = SEP.join([out_dir, '{dataset_gold_standard_pair}-eval', 'pr-curve-ensemble-nodes-per-algorithm-nodes.txt']), run: node_table = Evaluation.from_file(input.node_gold_standard_file).node_table - node_ensembles_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_files, input.dataset_file) - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, node_table, output.node_pr_curve_png, output.node_pr_curve_file, include_aggregate_algo_eval) + input_nodes = Dataset.from_file(input.dataset_file).get_input_nodes() + node_ensembles_dict= Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_files, input.dataset_file) + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, node_table, input_nodes, output.node_pr_curve_png, output.node_pr_curve_file, include_aggregate_algo_eval) rule evaluation_edge_dummy: input: diff --git a/spras/dataset.py b/spras/dataset.py index 330bdcffc..c06c70cb2 100644 --- a/spras/dataset.py +++ b/spras/dataset.py @@ -226,6 +226,15 @@ def contains_node_columns(self, col_names: list[str] | str): return False return True + def get_input_nodes(self) -> pd.DataFrame: + """ + Returns: a table listing the input nodes considered as starting points for pathway reconstruction algorithms, + restricted to nodes that have at least one of the specified attributes. + """ + input_node_columns = ["sources", "targets", "prize", "active"] # TODO: do we want to add dummy nodes? + input_nodes = Dataset.get_node_columns(self, col_names = input_node_columns) + return input_nodes + def get_other_files(self): return self.other_files.copy() diff --git a/spras/evaluation.py b/spras/evaluation.py index 2faa8ffc7..42da75cf1 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -15,6 +15,7 @@ ) from spras.analysis.ml import create_palette +from spras.dataset import Dataset from spras.interactome import ( convert_directed_to_undirected, convert_undirected_to_directed, @@ -32,12 +33,12 @@ class GoldStandardDict(TypedDict): class Evaluation: NODE_ID = 'NODEID' - label: str - datasets: list[str] - node_table: pd.DataFrame - mixed_edge_table: pd.DataFrame - undirected_edge_table: pd.DataFrame - directed_edge_table: pd.DataFrame + label: str # gold standard label + datasets: list[str] # datasets associated with the dataset_labels for the specific gold standard + node_table: pd.DataFrame # the node gold standard + mixed_edge_table: pd.DataFrame # the edge gold standard + undirected_edge_table: pd.DataFrame # the edge gold standard fully undirected + directed_edge_table: pd.DataFrame # the edge gold standard fully directed @staticmethod def merge_gold_standard_input(gs_dict: GoldStandardDict, gs_file: str | os.PathLike): @@ -185,7 +186,7 @@ def node_precision_and_recall(file_paths: Iterable[Union[str, PathLike]], node_t return pr_df @staticmethod - def visualize_precision_and_recall_plot(pr_df: pd.DataFrame, output_file: str | PathLike, output_png: str | PathLike, title: str): + def visualize_precision_and_recall_plot(pr_df: pd.DataFrame, input_nodes: pd.DataFrame, node_table: pd.DataFrame, output_file: str | PathLike, output_png: str | PathLike, title: str): """ Generates a scatter plot of precision and recall values for each pathway and saves both the plot and the data. @@ -196,6 +197,8 @@ def visualize_precision_and_recall_plot(pr_df: pd.DataFrame, output_file: str | @param pr_df: Dataframe of calculated precision and recall for each pathway file. Must include a preprocessed 'Algorithm' column. + @param input_nodes: the input nodes (sources, targets, prizes, actives) used for a specific dataset + @param node_table: the gold standard nodes @param output_file: the filename to save the precision and recall of each pathway @param output_png: the filename to plot the precision and recall of each pathway (not a PRC) @param title: The title to use for the plot @@ -221,6 +224,14 @@ def visualize_precision_and_recall_plot(pr_df: pd.DataFrame, output_file: str | label=algorithm.capitalize() ) + # input nodes baseline + gold_standard_nodes = set(node_table[Evaluation.NODE_ID]) + input_nodes_set = set(input_nodes['NODEID']) + input_nodes_tp = len(input_nodes_set & gold_standard_nodes) + precision_input = input_nodes_tp / len(input_nodes_set) + recall_input = input_nodes_tp / len(gold_standard_nodes) + plt.plot(recall_input, precision_input, color='red', markersize=12, marker='X', linestyle='None', label=f'Input Nodes (P={precision_input:.3f}, R={recall_input:.3f})') + plt.title(title) plt.xlabel('Recall') plt.ylabel('Precision') @@ -231,18 +242,26 @@ def visualize_precision_and_recall_plot(pr_df: pd.DataFrame, output_file: str | plt.savefig(output_png) plt.close() - # save dataframe + # save dataframe with input node baseline pr_df.drop(columns=['Algorithm'], inplace=True) + input_nodes_row = pd.DataFrame({ + 'Pathway': ['Input Nodes'], + 'Precision': [precision_input], + 'Recall': [recall_input], + }) + pr_df = pd.concat([pr_df, input_nodes_row], ignore_index=True) pr_df.to_csv(output_file, sep='\t', index=False) @staticmethod - def precision_and_recall_per_pathway(pr_df: pd.DataFrame, output_file: str | PathLike, output_png: str | PathLike, aggregate_per_algorithm: bool = False): + def precision_and_recall_per_pathway(pr_df: pd.DataFrame, input_nodes: pd.DataFrame, node_table: pd.DataFrame, output_file: str | PathLike, output_png: str | PathLike, aggregate_per_algorithm: bool = False): """ Function for visualizing per pathway precision and recall across all algorithms. Each point in the plot represents a single pathway reconstruction. If `aggregate_per_algorithm` is set to True, the plot is restricted to a single algorithm and titled accordingly. @param pr_df: Dataframe of calculated precision and recall for each pathway file + @param input_nodes: the input nodes (sources, targets, prizes, actives) used for a specific dataset + @param node_table: the gold standard nodes @param output_file: the filename to save the precision and recall of each pathway @param output_png: the filename to plot the precision and recall of each pathway (not a PRC) @param aggregate_per_algorithm: Boolean indicating if function is used per algorithm (Default False) @@ -257,7 +276,7 @@ def precision_and_recall_per_pathway(pr_df: pd.DataFrame, output_file: str | Pat else: title = "Precision and Recall Plot Per Pathway Per Algorithm" - Evaluation.visualize_precision_and_recall_plot(pr_df, output_file, output_png, title) + Evaluation.visualize_precision_and_recall_plot(pr_df, input_nodes, node_table, output_file, output_png, title) else: # this block should never be reached — having 0 pathways implies that no algorithms or parameter combinations were run, @@ -265,7 +284,7 @@ def precision_and_recall_per_pathway(pr_df: pd.DataFrame, output_file: str | Pat raise ValueError("No pathways were provided to evaluate and visulize on. This likely means no algorithms or parameter combinations were run.") @staticmethod - def precision_and_recall_pca_chosen_pathway(pr_df: pd.DataFrame, output_file: str | PathLike, output_png: str | PathLike, aggregate_per_algorithm: bool = False): + def precision_and_recall_pca_chosen_pathway(pr_df: pd.DataFrame, input_nodes: pd.DataFrame, node_table: pd.DataFrame, output_file: str | PathLike, output_png: str | PathLike, aggregate_per_algorithm: bool = False): """ Function for visualizing the precision and recall of the single parameter combination selected via PCA, @@ -274,6 +293,8 @@ def precision_and_recall_pca_chosen_pathway(pr_df: pd.DataFrame, output_file: st is True, the plot includes a pca chosen pathway per algorithm and titled accordingly. @param pr_df: Dataframe of calculated precision and recall for each pathway file + @param input_nodes: the input nodes (sources, targets, prizes, actives) used for a specific dataset + @param node_table: the gold standard nodes @param output_file: the filename to save the precision and recall of each pathway @param output_png: the filename to plot the precision and recall of each pathway (not a PRC) @param aggregate_per_algorithm: Boolean indicating if function is used per algorithm (Default False) @@ -283,13 +304,12 @@ def precision_and_recall_pca_chosen_pathway(pr_df: pd.DataFrame, output_file: st if not pr_df.empty: pr_df['Algorithm'] = pr_df['Pathway'].apply(lambda p: Path(p).parent.name.split('-')[1]) pr_df.sort_values(by=['Recall', 'Pathway'], axis=0, ascending=True, inplace=True) - if aggregate_per_algorithm: title = "PCA-Chosen Pathway Per Algorithm Precision and Recall Plot" else: title = "PCA-Chosen Pathway Across All Algorithms Precision and Recall Plot" - Evaluation.visualize_precision_and_recall_plot(pr_df, output_file, output_png, title) + Evaluation.visualize_precision_and_recall_plot(pr_df, input_nodes, node_table, output_file, output_png, title) else: # Edge case: if all algorithms chosen use only 1 parameter combination @@ -384,12 +404,12 @@ def edge_frequency_node_ensemble(node_table: pd.DataFrame, ensemble_files: Itera @param node_table: dataFrame of gold standard nodes (column: NODEID) @param ensemble_files: list of file paths containing edge ensemble outputs @param dataset_file: path to the dataset file used to load the interactome - @return: dictionary mapping each ensemble source to its node ensemble DataFrame + @return: dictionary mapping each ensemble source to its node ensemble DataFrame and the input nodes (sources, targets, prizes, actives) """ node_ensembles_dict = dict() - pickle = Evaluation.from_file(dataset_file) + pickle = Dataset.from_file(dataset_file) interactome = pickle.get_interactome() if interactome.empty: @@ -429,8 +449,9 @@ def edge_frequency_node_ensemble(node_table: pd.DataFrame, ensemble_files: Itera return node_ensembles_dict @staticmethod - def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.DataFrame, output_png: str | PathLike, - output_file: str | PathLike, aggregate_per_algorithm: bool = False): + def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.DataFrame, input_nodes: pd.DataFrame, + output_png: str | PathLike, output_file: str | PathLike, + aggregate_per_algorithm: bool = False): """ Plots precision-recall (PR) curves for a set of node ensembles evaluated against a gold standard. @@ -440,12 +461,14 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da @param node_ensembles: dict of the pre-computed node_ensemble(s) @param node_table: gold standard nodes + @param input_nodes: the input nodes (sources, targets, prizes, actives) used for a specific dataset @param output_png: filename to save the precision and recall curves as a .png image @param output_file: filename to save the precision, recall, threshold values, average precision, and baseline average precision @param aggregate_per_algorithm: Boolean indicating if function is used per algorithm (Default False) """ gold_standard_nodes = set(node_table[Evaluation.NODE_ID]) + input_nodes_set = set(input_nodes['NODEID']) # make color palette per ensemble label name label_names = list(node_ensembles.keys()) @@ -455,53 +478,76 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da prc_dfs = [] metric_dfs = [] - baseline = None + input_ap = None for label, node_ensemble in node_ensembles.items(): - if not node_ensemble.empty: - y_true = [1 if node in gold_standard_nodes else 0 for node in node_ensemble['Node']] - y_scores = node_ensemble['Frequency'].tolist() - precision, recall, thresholds = precision_recall_curve(y_true, y_scores) - # avg precision summarizes a precision-recall curve as the weighted mean of precisions achieved at each threshold - avg_precision = average_precision_score(y_true, y_scores) - - # only set baseline precision once - # the same for every algorithm per dataset/goldstandard pair - if baseline is None: - baseline = np.sum(y_true) / len(y_true) - plt.axhline(y=baseline, color='black', linestyle='--', label=f'Baseline: {baseline:.4f}') - - plt.plot(recall, precision, color=color_palette[label], marker='o', - label=f'{label.capitalize()} (AP: {avg_precision:.4f})') - - # Dropping last elements because scikit-learn adds (1, 0) to precision/recall for plotting, not tied to real thresholds - # https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_curve.html#sklearn.metrics.precision_recall_curve:~:text=Returns%3A-,precision,predictions%20with%20score%20%3E%3D%20thresholds%5Bi%5D%20and%20the%20last%20element%20is%200.,-thresholds - prc_data = { - 'Threshold': thresholds, - 'Precision': precision[:-1], - 'Recall': recall[:-1], - } - - metric_data = { - 'Average_Precision': [avg_precision], - } - - ensemble_source = label.capitalize() if label != 'ensemble' else 'Aggregated' - prc_data = {'Ensemble_Source': [ensemble_source] * len(thresholds), **prc_data} - metric_data = {'Ensemble_Source': [ensemble_source], **metric_data} - - prc_df = pd.DataFrame.from_dict(prc_data) - prc_dfs.append(prc_df) - metric_df = pd.DataFrame.from_dict(metric_data) - metric_dfs.append(metric_df) - - else: + if node_ensemble.empty: raise ValueError( - "Cannot compute PR curve: the ensemble network is empty." - f"This should not happen unless the input network for pathway reconstruction is empty." + "Cannot compute PR curve: the ensemble network is empty. " + "This should not happen unless the input network for pathway reconstruction is empty." ) + y_true = [1 if node in gold_standard_nodes else 0 for node in node_ensemble['Node']] + y_scores = node_ensemble['Frequency'].tolist() + + # Different baselines + # Computed once; identical for every algorithm on a given dataset/gold-standard pair. + if baseline is None and input_ap is None: + universe_size = len(y_scores) + + # baseline = |gold_standard| / |universe|: random-predictor precision + baseline = np.sum(y_true) / universe_size + plt.axhline(y=baseline, color='red', linestyle='--', + label=f'Baseline (P={baseline:.4f})') + + # Input nodes PR curve: a 2-point curve built the same way as the algorithm + # ensembles, but with a synthetic frequency of 1 for input nodes and 0 for + # everything else. Returns exactly two operating points: predicting only + # the input nodes as positive, and predicting the full universe as positive + input_node_ensemble = node_ensemble[['Node']].copy() # the full interactome is in this already + input_node_ensemble['Frequency'] = input_node_ensemble['Node'].isin(input_nodes_set).astype(float) + + input_y_true = [1 if node in gold_standard_nodes else 0 for node in input_node_ensemble['Node']] + input_y_scores = input_node_ensemble['Frequency'].tolist() + + input_precision, input_recall, input_thresholds = precision_recall_curve(input_y_true, input_y_scores) + input_ap = average_precision_score(input_y_true, input_y_scores) + + plt.plot(input_recall, input_precision, color='red', marker='s', linestyle='--', + label=f'Input Nodes (AP: {input_ap:.4f})') + + prc_dfs.append(pd.DataFrame({ + 'Ensemble_Source': ['Input Nodes'] * len(input_thresholds), + 'Threshold': input_thresholds, + 'Precision': input_precision[:-1], + 'Recall': input_recall[:-1], + })) + metric_dfs.append(pd.DataFrame({ + 'Ensemble_Source': ['Input Nodes'], + 'Average_Precision': [input_ap], + })) + + # calculate and plot prc for node_ensemble + precision, recall, thresholds = precision_recall_curve(y_true, y_scores) + # avg precision summarizes a PR curve as the weighted mean of precisions achieved at each threshold + avg_precision = average_precision_score(y_true, y_scores) + plt.plot(recall, precision, color=color_palette[label], marker='o', label=f'{label.capitalize()} (AP: {avg_precision:.4f})') + + # Drop the last precision/recall element: sklearn appends (1, 0) for plotting, not tied to a real threshold. + # https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_curve.html + ensemble_source = label.capitalize() if label != 'ensemble' else 'Aggregated' + prc_dfs.append(pd.DataFrame({ + 'Ensemble_Source': [ensemble_source] * len(thresholds), + 'Threshold': thresholds, + 'Precision': precision[:-1], + 'Recall': recall[:-1], + })) + metric_dfs.append(pd.DataFrame({ + 'Ensemble_Source': [ensemble_source], + 'Average_Precision': [avg_precision], + })) + if aggregate_per_algorithm: plt.title('Precision-Recall Curve Per Algorithm Specific Ensemble') else: @@ -520,10 +566,21 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da combined_metrics_df = pd.concat(metric_dfs, ignore_index=True) combined_metrics_df['Baseline'] = baseline - # merge dfs and NaN out metric values except for first row of each Ensemble_Source - complete_df = combined_prc_df.merge(combined_metrics_df, on='Ensemble_Source', how='left') - not_last_rows = complete_df.duplicated(subset='Ensemble_Source', keep='first') - complete_df.loc[not_last_rows, ['Average_Precision', 'Baseline']] = None + # merge curves with per-source metrics, then add the input-nodes baseline as its own row + complete_df = ( + combined_prc_df + .merge(combined_metrics_df, on='Ensemble_Source', how='left') + ) + + # keep Average_Precision and Baseline only on the first row of each Ensemble_Source + not_first_rows = complete_df.duplicated(subset='Ensemble_Source', keep='first') + complete_df.loc[not_first_rows, ['Average_Precision', 'Baseline']] = None + + # save df + complete_df.sort_values( + by='Ensemble_Source', + inplace=True + ) complete_df.to_csv(output_file, index=False, sep='\t') @staticmethod diff --git a/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt b/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt index c9f6561c6..089b2ffa3 100644 --- a/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt +++ b/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt @@ -1,2 +1,4 @@ Ensemble_Source Threshold Precision Recall Average_Precision Baseline Aggregated 0.0 0.15384615384615385 1.0 0.15384615384615385 0.15384615384615385 +Input Nodes 0.0 0.15384615384615385 1.0 0.2403846153846154 0.15384615384615385 +Input Nodes 1.0 0.5 0.25 diff --git a/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt b/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt index b0e50594e..76b9e68ff 100644 --- a/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt +++ b/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt @@ -3,3 +3,5 @@ Aggregated 0.0 0.15384615384615385 1.0 0.6666666666666666 0.15384615384615385 Aggregated 0.01 0.6666666666666666 1.0 Aggregated 0.5 0.75 0.75 Aggregated 0.66 0.5 0.25 +Input Nodes 0.0 0.15384615384615385 1.0 0.2403846153846154 0.15384615384615385 +Input Nodes 1.0 0.5 0.25 diff --git a/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt b/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt index 630a89ceb..6313978a7 100644 --- a/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt +++ b/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt @@ -8,3 +8,5 @@ Ensemble2 0.01 0.6666666666666666 1.0 Ensemble2 0.5 0.75 0.75 Ensemble2 0.66 0.5 0.25 Ensemble3 0.0 0.15384615384615385 1.0 0.15384615384615385 0.15384615384615385 +Input Nodes 0.0 0.15384615384615385 1.0 0.2403846153846154 0.15384615384615385 +Input Nodes 1.0 0.5 0.25 diff --git a/test/evaluate/expected/expected-pr-per-pathway-empty.txt b/test/evaluate/expected/expected-pr-per-pathway-empty.txt index 6c97ff7eb..da1068b5e 100644 --- a/test/evaluate/expected/expected-pr-per-pathway-empty.txt +++ b/test/evaluate/expected/expected-pr-per-pathway-empty.txt @@ -1,2 +1,3 @@ Pathway Precision Recall test/evaluate/input/data-test-params-empty/pathway.txt 0.0 0.0 +Input Nodes 0.5 0.25 diff --git a/test/evaluate/expected/expected-pr-per-pathway-pca-chosen.txt b/test/evaluate/expected/expected-pr-per-pathway-pca-chosen.txt index 97c13ebc3..74614f0aa 100644 --- a/test/evaluate/expected/expected-pr-per-pathway-pca-chosen.txt +++ b/test/evaluate/expected/expected-pr-per-pathway-pca-chosen.txt @@ -1,2 +1,3 @@ Pathway Precision Recall test/evaluate/input/data-test-params-123/pathway.txt 0.6666666666666666 0.5 +Input Nodes 0.5 0.25 \ No newline at end of file diff --git a/test/evaluate/expected/expected-pr-per-pathway.txt b/test/evaluate/expected/expected-pr-per-pathway.txt index 08732918b..de1c9615f 100644 --- a/test/evaluate/expected/expected-pr-per-pathway.txt +++ b/test/evaluate/expected/expected-pr-per-pathway.txt @@ -3,3 +3,4 @@ test/evaluate/input/data-test-params-456/pathway.txt 0.0 0.0 test/evaluate/input/data-test-params-empty/pathway.txt 0.0 0.0 test/evaluate/input/data-test-params-123/pathway.txt 0.6666666666666666 0.5 test/evaluate/input/data-test-params-789/pathway.txt 1.0 0.75 +Input Nodes 0.5 0.25 diff --git a/test/evaluate/input/input-nodes.txt b/test/evaluate/input/input-nodes.txt index 1ffd8bdff..ec0e4058f 100644 --- a/test/evaluate/input/input-nodes.txt +++ b/test/evaluate/input/input-nodes.txt @@ -1,3 +1,4 @@ NODEID prize active dummy sources targets -N -C 5.7 True True +N +C 5.7 True True +A 5 True True \ No newline at end of file diff --git a/test/evaluate/test_evaluate.py b/test/evaluate/test_evaluate.py index 198c2cc50..2995a557d 100644 --- a/test/evaluate/test_evaluate.py +++ b/test/evaluate/test_evaluate.py @@ -45,9 +45,11 @@ def test_node_precision_recall_per_pathway(self): output_png = Path(OUT_DIR + 'pr-per-pathway.png') output_file.unlink(missing_ok=True) output_png.unlink(missing_ok=True) + input_data = OUT_DIR + 'data.pickle' + input_nodes = Dataset.from_file(input_data).get_input_nodes() pr_df = Evaluation.node_precision_and_recall(file_paths, GS_NODE_TABLE) - Evaluation.precision_and_recall_per_pathway(pr_df, output_file, output_png, True) + Evaluation.precision_and_recall_per_pathway(pr_df, input_nodes, GS_NODE_TABLE, output_file, output_png, True) output = pd.read_csv(output_file, sep='\t', header=0).round(8) expected = pd.read_csv(EXPECT_DIR + 'expected-pr-per-pathway.txt', sep='\t', header=0).round(8) @@ -62,9 +64,11 @@ def test_node_precision_recall_per_pathway_empty(self): output_png = Path(OUT_DIR + 'pr-per-pathway-empty.png') output_file.unlink(missing_ok=True) output_png.unlink(missing_ok=True) + input_data = OUT_DIR + 'data.pickle' + input_nodes = Dataset.from_file(input_data).get_input_nodes() pr_df = Evaluation.node_precision_and_recall(file_paths, GS_NODE_TABLE) - Evaluation.precision_and_recall_per_pathway(pr_df, output_file, output_png, True) + Evaluation.precision_and_recall_per_pathway(pr_df, input_nodes, GS_NODE_TABLE, output_file, output_png, True) output = pd.read_csv(output_file, sep='\t', header=0).round(8) expected = pd.read_csv(EXPECT_DIR + 'expected-pr-per-pathway-empty.txt', sep='\t', header=0).round(8) @@ -75,21 +79,25 @@ def test_node_precision_recall_per_pathway_empty(self): def test_node_precision_recall_per_pathway_not_provided(self): output_file = OUT_DIR + 'pr-per-pathway-not-provided.txt' output_png = OUT_DIR + 'pr-per-pathway-not-provided.png' + input_data = OUT_DIR + 'data.pickle' + input_nodes = Dataset.from_file(input_data).get_input_nodes() file_paths = [] pr_df = Evaluation.node_precision_and_recall(file_paths, GS_NODE_TABLE) with pytest.raises(ValueError): - Evaluation.precision_and_recall_per_pathway(pr_df, output_file, output_png) + Evaluation.precision_and_recall_per_pathway(pr_df, input_nodes, GS_NODE_TABLE, output_file, output_png) def test_node_precision_recall_pca_chosen_pathway_not_provided(self): output_file = Path( OUT_DIR + 'pr-per-pathway-pca-chosen-not-provided.txt') output_file.unlink(missing_ok=True) output_png = Path(OUT_DIR + 'pr-per-pathway-pca-chosen-not-provided.png') output_png.unlink(missing_ok=True) + input_data = OUT_DIR + 'data.pickle' + input_nodes = Dataset.from_file(input_data).get_input_nodes() file_paths = [] pr_df = Evaluation.node_precision_and_recall(file_paths, GS_NODE_TABLE) - Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, output_file, output_png) + Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, input_nodes, GS_NODE_TABLE, output_file, output_png) output = pd.read_csv(output_file, sep='\t', header=0).round(8) expected = pd.read_csv(EXPECT_DIR + 'expected-pr-pca-chosen-not-provided.txt', sep='\t', header=0).round(8) @@ -104,6 +112,8 @@ def test_node_precision_recall_pca_chosen_pathway(self): output_png.unlink(missing_ok=True) output_coordinates = Path(OUT_DIR + 'pca-coordinates.tsv') output_coordinates.unlink(missing_ok=True) + input_data = OUT_DIR + 'data.pickle' + input_nodes = Dataset.from_file(input_data).get_input_nodes() file_paths = [INPUT_DIR + 'data-test-params-123/pathway.txt', INPUT_DIR + 'data-test-params-456/pathway.txt', INPUT_DIR + 'data-test-params-789/pathway.txt', INPUT_DIR + 'data-test-params-empty/pathway.txt'] @@ -114,8 +124,7 @@ def test_node_precision_recall_pca_chosen_pathway(self): pathway = Evaluation.pca_chosen_pathway([output_coordinates], SUMMARY_FILE, INPUT_DIR) pr_df = Evaluation.node_precision_and_recall(pathway, GS_NODE_TABLE) - Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, output_file, output_png, True) - + Evaluation.precision_and_recall_pca_chosen_pathway(pr_df, input_nodes, GS_NODE_TABLE, output_file, output_png, True) chosen = pd.read_csv(output_file, sep='\t', header=0).round(8) expected = pd.read_csv(EXPECT_DIR + 'expected-pr-per-pathway-pca-chosen.txt', sep='\t', header=0).round(8) @@ -127,8 +136,9 @@ def test_node_ensemble(self): out_path_file = Path(OUT_DIR + 'node-ensemble.csv') out_path_file.unlink(missing_ok=True) ensemble_network = [INPUT_DIR + 'ensemble-network.tsv'] - input_network = OUT_DIR + 'data.pickle' - node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_network, input_network) + input_data = OUT_DIR + 'data.pickle' + node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_network, input_data) + print(node_ensemble_dict) node_ensemble_dict['ensemble'].to_csv(out_path_file, sep='\t', index=False) assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-node-ensemble.csv', shallow=False) @@ -136,9 +146,8 @@ def test_empty_node_ensemble(self): out_path_file = Path(OUT_DIR + 'empty-node-ensemble.csv') out_path_file.unlink(missing_ok=True) empty_ensemble_network = [INPUT_DIR + 'empty-ensemble-network.tsv'] - input_network = OUT_DIR + 'data.pickle' - node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, empty_ensemble_network, - input_network) + input_data = OUT_DIR + 'data.pickle' + node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, empty_ensemble_network, input_data) node_ensemble_dict['empty'].to_csv(out_path_file, sep='\t', index=False) assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-empty-node-ensemble.csv', shallow=False) @@ -148,8 +157,8 @@ def test_multiple_node_ensemble(self): out_path_empty_file = Path(OUT_DIR + 'empty-node-ensemble.csv') out_path_empty_file.unlink(missing_ok=True) ensemble_networks = [INPUT_DIR + 'ensemble-network.tsv', INPUT_DIR + 'empty-ensemble-network.tsv'] - input_network = OUT_DIR + 'data.pickle' - node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_networks, input_network) + input_data = OUT_DIR + 'data.pickle' + node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_networks, input_data) node_ensemble_dict['ensemble'].to_csv(out_path_file, sep='\t', index=False) assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-node-ensemble.csv', shallow=False) node_ensemble_dict['empty'].to_csv(out_path_empty_file, sep='\t', index=False) @@ -160,9 +169,11 @@ def test_precision_recall_curve_ensemble_nodes(self): out_path_png.unlink(missing_ok=True) out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) + input_data = OUT_DIR + 'data.pickle' + input_nodes = Dataset.from_file(input_data).get_input_nodes() ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble': ensemble_file} - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, out_path_png, + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_nodes, out_path_png, out_path_file) assert out_path_png.exists() assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-pr-curve-ensemble-nodes.txt', shallow=False) @@ -172,9 +183,11 @@ def test_precision_recall_curve_ensemble_nodes_empty(self): out_path_png.unlink(missing_ok=True) out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes-empty.txt') out_path_file.unlink(missing_ok=True) + input_data = OUT_DIR + 'data.pickle' + input_nodes = Dataset.from_file(input_data).get_input_nodes() empty_ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble-empty.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble': empty_ensemble_file} - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, out_path_png, + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_nodes, out_path_png, out_path_file) assert out_path_png.exists() assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-pr-curve-ensemble-nodes-empty.txt', shallow=False) @@ -184,10 +197,12 @@ def test_precision_recall_curve_multiple_ensemble_nodes(self): out_path_png.unlink(missing_ok=True) out_path_file = Path(OUT_DIR + 'pr-curve-multiple-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) + input_data = OUT_DIR + 'data.pickle' + input_nodes = Dataset.from_file(input_data).get_input_nodes() ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) empty_ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble-empty.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble1': ensemble_file, 'ensemble2': ensemble_file, 'ensemble3': empty_ensemble_file} - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, out_path_png, + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_nodes, out_path_png, out_path_file, True) assert out_path_png.exists() assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-pr-curve-multiple-ensemble-nodes.txt', shallow=False)