diff --git a/src/pmotools/pmo_engine/pmo_exporter.py b/src/pmotools/pmo_engine/pmo_exporter.py index 361ebd7..66238c7 100644 --- a/src/pmotools/pmo_engine/pmo_exporter.py +++ b/src/pmotools/pmo_engine/pmo_exporter.py @@ -121,10 +121,14 @@ def export_specimen_meta_table(pmodata, separator: str = ",") -> pd.DataFrame: for specimen in pmodata["specimen_info"]: export_row = {} for key, value in specimen.items(): - if "project_id" == key: + if "project_id" == key and "project_info" in pmodata: export_row["project_name"] = pmodata["project_info"][value][ "project_name" ] + elif "project_id" == key: + # project_info is optional as of schema v1.1.0; keep the raw id + # rather than failing to resolve a name that isn't available + export_row[key] = value elif PMOExporter._is_primitive(value): export_row[key] = value elif PMOExporter._is_primitive_list(value): @@ -145,10 +149,14 @@ def export_library_sample_meta_table(pmodata, separator: str = ",") -> pd.DataFr for library_sample in pmodata["library_sample_info"]: export_row = {} for key, value in library_sample.items(): - if "sequencing_info_id" == key: + if "sequencing_info_id" == key and "sequencing_info" in pmodata: export_row["sequencing_info_name"] = pmodata["sequencing_info"][ value ]["sequencing_info_name"] + elif "sequencing_info_id" == key: + # sequencing_info is optional as of schema v1.1.0; keep the raw id + # rather than failing to resolve a name that isn't available + export_row[key] = value elif "specimen_id" == key: export_row["specimen_name"] = pmodata["specimen_info"][value][ "specimen_name" diff --git a/src/pmotools/pmo_engine/pmo_processor.py b/src/pmotools/pmo_engine/pmo_processor.py index cef95fb..c452916 100644 --- a/src/pmotools/pmo_engine/pmo_processor.py +++ b/src/pmotools/pmo_engine/pmo_processor.py @@ -20,6 +20,10 @@ def get_index_key_of_bioinformatics_run_names(pmodata): :return: a dictionary of indexes keyed by bioinformatics_run_name """ ret = {} + # bioinformatics_run_info is optional as of schema v1.1.0; a PMO without it + # simply has no run-name to index mapping + if "bioinformatics_run_info" not in pmodata: + return ret for idx, bioinformatics_run in enumerate(pmodata["bioinformatics_run_info"]): ret[bioinformatics_run["bioinformatics_run_name"]] = idx return ret @@ -1073,23 +1077,29 @@ def extract_from_pmo_with_read_filter(pmodata, read_filter: float): pmo_out = { "pmo_header": copy.deepcopy(pmodata["pmo_header"]), "panel_info": copy.deepcopy(pmodata["panel_info"]), - "sequencing_info": copy.deepcopy(pmodata["sequencing_info"]), "target_info": copy.deepcopy(pmodata["target_info"]), "specimen_info": copy.deepcopy(pmodata["specimen_info"]), "library_sample_info": copy.deepcopy(pmodata["library_sample_info"]), - "project_info": copy.deepcopy(pmodata["project_info"]), - "targeted_genomes": copy.deepcopy(pmodata["targeted_genomes"]), "representative_microhaplotypes": copy.deepcopy( pmodata["representative_microhaplotypes"] ), - "bioinformatics_methods_info": copy.deepcopy( - pmodata["bioinformatics_methods_info"] - ), - "bioinformatics_run_info": copy.deepcopy( - pmodata["bioinformatics_run_info"] - ), "detected_microhaplotypes": [], } + # only copy optional sections if they are present (no longer required by the schema) + if "sequencing_info" in pmodata: + pmo_out["sequencing_info"] = copy.deepcopy(pmodata["sequencing_info"]) + if "project_info" in pmodata: + pmo_out["project_info"] = copy.deepcopy(pmodata["project_info"]) + if "targeted_genomes" in pmodata: + pmo_out["targeted_genomes"] = copy.deepcopy(pmodata["targeted_genomes"]) + if "bioinformatics_methods_info" in pmodata: + pmo_out["bioinformatics_methods_info"] = copy.deepcopy( + pmodata["bioinformatics_methods_info"] + ) + if "bioinformatics_run_info" in pmodata: + pmo_out["bioinformatics_run_info"] = copy.deepcopy( + pmodata["bioinformatics_run_info"] + ) # if has optional read_counts_by_stage then add as well # if does contain, @todo consider updating with new counts now that a filter has been applied diff --git a/tests/data/minimum_Furstenau2025_PMO.json.gz b/tests/data/minimum_Furstenau2025_PMO.json.gz new file mode 100644 index 0000000..b87e2e6 Binary files /dev/null and b/tests/data/minimum_Furstenau2025_PMO.json.gz differ diff --git a/tests/test_pmo_engine/test_pmo_exporter.py b/tests/test_pmo_engine/test_pmo_exporter.py index 1984bfa..89c8dfa 100755 --- a/tests/test_pmo_engine/test_pmo_exporter.py +++ b/tests/test_pmo_engine/test_pmo_exporter.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import copy import gzip import hashlib import os @@ -38,6 +39,15 @@ def setUp(self): "rt", ) as f: self.minimum_fields_v1_1_0_pmo_data = json.load(f) + # a real, schema-valid v1.1.0 PMO with only the required top-level sections + with gzip.open( + os.path.join( + os.path.dirname(self.working_dir), + "data/minimum_Furstenau2025_PMO.json.gz", + ), + "rt", + ) as f: + self.minimum_v1_1_0_pmo_data = json.load(f) def tearDown(self): self.test_dir.cleanup() @@ -245,6 +255,28 @@ def test_export_library_sample_meta_table(self): md5sum_of_fnp(os.path.join(self.test_dir.name, "library_sample_table.csv")), ) + def test_export_meta_tables_without_optional_cross_ref_sections(self): + # project_info and sequencing_info are optional as of v1.1.0; the specimen and + # library_sample meta-table exporters must not raise a raw KeyError when a + # referencing id is present but the referenced optional section is absent + self.assertNotIn("project_info", self.minimum_v1_1_0_pmo_data) + self.assertNotIn("sequencing_info", self.minimum_v1_1_0_pmo_data) + # a clean minimal PMO (no dangling ids) should export fine + PMOExporter.export_specimen_meta_table(self.minimum_v1_1_0_pmo_data) + PMOExporter.export_library_sample_meta_table(self.minimum_v1_1_0_pmo_data) + + # inject the referencing ids without the optional sections (referential dangling) + # the raw id is kept rather than resolving an unavailable name + dangling = copy.deepcopy(self.minimum_v1_1_0_pmo_data) + dangling["specimen_info"][0]["project_id"] = 0 + dangling["library_sample_info"][0]["sequencing_info_id"] = 0 + spec_table = PMOExporter.export_specimen_meta_table(dangling) + self.assertIn("project_id", spec_table.columns) + self.assertNotIn("project_name", spec_table.columns) + library_table = PMOExporter.export_library_sample_meta_table(dangling) + self.assertIn("sequencing_info_id", library_table.columns) + self.assertNotIn("sequencing_info_name", library_table.columns) + def test_export_sequencing_info_meta_table(self): sequencing_info_table = PMOExporter.export_sequencing_info_meta_table( self.small_example_pmo_data diff --git a/tests/test_pmo_engine/test_pmo_processor.py b/tests/test_pmo_engine/test_pmo_processor.py index 2c4a41b..2a48ba8 100755 --- a/tests/test_pmo_engine/test_pmo_processor.py +++ b/tests/test_pmo_engine/test_pmo_processor.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +import copy +import gzip import os import tempfile import unittest @@ -36,6 +38,16 @@ def setUp(self): ) ) as f: self.minimum_pmo_data = json.load(f) + # a real, schema-valid v1.1.0 PMO that contains only the required top-level + # sections (no sequencing_info/project_info/targeted_genomes/bioinformatics_*/ + # read_counts_by_stage) to guard against unguarded optional-section access + with gzip.open( + os.path.join( + os.path.dirname(self.working_dir), + "data/minimum_Furstenau2025_PMO.json.gz", + ) + ) as f: + self.minimum_v1_1_0_pmo_data = json.load(f) def tearDown(self): self.test_dir.cleanup() @@ -317,10 +329,52 @@ def test_extract_from_pmo_with_read_filter(self): ) with open(output_fnp, "w") as f: json.dump(pmo_data_filtered, f) - self.assertEqual("879b1a0c62a77a8fcc910a21d8ec9de6", md5sum_of_fnp(output_fnp)) + self.assertEqual("59e24e7efd4cd7822acde70828ad6119", md5sum_of_fnp(output_fnp)) checker = PMOChecker(self.pmo_jsonschema_data) checker.validate_pmo_json(pmo_data_filtered) + def test_extract_from_pmo_with_read_filter_missing_optional_sections(self): + # the v1.1.0 schema relaxed the required fields, so a PMO may omit these + # optional sections; the extracted PMO must not copy them in as null/empty keys + optional_sections = [ + "sequencing_info", + "project_info", + "targeted_genomes", + "bioinformatics_methods_info", + "bioinformatics_run_info", + "read_counts_by_stage", + ] + stripped_pmo_data = copy.deepcopy(self.combined_pmo_data) + for section in optional_sections: + stripped_pmo_data.pop(section, None) + pmo_data_filtered = PMOProcessor.extract_from_pmo_with_read_filter( + stripped_pmo_data, 1000 + ) + for section in optional_sections: + self.assertNotIn(section, pmo_data_filtered) + # the relaxed required fields are a v1.1.0 change, so validate against v1.1.0 + checker = PMOChecker( + load_schema("portable_microhaplotype_object_v1.1.0.schema.json") + ) + checker.validate_pmo_json(pmo_data_filtered) + + def test_bioinformatics_run_name_lookups_without_run_info(self): + # bioinformatics_run_info is optional as of v1.1.0; the run-name lookups + # must not raise a raw KeyError on a valid PMO that omits it + self.assertNotIn("bioinformatics_run_info", self.minimum_v1_1_0_pmo_data) + self.assertEqual( + {}, + PMOProcessor.get_index_key_of_bioinformatics_run_names( + self.minimum_v1_1_0_pmo_data + ), + ) + self.assertEqual( + [], + PMOProcessor.get_sorted_bioinformatics_run_names( + self.minimum_v1_1_0_pmo_data + ), + ) + def test_filter_pmo_by_target_ids(self): pmo_data_select_targets = PMOProcessor.filter_pmo_by_target_ids( self.combined_pmo_data, {1, 10, 11, 55}