Hotfix/various bug fixes

src/pmotools/pmo_engine/pmo_exporter.py

@@ -121,10 +121,14 @@ def export_specimen_meta_table(pmodata, separator: str = ",") -> pd.DataFrame:
         for specimen in pmodata["specimen_info"]:
             export_row = {}
             for key, value in specimen.items():
-                if "project_id" == key:
+                if "project_id" == key and "project_info" in pmodata:
                     export_row["project_name"] = pmodata["project_info"][value][
                         "project_name"
                     ]
+                elif "project_id" == key:
+                    # project_info is optional as of schema v1.1.0; keep the raw id
+                    # rather than failing to resolve a name that isn't available
+                    export_row[key] = value
                 elif PMOExporter._is_primitive(value):
                     export_row[key] = value
                 elif PMOExporter._is_primitive_list(value):
@@ -145,10 +149,14 @@ def export_library_sample_meta_table(pmodata, separator: str = ",") -> pd.DataFr
         for library_sample in pmodata["library_sample_info"]:
             export_row = {}
             for key, value in library_sample.items():
-                if "sequencing_info_id" == key:
+                if "sequencing_info_id" == key and "sequencing_info" in pmodata:
                     export_row["sequencing_info_name"] = pmodata["sequencing_info"][
                         value
                     ]["sequencing_info_name"]
+                elif "sequencing_info_id" == key:
+                    # sequencing_info is optional as of schema v1.1.0; keep the raw id
+                    # rather than failing to resolve a name that isn't available
+                    export_row[key] = value
                 elif "specimen_id" == key:
                     export_row["specimen_name"] = pmodata["specimen_info"][value][
                         "specimen_name"

src/pmotools/pmo_engine/pmo_processor.py

@@ -20,6 +20,10 @@ def get_index_key_of_bioinformatics_run_names(pmodata):
         :return: a dictionary of indexes keyed by bioinformatics_run_name
         """
         ret = {}
+        # bioinformatics_run_info is optional as of schema v1.1.0; a PMO without it
+        # simply has no run-name to index mapping
+        if "bioinformatics_run_info" not in pmodata:
+            return ret
         for idx, bioinformatics_run in enumerate(pmodata["bioinformatics_run_info"]):
             ret[bioinformatics_run["bioinformatics_run_name"]] = idx
         return ret
@@ -1073,23 +1077,29 @@ def extract_from_pmo_with_read_filter(pmodata, read_filter: float):
         pmo_out = {
             "pmo_header": copy.deepcopy(pmodata["pmo_header"]),
             "panel_info": copy.deepcopy(pmodata["panel_info"]),
-            "sequencing_info": copy.deepcopy(pmodata["sequencing_info"]),
             "target_info": copy.deepcopy(pmodata["target_info"]),
             "specimen_info": copy.deepcopy(pmodata["specimen_info"]),
             "library_sample_info": copy.deepcopy(pmodata["library_sample_info"]),
-            "project_info": copy.deepcopy(pmodata["project_info"]),
-            "targeted_genomes": copy.deepcopy(pmodata["targeted_genomes"]),
             "representative_microhaplotypes": copy.deepcopy(
                 pmodata["representative_microhaplotypes"]
             ),
-            "bioinformatics_methods_info": copy.deepcopy(
-                pmodata["bioinformatics_methods_info"]
-            ),
-            "bioinformatics_run_info": copy.deepcopy(
-                pmodata["bioinformatics_run_info"]
-            ),
             "detected_microhaplotypes": [],
         }
+        # only copy optional sections if they are present (no longer required by the schema)
+        if "sequencing_info" in pmodata:
+            pmo_out["sequencing_info"] = copy.deepcopy(pmodata["sequencing_info"])
+        if "project_info" in pmodata:
+            pmo_out["project_info"] = copy.deepcopy(pmodata["project_info"])
+        if "targeted_genomes" in pmodata:
+            pmo_out["targeted_genomes"] = copy.deepcopy(pmodata["targeted_genomes"])
+        if "bioinformatics_methods_info" in pmodata:
+            pmo_out["bioinformatics_methods_info"] = copy.deepcopy(
+                pmodata["bioinformatics_methods_info"]
+            )
+        if "bioinformatics_run_info" in pmodata:
+            pmo_out["bioinformatics_run_info"] = copy.deepcopy(
+                pmodata["bioinformatics_run_info"]
+            )
         # if has optional read_counts_by_stage then add as well
         # if does contain, @todo consider updating with new counts now that a filter has been applied
 

tests/test_pmo_engine/test_pmo_exporter.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import copy
 import gzip
 import hashlib
 import os
@@ -38,6 +39,15 @@ def setUp(self):
             "rt",
         ) as f:
             self.minimum_fields_v1_1_0_pmo_data = json.load(f)
+        # a real, schema-valid v1.1.0 PMO with only the required top-level sections
+        with gzip.open(
+            os.path.join(
+                os.path.dirname(self.working_dir),
+                "data/minimum_Furstenau2025_PMO.json.gz",
+            ),
+            "rt",
+        ) as f:
+            self.minimum_v1_1_0_pmo_data = json.load(f)
 
     def tearDown(self):
         self.test_dir.cleanup()
@@ -245,6 +255,28 @@ def test_export_library_sample_meta_table(self):
             md5sum_of_fnp(os.path.join(self.test_dir.name, "library_sample_table.csv")),
         )
 
+    def test_export_meta_tables_without_optional_cross_ref_sections(self):
+        # project_info and sequencing_info are optional as of v1.1.0; the specimen and
+        # library_sample meta-table exporters must not raise a raw KeyError when a
+        # referencing id is present but the referenced optional section is absent
+        self.assertNotIn("project_info", self.minimum_v1_1_0_pmo_data)
+        self.assertNotIn("sequencing_info", self.minimum_v1_1_0_pmo_data)
+        # a clean minimal PMO (no dangling ids) should export fine
+        PMOExporter.export_specimen_meta_table(self.minimum_v1_1_0_pmo_data)
+        PMOExporter.export_library_sample_meta_table(self.minimum_v1_1_0_pmo_data)
+
+        # inject the referencing ids without the optional sections (referential dangling)
+        # the raw id is kept rather than resolving an unavailable name
+        dangling = copy.deepcopy(self.minimum_v1_1_0_pmo_data)
+        dangling["specimen_info"][0]["project_id"] = 0
+        dangling["library_sample_info"][0]["sequencing_info_id"] = 0
+        spec_table = PMOExporter.export_specimen_meta_table(dangling)
+        self.assertIn("project_id", spec_table.columns)
+        self.assertNotIn("project_name", spec_table.columns)
+        library_table = PMOExporter.export_library_sample_meta_table(dangling)
+        self.assertIn("sequencing_info_id", library_table.columns)
+        self.assertNotIn("sequencing_info_name", library_table.columns)
+
     def test_export_sequencing_info_meta_table(self):
         sequencing_info_table = PMOExporter.export_sequencing_info_meta_table(
             self.small_example_pmo_data

tests/test_pmo_engine/test_pmo_processor.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 
+import copy
+import gzip
 import os
 import tempfile
 import unittest
@@ -36,6 +38,16 @@ def setUp(self):
             )
         ) as f:
             self.minimum_pmo_data = json.load(f)
+        # a real, schema-valid v1.1.0 PMO that contains only the required top-level
+        # sections (no sequencing_info/project_info/targeted_genomes/bioinformatics_*/
+        # read_counts_by_stage) to guard against unguarded optional-section access
+        with gzip.open(
+            os.path.join(
+                os.path.dirname(self.working_dir),
+                "data/minimum_Furstenau2025_PMO.json.gz",
+            )
+        ) as f:
+            self.minimum_v1_1_0_pmo_data = json.load(f)
 
     def tearDown(self):
         self.test_dir.cleanup()
@@ -317,10 +329,52 @@ def test_extract_from_pmo_with_read_filter(self):
         )
         with open(output_fnp, "w") as f:
             json.dump(pmo_data_filtered, f)
-        self.assertEqual("879b1a0c62a77a8fcc910a21d8ec9de6", md5sum_of_fnp(output_fnp))
+        self.assertEqual("59e24e7efd4cd7822acde70828ad6119", md5sum_of_fnp(output_fnp))
         checker = PMOChecker(self.pmo_jsonschema_data)
         checker.validate_pmo_json(pmo_data_filtered)
 
+    def test_extract_from_pmo_with_read_filter_missing_optional_sections(self):
+        # the v1.1.0 schema relaxed the required fields, so a PMO may omit these
+        # optional sections; the extracted PMO must not copy them in as null/empty keys
+        optional_sections = [
+            "sequencing_info",
+            "project_info",
+            "targeted_genomes",
+            "bioinformatics_methods_info",
+            "bioinformatics_run_info",
+            "read_counts_by_stage",
+        ]
+        stripped_pmo_data = copy.deepcopy(self.combined_pmo_data)
+        for section in optional_sections:
+            stripped_pmo_data.pop(section, None)
+        pmo_data_filtered = PMOProcessor.extract_from_pmo_with_read_filter(
+            stripped_pmo_data, 1000
+        )
+        for section in optional_sections:
+            self.assertNotIn(section, pmo_data_filtered)
+        # the relaxed required fields are a v1.1.0 change, so validate against v1.1.0
+        checker = PMOChecker(
+            load_schema("portable_microhaplotype_object_v1.1.0.schema.json")
+        )
+        checker.validate_pmo_json(pmo_data_filtered)
+
+    def test_bioinformatics_run_name_lookups_without_run_info(self):
+        # bioinformatics_run_info is optional as of v1.1.0; the run-name lookups
+        # must not raise a raw KeyError on a valid PMO that omits it
+        self.assertNotIn("bioinformatics_run_info", self.minimum_v1_1_0_pmo_data)
+        self.assertEqual(
+            {},
+            PMOProcessor.get_index_key_of_bioinformatics_run_names(
+                self.minimum_v1_1_0_pmo_data
+            ),
+        )
+        self.assertEqual(
+            [],
+            PMOProcessor.get_sorted_bioinformatics_run_names(
+                self.minimum_v1_1_0_pmo_data
+            ),
+        )
+
     def test_filter_pmo_by_target_ids(self):
         pmo_data_select_targets = PMOProcessor.filter_pmo_by_target_ids(
             self.combined_pmo_data, {1, 10, 11, 55}