From 91a16c9616d85c845eced7d987d49530039572b8 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 10 Apr 2026 17:31:50 +0100 Subject: [PATCH 01/22] Separate the bulk of the '_analyse' logic from the while loop it is in; '_analyse' is now run in the while loop from '_analyse_in_thread'; grouped SPA/tomo-specific Analyser attributes in a separate block --- src/murfey/client/analyser.py | 338 +++++++++++++++++----------------- 1 file changed, 168 insertions(+), 170 deletions(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index 88ee0721..dd15ec53 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -56,23 +56,14 @@ def __init__( ): super().__init__() self._basepath = basepath_local.absolute() + self._token = token + self._environment = environment self._limited = limited self._experiment_type = "" self._acquisition_software = "" - self._extension: str = "" - self._unseen_xml: list = [] self._context: Context | None = None - self._batch_store: dict = {} - self._environment = environment - self._force_mdoc_metadata = force_mdoc_metadata - self._token = token - self._serialem = serialem - self.parameters_model: ( - Type[ProcessingParametersSPA] | Type[ProcessingParametersTomo] | None - ) = None - self.queue: queue.Queue = queue.Queue() - self.thread = threading.Thread(name="Analyser", target=self._analyse) + self.thread = threading.Thread(name="Analyser", target=self._analyse_in_thread) self._stopping = False self._halt_thread = False self._murfey_config = ( @@ -85,6 +76,17 @@ def __init__( else {} ) + # SPA & Tomo-specific attributes + self._extension: str = "" + self._unseen_xml: list = [] + self._batch_store: dict = {} + self._force_mdoc_metadata = force_mdoc_metadata + self._mdoc_for_reading: Path | None = None + self._serialem = serialem + self.parameters_model: ( + Type[ProcessingParametersSPA] | Type[ProcessingParametersTomo] | None + ) = None + def __repr__(self) -> str: return f"" @@ -334,9 +336,8 @@ def post_transfer(self, transferred_file: Path): f"An exception was encountered post transfer: {e}", exc_info=True ) - def _analyse(self): + def _analyse_in_thread(self): logger.info("Analyser thread started") - mdoc_for_reading = None while not self._halt_thread: transferred_file = self.queue.get() transferred_file = ( @@ -347,185 +348,182 @@ def _analyse(self): if not transferred_file: self._halt_thread = True continue - if self._limited: - if ( - "Metadata" in transferred_file.parts - or transferred_file.name == "EpuSession.dm" - and not self._context - ): - if not (context := _get_context("SPAMetadataContext")): - continue - self._context = context.load()( - "epu", - self._basepath, - self._murfey_config, - self._token, - ) - elif ( - "Batch" in transferred_file.parts - or "SearchMaps" in transferred_file.parts - or transferred_file.name == "Session.dm" - and not self._context - ): - if not (context := _get_context("TomographyMetadataContext")): - continue - self._context = context.load()( - "tomo", - self._basepath, - self._murfey_config, - self._token, - ) - self.post_transfer(transferred_file) - else: - dc_metadata = {} - if not self._serialem and ( - self._force_mdoc_metadata - and transferred_file.suffix == ".mdoc" - or mdoc_for_reading - ): - if self._context: - try: - dc_metadata = self._context.gather_metadata( - mdoc_for_reading or transferred_file, - environment=self._environment, - ) - except KeyError as e: - logger.error( - f"Metadata gathering failed with a key error for key: {e.args[0]}" - ) - raise e - if not dc_metadata: - mdoc_for_reading = None - elif transferred_file.suffix == ".mdoc": - mdoc_for_reading = transferred_file - if not self._context: - if not self._find_extension(transferred_file): - logger.debug(f"No extension found for {transferred_file}") - continue - if not self._find_context(transferred_file): - logger.debug( - f"Couldn't find context for {str(transferred_file)!r}" + self._analyse(transferred_file) + self.queue.task_done() + logger.debug("Analyer thread has stopped analysing incoming files") + self.notify(final=True) + + def _analyse(self, transferred_file: Path): + if self._limited: + if ( + "Metadata" in transferred_file.parts + or transferred_file.name == "EpuSession.dm" + and not self._context + ): + if not (context := _get_context("SPAMetadataContext")): + return + self._context = context.load()( + "epu", + self._basepath, + self._murfey_config, + self._token, + ) + elif ( + "Batch" in transferred_file.parts + or "SearchMaps" in transferred_file.parts + or transferred_file.name == "Session.dm" + and not self._context + ): + if not (context := _get_context("TomographyMetadataContext")): + return + self._context = context.load()( + "tomo", + self._basepath, + self._murfey_config, + self._token, + ) + self.post_transfer(transferred_file) + else: + dc_metadata = {} + if not self._serialem and ( + self._force_mdoc_metadata + and transferred_file.suffix == ".mdoc" + or self._mdoc_for_reading + ): + if self._context: + try: + dc_metadata = self._context.gather_metadata( + self._mdoc_for_reading or transferred_file, + environment=self._environment, ) - self.queue.task_done() - continue - elif self._extension: - logger.info( - f"Context found successfully for {transferred_file}" + except KeyError as e: + logger.error( + f"Metadata gathering failed with a key error for key: {e.args[0]}" ) - try: + raise e + if not dc_metadata: + self._mdoc_for_reading = None + elif transferred_file.suffix == ".mdoc": + self._mdoc_for_reading = transferred_file + if not self._context: + if not self._find_extension(transferred_file): + logger.debug(f"No extension found for {transferred_file}") + return + if not self._find_context(transferred_file): + logger.debug(f"Couldn't find context for {str(transferred_file)!r}") + return + elif self._extension: + logger.info(f"Context found successfully for {transferred_file}") + try: + if self._context is not None: self._context.post_first_transfer( transferred_file, environment=self._environment, ) - except Exception as e: - logger.error(f"Exception encountered: {e}") - if "AtlasContext" not in str(self._context): - if not dc_metadata: - try: + except Exception as e: + logger.error(f"Exception encountered: {e}") + if "AtlasContext" not in str(self._context): + if not dc_metadata: + try: + if self._context is not None: dc_metadata = self._context.gather_metadata( self._xml_file(transferred_file), environment=self._environment, ) - except NotImplementedError: - dc_metadata = {} - except KeyError as e: - logger.error( - f"Metadata gathering failed with a key error for key: {e.args[0]}" - ) - raise e - except ValueError as e: - logger.error( - f"Metadata gathering failed with a value error: {e}" - ) - if not dc_metadata or not self._force_mdoc_metadata: - self._unseen_xml.append(transferred_file) - else: - self._unseen_xml = [] - if dc_metadata.get("file_extension"): - self._extension = dc_metadata["file_extension"] - else: - dc_metadata["file_extension"] = self._extension - dc_metadata["acquisition_software"] = ( - self._context._acquisition_software - ) - self.notify(dc_metadata) - - # Contexts that can be immediately posted without additional work - elif "CLEMContext" in str(self._context): - logger.debug( - f"File {transferred_file.name!r} is part of CLEM workflow" - ) - self.post_transfer(transferred_file) - elif "FIBContext" in str(self._context): - logger.debug( - f"File {transferred_file.name!r} is part of the FIB workflow" - ) - self.post_transfer(transferred_file) - elif "SXTContext" in str(self._context): - logger.debug(f"File {transferred_file.name!r} is an SXT file") - self.post_transfer(transferred_file) - elif "AtlasContext" in str(self._context): - logger.debug(f"File {transferred_file.name!r} is part of the atlas") - self.post_transfer(transferred_file) - - # Handle files with tomography and SPA context differently - elif not self._extension or self._unseen_xml: - if not self._find_extension(transferred_file): - logger.error(f"No extension found for {transferred_file}") - continue - if self._extension: - logger.info( - f"Extension found successfully for {transferred_file}" - ) - try: - self._context.post_first_transfer( - transferred_file, - environment=self._environment, - ) - except Exception as e: - logger.error(f"Exception encountered: {e}") - if not dc_metadata: - try: - dc_metadata = self._context.gather_metadata( - mdoc_for_reading - or self._xml_file(transferred_file), - environment=self._environment, - ) + except NotImplementedError: + dc_metadata = {} except KeyError as e: logger.error( f"Metadata gathering failed with a key error for key: {e.args[0]}" ) raise e + except ValueError as e: + logger.error( + f"Metadata gathering failed with a value error: {e}" + ) if not dc_metadata or not self._force_mdoc_metadata: - mdoc_for_reading = None self._unseen_xml.append(transferred_file) - if dc_metadata: + else: self._unseen_xml = [] if dc_metadata.get("file_extension"): self._extension = dc_metadata["file_extension"] else: dc_metadata["file_extension"] = self._extension - dc_metadata["acquisition_software"] = ( - self._context._acquisition_software + if self._context is not None: + dc_metadata["acquisition_software"] = ( + self._context._acquisition_software + ) + self.notify(dc_metadata) + + # Contexts that can be immediately posted without additional work + elif "CLEMContext" in str(self._context): + logger.debug(f"File {transferred_file.name!r} is part of CLEM workflow") + self.post_transfer(transferred_file) + elif "FIBContext" in str(self._context): + logger.debug( + f"File {transferred_file.name!r} is part of the FIB workflow" + ) + self.post_transfer(transferred_file) + elif "SXTContext" in str(self._context): + logger.debug(f"File {transferred_file.name!r} is an SXT file") + self.post_transfer(transferred_file) + elif "AtlasContext" in str(self._context): + logger.debug(f"File {transferred_file.name!r} is part of the atlas") + self.post_transfer(transferred_file) + + # Handle files with tomography and SPA context differently + elif not self._extension or self._unseen_xml: + if not self._find_extension(transferred_file): + logger.error(f"No extension found for {transferred_file}") + return + if self._extension: + logger.info(f"Extension found successfully for {transferred_file}") + try: + self._context.post_first_transfer( + transferred_file, + environment=self._environment, + ) + except Exception as e: + logger.error(f"Exception encountered: {e}") + if not dc_metadata: + try: + dc_metadata = self._context.gather_metadata( + self._mdoc_for_reading + or self._xml_file(transferred_file), + environment=self._environment, ) - self.notify(dc_metadata) - elif any( - context in str(self._context) - for context in ( - "SPAContext", - "SPAMetadataContext", - "TomographyContext", - "TomographyMetadataContext", - ) - ): - context = str(self._context).split(" ")[0].split(".")[-1] - logger.debug( - f"Transferring file {str(transferred_file)} with context {context!r}" - ) - self.post_transfer(transferred_file) - self.queue.task_done() - logger.debug("Analyer thread has stopped analysing incoming files") - self.notify(final=True) + except KeyError as e: + logger.error( + f"Metadata gathering failed with a key error for key: {e.args[0]}" + ) + raise e + if not dc_metadata or not self._force_mdoc_metadata: + self._mdoc_for_reading = None + self._unseen_xml.append(transferred_file) + if dc_metadata: + self._unseen_xml = [] + if dc_metadata.get("file_extension"): + self._extension = dc_metadata["file_extension"] + else: + dc_metadata["file_extension"] = self._extension + dc_metadata["acquisition_software"] = ( + self._context._acquisition_software + ) + self.notify(dc_metadata) + elif any( + context in str(self._context) + for context in ( + "SPAContext", + "SPAMetadataContext", + "TomographyContext", + "TomographyMetadataContext", + ) + ): + context = str(self._context).split(" ")[0].split(".")[-1] + logger.debug( + f"Transferring file {str(transferred_file)} with context {context!r}" + ) + self.post_transfer(transferred_file) def _xml_file(self, data_file: Path) -> Path: if not self._environment: From ef1923ab5d2321ef8f956818a8e46d2fc9829559 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 10 Apr 2026 17:42:18 +0100 Subject: [PATCH 02/22] Explicitly determine which contexts to permit to enter the DC metadata-parsing logic block --- src/murfey/client/analyser.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index dd15ec53..4049f2a0 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -422,7 +422,15 @@ def _analyse(self, transferred_file: Path): ) except Exception as e: logger.error(f"Exception encountered: {e}") - if "AtlasContext" not in str(self._context): + if any( + context in str(self._context) + for context in ( + "SPAContext", + "SPAMetadataContext", + "TomographyContext", + "TomographyMetadataContext", + ) + ): if not dc_metadata: try: if self._context is not None: From e348b9b1d09e3517e2c1ae6425a223a7439e69c1 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 13 Apr 2026 13:24:43 +0100 Subject: [PATCH 03/22] Updated type hinting and rearranged functions in 'murfey.client.context' --- src/murfey/client/context.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/murfey/client/context.py b/src/murfey/client/context.py index c64f713c..73172e62 100644 --- a/src/murfey/client/context.py +++ b/src/murfey/client/context.py @@ -3,7 +3,7 @@ import logging from importlib.metadata import entry_points from pathlib import Path -from typing import Any, List, NamedTuple +from typing import Any, NamedTuple, OrderedDict import xmltodict @@ -209,12 +209,6 @@ def ensure_dcg_exists( return dcg_tag -class ProcessingParameter(NamedTuple): - name: str - label: str - default: Any = None - - def detect_acquisition_software(dir_for_transfer: Path) -> str: glob = dir_for_transfer.glob("*") for f in glob: @@ -225,9 +219,15 @@ def detect_acquisition_software(dir_for_transfer: Path) -> str: return "" +class ProcessingParameter(NamedTuple): + name: str + label: str + default: Any = None + + class Context: - user_params: List[ProcessingParameter] = [] - metadata_params: List[ProcessingParameter] = [] + user_params: list[ProcessingParameter] = [] + metadata_params: list[ProcessingParameter] = [] def __init__(self, name: str, acquisition_software: str, token: str): self._acquisition_software = acquisition_software @@ -256,7 +256,7 @@ def post_first_transfer( def gather_metadata( self, metadata_file: Path, environment: MurfeyInstanceEnvironment | None = None - ): + ) -> OrderedDict | None: raise NotImplementedError( f"gather_metadata must be declared in derived class to be used: {self}" ) From a410fa6412f714168eb9b0dfb2ea410230fe0eb9 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 13 Apr 2026 13:29:06 +0100 Subject: [PATCH 04/22] Adjusted logic for '_analyse' so that context-specific metadata processing is done within the relevant Context blocks --- src/murfey/client/analyser.py | 198 ++++++++++++++-------------------- 1 file changed, 79 insertions(+), 119 deletions(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index 4049f2a0..a9164c30 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -14,7 +14,7 @@ import threading from importlib.metadata import entry_points from pathlib import Path -from typing import Type +from typing import OrderedDict, Type from murfey.client.context import Context from murfey.client.destinations import find_longest_data_directory @@ -384,87 +384,24 @@ def _analyse(self, transferred_file: Path): ) self.post_transfer(transferred_file) else: - dc_metadata = {} + # Logic that doesn't require context determination if not self._serialem and ( - self._force_mdoc_metadata - and transferred_file.suffix == ".mdoc" - or self._mdoc_for_reading + self._force_mdoc_metadata and transferred_file.suffix == ".mdoc" ): - if self._context: - try: - dc_metadata = self._context.gather_metadata( - self._mdoc_for_reading or transferred_file, - environment=self._environment, - ) - except KeyError as e: - logger.error( - f"Metadata gathering failed with a key error for key: {e.args[0]}" - ) - raise e - if not dc_metadata: - self._mdoc_for_reading = None - elif transferred_file.suffix == ".mdoc": - self._mdoc_for_reading = transferred_file - if not self._context: - if not self._find_extension(transferred_file): - logger.debug(f"No extension found for {transferred_file}") - return + self._mdoc_for_reading = transferred_file + + # Try and determine context, and notify once when context is found + if self._context is None: + # Exit early if the file can't be used to determine the context if not self._find_context(transferred_file): logger.debug(f"Couldn't find context for {str(transferred_file)!r}") return - elif self._extension: - logger.info(f"Context found successfully for {transferred_file}") - try: - if self._context is not None: - self._context.post_first_transfer( - transferred_file, - environment=self._environment, - ) - except Exception as e: - logger.error(f"Exception encountered: {e}") - if any( - context in str(self._context) - for context in ( - "SPAContext", - "SPAMetadataContext", - "TomographyContext", - "TomographyMetadataContext", - ) - ): - if not dc_metadata: - try: - if self._context is not None: - dc_metadata = self._context.gather_metadata( - self._xml_file(transferred_file), - environment=self._environment, - ) - except NotImplementedError: - dc_metadata = {} - except KeyError as e: - logger.error( - f"Metadata gathering failed with a key error for key: {e.args[0]}" - ) - raise e - except ValueError as e: - logger.error( - f"Metadata gathering failed with a value error: {e}" - ) - if not dc_metadata or not self._force_mdoc_metadata: - self._unseen_xml.append(transferred_file) - else: - self._unseen_xml = [] - if dc_metadata.get("file_extension"): - self._extension = dc_metadata["file_extension"] - else: - dc_metadata["file_extension"] = self._extension - if self._context is not None: - dc_metadata["acquisition_software"] = ( - self._context._acquisition_software - ) - self.notify(dc_metadata) - - # Contexts that can be immediately posted without additional work - elif "CLEMContext" in str(self._context): + else: + logger.info(f"Context found successfully using {transferred_file}") + + # Trigger processing or metadata parsing according to the context + # Go through the straightforward ones first + if "CLEMContext" in str(self._context): logger.debug(f"File {transferred_file.name!r} is part of CLEM workflow") self.post_transfer(transferred_file) elif "FIBContext" in str(self._context): @@ -480,59 +417,82 @@ def _analyse(self, transferred_file: Path): self.post_transfer(transferred_file) # Handle files with tomography and SPA context differently - elif not self._extension or self._unseen_xml: - if not self._find_extension(transferred_file): - logger.error(f"No extension found for {transferred_file}") - return - if self._extension: - logger.info(f"Extension found successfully for {transferred_file}") + elif ( + any( + context in str(self._context) + for context in ( + "SPAContext", + "SPAMetadataContext", + "TomographyContext", + "TomographyMetadataContext", + ) + ) + and self._context is not None + ): + context = str(self._context).split(" ")[0].split(".")[-1] + + dc_metadata: OrderedDict | None = None + if not self._serialem and ( + self._force_mdoc_metadata + and transferred_file.suffix == ".mdoc" + or self._mdoc_for_reading + ): try: - self._context.post_first_transfer( - transferred_file, + dc_metadata = self._context.gather_metadata( + self._mdoc_for_reading or transferred_file, environment=self._environment, ) - except Exception as e: - logger.error(f"Exception encountered: {e}") + except KeyError as e: + logger.error( + f"Metadata gathering failed with a key error for key: " + f"{e.args[0]}" + ) + raise e + # Set the mdoc field to None if no metadata was found if not dc_metadata: - try: - dc_metadata = self._context.gather_metadata( - self._mdoc_for_reading - or self._xml_file(transferred_file), - environment=self._environment, - ) - except KeyError as e: - logger.error( - f"Metadata gathering failed with a key error for key: {e.args[0]}" - ) - raise e - if not dc_metadata or not self._force_mdoc_metadata: self._mdoc_for_reading = None - self._unseen_xml.append(transferred_file) - if dc_metadata: - self._unseen_xml = [] - if dc_metadata.get("file_extension"): - self._extension = dc_metadata["file_extension"] - else: - dc_metadata["file_extension"] = self._extension - dc_metadata["acquisition_software"] = ( - self._context._acquisition_software + + if not self._extension or self._unseen_xml: + # Early return if no extension was found + if not self._find_extension(transferred_file): + logger.warning(f"No extension found for {transferred_file}") + return + else: + logger.info( + f"Extension found successfully for {transferred_file}" ) - self.notify(dc_metadata) - elif any( - context in str(self._context) - for context in ( - "SPAContext", - "SPAMetadataContext", - "TomographyContext", - "TomographyMetadataContext", - ) - ): - context = str(self._context).split(" ")[0].split(".")[-1] + logger.debug( f"Transferring file {str(transferred_file)} with context {context!r}" ) self.post_transfer(transferred_file) + if not dc_metadata and transferred_file.suffix != ".mdoc": + try: + dc_metadata = self._context.gather_metadata( + self._mdoc_for_reading or self._xml_file(transferred_file), + environment=self._environment, + ) + except KeyError as e: + logger.error( + f"Metadata gathering failed with a key error for key: {e.args[0]}" + ) + raise e + if not dc_metadata or not self._force_mdoc_metadata: + self._mdoc_for_reading = None + self._unseen_xml.append(transferred_file) + if dc_metadata: + self._unseen_xml = [] + if dc_metadata.get("file_extension"): + self._extension = dc_metadata["file_extension"] + else: + dc_metadata["file_extension"] = self._extension + dc_metadata["acquisition_software"] = ( + self._context._acquisition_software + ) + self.notify(dc_metadata) + return + def _xml_file(self, data_file: Path) -> Path: if not self._environment: return data_file.with_suffix(".xml") From b848a97dfc460ca12a416098d0366e7938b4344f Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 13 Apr 2026 13:33:06 +0100 Subject: [PATCH 05/22] Typo --- src/murfey/client/analyser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index a9164c30..44e80a8e 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -350,7 +350,7 @@ def _analyse_in_thread(self): continue self._analyse(transferred_file) self.queue.task_done() - logger.debug("Analyer thread has stopped analysing incoming files") + logger.debug("Analyser thread has stopped analysing incoming files") self.notify(final=True) def _analyse(self, transferred_file: Path): From 4cb78c512c55b361f576af7f381b4333d6947da3 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 13 Apr 2026 13:57:11 +0100 Subject: [PATCH 06/22] Streamlined data structure for example files used in context test --- tests/client/test_analyser.py | 131 ++++++++++++++-------------------- 1 file changed, 52 insertions(+), 79 deletions(-) diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index 3c21b380..eb3bb00a 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -3,105 +3,78 @@ import pytest from murfey.client.analyser import Analyser -from murfey.client.contexts.atlas import AtlasContext -from murfey.client.contexts.clem import CLEMContext -from murfey.client.contexts.fib import FIBContext from murfey.client.contexts.spa import SPAContext -from murfey.client.contexts.spa_metadata import SPAMetadataContext -from murfey.client.contexts.sxt import SXTContext from murfey.client.contexts.tomo import TomographyContext -from murfey.client.contexts.tomo_metadata import TomographyMetadataContext from murfey.util.models import ProcessingParametersSPA, ProcessingParametersTomo -example_files = [ - # Tomography - ["visit/Position_1_001_0.0_20250715_012434_fractions.tiff", TomographyContext], - ["visit/Position_1_2_002_3.0_20250715_012434_Fractions.mrc", TomographyContext], - ["visit/Position_1_2_003_6.0_20250715_012434_EER.eer", TomographyContext], - ["visit/name1_004_9.0_20250715_012434_fractions.tiff", TomographyContext], - ["visit/Position_1_[30.0].tiff", TomographyContext], - ["visit/Position_1.mdoc", TomographyContext], - ["visit/name1_2.mdoc", TomographyContext], - # Tomography metadata - ["visit/Session.dm", TomographyMetadataContext], - ["visit/SearchMaps/SearchMap.xml", TomographyMetadataContext], - ["visit/Batch/BatchPositionsList.xml", TomographyMetadataContext], - ["visit/Thumbnails/file.mrc", TomographyMetadataContext], - # SPA - ["visit/FoilHole_01234_fractions.tiff", SPAContext], - ["visit/FoilHole_01234_EER.eer", SPAContext], - # SPA metadata - ["atlas/atlas.mrc", AtlasContext], - ["visit/EpuSession.dm", SPAMetadataContext], - ["visit/Metadata/GridSquare.dm", SPAMetadataContext], - # CLEM LIF file - ["visit/images/test_file.lif", CLEMContext], - # CLEM TIFF files - [ +example_files = { + "CLEMContext": [ + # CLEM LIF file + "visit/images/test_file.lif", + # CLEM TIFF files "visit/images/2024_03_14_12_34_56--Project001/grid1/Position 12--Z02--C01.tif", - CLEMContext, - ], - [ "visit/images/2024_03_14_12_34_56--Project001/grid1/Position 12_Lng_LVCC--Z02--C01.tif", - CLEMContext, - ], - [ "visit/images/2024_03_14_12_34_56--Project001/grid1/Series001--Z00--C00.tif", - CLEMContext, - ], - [ "visit/images/2024_03_14_12_34_56--Project001/grid1/Series001_Lng_LVCC--Z00--C00.tif", - CLEMContext, - ], - # CLEM TIFF file accompanying metadata - [ + # CLEM TIFF file accompanying metadata "visit/images/2024_03_14_12_34_56--Project001/grid1/Metadata/Position 12.xlif", - CLEMContext, - ], - [ "visit/images/2024_03_14_12_34_56--Project001/grid1/Metadata/Position 12_Lng_LVCC.xlif", - CLEMContext, - ], - [ "visit/images/2024_03_14_12_34_56--Project001/grid1/Position 12/Metadata/Position 12_histo.xlif", - CLEMContext, - ], - [ "visit/images/2024_03_14_12_34_56--Project001/grid1/Position 12/Metadata/Position 12_Lng_LVCC_histo.xlif", - CLEMContext, - ], - [ "visit/images/2024_03_14_12_34_56--Project001/grid1/Metadata/Series001.xlif", - CLEMContext, - ], - [ "visit/images/2024_03_14_12_34_56--Project001/grid1/Metadata/Series001_Lng_LVCC.xlif", - CLEMContext, ], - # FIB Autotem files - ["visit/autotem/visit/ProjectData.dat", FIBContext], - ["visit/autotem/visit/Sites/Lamella/SetupImages/Preparation.tif", FIBContext], - [ + "FIBContext": [ + # FIB Autotem files + "visit/autotem/visit/ProjectData.dat", + "visit/autotem/visit/Sites/Lamella/SetupImages/Preparation.tif", "visit/autotem/visit/Sites/Lamella (2)//DCImages/DCM_2026-03-09-23-45-40.926/2026-03-09-23-48-43-Finer-Milling-dc_rescan-image-.png", - FIBContext, - ], - # FIB Maps files - ["visit/maps/visit/EMproject.emxml", FIBContext], - [ + # FIB Maps files + "visit/maps/visit/EMproject.emxml", "visit/maps/visit/LayersData/Layer/Electron Snapshot/Electron Snapshot.tiff", - FIBContext, - ], - [ "visit/maps/visit/LayersData/Layer/Electron Snapshot (2)/Electron Snapshot (2).tiff", - FIBContext, ], - # Soft x-ray tomography - ["visit/tomo__tag_ROI10_area1_angle-60to60@1.5_1sec_251p.txrm", SXTContext], - ["visit/X-ray_mosaic_ROI2.xrm", SXTContext], -] + "SXTContext": [ + "visit/tomo__tag_ROI10_area1_angle-60to60@1.5_1sec_251p.txrm", + "visit/X-ray_mosaic_ROI2.xrm", + ], + "AtlasContext": [ + "atlas/atlas.mrc", + ], + "TomographyContext": [ + "visit/Position_1_001_0.0_20250715_012434_fractions.tiff", + "visit/Position_1_2_002_3.0_20250715_012434_Fractions.mrc", + "visit/Position_1_2_003_6.0_20250715_012434_EER.eer", + "visit/name1_004_9.0_20250715_012434_fractions.tiff", + "visit/Position_1_[30.0].tiff", + "visit/Position_1.mdoc", + "visit/name1_2.mdoc", + ], + "TomographyMetadataContext": [ + "visit/Session.dm", + "visit/SearchMaps/SearchMap.xml", + "visit/Batch/BatchPositionsList.xml", + "visit/Thumbnails/file.mrc", + ], + "SPAContext": [ + "visit/FoilHole_01234_fractions.tiff", + "visit/FoilHole_01234_EER.eer", + ], + "SPAMetadataContext": [ + "visit/EpuSession.dm", + "visit/Metadata/GridSquare.dm", + ], +} -@pytest.mark.parametrize("file_and_context", example_files) +@pytest.mark.parametrize( + "file_and_context", + [ + [file, context] + for context, file_list in example_files.items() + for file in file_list + ], +) def test_find_context(file_and_context, tmp_path): # Unpack parametrised variables file_name, context = file_and_context @@ -111,7 +84,7 @@ def test_find_context(file_and_context, tmp_path): # Check that the results are as expected assert analyser._find_context(tmp_path / file_name) - assert isinstance(analyser._context, context) + assert analyser._context is not None and context in str(analyser._context) # Checks for the specific workflow contexts if isinstance(analyser._context, TomographyContext): From ee8a1750f9f65836e772ea599d6151925a909d32 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 13 Apr 2026 13:58:14 +0100 Subject: [PATCH 07/22] Add stubs for tests for the different supported workflows --- tests/client/test_analyser.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index eb3bb00a..fd5e252e 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -141,3 +141,23 @@ def test_analyser_epu_determination(tmp_path): analyser.queue.put(tomo_file) analyser.stop() assert analyser._context._acquisition_software == "epu" + + +def test_analyse_clem(): + pass + + +def test_analyse_fib(): + pass + + +def test_analyse_sxt(): + pass + + +def test_analyse_spa(): + pass + + +def test_analyse_tomo(): + pass From 7852f0421db60db6edeb3f2489a6c4f49123a24e Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 13 Apr 2026 14:44:54 +0100 Subject: [PATCH 08/22] Added tests for '_analyse' for the CLEM, FIB, and SXT workflows, as well as for contextless files --- tests/client/test_analyser.py | 118 ++++++++++++++++++++++++++++++++-- 1 file changed, 112 insertions(+), 6 deletions(-) diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index fd5e252e..c10690e3 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -1,6 +1,9 @@ from __future__ import annotations +from pathlib import Path + import pytest +from pytest_mock import MockerFixture from murfey.client.analyser import Analyser from murfey.client.contexts.spa import SPAContext @@ -143,16 +146,119 @@ def test_analyser_epu_determination(tmp_path): assert analyser._context._acquisition_software == "epu" -def test_analyse_clem(): - pass +@pytest.mark.parametrize("test_file", contextless_files) +def test_analyse_no_context( + test_file: str, + mocker: MockerFixture, + tmp_path: Path, +): + # Mock the 'post_transfer' class function + mock_post_transfer = mocker.patch.object(Analyser, "post_transfer") + spy_find_context = mocker.spy(Analyser, "_find_context") + # Initialise the Analyser + analyser = Analyser(tmp_path, "") + analyser._analyse(tmp_path / test_file) -def test_analyse_fib(): - pass + # "_find_context" should be called + assert spy_find_context.call_count == 1 + # "post_transfer" should not be called + mock_post_transfer.assert_not_called() -def test_analyse_sxt(): - pass + +def test_analyse_clem( + mocker: MockerFixture, + tmp_path: Path, +): + # Gather example files related to the CLEM workflow + test_files = [ + file + for context, file_list in example_files.items() + for file in file_list + if context == "CLEMContext" and not file.endswith(".lif") + ] + + # Mock the 'post_transfer' class function + mock_post_transfer = mocker.patch.object(Analyser, "post_transfer") + spy_find_context = mocker.spy(Analyser, "_find_context") + + # Initialise the Analyser + analyser = Analyser(tmp_path, "") + for file in test_files: + analyser._analyse(tmp_path / file) + + # "_find_context" should be called only once + assert spy_find_context.call_count == 1 + assert analyser._context is not None and "CLEMContext" in str(analyser._context) + + # "_post_transfer" should be called on every one of these files + assert mock_post_transfer.call_count == len(test_files) + + +@pytest.mark.parametrize( + "test_params", + # Test the "autotem" and "maps" workflows separately + [ + [software, [file for file in file_list if software in file]] + for software in ("autotem", "maps") + for context, file_list in example_files.items() + if context == "FIBContext" + ], +) +def test_analyse_fib( + mocker: MockerFixture, + test_params: tuple[str, list[str]], + tmp_path: Path, +): + # Unpack test params + software, test_files = test_params + + # Mock the 'post_transfer' class function + mock_post_transfer = mocker.patch.object(Analyser, "post_transfer") + spy_find_context = mocker.spy(Analyser, "_find_context") + + # Initialise the Analyser + analyser = Analyser(tmp_path, "") + for file in test_files: + analyser._analyse(tmp_path / file) + + # "_find_context" should be called only once + assert spy_find_context.call_count == 1 + assert analyser._context is not None and "FIBContext" in str(analyser._context) + assert analyser._context._acquisition_software == software + + # "_post_transfer" should be called on every one of these files + assert mock_post_transfer.call_count == len(test_files) + + +def test_analyse_sxt( + mocker: MockerFixture, + tmp_path: Path, +): + # Load the example files corresponding to the SXT workflow + test_files = [ + file + for context, file_list in example_files.items() + for file in file_list + if context == "SXTContext" + ] + + # Mock the 'post_transfer' class function + mock_post_transfer = mocker.patch.object(Analyser, "post_transfer") + spy_find_context = mocker.spy(Analyser, "_find_context") + + # Initialise the Analyser + analyser = Analyser(tmp_path, "") + for file in test_files: + analyser._analyse(tmp_path / file) + + # "_find_context" should be called only once + assert spy_find_context.call_count == 1 + assert analyser._context is not None and "SXTContext" in str(analyser._context) + + # "_post_transfer" should be called on every one of these files + assert mock_post_transfer.call_count == len(test_files) def test_analyse_spa(): From 00508df6ab57a45a8bafb9fc9e29e11b9193819b Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 13 Apr 2026 16:38:39 +0100 Subject: [PATCH 09/22] Added tests for the 'limited' logic block in '_analyse' --- tests/client/test_analyser.py | 36 +++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index c10690e3..65d57c44 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -261,6 +261,42 @@ def test_analyse_sxt( assert mock_post_transfer.call_count == len(test_files) +@pytest.mark.parametrize( + "test_params", + [ + ["SPAMetadataContext", ["AtlasContext", "SPAContext"]], + ["TomographyMetadataContext", ["AtlasContext", "SPAContext"]], + ], +) +def test_analyse_limited( + mocker: MockerFixture, tmp_path: Path, test_params: tuple[str, list[str]] +): + # Unpack test params + expected_context, other_contexts = test_params + + # Load example files related to the CLEM + test_files = [ + file + for context, file_list in example_files.items() + for file in file_list + if context in (expected_context, *other_contexts) + ] + + # Mock the 'post_transfer' class function + mock_post_transfer = mocker.patch.object(Analyser, "post_transfer") + + # Initialise the Analyser + analyser = Analyser(tmp_path, "", limited=True) + for file in test_files: + analyser._analyse(tmp_path / file) + + # "_find_context" should be called only once + assert analyser._context is not None and expected_context in str(analyser._context) + + # "_post_transfer" should be called on every one of these files + assert mock_post_transfer.call_count == len(test_files) + + def test_analyse_spa(): pass From a76e2d0a31b3bc4780c92cdd601a175c7796db14 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 13 Apr 2026 18:14:16 +0100 Subject: [PATCH 10/22] Added test for atlas analysis and SPA analysis --- tests/client/test_analyser.py | 64 +++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index 65d57c44..242ef52a 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -297,8 +297,68 @@ def test_analyse_limited( assert mock_post_transfer.call_count == len(test_files) -def test_analyse_spa(): - pass +def test_analyse_atlas( + mocker: MockerFixture, + tmp_path: Path, +): + test_files = [ + file + for context, file_list in example_files.items() + for file in file_list + if context == "AtlasContext" + ] + + # Mock the 'post_transfer' class function + mock_post_transfer = mocker.patch.object(Analyser, "post_transfer") + spy_find_context = mocker.spy(Analyser, "_find_context") + + # Initialise the Analyser + analyser = Analyser(tmp_path, "", force_mdoc_metadata=True) + for file in test_files: + analyser._analyse(tmp_path / file) + + # Context should be set + assert analyser._context is not None and "AtlasContext" in str(analyser._context) + + # "_find_context" should be called once + assert spy_find_context.call_count == 1 + + # "post_transfer" should be called on all files + assert mock_post_transfer.call_count == len(test_files) + + +@pytest.mark.parametrize( + "context_to_test", + ("SPAContext", "SPAMetadataContext"), +) +def test_analyse_spa( + mocker: MockerFixture, + context_to_test: str, + tmp_path: Path, +): + test_files = [ + file + for context, file_list in example_files.items() + for file in file_list + if context == context_to_test + ] + + # Mock the 'post_transfer' class function + mock_post_transfer = mocker.patch.object(Analyser, "post_transfer") + spy_find_context = mocker.spy(Analyser, "_find_context") + spy_find_extension = mocker.spy(Analyser, "_find_extension") + + # Initialise the Analyser + analyser = Analyser(tmp_path, "", force_mdoc_metadata=True) + for file in test_files: + analyser._analyse(tmp_path / file) + + assert spy_find_context.call_count == 1 + assert analyser._context is not None and context_to_test in str(analyser._context) + if context_to_test == "SPAContext": + assert spy_find_extension.call_count > 0 + assert analyser._extension in (".eer", ".tiff") + mock_post_transfer.assert_called() def test_analyse_tomo(): From fb07598e10b3487202eefa8dac755c7073b2b477 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Apr 2026 11:23:45 +0100 Subject: [PATCH 11/22] Add preliminary test for '_analyse' for tomo workflow --- tests/client/test_analyser.py | 45 +++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index 242ef52a..534161ad 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -308,7 +308,7 @@ def test_analyse_atlas( if context == "AtlasContext" ] - # Mock the 'post_transfer' class function + # Set up mocks and spies mock_post_transfer = mocker.patch.object(Analyser, "post_transfer") spy_find_context = mocker.spy(Analyser, "_find_context") @@ -343,7 +343,7 @@ def test_analyse_spa( if context == context_to_test ] - # Mock the 'post_transfer' class function + # Set up mocks and spies mock_post_transfer = mocker.patch.object(Analyser, "post_transfer") spy_find_context = mocker.spy(Analyser, "_find_context") spy_find_extension = mocker.spy(Analyser, "_find_extension") @@ -361,5 +361,42 @@ def test_analyse_spa( mock_post_transfer.assert_called() -def test_analyse_tomo(): - pass +@pytest.mark.parametrize( + "context_to_test", + ("TomographyContext", "TomographyMetadataContext"), +) +def test_analyse_tomo( + mocker: MockerFixture, + context_to_test: str, + tmp_path: Path, +): + test_files = [ + file + for context, file_list in example_files.items() + for file in file_list + if context == context_to_test + ] + + # Set up mocks and spies + mock_post_transfer = mocker.patch.object(Analyser, "post_transfer") + if context_to_test == "TomographyContext": + mock_gather_metadata = mocker.patch( + "murfey.client.contexts.tomo.TomographyContext.gather_metadata" + ) + else: + mock_gather_metadata = mocker.patch( + "murfey.client.contexts.tomo_metadata.TomographyMetadataContext.gather_metadata" + ) + mock_gather_metadata.return_value = {"dummy": "dummy"} + + spy_find_context = mocker.spy(Analyser, "_find_context") + spy_find_extension = mocker.spy(Analyser, "_find_extension") + + # Initialise the Analyser + analyser = Analyser(tmp_path, "", force_mdoc_metadata=True) + for file in test_files: + analyser._analyse(tmp_path / file) + + assert spy_find_context.call_count == 1 + assert spy_find_extension.call_count > 0 + mock_post_transfer.assert_called() From 7e00b7e6e5ebe9a081b8f43990cae2075756ca87 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Apr 2026 13:45:41 +0100 Subject: [PATCH 12/22] Added test for the '_find_extension' class function --- tests/client/test_analyser.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index 534161ad..ff94e73b 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -70,6 +70,23 @@ } +@pytest.mark.parametrize( + "test_file", + [ + file + for file_list in example_files.values() + for file in file_list + for suffix in (".mrc", ".tiff", ".tif", ".eer", ".lif", ".txrm", ".xrm") + if file.endswith(suffix) + ], +) +def test_find_extension(test_file: str, tmp_path: Path): + analyser = Analyser(basepath_local=tmp_path, token="") + # Pass the file to the function, and check the outputs are as expected + assert analyser._find_extension(tmp_path / test_file) + assert test_file.endswith(analyser._extension) + + @pytest.mark.parametrize( "file_and_context", [ @@ -82,14 +99,14 @@ def test_find_context(file_and_context, tmp_path): # Unpack parametrised variables file_name, context = file_and_context - # Pass the file to the Analyser; add environment as needed + # Set up the Analyser analyser = Analyser(basepath_local=tmp_path, token="") - # Check that the results are as expected + # Pass the file to the function, and check that outputs are as expected assert analyser._find_context(tmp_path / file_name) assert analyser._context is not None and context in str(analyser._context) - # Checks for the specific workflow contexts + # Additional checks for specific contexts if isinstance(analyser._context, TomographyContext): assert analyser.parameters_model == ProcessingParametersTomo if isinstance(analyser._context, SPAContext): From 884bbc02417af6a46ea01afcbd01be85bd24d4ab Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Apr 2026 15:32:50 +0100 Subject: [PATCH 13/22] Missing brackets around the 'or' clauses in the 'if self._limited' block of logic --- src/murfey/client/analyser.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index 44e80a8e..b9107060 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -358,8 +358,7 @@ def _analyse(self, transferred_file: Path): if ( "Metadata" in transferred_file.parts or transferred_file.name == "EpuSession.dm" - and not self._context - ): + ) and not self._context: if not (context := _get_context("SPAMetadataContext")): return self._context = context.load()( @@ -372,8 +371,7 @@ def _analyse(self, transferred_file: Path): "Batch" in transferred_file.parts or "SearchMaps" in transferred_file.parts or transferred_file.name == "Session.dm" - and not self._context - ): + ) and not self._context: if not (context := _get_context("TomographyMetadataContext")): return self._context = context.load()( From b912c2ef7252747a27da79323e9f9c064d6bbdcc Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Apr 2026 17:33:15 +0100 Subject: [PATCH 14/22] Updated Context names and removed deprecated logic for parsing XML metadata from the TomographyContext --- src/murfey/client/contexts/atlas.py | 2 +- src/murfey/client/contexts/clem.py | 2 +- src/murfey/client/contexts/fib.py | 2 +- src/murfey/client/contexts/spa.py | 2 +- src/murfey/client/contexts/spa_metadata.py | 2 +- src/murfey/client/contexts/sxt.py | 2 +- src/murfey/client/contexts/tomo.py | 38 ++------------------- src/murfey/client/contexts/tomo_metadata.py | 2 +- 8 files changed, 9 insertions(+), 43 deletions(-) diff --git a/src/murfey/client/contexts/atlas.py b/src/murfey/client/contexts/atlas.py index 6269a350..563cc8a8 100644 --- a/src/murfey/client/contexts/atlas.py +++ b/src/murfey/client/contexts/atlas.py @@ -19,7 +19,7 @@ def __init__( machine_config: dict, token: str, ): - super().__init__("Atlas", acquisition_software, token) + super().__init__("AtlasContext", acquisition_software, token) self._basepath = basepath self._machine_config = machine_config diff --git a/src/murfey/client/contexts/clem.py b/src/murfey/client/contexts/clem.py index e3827c42..e4f23441 100644 --- a/src/murfey/client/contexts/clem.py +++ b/src/murfey/client/contexts/clem.py @@ -87,7 +87,7 @@ def __init__( machine_config: dict, token: str, ): - super().__init__("CLEM", acquisition_software, token) + super().__init__("CLEMContext", acquisition_software, token) self._basepath = basepath self._machine_config = machine_config # CLEM contexts for "auto-save" acquisition mode diff --git a/src/murfey/client/contexts/fib.py b/src/murfey/client/contexts/fib.py index d1cb43fa..f23ddf37 100644 --- a/src/murfey/client/contexts/fib.py +++ b/src/murfey/client/contexts/fib.py @@ -81,7 +81,7 @@ def __init__( machine_config: dict, token: str, ): - super().__init__("FIB", acquisition_software, token) + super().__init__("FIBContext", acquisition_software, token) self._basepath = basepath self._machine_config = machine_config self._milling: dict[int, list[MillingProgress]] = {} diff --git a/src/murfey/client/contexts/spa.py b/src/murfey/client/contexts/spa.py index 9bff4f19..78f98111 100644 --- a/src/murfey/client/contexts/spa.py +++ b/src/murfey/client/contexts/spa.py @@ -82,7 +82,7 @@ def __init__( machine_config: dict, token: str, ): - super().__init__("SPA", acquisition_software, token) + super().__init__("SPAContext", acquisition_software, token) self._basepath = basepath self._machine_config = machine_config self._processing_job_stash: dict = {} diff --git a/src/murfey/client/contexts/spa_metadata.py b/src/murfey/client/contexts/spa_metadata.py index 34a58c92..25ef0f05 100644 --- a/src/murfey/client/contexts/spa_metadata.py +++ b/src/murfey/client/contexts/spa_metadata.py @@ -81,7 +81,7 @@ def __init__( machine_config: dict, token: str, ): - super().__init__("SPA_metadata", acquisition_software, token) + super().__init__("SPAMetadataContext", acquisition_software, token) self._basepath = basepath self._machine_config = machine_config diff --git a/src/murfey/client/contexts/sxt.py b/src/murfey/client/contexts/sxt.py index 8a3a4b43..95964350 100644 --- a/src/murfey/client/contexts/sxt.py +++ b/src/murfey/client/contexts/sxt.py @@ -28,7 +28,7 @@ def __init__( machine_config: dict, token: str, ): - super().__init__("SXT", acquisition_software, token) + super().__init__("SXTContext", acquisition_software, token) self._basepath = basepath self._machine_config = machine_config diff --git a/src/murfey/client/contexts/tomo.py b/src/murfey/client/contexts/tomo.py index 5dba3cbb..f4115007 100644 --- a/src/murfey/client/contexts/tomo.py +++ b/src/murfey/client/contexts/tomo.py @@ -5,8 +5,6 @@ from threading import RLock from typing import Callable, Dict, List, OrderedDict -import xmltodict - import murfey.util.eer from murfey.client.context import Context, ProcessingParameter, ensure_dcg_exists from murfey.client.instance_environment import ( @@ -84,7 +82,7 @@ def __init__( machine_config: dict, token: str, ): - super().__init__("Tomography", acquisition_software, token) + super().__init__("TomographyContext", acquisition_software, token) self._basepath = basepath self._machine_config = machine_config self._tilt_series: Dict[str, List[Path]] = {} @@ -550,7 +548,7 @@ def post_first_transfer( def gather_metadata( self, metadata_file: Path, environment: MurfeyInstanceEnvironment | None = None ) -> OrderedDict: - if metadata_file.suffix not in (".mdoc", ".xml"): + if metadata_file.suffix != ".mdoc": raise ValueError( f"Tomography gather_metadata method expected xml or mdoc file not {metadata_file.name}" ) @@ -558,38 +556,6 @@ def gather_metadata( if not metadata_file.is_file(): logger.debug(f"Metadata file {metadata_file} not found") return OrderedDict({}) - if metadata_file.suffix == ".xml": - with open(metadata_file, "r") as xml: - try: - for_parsing = xml.read() - except Exception: - logger.warning( - f"Failed to parse file {metadata_file}", exc_info=True - ) - return OrderedDict({}) - data = xmltodict.parse(for_parsing) - try: - metadata: OrderedDict = OrderedDict({}) - metadata["experiment_type"] = "tomography" - metadata["voltage"] = 300 - metadata["image_size_x"] = data["Acquisition"]["Info"]["ImageSize"][ - "Width" - ] - metadata["image_size_y"] = data["Acquisition"]["Info"]["ImageSize"][ - "Height" - ] - metadata["pixel_size_on_image"] = float( - data["Acquisition"]["Info"]["SensorPixelSize"]["Height"] - ) - metadata["motion_corr_binning"] = 1 - metadata["gain_ref"] = None - metadata["dose_per_frame"] = ( - environment.dose_per_frame if environment else None - ) - metadata["source"] = str(self._basepath) - except KeyError: - return OrderedDict({}) - return metadata with open(metadata_file, "r") as md: mdoc_data = get_global_data(md) num_blocks = get_num_blocks(md) diff --git a/src/murfey/client/contexts/tomo_metadata.py b/src/murfey/client/contexts/tomo_metadata.py index 0f67089e..09f2780d 100644 --- a/src/murfey/client/contexts/tomo_metadata.py +++ b/src/murfey/client/contexts/tomo_metadata.py @@ -26,7 +26,7 @@ def __init__( machine_config: dict, token: str, ): - super().__init__("Tomography_metadata", acquisition_software, token) + super().__init__("TomographyMetadataContext", acquisition_software, token) self._basepath = basepath self._machine_config = machine_config From 24fee4a993937253f50ab3bfe9ddf372faf35b85 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Apr 2026 17:44:35 +0100 Subject: [PATCH 15/22] Fixed broken test --- tests/client/contexts/test_atlas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/client/contexts/test_atlas.py b/tests/client/contexts/test_atlas.py index 915f9b3e..cda7cc2d 100644 --- a/tests/client/contexts/test_atlas.py +++ b/tests/client/contexts/test_atlas.py @@ -7,7 +7,7 @@ def test_atlas_context_initialisation(tmp_path): context = AtlasContext("tomo", tmp_path, {}, "token") - assert context.name == "Atlas" + assert context.name == "AtlasContext" assert context._acquisition_software == "tomo" assert context._basepath == tmp_path assert context._machine_config == {} From a51a5b4fa3d39e361312b916b7215617fb4cdf6f Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Apr 2026 18:01:31 +0100 Subject: [PATCH 16/22] Major rewrite of the '_analyse' function: * Disentangled the processing logic for the 4 SPA- and Tomography-related contexts * Replaced matching Contexts using 'str(self._context)' with 'self._context.name' * Replaced giant if-else blocks with match-case logic * Use explicit returns --- src/murfey/client/analyser.py | 222 +++++++++++++++++----------------- 1 file changed, 108 insertions(+), 114 deletions(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index b9107060..e06241d2 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -14,7 +14,7 @@ import threading from importlib.metadata import entry_points from pathlib import Path -from typing import OrderedDict, Type +from typing import Type from murfey.client.context import Context from murfey.client.destinations import find_longest_data_directory @@ -78,10 +78,10 @@ def __init__( # SPA & Tomo-specific attributes self._extension: str = "" - self._unseen_xml: list = [] - self._batch_store: dict = {} - self._force_mdoc_metadata = force_mdoc_metadata - self._mdoc_for_reading: Path | None = None + self._processing_params_found: bool = ( + False # Have the processing parameters been collected from the metadata? + ) + # self._force_mdoc_metadata = force_mdoc_metadata # Seems deprecated self._serialem = serialem self.parameters_model: ( Type[ProcessingParametersSPA] | Type[ProcessingParametersTomo] | None @@ -94,10 +94,6 @@ def _find_extension(self, file_path: Path) -> bool: """ Identifies the file extension and stores that information in the class. """ - if "atlas" in file_path.parts: - self._extension = file_path.suffix - return True - if ( required_substrings := self._murfey_config.get( "data_required_substrings", {} @@ -125,14 +121,6 @@ def _find_extension(self, file_path: Path) -> bool: if subframe_path := mdoc_data_block.get("SubFramePath"): self._extension = Path(subframe_path).suffix return True - # Check for LIF files and TXRM files separately - elif ( - file_path.suffix == ".lif" - or file_path.suffix == ".txrm" - or file_path.suffix == ".xrm" - ): - self._extension = file_path.suffix - return True return False def _find_context(self, file_path: Path) -> bool: @@ -337,6 +325,11 @@ def post_transfer(self, transferred_file: Path): ) def _analyse_in_thread(self): + """ + Class function that will be executed by the '_thread' attribute. It will + execute a while-loop in which is takes files of the queue and feeds them + into the '_analyse' class function until '_halt_thread' is set to True. + """ logger.info("Analyser thread started") while not self._halt_thread: transferred_file = self.queue.get() @@ -354,13 +347,18 @@ def _analyse_in_thread(self): self.notify(final=True) def _analyse(self, transferred_file: Path): + """ + Class function that is called by '_analyse_in_thread'. It will identify + the Context class to use based on the files inspected, then run different + processing logic based on the context that was established. + """ if self._limited: if ( "Metadata" in transferred_file.parts or transferred_file.name == "EpuSession.dm" ) and not self._context: if not (context := _get_context("SPAMetadataContext")): - return + return None self._context = context.load()( "epu", self._basepath, @@ -373,7 +371,7 @@ def _analyse(self, transferred_file: Path): or transferred_file.name == "Session.dm" ) and not self._context: if not (context := _get_context("TomographyMetadataContext")): - return + return None self._context = context.load()( "tomo", self._basepath, @@ -382,114 +380,110 @@ def _analyse(self, transferred_file: Path): ) self.post_transfer(transferred_file) else: - # Logic that doesn't require context determination - if not self._serialem and ( - self._force_mdoc_metadata and transferred_file.suffix == ".mdoc" - ): - self._mdoc_for_reading = transferred_file - # Try and determine context, and notify once when context is found if self._context is None: # Exit early if the file can't be used to determine the context if not self._find_context(transferred_file): logger.debug(f"Couldn't find context for {str(transferred_file)!r}") - return + return None else: logger.info(f"Context found successfully using {transferred_file}") - # Trigger processing or metadata parsing according to the context - # Go through the straightforward ones first - if "CLEMContext" in str(self._context): - logger.debug(f"File {transferred_file.name!r} is part of CLEM workflow") - self.post_transfer(transferred_file) - elif "FIBContext" in str(self._context): - logger.debug( - f"File {transferred_file.name!r} is part of the FIB workflow" - ) - self.post_transfer(transferred_file) - elif "SXTContext" in str(self._context): - logger.debug(f"File {transferred_file.name!r} is an SXT file") - self.post_transfer(transferred_file) - elif "AtlasContext" in str(self._context): - logger.debug(f"File {transferred_file.name!r} is part of the atlas") - self.post_transfer(transferred_file) - - # Handle files with tomography and SPA context differently - elif ( - any( - context in str(self._context) - for context in ( - "SPAContext", - "SPAMetadataContext", - "TomographyContext", - "TomographyMetadataContext", - ) - ) - and self._context is not None - ): - context = str(self._context).split(" ")[0].split(".")[-1] - - dc_metadata: OrderedDict | None = None - if not self._serialem and ( - self._force_mdoc_metadata - and transferred_file.suffix == ".mdoc" - or self._mdoc_for_reading + # Extra if-block for MyPy to verify that the context is set by this point + if self._context is None: + logger.error("Failed to set context even after finding context") + return None + + # Trigger processing and metadata parsing according to the context + match self._context.name: + case ( + "CLEMContext" + | "FIBContext" + | "SPAMetadataContext" + | "SXTContext" + | "TomographyMetadataContext" ): - try: - dc_metadata = self._context.gather_metadata( - self._mdoc_for_reading or transferred_file, - environment=self._environment, - ) - except KeyError as e: - logger.error( - f"Metadata gathering failed with a key error for key: " - f"{e.args[0]}" - ) - raise e - # Set the mdoc field to None if no metadata was found - if not dc_metadata: - self._mdoc_for_reading = None - - if not self._extension or self._unseen_xml: - # Early return if no extension was found - if not self._find_extension(transferred_file): - logger.warning(f"No extension found for {transferred_file}") - return - else: + logger.debug( + f"File {transferred_file.name!r} transferred with context {self._context.name}" + ) + self.post_transfer(transferred_file) + case "SPAContext": + logger.debug(f"File {transferred_file.name!r} is part of the atlas") + self.post_transfer(transferred_file) + + # Find extension + if not self._extension: + if not self._find_extension(transferred_file): + logger.warning(f"No extension found for {transferred_file}") + return None logger.info( f"Extension found successfully for {transferred_file}" ) - - logger.debug( - f"Transferring file {str(transferred_file)} with context {context!r}" - ) - self.post_transfer(transferred_file) - - if not dc_metadata and transferred_file.suffix != ".mdoc": - try: - dc_metadata = self._context.gather_metadata( - self._mdoc_for_reading or self._xml_file(transferred_file), - environment=self._environment, - ) - except KeyError as e: - logger.error( - f"Metadata gathering failed with a key error for key: {e.args[0]}" + if not self._processing_params_found: + # Try and gather the metadata from each file passing through + # Once gathered, set the attribute to True and don't repeat again + try: + dc_metadata = self._context.gather_metadata( + self._xml_file(transferred_file), + environment=self._environment, + ) + except (KeyError, ValueError) as e: + logger.error( + f"Metadata gathering failed with the following error: {e}" + ) + dc_metadata = None + if dc_metadata: + self._processing_params_found = True + if dc_metadata.get("file_extension"): + self._extension = dc_metadata["file_extension"] + else: + dc_metadata["file_extension"] = self._extension + dc_metadata["acquisition_software"] = ( + self._context._acquisition_software + ) + self.notify(dc_metadata) + + case "TomographyContext": + logger.debug(f"File {transferred_file.name!r} is part of the atlas") + self.post_transfer(transferred_file) + + # Find extension + if not self._extension: + if not self._find_extension(transferred_file): + logger.warning(f"No extension found for {transferred_file}") + return None + logger.info( + f"Extension found successfully for {transferred_file}" ) - raise e - if not dc_metadata or not self._force_mdoc_metadata: - self._mdoc_for_reading = None - self._unseen_xml.append(transferred_file) - if dc_metadata: - self._unseen_xml = [] - if dc_metadata.get("file_extension"): - self._extension = dc_metadata["file_extension"] - else: - dc_metadata["file_extension"] = self._extension - dc_metadata["acquisition_software"] = ( - self._context._acquisition_software - ) - self.notify(dc_metadata) - return + if ( + not self._processing_params_found + and transferred_file.suffix == ".mdoc" + ): + # Try and gather the metadata from a passing .mdoc file + # When gathered, set the attribute to True and don't repeat again + try: + dc_metadata = self._context.gather_metadata( + transferred_file, + environment=self._environment, + ) + except (KeyError, ValueError) as e: + logger.error( + f"Metadata gathering failed with the following error: {e}" + ) + dc_metadata = None + if dc_metadata: + self._processing_params_found = True + if dc_metadata.get("file_extension"): + self._extension = dc_metadata["file_extension"] + else: + dc_metadata["file_extension"] = self._extension + dc_metadata["acquisition_software"] = ( + self._context._acquisition_software + ) + self.notify(dc_metadata) + case _: + logger.warning(f"Unknown context provided: {str(self._context)}") + return None def _xml_file(self, data_file: Path) -> Path: if not self._environment: From 76b529db7526234e4f66933c6d615a9cb43ef5c8 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Apr 2026 18:23:50 +0100 Subject: [PATCH 17/22] Forgot to include 'AtlasContext' in match-case logic --- src/murfey/client/analyser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index e06241d2..d1e53817 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -397,7 +397,8 @@ def _analyse(self, transferred_file: Path): # Trigger processing and metadata parsing according to the context match self._context.name: case ( - "CLEMContext" + "AtlasContext" + | "CLEMContext" | "FIBContext" | "SPAMetadataContext" | "SXTContext" From 0373096c21cd7f37b12f1a30d8669a4e513844f0 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Apr 2026 18:33:02 +0100 Subject: [PATCH 18/22] No need for else-block --- src/murfey/client/analyser.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index d1e53817..79c6ab28 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -386,8 +386,7 @@ def _analyse(self, transferred_file: Path): if not self._find_context(transferred_file): logger.debug(f"Couldn't find context for {str(transferred_file)!r}") return None - else: - logger.info(f"Context found successfully using {transferred_file}") + logger.info(f"Context found successfully using {transferred_file}") # Extra if-block for MyPy to verify that the context is set by this point if self._context is None: From d2122ad4e151b0caad11bc89122d386479cf490d Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Apr 2026 18:36:19 +0100 Subject: [PATCH 19/22] Updated Analyser tests after rewrite --- tests/client/test_analyser.py | 107 +++++++++++++++++++--------------- 1 file changed, 59 insertions(+), 48 deletions(-) diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index ff94e73b..7bab99ea 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -1,13 +1,12 @@ from __future__ import annotations from pathlib import Path +from unittest.mock import mock_open import pytest from pytest_mock import MockerFixture from murfey.client.analyser import Analyser -from murfey.client.contexts.spa import SPAContext -from murfey.client.contexts.tomo import TomographyContext from murfey.util.models import ProcessingParametersSPA, ProcessingParametersTomo example_files = { @@ -76,15 +75,30 @@ file for file_list in example_files.values() for file in file_list - for suffix in (".mrc", ".tiff", ".tif", ".eer", ".lif", ".txrm", ".xrm") + for suffix in (".mrc", ".tiff", ".tif", ".eer", ".mdoc") if file.endswith(suffix) ], ) -def test_find_extension(test_file: str, tmp_path: Path): - analyser = Analyser(basepath_local=tmp_path, token="") +def test_find_extension( + mocker: MockerFixture, + test_file: str, + tmp_path: Path, +): + # Mock the functions used to open a .mdoc file to return a dummy file path + m = mock_open(read_data="dummy data") + mocker.patch("murfey.client.analyser.open", m) + mocker.patch( + "murfey.client.analyser.get_block", + return_value={"SubFramePath": "/path/to/test_file.tiff"}, + ) + # Pass the file to the function, and check the outputs are as expected + analyser = Analyser(basepath_local=tmp_path, token="") assert analyser._find_extension(tmp_path / test_file) - assert test_file.endswith(analyser._extension) + if not test_file.endswith(".mdoc"): + assert test_file.endswith(analyser._extension) + else: + assert analyser._extension == ".tiff" @pytest.mark.parametrize( @@ -107,9 +121,9 @@ def test_find_context(file_and_context, tmp_path): assert analyser._context is not None and context in str(analyser._context) # Additional checks for specific contexts - if isinstance(analyser._context, TomographyContext): + if analyser._context is not None and analyser._context.name == "TomographyContext": assert analyser.parameters_model == ProcessingParametersTomo - if isinstance(analyser._context, SPAContext): + if analyser._context is not None and analyser._context.name == "SPAContext": assert analyser.parameters_model == ProcessingParametersSPA @@ -279,24 +293,23 @@ def test_analyse_sxt( @pytest.mark.parametrize( - "test_params", + "context_to_test", [ - ["SPAMetadataContext", ["AtlasContext", "SPAContext"]], - ["TomographyMetadataContext", ["AtlasContext", "SPAContext"]], + "SPAMetadataContext", + "TomographyMetadataContext", ], ) def test_analyse_limited( - mocker: MockerFixture, tmp_path: Path, test_params: tuple[str, list[str]] + mocker: MockerFixture, + context_to_test: str, + tmp_path: Path, ): - # Unpack test params - expected_context, other_contexts = test_params - # Load example files related to the CLEM test_files = [ file for context, file_list in example_files.items() for file in file_list - if context in (expected_context, *other_contexts) + if context in context_to_test ] # Mock the 'post_transfer' class function @@ -308,21 +321,36 @@ def test_analyse_limited( analyser._analyse(tmp_path / file) # "_find_context" should be called only once - assert analyser._context is not None and expected_context in str(analyser._context) + assert analyser._context is not None and analyser._context.name == context_to_test # "_post_transfer" should be called on every one of these files assert mock_post_transfer.call_count == len(test_files) -def test_analyse_atlas( +@pytest.mark.parametrize( + "context_to_test", + [ + "AtlasContext", + "CLEMContext", + "FIBContext", + "SPAMetadataContext", + "SXTContext", + "TomographyMetadataContext", + ], +) +def test_analyse_generic( mocker: MockerFixture, + context_to_test: str, tmp_path: Path, ): + """ + Tests the Contexts which has straightforward processing logic. + """ test_files = [ file for context, file_list in example_files.items() for file in file_list - if context == "AtlasContext" + if context == context_to_test ] # Set up mocks and spies @@ -334,30 +362,25 @@ def test_analyse_atlas( for file in test_files: analyser._analyse(tmp_path / file) - # Context should be set - assert analyser._context is not None and "AtlasContext" in str(analyser._context) - # "_find_context" should be called once assert spy_find_context.call_count == 1 + # Context should be set + assert analyser._context is not None and analyser._context.name == context_to_test + # "post_transfer" should be called on all files assert mock_post_transfer.call_count == len(test_files) -@pytest.mark.parametrize( - "context_to_test", - ("SPAContext", "SPAMetadataContext"), -) def test_analyse_spa( mocker: MockerFixture, - context_to_test: str, tmp_path: Path, ): test_files = [ file for context, file_list in example_files.items() for file in file_list - if context == context_to_test + if context == "SPAContext" ] # Set up mocks and spies @@ -371,41 +394,29 @@ def test_analyse_spa( analyser._analyse(tmp_path / file) assert spy_find_context.call_count == 1 - assert analyser._context is not None and context_to_test in str(analyser._context) - if context_to_test == "SPAContext": - assert spy_find_extension.call_count > 0 - assert analyser._extension in (".eer", ".tiff") - mock_post_transfer.assert_called() + assert analyser._context is not None and analyser._context.name == "SPAContext" + assert spy_find_extension.call_count > 0 + assert analyser._extension in (".eer", ".tiff") + mock_post_transfer.assert_called() -@pytest.mark.parametrize( - "context_to_test", - ("TomographyContext", "TomographyMetadataContext"), -) def test_analyse_tomo( mocker: MockerFixture, - context_to_test: str, tmp_path: Path, ): test_files = [ file for context, file_list in example_files.items() for file in file_list - if context == context_to_test + if context == "TomographyContext" ] # Set up mocks and spies mock_post_transfer = mocker.patch.object(Analyser, "post_transfer") - if context_to_test == "TomographyContext": - mock_gather_metadata = mocker.patch( - "murfey.client.contexts.tomo.TomographyContext.gather_metadata" - ) - else: - mock_gather_metadata = mocker.patch( - "murfey.client.contexts.tomo_metadata.TomographyMetadataContext.gather_metadata" - ) + mock_gather_metadata = mocker.patch( + "murfey.client.contexts.tomo.TomographyContext.gather_metadata" + ) mock_gather_metadata.return_value = {"dummy": "dummy"} - spy_find_context = mocker.spy(Analyser, "_find_context") spy_find_extension = mocker.spy(Analyser, "_find_extension") From 2575978058bef4f516b4367290cd2b6288d0ecec Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Apr 2026 19:03:55 +0100 Subject: [PATCH 20/22] Expanded '_analyse' tests for the SPA and Tomo contexts --- tests/client/test_analyser.py | 58 ++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index 7bab99ea..5deda832 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -372,8 +372,10 @@ def test_analyse_generic( assert mock_post_transfer.call_count == len(test_files) +@pytest.mark.parametrize("has_extension", [True, False]) def test_analyse_spa( mocker: MockerFixture, + has_extension: bool, tmp_path: Path, ): test_files = [ @@ -384,24 +386,46 @@ def test_analyse_spa( ] # Set up mocks and spies + mock_metadata = { + "dummy": "dummy", + } + if has_extension: + mock_metadata["file_extension"] = ".tiff" + mock_post_transfer = mocker.patch.object(Analyser, "post_transfer") + mock_gather_metadata = mocker.patch( + "murfey.client.contexts.spa.SPAContext.gather_metadata", + return_value=mock_metadata, + ) + mock_notify = mocker.patch.object(Analyser, "notify") spy_find_context = mocker.spy(Analyser, "_find_context") spy_find_extension = mocker.spy(Analyser, "_find_extension") # Initialise the Analyser - analyser = Analyser(tmp_path, "", force_mdoc_metadata=True) + analyser = Analyser(tmp_path, "") for file in test_files: analyser._analyse(tmp_path / file) assert spy_find_context.call_count == 1 assert analyser._context is not None and analyser._context.name == "SPAContext" - assert spy_find_extension.call_count > 0 - assert analyser._extension in (".eer", ".tiff") mock_post_transfer.assert_called() + assert spy_find_extension.call_count > 0 + assert analyser._extension == ".tiff" + mock_gather_metadata.assert_called() + mock_notify.assert_called_with( + { + "dummy": "dummy", + "file_extension": analyser._extension, + "acquisition_software": analyser._context._acquisition_software, + } + ) + +@pytest.mark.parametrize("has_extension", [True, False]) def test_analyse_tomo( mocker: MockerFixture, + has_extension: bool, tmp_path: Path, ): test_files = [ @@ -412,19 +436,39 @@ def test_analyse_tomo( ] # Set up mocks and spies + mock_metadata = { + "dummy": "dummy", + } + if has_extension: + mock_metadata["file_extension"] = ".tiff" + mock_post_transfer = mocker.patch.object(Analyser, "post_transfer") mock_gather_metadata = mocker.patch( - "murfey.client.contexts.tomo.TomographyContext.gather_metadata" + "murfey.client.contexts.tomo.TomographyContext.gather_metadata", + return_value=mock_metadata, ) - mock_gather_metadata.return_value = {"dummy": "dummy"} + mock_notify = mocker.patch.object(Analyser, "notify") spy_find_context = mocker.spy(Analyser, "_find_context") spy_find_extension = mocker.spy(Analyser, "_find_extension") # Initialise the Analyser - analyser = Analyser(tmp_path, "", force_mdoc_metadata=True) + analyser = Analyser(tmp_path, "") for file in test_files: analyser._analyse(tmp_path / file) assert spy_find_context.call_count == 1 - assert spy_find_extension.call_count > 0 + assert ( + analyser._context is not None and analyser._context.name == "TomographyContext" + ) mock_post_transfer.assert_called() + + assert spy_find_extension.call_count > 0 + assert analyser._extension == ".tiff" + mock_gather_metadata.assert_called() + mock_notify.assert_called_with( + { + "dummy": "dummy", + "file_extension": analyser._extension, + "acquisition_software": analyser._context._acquisition_software, + } + ) From f8ffd2c1cff7ef10375250856a7e8dcae376427c Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 15 Apr 2026 09:27:05 +0100 Subject: [PATCH 21/22] Fixed typo in FIBContext AutoTEM workflow --- src/murfey/client/contexts/fib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/client/contexts/fib.py b/src/murfey/client/contexts/fib.py index f23ddf37..f90f17c0 100644 --- a/src/murfey/client/contexts/fib.py +++ b/src/murfey/client/contexts/fib.py @@ -189,7 +189,7 @@ def post_transfer( sites = metadata["AutoTEM"]["Project"]["Sites"]["Site"] for site in sites: number = _number_from_name(site["Name"]) - milling_angle = site["Workflow"]["Recipe"][0]["Activites"][ + milling_angle = site["Workflow"]["Recipe"][0]["Activities"][ "MillingAngleActivity" ].get("MillingAngle") if self._lamellae.get(number) and milling_angle: From 829bd48883337b4a6691d7a447b08d35909a20f0 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 15 Apr 2026 16:46:55 +0100 Subject: [PATCH 22/22] Typo in documentation --- src/murfey/client/analyser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index 79c6ab28..bb8ec39a 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -327,8 +327,8 @@ def post_transfer(self, transferred_file: Path): def _analyse_in_thread(self): """ Class function that will be executed by the '_thread' attribute. It will - execute a while-loop in which is takes files of the queue and feeds them - into the '_analyse' class function until '_halt_thread' is set to True. + execute a while-loop where it takes files off the queue and feeds them to + the '_analyse' class function until '_halt_thread' is set to True. """ logger.info("Analyser thread started") while not self._halt_thread: