From 5147daf728764dbe10b5a0a782ff943da23ffd9c Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 12 Dec 2024 16:11:16 +0000 Subject: [PATCH 1/9] Moved 'TiffSeriesInfo' Pydantic model to the CLEM API page --- src/murfey/server/api/clem.py | 15 +++++++++++---- src/murfey/util/models.py | 13 ------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/murfey/server/api/clem.py b/src/murfey/server/api/clem.py index 17cdd6f1d..6dbbb12f5 100644 --- a/src/murfey/server/api/clem.py +++ b/src/murfey/server/api/clem.py @@ -2,13 +2,15 @@ import re import traceback +from ast import literal_eval from importlib.metadata import EntryPoint # type hinting only from logging import getLogger from pathlib import Path -from typing import Optional, Type, Union +from typing import Literal, Optional, Type, Union from backports.entry_points_selectable import entry_points from fastapi import APIRouter +from pydantic import BaseModel, validator from sqlalchemy.exc import NoResultFound from sqlmodel import Session, select @@ -23,7 +25,6 @@ CLEMTIFFFile, ) from murfey.util.db import Session as MurfeySession -from murfey.util.models import TIFFSeriesInfo # Set up logger logger = getLogger("murfey.server.api.clem") @@ -622,7 +623,7 @@ def register_image_stack( "/sessions/{session_id}/clem/preprocessing/process_raw_lifs" ) # API posts to this URL def process_raw_lifs( - session_id: int, # Used by the decorator + session_id: int, lif_file: Path, db: Session = murfey_db, ): @@ -654,9 +655,15 @@ def process_raw_lifs( return True +class TIFFSeriesInfo(BaseModel): + series_name: str + tiff_files: list[Path] + series_metadata: Path + + @router.post("/sessions/{session_id}/clem/preprocessing/process_raw_tiffs") def process_raw_tiffs( - session_id: int, # Used by the decorator + session_id: int, tiff_info: TIFFSeriesInfo, db: Session = murfey_db, ): diff --git a/src/murfey/util/models.py b/src/murfey/util/models.py index 97fedb610..911eae8e5 100644 --- a/src/murfey/util/models.py +++ b/src/murfey/util/models.py @@ -147,19 +147,6 @@ class FractionationParameters(BaseModel): fractionation_file_name: str = "eer_fractionation.txt" -""" -Cryo-CLEM -========= -Models related to the cryo-CLEM workflow. -""" - - -class TIFFSeriesInfo(BaseModel): - series_name: str - tiff_files: List[Path] - series_metadata: Path - - """ FIB === From 324531bc6088e9e1885dd23e269b94cb58743106 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 12 Dec 2024 16:12:37 +0000 Subject: [PATCH 2/9] Added endpoint to enable manual triggering of 'align-and-merge' step of CLEM workflow for a given image series --- src/murfey/server/api/clem.py | 67 +++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/src/murfey/server/api/clem.py b/src/murfey/server/api/clem.py index 6dbbb12f5..d0ec64f14 100644 --- a/src/murfey/server/api/clem.py +++ b/src/murfey/server/api/clem.py @@ -694,3 +694,70 @@ def process_raw_tiffs( messenger=_transport_object, ) return True + + +class AlignAndMergeParams(BaseModel): + # Processing parameters + series_name: str + images: list[Path] + metadata: Path + # Optional processing parameters + align_self: Optional[str] = None + flatten: Optional[Literal["min", "max", "mean"]] = "mean" + align_across: Optional[str] = None + + @validator( + "images", + pre=True, + ) + def parse_stringified_list(cls, value): + if isinstance(value, str): + try: + eval_result = literal_eval(value) + if isinstance(eval_result, list): + parent_tiffs = [Path(p) for p in eval_result] + return parent_tiffs + except (SyntaxError, ValueError): + raise ValueError("Unable to parse input") + # Return value as-is; if it fails, it fails + return value + + +@router.post("/sessions/{session_id}/clem/preprocessing/align_and_merge_stacks") +def align_and_merge_stacks( + session_id: int, + align_and_merge_params: AlignAndMergeParams, + db: Session = murfey_db, +): + try: + # Try and load relevant Murfey workflow + workflow: EntryPoint = list( + entry_points().select(group="murfey.workflows", name="clem.align_and_merge") + )[0] + except IndexError: + raise RuntimeError("The relevant Murfey workflow was not found") + + # Get instrument name from the database to load the correct config file + session_row: MurfeySession = db.exec( + select(MurfeySession).where(MurfeySession.id == session_id) + ).one() + instrument_name = session_row.instrument_name + + # Pass arguments to correct workflow + workflow.load()( + # Match the arguments found in murfey.workflows.clem.align_and_merge + # Session parameters + session_id=session_id, + instrument_name=instrument_name, + # Processing parameters + series_name=align_and_merge_params.series_name, + images=align_and_merge_params.images, + metadata=align_and_merge_params.metadata, + # Optional processing parameters + align_self=align_and_merge_params.align_self, + flatten=align_and_merge_params.flatten, + align_across=align_and_merge_params.align_across, + # Optional session parameters + messenger=_transport_object, + ) + return True From ce9d13b49b860e1146bc6fed792fdabe2d489275 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 12 Dec 2024 16:37:13 +0000 Subject: [PATCH 3/9] Enabled image alignment when triggering 'align-and-merge' job in CLEM workflow --- .../workflows/clem/register_preprocessing_results.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/murfey/workflows/clem/register_preprocessing_results.py b/src/murfey/workflows/clem/register_preprocessing_results.py index 35d4b7d9e..524bf2f21 100644 --- a/src/murfey/workflows/clem/register_preprocessing_results.py +++ b/src/murfey/workflows/clem/register_preprocessing_results.py @@ -187,9 +187,9 @@ def register_lif_preprocessing_result( series_name=result.series_name, images=image_stacks, metadata=result.metadata, - align_self=None, + align_self="enabled", flatten="mean", - align_across=None, + align_across="enabled", messenger=_transport_object, ) if cluster_response is False: @@ -369,9 +369,9 @@ def register_tiff_preprocessing_result( series_name=result.series_name, images=image_stacks, metadata=result.metadata, - align_self=None, + align_self="enabled", flatten="mean", - align_across=None, + align_across="enabled", messenger=_transport_object, ) if cluster_response is False: From 23bd2b6ce50c077bb1cbb2e9b911741a84f0655a Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 13 Dec 2024 00:04:23 +0000 Subject: [PATCH 4/9] Added option to crop image stacks and process only central frames --- src/murfey/server/api/clem.py | 2 ++ src/murfey/workflows/clem/align_and_merge.py | 2 ++ src/murfey/workflows/clem/register_preprocessing_results.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/murfey/server/api/clem.py b/src/murfey/server/api/clem.py index d0ec64f14..7d729e3bb 100644 --- a/src/murfey/server/api/clem.py +++ b/src/murfey/server/api/clem.py @@ -702,6 +702,7 @@ class AlignAndMergeParams(BaseModel): images: list[Path] metadata: Path # Optional processing parameters + crop_to_n_frames: Optional[int] = None align_self: Optional[str] = None flatten: Optional[Literal["min", "max", "mean"]] = "mean" align_across: Optional[str] = None @@ -754,6 +755,7 @@ def align_and_merge_stacks( images=align_and_merge_params.images, metadata=align_and_merge_params.metadata, # Optional processing parameters + crop_to_n_frames=align_and_merge_params.crop_to_n_frames, align_self=align_and_merge_params.align_self, flatten=align_and_merge_params.flatten, align_across=align_and_merge_params.align_across, diff --git a/src/murfey/workflows/clem/align_and_merge.py b/src/murfey/workflows/clem/align_and_merge.py index 5c02e6e0f..3ffd796d4 100644 --- a/src/murfey/workflows/clem/align_and_merge.py +++ b/src/murfey/workflows/clem/align_and_merge.py @@ -25,6 +25,7 @@ def submit_cluster_request( images: list[Path], metadata: Path, # Optional processing parameters + crop_to_n_frames: Optional[int] = None, align_self: Optional[str] = None, flatten: Optional[Literal["min", "max", "mean"]] = "mean", align_across: Optional[str] = None, @@ -64,6 +65,7 @@ def submit_cluster_request( "series_name": series_name, "images": [str(file) for file in images], "metadata": str(metadata), + "crop_to_n_frames": crop_to_n_frames, "align_self": align_self, "flatten": flatten, "align_across": align_across, diff --git a/src/murfey/workflows/clem/register_preprocessing_results.py b/src/murfey/workflows/clem/register_preprocessing_results.py index 524bf2f21..5ce118901 100644 --- a/src/murfey/workflows/clem/register_preprocessing_results.py +++ b/src/murfey/workflows/clem/register_preprocessing_results.py @@ -187,6 +187,7 @@ def register_lif_preprocessing_result( series_name=result.series_name, images=image_stacks, metadata=result.metadata, + crop_to_n_frames=50, align_self="enabled", flatten="mean", align_across="enabled", @@ -369,6 +370,7 @@ def register_tiff_preprocessing_result( series_name=result.series_name, images=image_stacks, metadata=result.metadata, + crop_to_n_frames=50, align_self="enabled", flatten="mean", align_across="enabled", From b774a6563040adacef8dc6e5f22b0a782b40f56e Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 13 Dec 2024 18:21:57 +0000 Subject: [PATCH 5/9] CLEM image alignment and merging should be considered a processing step instead of preprocessing --- src/murfey/server/api/clem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/server/api/clem.py b/src/murfey/server/api/clem.py index 7d729e3bb..87354c13d 100644 --- a/src/murfey/server/api/clem.py +++ b/src/murfey/server/api/clem.py @@ -724,7 +724,7 @@ def parse_stringified_list(cls, value): return value -@router.post("/sessions/{session_id}/clem/preprocessing/align_and_merge_stacks") +@router.post("/sessions/{session_id}/clem/processing/align_and_merge_stacks") def align_and_merge_stacks( session_id: int, align_and_merge_params: AlignAndMergeParams, From c28433fc3d437ed4dac3cda28e513ceff733c818 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 18 Dec 2024 15:49:28 +0000 Subject: [PATCH 6/9] Updated input parameters for the align and merge CLEM workflow, and updated validation checks accordingly --- src/murfey/server/api/clem.py | 6 +++--- src/murfey/workflows/clem/align_and_merge.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/murfey/server/api/clem.py b/src/murfey/server/api/clem.py index 87354c13d..606e18496 100644 --- a/src/murfey/server/api/clem.py +++ b/src/murfey/server/api/clem.py @@ -703,9 +703,9 @@ class AlignAndMergeParams(BaseModel): metadata: Path # Optional processing parameters crop_to_n_frames: Optional[int] = None - align_self: Optional[str] = None - flatten: Optional[Literal["min", "max", "mean"]] = "mean" - align_across: Optional[str] = None + align_self: Literal["enabled", ""] = "" + flatten: Literal["mean", "min", "max", ""] = "" + align_across: Literal["enabled", ""] = "" @validator( "images", diff --git a/src/murfey/workflows/clem/align_and_merge.py b/src/murfey/workflows/clem/align_and_merge.py index 3ffd796d4..c5df9fecf 100644 --- a/src/murfey/workflows/clem/align_and_merge.py +++ b/src/murfey/workflows/clem/align_and_merge.py @@ -26,9 +26,9 @@ def submit_cluster_request( metadata: Path, # Optional processing parameters crop_to_n_frames: Optional[int] = None, - align_self: Optional[str] = None, - flatten: Optional[Literal["min", "max", "mean"]] = "mean", - align_across: Optional[str] = None, + align_self: Literal["enabled", ""] = "", + flatten: Literal["mean", "min", "max", ""] = "mean", + align_across: Literal["enabled", ""] = "", # Optional session parameters messenger: Optional[TransportManager] = None, ): From e6aa5cd0210d0f951e7bc87dbef5d11a3f9cefdf Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 18 Dec 2024 15:55:19 +0000 Subject: [PATCH 7/9] Defined align and merge processing parameters separately to be passed to both the LIF and TIFF workflows --- .../clem/register_preprocessing_results.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/murfey/workflows/clem/register_preprocessing_results.py b/src/murfey/workflows/clem/register_preprocessing_results.py index 5ce118901..8f7562575 100644 --- a/src/murfey/workflows/clem/register_preprocessing_results.py +++ b/src/murfey/workflows/clem/register_preprocessing_results.py @@ -12,6 +12,7 @@ import traceback from ast import literal_eval from pathlib import Path +from typing import Literal, Optional from pydantic import BaseModel, validator from sqlmodel import Session, select @@ -31,6 +32,13 @@ logger = logging.getLogger("murfey.workflows.clem.register_preprocessing_results") +# Define align and merge parameters here +crop_to_n_frames: Optional[int] = 50 +align_self: Literal["enabled", ""] = "enabled" +flatten: Literal["mean", "min", "max", ""] = "mean" +align_across: Literal["enabled", ""] = "enabled" + + class LIFPreprocessingResult(BaseModel): image_stack: Path metadata: Path @@ -187,10 +195,10 @@ def register_lif_preprocessing_result( series_name=result.series_name, images=image_stacks, metadata=result.metadata, - crop_to_n_frames=50, - align_self="enabled", - flatten="mean", - align_across="enabled", + crop_to_n_frames=crop_to_n_frames, + align_self=align_self, + flatten=flatten, + align_across=align_across, messenger=_transport_object, ) if cluster_response is False: @@ -370,10 +378,10 @@ def register_tiff_preprocessing_result( series_name=result.series_name, images=image_stacks, metadata=result.metadata, - crop_to_n_frames=50, - align_self="enabled", - flatten="mean", - align_across="enabled", + crop_to_n_frames=crop_to_n_frames, + align_self=align_self, + flatten=flatten, + align_across=align_across, messenger=_transport_object, ) if cluster_response is False: From d43ca95313d62a38a4e686608f9c4a4e79047b3d Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 19 Dec 2024 12:16:01 +0000 Subject: [PATCH 8/9] Renamed spa_params to processing_params, to store default processing parameters in --- src/murfey/cli/inject_spa_processing.py | 2 +- src/murfey/server/__init__.py | 2 +- src/murfey/server/api/__init__.py | 2 +- src/murfey/server/demo_api.py | 2 +- .../util/{spa_params.py => processing_params.py} | 12 ++++++++++++ 5 files changed, 16 insertions(+), 4 deletions(-) rename src/murfey/util/{spa_params.py => processing_params.py} (56%) diff --git a/src/murfey/cli/inject_spa_processing.py b/src/murfey/cli/inject_spa_processing.py index 2b14df9d9..2294835a5 100644 --- a/src/murfey/cli/inject_spa_processing.py +++ b/src/murfey/cli/inject_spa_processing.py @@ -21,7 +21,7 @@ SPAFeedbackParameters, SPARelionParameters, ) -from murfey.util.spa_params import default_spa_parameters +from murfey.util.processing_params import default_spa_parameters def run(): diff --git a/src/murfey/server/__init__.py b/src/murfey/server/__init__.py index 0c795ac6c..07c50fade 100644 --- a/src/murfey/server/__init__.py +++ b/src/murfey/server/__init__.py @@ -56,7 +56,7 @@ get_microscope, get_security_config, ) -from murfey.util.spa_params import default_spa_parameters +from murfey.util.processing_params import default_spa_parameters from murfey.util.state import global_state try: diff --git a/src/murfey/server/api/__init__.py b/src/murfey/server/api/__init__.py index 86e140998..70cdbe716 100644 --- a/src/murfey/server/api/__init__.py +++ b/src/murfey/server/api/__init__.py @@ -105,7 +105,7 @@ TiltSeriesInfo, Visit, ) -from murfey.util.spa_params import default_spa_parameters +from murfey.util.processing_params import default_spa_parameters from murfey.util.state import global_state log = logging.getLogger("murfey.server.api") diff --git a/src/murfey/server/demo_api.py b/src/murfey/server/demo_api.py index eab41858a..7f1221e3c 100644 --- a/src/murfey/server/demo_api.py +++ b/src/murfey/server/demo_api.py @@ -90,7 +90,7 @@ TiltSeriesInfo, Visit, ) -from murfey.util.spa_params import default_spa_parameters +from murfey.util.processing_params import default_spa_parameters from murfey.util.state import global_state log = logging.getLogger("murfey.server.demo_api") diff --git a/src/murfey/util/spa_params.py b/src/murfey/util/processing_params.py similarity index 56% rename from src/murfey/util/spa_params.py rename to src/murfey/util/processing_params.py index 3cc130be8..66ea481c0 100644 --- a/src/murfey/util/spa_params.py +++ b/src/murfey/util/processing_params.py @@ -1,6 +1,18 @@ +from typing import Literal, Optional + from pydantic import BaseModel +class CLEMAlignAndMergeParameters(BaseModel): + crop_to_n_frames: Optional[int] = 50 + align_self: Literal["enabled", ""] = "enabled" + flatten: Literal["mean", "min", "max", ""] = "mean" + align_across: Literal["enabled", ""] = "enabled" + + +default_clem_align_and_merge_parameters = CLEMAlignAndMergeParameters() + + class SPAParameters(BaseModel): nr_iter_2d: int = 25 nr_iter_3d: int = 25 From 8bd08be0dfa66ad4654663600c7440211cac67ce Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 19 Dec 2024 12:16:59 +0000 Subject: [PATCH 9/9] Moved default processing parameters for align-and-merge CLEM workflow to 'processing_params' --- .../clem/register_preprocessing_results.py | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/src/murfey/workflows/clem/register_preprocessing_results.py b/src/murfey/workflows/clem/register_preprocessing_results.py index 8f7562575..202cc740a 100644 --- a/src/murfey/workflows/clem/register_preprocessing_results.py +++ b/src/murfey/workflows/clem/register_preprocessing_results.py @@ -12,7 +12,6 @@ import traceback from ast import literal_eval from pathlib import Path -from typing import Literal, Optional from pydantic import BaseModel, validator from sqlmodel import Session, select @@ -26,19 +25,15 @@ CLEMTIFFFile, ) from murfey.util.db import Session as MurfeySession +from murfey.util.processing_params import ( + default_clem_align_and_merge_parameters as processing_params, +) from murfey.workflows.clem import get_db_entry from murfey.workflows.clem.align_and_merge import submit_cluster_request logger = logging.getLogger("murfey.workflows.clem.register_preprocessing_results") -# Define align and merge parameters here -crop_to_n_frames: Optional[int] = 50 -align_self: Literal["enabled", ""] = "enabled" -flatten: Literal["mean", "min", "max", ""] = "mean" -align_across: Literal["enabled", ""] = "enabled" - - class LIFPreprocessingResult(BaseModel): image_stack: Path metadata: Path @@ -195,10 +190,10 @@ def register_lif_preprocessing_result( series_name=result.series_name, images=image_stacks, metadata=result.metadata, - crop_to_n_frames=crop_to_n_frames, - align_self=align_self, - flatten=flatten, - align_across=align_across, + crop_to_n_frames=processing_params.crop_to_n_frames, + align_self=processing_params.align_self, + flatten=processing_params.flatten, + align_across=processing_params.align_across, messenger=_transport_object, ) if cluster_response is False: @@ -378,10 +373,10 @@ def register_tiff_preprocessing_result( series_name=result.series_name, images=image_stacks, metadata=result.metadata, - crop_to_n_frames=crop_to_n_frames, - align_self=align_self, - flatten=flatten, - align_across=align_across, + crop_to_n_frames=processing_params.crop_to_n_frames, + align_self=processing_params.align_self, + flatten=processing_params.flatten, + align_across=processing_params.align_across, messenger=_transport_object, ) if cluster_response is False: