From bc15f7eb0c8af1f545584d628b56f561909436d2 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 11 Dec 2025 17:58:13 +0000 Subject: [PATCH 01/28] First pass of removing files and attributes related to the Murfey TUI app --- pyproject.toml | 2 - src/murfey/client/customlogging.py | 24 - src/murfey/client/rsync.py | 13 - src/murfey/client/tui/__init__.py | 0 src/murfey/client/tui/app.py | 860 ------------------ src/murfey/client/tui/controller.css | 524 ----------- src/murfey/client/tui/launcher.css | 25 - src/murfey/client/tui/main.py | 353 -------- src/murfey/client/tui/progress.py | 159 ---- src/murfey/client/tui/screens.py | 1260 -------------------------- src/murfey/client/tui/status_bar.py | 67 -- src/murfey/client/watchdir.py | 12 +- tests/client/test_watchdir.py | 1 - tests/client/tui/test_main.py | 71 -- 14 files changed, 1 insertion(+), 3370 deletions(-) delete mode 100644 src/murfey/client/tui/__init__.py delete mode 100644 src/murfey/client/tui/app.py delete mode 100644 src/murfey/client/tui/controller.css delete mode 100644 src/murfey/client/tui/launcher.css delete mode 100644 src/murfey/client/tui/main.py delete mode 100644 src/murfey/client/tui/progress.py delete mode 100644 src/murfey/client/tui/screens.py delete mode 100644 src/murfey/client/tui/status_bar.py diff --git a/pyproject.toml b/pyproject.toml index 826df4c94..5b1c46a82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,6 @@ Documentation = "https://github.com/DiamondLightSource/python-murfey" GitHub = "https://github.com/DiamondLightSource/python-murfey" [project.scripts] "murfey.add_user" = "murfey.cli.add_user:run" -"murfey.client" = "murfey.client.tui.main:run" "murfey.create_db" = "murfey.cli.create_db:run" "murfey.db_sql" = "murfey.cli.murfey_db_sql:run" "murfey.decrypt_password" = "murfey.cli.decrypt_db_password:run" @@ -121,7 +120,6 @@ include-package-data = true zip-safe = false [tool.setuptools.package-data] -"murfey.client.tui" = ["*.css"] "murfey.util" = ["route_manifest.yaml"] [tool.setuptools.packages.find] diff --git a/src/murfey/client/customlogging.py b/src/murfey/client/customlogging.py index 5ced13e15..567cf7b7e 100644 --- a/src/murfey/client/customlogging.py +++ b/src/murfey/client/customlogging.py @@ -3,10 +3,6 @@ import json import logging -from rich.logging import RichHandler - -from murfey.client.tui.screens import LogBook - logger = logging.getLogger("murfey.client.customlogging") @@ -30,23 +26,3 @@ def emit(self, record): self._callback(self.prepare(record)) except Exception: self.handleError(record) - - -class DirectableRichHandler(RichHandler): - def __init__(self, text_log: LogBook | None = None, **kwargs): - super().__init__(**kwargs) - self.text_log = text_log - self.redirect = False - self._last_time = None - - def emit(self, record): - try: - if self.text_log: - message = self.format(record) - message_renderable = self.render_message(record, message) - log_renderable = self.render( - record=record, traceback=None, message_renderable=message_renderable - ) - self.text_log.post_message(self.text_log.Log(log_renderable)) - except Exception: - self.handleError(record) diff --git a/src/murfey/client/rsync.py b/src/murfey/client/rsync.py index b59321471..f9729cf0b 100644 --- a/src/murfey/client/rsync.py +++ b/src/murfey/client/rsync.py @@ -19,7 +19,6 @@ from typing import Awaitable, Callable, List, NamedTuple from urllib.parse import ParseResult -from murfey.client.tui.status_bar import StatusBar from murfey.util.client import Observer logger = logging.getLogger("murfey.client.rsync") @@ -59,7 +58,6 @@ def __init__( server_url: ParseResult, stop_callback: Callable = lambda *args, **kwargs: None, local: bool = False, - status_bar: StatusBar | None = None, do_transfer: bool = True, remove_files: bool = False, required_substrings_for_removal: List[str] = [], @@ -107,7 +105,6 @@ def __init__( ) self._stopping = False self._halt_thread = False - self._statusbar = status_bar def __repr__(self) -> str: return f" str: def from_rsyncer(cls, rsyncer: RSyncer, **kwargs): kwarguments_from_rsyncer = { "local": rsyncer._local, - "status_bar": rsyncer._statusbar, "do_transfer": rsyncer._do_transfer, "remove_files": rsyncer._remove_files, "notify": rsyncer._notify, } kwarguments_from_rsyncer.update(kwargs) assert isinstance(kwarguments_from_rsyncer["local"], bool) - if kwarguments_from_rsyncer["status_bar"] is not None: - assert isinstance(kwarguments_from_rsyncer["status_bar"], StatusBar) assert isinstance(kwarguments_from_rsyncer["do_transfer"], bool) assert isinstance(kwarguments_from_rsyncer["remove_files"], bool) assert isinstance(kwarguments_from_rsyncer["notify"], bool) @@ -134,7 +128,6 @@ def from_rsyncer(cls, rsyncer: RSyncer, **kwargs): rsyncer._rsync_module, rsyncer._server_url, local=kwarguments_from_rsyncer["local"], - status_bar=kwarguments_from_rsyncer["status_bar"], do_transfer=kwarguments_from_rsyncer["do_transfer"], remove_files=kwarguments_from_rsyncer["remove_files"], notify=kwarguments_from_rsyncer["notify"], @@ -398,12 +391,6 @@ def parse_stdout(line: str): return self._files_transferred += 1 - if self._statusbar: - with self._statusbar.lock: - self._statusbar.transferred = [ - self._statusbar.transferred[0] + 1, - self._statusbar.transferred[1], - ] current_outstanding = self.queue.unfinished_tasks - ( self._files_transferred - previously_transferred ) diff --git a/src/murfey/client/tui/__init__.py b/src/murfey/client/tui/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/murfey/client/tui/app.py b/src/murfey/client/tui/app.py deleted file mode 100644 index 3ab2cb616..000000000 --- a/src/murfey/client/tui/app.py +++ /dev/null @@ -1,860 +0,0 @@ -from __future__ import annotations - -import logging -import subprocess -from datetime import datetime -from functools import partial -from pathlib import Path -from queue import Queue -from typing import Awaitable, Callable, Dict, List, OrderedDict, TypeVar -from urllib.parse import urlparse - -from textual.app import App -from textual.reactive import reactive -from textual.widgets import Button, Input - -from murfey.client.analyser import Analyser -from murfey.client.context import ensure_dcg_exists -from murfey.client.contexts.spa import SPAModularContext -from murfey.client.contexts.tomo import TomographyContext -from murfey.client.destinations import determine_default_destination -from murfey.client.instance_environment import MurfeyInstanceEnvironment -from murfey.client.rsync import RSyncer, RSyncerUpdate, TransferResult -from murfey.client.tui.screens import ( - ConfirmScreen, - InputResponse, - MainScreen, - ProcessingForm, - SessionSelection, - VisitCreation, - VisitSelection, - WaitingScreen, -) -from murfey.client.tui.status_bar import StatusBar -from murfey.client.watchdir import DirWatcher -from murfey.client.watchdir_multigrid import MultigridDirWatcher -from murfey.util import posix_path -from murfey.util.client import ( - capture_delete, - capture_get, - capture_post, - get_machine_config_client, - read_config, - set_default_acquisition_output, -) - -log = logging.getLogger("murfey.tui.app") - -ReactiveType = TypeVar("ReactiveType") - -token = read_config()["Murfey"].get("token", "") -instrument_name = read_config()["Murfey"].get("instrument_name", "") - - -class MurfeyTUI(App): - CSS_PATH = "controller.css" - processing_btn: Button - processing_form: ProcessingForm - hover: List[str] - visits: List[str] - rsync_processes: Dict[Path, RSyncer] = {} - analysers: Dict[Path, Analyser] = {} - _form_values: dict = reactive({}) - - def __init__( - self, - environment: MurfeyInstanceEnvironment | None = None, - visits: List[str] | None = None, - queues: Dict[str, Queue] | None = None, - status_bar: StatusBar | None = None, - dummy_dc: bool = True, - do_transfer: bool = True, - gain_ref: Path | None = None, - redirected_logger=None, - force_mdoc_metadata: bool = False, - processing_enabled: bool = True, - skip_existing_processing: bool = False, - **kwargs, - ): - super().__init__(**kwargs) - self._environment = environment or MurfeyInstanceEnvironment( - urlparse("http://localhost:8000") - ) - self._environment.gain_ref = str(gain_ref) - self._sources = self._environment.sources or [Path(".")] - self._url = self._environment.url - self._default_destinations = self._environment.default_destinations - self.visits = visits or [] - self._queues = queues or {} - self._statusbar = status_bar or StatusBar() - self._request_destinations = False - self._register_dc: bool | None = None - self._tmp_responses: List[dict] = [] - self._visit = "" - self._dc_metadata: dict = {} - self._dummy_dc = dummy_dc - self._do_transfer = do_transfer - self._data_collection_form_complete = False - self._form_readable_labels: dict = {} - self._redirected_logger = redirected_logger - self._multigrid = False - self._processing_enabled = processing_enabled - self._multigrid_watcher: MultigridDirWatcher | None = None - self._force_mdoc_metadata = force_mdoc_metadata - self._skip_existing_processing = skip_existing_processing - self._machine_config = get_machine_config_client( - str(self._environment.url.geturl()), - token, - instrument_name=self._environment.instrument_name, - demo=self._environment.demo, - ) - self._data_suffixes = (".mrc", ".tiff", ".tif", ".eer") - self._data_substrings = [ - s - for val in self._machine_config["data_required_substrings"].values() - for ds in val.values() - for s in ds - ] - self.install_screen(MainScreen(), "main") - - def _launch_multigrid_watcher( - self, source: Path, destination_overrides: Dict[Path, str] | None = None - ): - log.info(f"Launching multigrid watcher for source {source}") - machine_config = get_machine_config_client( - str(self._environment.url.geturl()), - token, - instrument_name=self._environment.instrument_name, - demo=self._environment.demo, - ) - self._multigrid_watcher = MultigridDirWatcher( - source, - machine_config, - skip_existing_processing=self._skip_existing_processing, - ) - self._multigrid_watcher.subscribe( - partial( - self._start_rsyncer_multigrid, - destination_overrides=destination_overrides or {}, - ) - ) - self._multigrid_watcher.start() - - def _start_rsyncer_multigrid( - self, - source: Path, - extra_directory: str = "", - include_mid_path: bool = True, - use_suggested_path: bool = True, - destination_overrides: Dict[Path, str] | None = None, - remove_files: bool = False, - analyse: bool = True, - limited: bool = False, - **kwargs, - ): - log.info(f"starting multigrid rsyncer: {source}") - destination_overrides = destination_overrides or {} - machine_data = capture_get( - base_url=str(self._environment.url.geturl()), - router_name="session_control.router", - function_name="machine_info_by_instrument", - token=token, - instrument_name=instrument_name, - ).json() - if destination_overrides.get(source): - destination = destination_overrides[source] + f"/{extra_directory}" - else: - for k, v in destination_overrides.items(): - if Path(v).name in source.parts: - destination = str(k / extra_directory) - break - else: - self._environment.default_destinations[source] = ( - f"{datetime.now().year}" - ) - destination = determine_default_destination( - self._visit, - source, - self._default_destinations[source], - self._environment, - self.analysers, - token, - touch=True, - extra_directory=extra_directory, - include_mid_path=include_mid_path, - use_suggested_path=use_suggested_path, - ) - self._environment.sources.append(source) - self._start_rsyncer( - source, - destination, - force_metadata=self._processing_enabled, - # analyse=not extra_directory and use_suggested_path and analyse, - analyse=analyse, - remove_files=remove_files, - limited=limited, - transfer=machine_data.get("data_transfer_enabled", True), - rsync_url=machine_data.get("rsync_url", ""), - ) - - def _start_rsyncer( - self, - source: Path, - destination: str, - visit_path: str = "", - force_metadata: bool = False, - analyse: bool = True, - remove_files: bool = False, - limited: bool = False, - transfer: bool = True, - rsync_url: str = "", - ): - log.info(f"starting rsyncer: {source}") - if transfer: - # Always make sure the destination directory exists - capture_post( - base_url=str(self._url.geturl()), - router_name="file_io_instrument.router", - function_name="make_rsyncer_destination", - token=token, - session_id=self._environment.murfey_session, - data={"destination": destination}, - ) - if self._environment: - self._environment.default_destinations[source] = destination - if self._environment.gain_ref and visit_path: - # Set up rsync command - rsync_cmd = [ - "rsync", - f"{posix_path(Path(self._environment.gain_ref))!r}", - f"{self._url.hostname}::{self._machine_config.get('rsync_module', 'data')}/{visit_path}/processing", - ] - # Encase in bash shell - cmd = [ - "bash", - "-c", - " ".join(rsync_cmd), - ] - # Run rsync subprocess - gain_rsync = subprocess.run(cmd) - if gain_rsync.returncode: - log.warning( - f"Gain reference file {posix_path(Path(self._environment.gain_ref))!r} was not successfully transferred to {visit_path}/processing" - ) - if transfer: - self.rsync_processes[source] = RSyncer( - source, - basepath_remote=Path(destination), - rsync_module=self._machine_config.get("rsync_module", "data"), - server_url=urlparse(rsync_url) if rsync_url else self._url, - # local=self._environment.demo, - status_bar=self._statusbar, - do_transfer=self._do_transfer, - required_substrings_for_removal=self._data_substrings, - remove_files=remove_files, - ) - - def rsync_result(update: RSyncerUpdate): - if not update.base_path: - raise ValueError("No base path from rsyncer update") - if not self.rsync_processes.get(update.base_path): - raise ValueError("TUI rsync process does not exist") - if update.outcome is TransferResult.SUCCESS: - log.debug( - f"Succesfully transferred file {str(update.file_path)!r} ({update.file_size} bytes)" - ) - # pass - else: - log.warning(f"Failed to transfer file {str(update.file_path)!r}") - self.rsync_processes[update.base_path].enqueue(update.file_path) - - self.rsync_processes[source].subscribe(rsync_result) - self.rsync_processes[source].subscribe( - partial( - self._increment_transferred_files_prometheus, - destination=destination, - source=str(source), - ) - ) - self.rsync_processes[source].subscribe( - partial( - self._increment_transferred_files, - destination=destination, - source=str(source), - ), - secondary=True, - ) - rsyncer_data = { - "source": str(source), - "destination": destination, - "session_id": self._environment.murfey_session, - "transferring": self._do_transfer, - } - capture_post( - base_url=str(self._url.geturl()), - router_name="session_control.router", - function_name="register_rsyncer", - token=token, - session_id=self._environment.murfey_session, - data=rsyncer_data, - ) - - self._environment.watchers[source] = DirWatcher(source, settling_time=30) - - if not self.analysers.get(source) and analyse: - log.info(f"Starting analyser for {source}") - self.analysers[source] = Analyser( - source, - token, - environment=self._environment if not self._dummy_dc else None, - force_mdoc_metadata=self._force_mdoc_metadata, - limited=limited, - ) - if force_metadata: - self.analysers[source].subscribe( - partial(self._start_dc, from_form=True) - ) - else: - self.analysers[source].subscribe(self._data_collection_form) - self.analysers[source].start() - if transfer: - self.rsync_processes[source].subscribe(self.analysers[source].enqueue) - - if transfer: - self.rsync_processes[source].start() - - if self._environment: - if self._environment.watchers.get(source): - if transfer: - self._environment.watchers[source].subscribe( - self.rsync_processes[source].enqueue - ) - else: - - def _rsync_update_converter(p: Path) -> None: - self.analysers[source].enqueue( - RSyncerUpdate( - file_path=p, - file_size=0, - outcome=TransferResult.SUCCESS, - transfer_total=0, - queue_size=0, - base_path=source, - ) - ) - return None - - self._environment.watchers[source].subscribe( - _rsync_update_converter - ) - self._environment.watchers[source].subscribe( - partial( - self._increment_file_count, - destination=destination, - source=str(source), - ), - secondary=True, - ) - self._environment.watchers[source].start() - - def _increment_file_count( - self, observed_files: List[Path], source: str, destination: str - ): - if len(observed_files): - num_data_files = len( - [ - f - for f in observed_files - if f.suffix in self._data_suffixes - and any(substring in f.name for substring in self._data_substrings) - ] - ) - data = { - "source": source, - "destination": destination, - "session_id": self._environment.murfey_session, - "increment_count": len(observed_files), - "increment_data_count": num_data_files, - } - capture_post( - base_url=str(self._url.geturl()), - router_name="prometheus.router", - function_name="increment_rsync_file_count", - token=token, - visit_name=self._visit, - data=data, - ) - - # Prometheus can handle higher traffic so update for every transferred file rather - # than batching as we do for the Murfey database updates in _increment_transferred_files - def _increment_transferred_files_prometheus( - self, update: RSyncerUpdate, source: str, destination: str - ): - if update.outcome is TransferResult.SUCCESS: - data_files = ( - [update] - if update.file_path.suffix in self._data_suffixes - and any( - substring in update.file_path.name - for substring in self._data_substrings - ) - else [] - ) - data = { - "source": source, - "destination": destination, - "session_id": self._environment.murfey_session, - "increment_count": 1, - "bytes": update.file_size, - "increment_data_count": len(data_files), - "data_bytes": sum(f.file_size for f in data_files), - } - capture_post( - base_url=str(self._url.geturl()), - router_name="prometheus.router", - function_name="increment_rsync_transferred_files_prometheus", - token=token, - visit_name=self._visit, - data=data, - ) - - def _increment_transferred_files( - self, updates: List[RSyncerUpdate], source: str, destination: str - ): - checked_updates = [ - update for update in updates if update.outcome is TransferResult.SUCCESS - ] - if not checked_updates: - return - data_files = [ - u - for u in updates - if u.file_path.suffix in self._data_suffixes - and any( - substring in u.file_path.name for substring in self._data_substrings - ) - ] - data = { - "source": source, - "destination": destination, - "session_id": self._environment.murfey_session, - "increment_count": len(checked_updates), - "bytes": sum(f.file_size for f in checked_updates), - "increment_data_count": len(data_files), - "data_bytes": sum(f.file_size for f in data_files), - } - capture_post( - base_url=str(self._url.geturl()), - router_name="prometheus.router", - function_name="increment_rsync_transferred_files", - token=token, - visit_name=self._visit, - data=data, - ) - - def _set_register_dc(self, response: str): - if response == "y": - self._register_dc = True - for r in self._tmp_responses: - self._queues["input"].put_nowait( - InputResponse( - question="Data collection parameters:", - form=r.get("form", OrderedDict({})), - model=getattr(self.analyser, "parameters_model", None), - callback=self.app._start_dc_confirm_prompt, - ) - ) - self._dc_metadata = r.get("form", OrderedDict({})) - elif response == "n": - self._register_dc = False - self._tmp_responses = [] - - def _data_collection_form(self, response: dict): - log.info("data collection form ready") - if self._data_collection_form_complete: - return - if self._register_dc and response.get("form"): - self._form_values = {k: str(v) for k, v in response.get("form", {}).items()} - log.info( - f"gain reference is set to {self._form_values.get('gain_ref')}, {self._environment.gain_ref}" - ) - if self._form_values.get("gain_ref") in (None, "None"): - self._form_values["gain_ref"] = self._environment.gain_ref - self.processing_btn.disabled = False - self._data_collection_form_complete = True - elif self._register_dc is None: - self._tmp_responses.append(response) - self._data_collection_form_complete = True - - def _start_dc_confirm_prompt(self, json: dict): - self._queues["input"].put_nowait( - InputResponse( - question="Would you like to start processing with chosen parameters?", - allowed_responses=["y", "n"], - callback=partial(self._start_dc_confirm, json=json), - ) - ) - - def _start_dc(self, metadata_json, from_form: bool = False): - if self._dummy_dc: - return - # for multigrid the analyser sends the message straight to _start_dc by-passing user input - # it is then necessary to extract the data from the message - if from_form: - metadata_json = metadata_json.get("form", {}) - metadata_json = { - k: v if v is None else str(v) for k, v in metadata_json.items() - } - self._environment.dose_per_frame = metadata_json.get("dose_per_frame") - self._environment.gain_ref = metadata_json.get("gain_ref") - self._environment.symmetry = metadata_json.get("symmetry") - self._environment.eer_fractionation = metadata_json.get("eer_fractionation") - source = Path(metadata_json["source"]) - context = self.analysers[source]._context - if context: - context.data_collection_parameters = { - k: None if v == "None" else v for k, v in metadata_json.items() - } - if isinstance(context, TomographyContext): - source = Path(metadata_json["source"]) - context.register_tomography_data_collections( - file_extension=metadata_json["file_extension"], - image_directory=str(self._environment.default_destinations[source]), - environment=self._environment, - ) - - log.info("Registering tomography processing parameters") - if context.data_collection_parameters.get("num_eer_frames"): - eer_response = capture_post( - base_url=str(self.app._environment.url.geturl()), - router_name="file_io_instrument.router", - function_name="write_eer_fractionation_file", - token=token, - visit_name=self.app._environment.visit, - session_id=self.app._environment.murfey_session, - data={ - "num_frames": context.data_collection_parameters[ - "num_eer_frames" - ], - "fractionation": self.app._environment.eer_fractionation, - "dose_per_frame": self.app._environment.dose_per_frame, - "fractionation_file_name": "eer_fractionation_tomo.txt", - }, - ) - eer_fractionation_file = eer_response.json()["eer_fractionation_file"] - metadata_json.update({"eer_fractionation_file": eer_fractionation_file}) - capture_post( - base_url=str(self.app._environment.url.geturl()), - router_name="workflow.tomo_router", - function_name="register_tomo_proc_params", - token=token, - session_id=self.app._environment.murfey_session, - data=metadata_json, - ) - capture_post( - base_url=str(self.app._environment.url.geturl()), - router_name="workflow.tomo_router", - function_name="flush_tomography_processing", - token=token, - visit_name=self._visit, - session_id=self.app._environment.murfey_session, - data={"rsync_source": str(source)}, - ) - log.info("Tomography processing flushed") - elif isinstance(context, SPAModularContext): - if self._environment.visit in source.parts: - metadata_source = source - else: - metadata_source_as_str = ( - "/".join(source.parts[:-2]) - + f"/{self._environment.visit}/" - + source.parts[-2] - ) - metadata_source = Path(metadata_source_as_str.replace("//", "/")) - ensure_dcg_exists( - collection_type="spa", - metadata_source=metadata_source, - environment=self._environment, - token=self.token, - ) - if from_form: - data = { - "voltage": metadata_json["voltage"], - "pixel_size_on_image": metadata_json["pixel_size_on_image"], - "experiment_type": metadata_json["experiment_type"], - "image_size_x": metadata_json["image_size_x"], - "image_size_y": metadata_json["image_size_y"], - "file_extension": metadata_json["file_extension"], - "acquisition_software": metadata_json["acquisition_software"], - "image_directory": str( - self._environment.default_destinations[source] - ), - "tag": str(source), - "source": str(source), - "magnification": metadata_json["magnification"], - "total_exposed_dose": metadata_json.get("total_exposed_dose"), - "c2aperture": metadata_json.get("c2aperture"), - "exposure_time": metadata_json.get("exposure_time"), - "slit_width": metadata_json.get("slit_width"), - "phase_plate": metadata_json.get("phase_plate", False), - } - capture_post( - base_url=str(self._url.geturl()), - router_name="workflow.router", - function_name="start_dc", - token=token, - visit_name=self._visit, - session_id=self._environment.murfey_session, - data=data, - ) - for recipe in ( - "em-spa-preprocess", - "em-spa-extract", - "em-spa-class2d", - "em-spa-class3d", - "em-spa-refine", - ): - capture_post( - base_url=str(self._url.geturl()), - router_name="workflow.router", - function_name="register_proc", - token=token, - visit_name=self._visit, - session_id=self._environment.murfey_session, - data={ - "tag": str(source), - "source": str(source), - "recipe": recipe, - }, - ) - log.info(f"Posting SPA processing parameters: {metadata_json}") - response = capture_post( - base_url=str(self.app._environment.url.geturl()), - router_name="workflow.spa_router", - function_name="register_spa_proc_params", - token=token, - session_id=self.app._environment.murfey_session, - data={ - **{ - k: None if v == "None" else v - for k, v in metadata_json.items() - }, - "tag": str(source), - }, - ) - if response is None: - log.error( - "Could not reach Murfey server to insert SPA processing parameters" - ) - return None - if not str(response.status_code).startswith("2"): - log.warning(f"{response.reason}") - capture_post( - base_url=str(self.app._environment.url.geturl()), - router_name="workflow.spa_router", - function_name="flush_spa_processing", - token=token, - visit_name=self.app._environment.visit, - session_id=self.app._environment.murfey_session, - data={"tag": str(source)}, - ) - - def _set_request_destination(self, response: str): - if response == "y": - self._request_destinations = True - - async def on_load(self, event): - self.bind("q", "quit", description="Quit", show=True) - self.bind("p", "process", description="Allow processing", show=True) - self.bind( - "d", "remove_session", description="Quit and remove session", show=True - ) - - def _install_processing_form(self): - self.processing_form = ProcessingForm(self._form_values) - self.install_screen(self.processing_form, "processing-form") - - def on_input_submitted(self, event: Input.Submitted): - event.input.has_focus = False - self.screen.focused = None - - async def on_button_pressed(self, event: Button.Pressed): - if event.button._id == "processing-btn": - self._install_processing_form() - self.push_screen("processing-form") - elif event.button._id == "new-visit-btn": - await self.reset() - - async def on_mount(self) -> None: - exisiting_sessions = capture_get( - base_url=str(self._environment.url.geturl()), - router_name="session_control.router", - function_name="get_sessions", - token=token, - ).json() - if self.visits: - self.install_screen(VisitSelection(self.visits), "visit-select-screen") - self.push_screen("visit-select-screen") - else: - self.install_screen(VisitCreation(), "visit-creation-screen") - self.push_screen("visit-creation-screen") - if exisiting_sessions: - self.install_screen( - SessionSelection( - [ - f"{s['session']['id']}: {s['session']['name']}" - for s in exisiting_sessions - ], - [ - f"{s['session']['id']}: {s['session']['name']}" - for s in exisiting_sessions - if s["clients"] - ], - ), - "session-select-screen", - ) - self.push_screen("session-select-screen") - else: - session_name = "Client connection" - resp = capture_post( - base_url=str(self._environment.url.geturl()), - router_name="session_control.router", - function_name="link_client_to_session", - token=token, - instrument_name=self._environment.instrument_name, - client_id=self._environment.client_id, - data={"session_id": None, "session_name": session_name}, - ) - if resp: - self._environment.murfey_session = resp.json() - - def on_log_book_log(self, message): - self.log_book.write(message.renderable) - - async def reset(self): - machine_config = get_machine_config_client( - str(self._environment.url.geturl()), - token, - instrument_name=self._environment.instrument_name, - demo=self._environment.demo, - ) - if self.rsync_processes and machine_config.get("allow_removal"): - sources = "\n".join(str(k) for k in self.rsync_processes.keys()) - prompt = f"Remove files from the following:\n {sources} \n" - rsync_instances = capture_get( - base_url=str(self._environment.url.geturl()), - router_name="session_control.router", - function_name="get_rsyncers_for_session", - token=token, - session_id=self._environment.murfey_session, - ).json() - prompt += f"Copied {sum(r['files_counted'] for r in rsync_instances)} / {sum(r['files_transferred'] for r in rsync_instances)}" - self.install_screen( - WaitingScreen(prompt, sum(r["files_counted"] for r in rsync_instances)), - "waiting", - ) - self.push_screen("waiting") - - async def action_quit(self) -> None: - log.info("quitting app") - - if self.rsync_processes: - for rp in self.rsync_processes.values(): - rp.stop() - if self.analysers: - for a in self.analysers.values(): - a.stop() - if self._multigrid_watcher: - self._multigrid_watcher.stop() - self.exit() - - async def action_remove_session(self) -> None: - capture_delete( - base_url=str(self._environment.url.geturl()), - router_name="session_control.router", - function_name="remove_session", - token=token, - session_id=self._environment.murfey_session, - ) - if self.rsync_processes: - for rp in self.rsync_processes.values(): - rp.stop() - if self.analysers: - for a in self.analysers.values(): - a.stop() - if self._multigrid_watcher: - self._multigrid_watcher.stop() - self.exit() - - def clean_up_quit(self) -> None: - capture_delete( - base_url=str(self._environment.url.geturl()), - router_name="session_control.router", - function_name="remove_session", - token=token, - session_id=self._environment.murfey_session, - ) - self.exit() - - async def action_clear(self) -> None: - machine_config = get_machine_config_client( - str(self._environment.url.geturl()), - token, - instrument_name=self._environment.instrument_name, - demo=self._environment.demo, - ) - if self.rsync_processes and machine_config.get("allow_removal"): - sources = "\n".join(str(k) for k in self.rsync_processes.keys()) - prompt = f"Remove files from the following: {sources}" - self.install_screen( - ConfirmScreen( - prompt, - pressed_callback=self._remove_data, - button_names={"launch": "Yes", "quit": "No"}, - ), - "clear-confirm", - ) - self.push_screen("clear-confirm") - - def _remove_data(self, listener: Callable[..., Awaitable[None] | None], **kwargs): - new_rsyncers = [] - if self.rsync_processes or self._environment.demo: - for k, rp in self.rsync_processes.items(): - rp.stop() - if self.analysers.get(k): - self.analysers[k].stop() - removal_rp = RSyncer.from_rsyncer(rp, remove_files=True, notify=False) - removal_rp.subscribe(listener) - new_rsyncers.append(removal_rp) - log.info( - f"Starting to remove data files {self._environment.demo}, {len(self.rsync_processes)}" - ) - for removal_rp in new_rsyncers: - removal_rp.start() - for f in k.absolute().glob("**/*"): - removal_rp.queue.put(f) - removal_rp.stop() - log.info(f"rsyncer {rp} rerun with removal") - capture_post( - base_url=str(self._environment.url.geturl()), - router_name="session_control.router", - function_name="register_processing_success_in_ispyb", - token=token, - session_id=self._environment.murfey_session, - ) - capture_delete( - base_url=str(self._environment.url.geturl()), - router_name="session_control.router", - function_name="remove_session", - token=token, - session_id=self._environment.murfey_session, - ) - self.exit() - - async def action_process(self) -> None: - self.processing_btn.disabled = False - - def _set_default_acquisition_directories(self, default_dir: Path): - set_default_acquisition_output( - default_dir, self._machine_config["software_settings_output_directories"] - ) diff --git a/src/murfey/client/tui/controller.css b/src/murfey/client/tui/controller.css deleted file mode 100644 index d88bad4f0..000000000 --- a/src/murfey/client/tui/controller.css +++ /dev/null @@ -1,524 +0,0 @@ -Screen { - layout: grid; - grid-size: 6; - padding: 2; - outline: hidden; -} - -LaunchScreen { - layout: grid; - grid-size: 3; - padding: 2; - border: hidden; -} - -GainReference { - layout: grid; - grid-size: 2; - padding: 2; - border: hidden; -} - -ConfirmScreen { - layout: grid; - grid-size: 2; - padding: 2; - border: hidden; -} - -UpstreamDownloads { - layout: grid; - grid-size: 2; - padding: 2; - border: hidden; -} - -WaitingScreen { - layout: grid; - grid_size: 4; - padding: 2; - border: hidden; -} - -SessionSelection { - layout: grid; - grid-size: 4; - border: hidden; -} - -VisitCreation { - layout: grid; - grid-size: 2; - border: hidden; -} - -VisitSelection { - layout: grid; - grid-size: 4; - border: hidden; -} - -DirectorySelection { - layout: grid; - grid-size: 2; - border: hidden; -} - -ProcessingForm { - layout: grid; - grid-size: 1; - border: hidden; -} - -DestinationSelect { - layout: grid; - grid-size: 7; - border: hidden; -} - -Input { - background: black; - border: solid teal; -} - -RadioSet { - width: 100%; -} - -#quit { - width: 100%; - height: 100%; - column-span: 1; - background: teal; - border: solid black; -} - -#quit:hover { - background: deeppink; -} - -#waiting-quit { - width: 100%; - height: 100%; - column-span: 2; - background: teal; - border: solid black; -} - -#waiting-quit:hover { - background: deeppink; -} - -#add { - width: 100%; - height: 100%; - column-span: 1; - background: teal; - border: solid black; -} - -#add:hover { - background: purple; -} - -#launch { - width: 100%; - height: 100%; - column-span: 1; - background: teal; - border: solid black; -} - -#launch:hover { - background: purple; -} - -#waiting-launch { - width: 100%; - height: 100%; - column-span: 2; - background: teal; - border: solid black; -} - -#waiting-launch:hover { - background: purple; -} - -#skip-gain { - width: 100%; - height: 100%; - column-span: 1; - background: teal; - border: solid black; -} - -#skip-gain:hover { - background: purple; -} - -#suggested-gain-ref { - width: 100%; - height: 90fr; - column-span: 2; - background: deeppink; - border: solid black; -} - -#suggested-gain-ref:hover { - background: purple; -} - -#suggested-gain-ref:focus { - background: darkslateblue; -} - -#clear { - width: 100%; - height: 20%; - column-span: 1; - background: teal; - border: solid black; -} - -#clear:hover { - background: purple; -} - -#dir-select { - width: 100%; - height: 100%; - column-span: 2; - row-span: 2; -} - -#launch-text { - width: 100%; - height: 100%; - column-span: 2; - text-align: center; -} - -#prompt { - width: 100%; - height: 100%; - column-span: 2; - text-align: center; -} - -#waiting-prompt { - width: 100%; - height: 100%; - column-span: 4; - text-align: center; -} - -#progress { - width: 100%; - height: 100%; - column-span: 4; - align: center middle; -} - -#input-grid { - width: 100%; - column-span: 2; - row-span: 1; -} - -.btn-visit { - width: 100%; - height: 90fr; - background: teal; - border: solid black; -} - -.btn-visit:hover { - background: purple; -} - -.btn-visit:focus { - background: darkslateblue; -} - -.btn-visit-create { - width: 100%; - height: 90fr; - column-span: 2; - background: teal; - border: solid black; -} - -.btn-visit-create:hover { - background: purple; -} - -.btn-visit-create:focus { - background: darkslateblue; -} - -.btn-session { - width: 100%; - height: 20%; - column-span: 4; - row-span: 3; - background: teal; - border: solid black; -} - -.btn-session:hover { - background: purple; -} - -.btn-session:focus { - background: darkslateblue; -} - -.btn-directory { - width: 100%; - height: 90fr; - background: teal; - border: solid black; -} - -.btn-directory:hover { - background: purple; -} - -.btn-directory:focus { - background: darkslateblue; -} - -#confirm-btn { - width: 100%; - background: teal; - border: solid black; -} - -#confirm-btn:hover { - background: purple; -} - -#confirm-btn:focus { - background: darkslateblue; -} - -#destination-btn { - width: 100%; - height: 90fr; - column-span: 7; - background: teal; - border: solid black; -} - -#destination-btn:hover { - background: purple; -} - -#destination-btn:focus { - background: darkslateblue; -} - -#processing-btn { - width: 100%; - height: 90fr; - column-span: 2; - background: teal; - border: solid black; -} - -#processing-btn:hover { - background: purple; -} - -#new-visit-btn { - width: 100%; - height: 90fr; - column-span: 1; - background: teal; - border: solid black; -} - -#new-visit-btn:hover { - background: purple; -} - -#input { - width: 100%; - height: 100%; - column-span: 2; - row-span: 1; - content-align: left middle; - text-style: bold; - background: blueviolet; -} - -#monitoring { - width: 100%; - height: 90fr; - column-span: 1; -} - -.monitoring-switch { - width: 75%; -} - -#input:hover { - background: blue; -} - -#info { - width: 100%; - height: 100%; - column-span: 3; - row-span: 5; - content-align: left bottom; - text-style: bold; - background: blueviolet; -} - -#select-visit { - width: 100%; - height: 100%; - column-span: 4; - row-span: 3; - content-align: left top; - background: blueviolet; -} - -#switch-visit { - width: 100%; - height: 100%; - column-span: 3; - row-span: 1; -} - -#label-visit { - width: 100%; - height: 100%; - column-span: 1; - row-span: 1; - content-align: center middle; - text-style: bold; -} - -#select-session { - width: 100%; - height: 100%; - column-span: 4; - row-span: 3; - background: blueviolet; -} - -#label-session { - width: 100%; - height: 100%; - column-span: 1; - row-span: 1; - content-align: center middle; - text-style: bold; -} - -#select-directory { - width: 100%; - height: 100%; - column-span: 4; - row-span: 3; - content-align: left top; - background: gray; -} - -#modality-select { - width: 100%; - height: 10%; -} - -#selected-directories { - width: 100%; - height: 65%; - column-span: 1; - row-span: 2; -} - -#switch-directory { - width: 100%; - height: 100%; - column-span: 3; - row-span: 1; -} - -#label-directory { - width: 100%; - height: 100%; - column-span: 1; - row-span: 1; - content-align: center middle; - text-style: bold; -} - -#input-form { - width: 100%; - height: 100%; - background: black; -} - -#selected-directories-vert { - width: 100%; - height: 100%; - row-span: 2; - background: black; -} - -#destination-holder { - width: 100%; - height: 100%; - column-span: 7; - row-span: 3; - background: black; -} - -#user-params { - width: 100%; - height: 100%; - column-span: 7; - row-span: 3; - background: black; -} - -.input-destination { - width: 100%; - background: black; -} - -.input-visit-name { - width: 100%; - height: 100%; - column-span: 2; - row-span: 1; - content-align: left middle; - text-style: bold; - background: blueviolet; -} - -#log_book { - width: 100%; - height: 100%; - column-span: 3; - row-span: 6; - content-align: right top; - text-style: bold; -} - -.label { - width: 100%; - content-align: center middle; -} - -.input { - width: 100%; -} - -#gain-select { - width: 100%; - column-span: 2; -} diff --git a/src/murfey/client/tui/launcher.css b/src/murfey/client/tui/launcher.css deleted file mode 100644 index a204572bd..000000000 --- a/src/murfey/client/tui/launcher.css +++ /dev/null @@ -1,25 +0,0 @@ -Screen { - layout: grid; - grid-size: 2; - padding: 2; - border: hidden; -} - -#quit { - width: 100%; - height: 100%; - column-span: 1; - background: aquamarine; -} - -#launch { - width: 100%; - height: 100%; - column-span: 1; -} - -#launch-text { - width: 100%; - height: 100%; - column-span: 2; -} diff --git a/src/murfey/client/tui/main.py b/src/murfey/client/tui/main.py deleted file mode 100644 index d5949879c..000000000 --- a/src/murfey/client/tui/main.py +++ /dev/null @@ -1,353 +0,0 @@ -from __future__ import annotations - -import argparse -import configparser -import logging -import os -import platform -import shutil -import sys -import time -import webbrowser -from pathlib import Path -from pprint import pprint -from queue import Queue -from typing import Literal -from urllib.parse import ParseResult, urlparse - -from rich.prompt import Confirm - -import murfey.client.update -import murfey.client.watchdir -import murfey.client.websocket -from murfey.client.customlogging import CustomHandler, DirectableRichHandler -from murfey.client.instance_environment import MurfeyInstanceEnvironment -from murfey.client.tui.app import MurfeyTUI -from murfey.client.tui.status_bar import StatusBar -from murfey.util.client import capture_get, read_config -from murfey.util.models import Visit - -log = logging.getLogger("murfey.client") - -token = read_config()["Murfey"].get("token", "") - - -def _get_visit_list(api_base: ParseResult, instrument_name: str): - proxy_path = api_base.path.rstrip("/") - get_visits_url = api_base._replace(path=f"{proxy_path}") - server_reply = capture_get( - base_url=str(get_visits_url.geturl()), - router_name="session_control.router", - function_name="get_current_visits", - token=token, - instrument_name=instrument_name, - ) - if server_reply.status_code != 200: - raise ValueError(f"Server unreachable ({server_reply.status_code})") - return [Visit.model_validate(v) for v in server_reply.json()] - - -def write_config(config: configparser.ConfigParser): - mcch = os.environ.get("MURFEY_CLIENT_CONFIG_HOME") - murfey_client_config_home = Path(mcch) if mcch else Path.home() - with open(murfey_client_config_home / ".murfey", "w") as configfile: - config.write(configfile) - - -def main_loop( - source_watchers: list[murfey.client.watchdir.DirWatcher], - appearance_time: float, - transfer_all: bool, -): - log.info( - f"Murfey {murfey.__version__} on Python {'.'.join(map(str, sys.version_info[0:3]))} entering main loop" - ) - if appearance_time > 0: - modification_time: float | None = time.time() - appearance_time * 3600 - else: - modification_time = None - while True: - for sw in source_watchers: - sw.scan(modification_time=modification_time, transfer_all=transfer_all) - time.sleep(15) - - -def _enable_webbrowser_in_cygwin(): - """Helper function to make webbrowser.open() work in CygWin""" - if "cygwin" in platform.system().lower() and shutil.which("cygstart"): - webbrowser.register("cygstart", None, webbrowser.GenericBrowser("cygstart")) - - -def _check_for_updates( - server: ParseResult, install_version: None | Literal[True] | str -): - if install_version is True: - # User requested installation of the newest version - try: - murfey.client.update.check(server, force=True) - print("\nYou are already running the newest version of Murfey") - exit() - except Exception as e: - exit(f"Murfey update check failed with {e}") - - if install_version: - # User requested installation of a specific version - if murfey.client.update.install_murfey(server, install_version): - print(f"\nMurfey has been updated to version {install_version}") - exit() - else: - exit("Error occurred while updating Murfey") - - # Otherwise run a routine update check to ensure client and server are compatible - try: - murfey.client.update.check(server) - except Exception as e: - print(f"Murfey update check failed with {e}") - - -def run(): - # Load client config and server information - config = read_config() - instrument_name = config["Murfey"]["instrument_name"] - try: - server_routing = config["ServerRouter"] - except KeyError: - server_routing = {} - server_routing_prefix_found = False - if server_routing: - for path_prefix, server in server_routing.items(): - if str(Path.cwd()).startswith(path_prefix): - known_server = server - server_routing_prefix_found = True - break - else: - known_server = None - else: - known_server = config["Murfey"].get("server") - - # Set up argument parser with dynamic defaults based on client config - parser = argparse.ArgumentParser(description="Start the Murfey client") - parser.add_argument( - "--server", - metavar="HOST:PORT", - type=str, - help=f"Murfey server to connect to ({known_server})", - default=known_server, - ) - parser.add_argument("--visit", help="Name of visit") - parser.add_argument( - "--source", help="Directory to transfer files from", type=Path, default="." - ) - parser.add_argument( - "--destination", - help="Directory to transfer files to (syntax: 'data/2022/cm31093-2/tmp/murfey')", - ) - parser.add_argument( - "--update", - metavar="VERSION", - nargs="?", - default=None, - const=True, - help="Update Murfey to the newest or to a specific version", - ) - parser.add_argument( - "--demo", - action="store_true", - ) - parser.add_argument( - "--appearance-time", - type=float, - default=-1, - help="Only consider top level directories that have appeared more recently than this many hours ago", - ) - parser.add_argument( - "--fake-dc", - action="store_true", - default=False, - help="Do not perform data collection related calls to API (avoids database inserts)", - ) - parser.add_argument( - "--time-based-transfer", - action="store_true", - help="Transfer new files", - ) - parser.add_argument( - "--no-transfer", - action="store_true", - help="Avoid actually transferring files", - ) - parser.add_argument( - "--debug", - action="store_true", - help="Turn on debugging logs", - ) - parser.add_argument( - "--local", - action="store_true", - default=False, - help="Perform rsync transfers locally rather than remotely", - ) - parser.add_argument( - "--ignore-mdoc-metadata", - action="store_true", - default=False, - help="Do not attempt to read metadata from all mdoc files", - ) - parser.add_argument( - "--remove-files", - action="store_true", - default=False, - help="Remove source files immediately after their transfer", - ) - parser.add_argument( - "--name", - type=str, - default="", - help="Name of Murfey session to be created", - ) - parser.add_argument( - "--skip-existing-processing", - action="store_true", - default=False, - help="Do not trigger processing for any data directories currently on disk (you may have started processing for them in a previous murfey run)", - ) - args = parser.parse_args() - - # Logic to exit early based on parsed args - if not args.server: - exit("Murfey server not set. Please run with --server host:port") - if not args.server.startswith(("http://", "https://")): - if "://" in args.server: - exit("Unknown server protocol. Only http:// and https:// are allowed") - args.server = f"http://{args.server}" - if args.remove_files: - remove_prompt = Confirm.ask( - f"Are you sure you want to remove files from {args.source or Path('.').absolute()}?" - ) - if not remove_prompt: - exit("Exiting") - - # If a new server URL is provided, save info to config file - murfey_url = urlparse(args.server, allow_fragments=False) - if args.server != known_server: - # New server specified. Verify that it is real - print(f"Attempting to connect to new server {args.server}") - try: - murfey.client.update.check(murfey_url, install=False) - except Exception as e: - exit(f"Could not reach Murfey server at {args.server!r} - {e}") - - # If server is reachable then update the configuration - config["Murfey"]["server"] = args.server - write_config(config) - - # If user requested installation of a specific or a newer version then - # make that happen, otherwise ensure client and server are compatible and - # update if necessary. - _check_for_updates(server=murfey_url, install_version=args.update) - - if args.no_transfer: - log.info("No files will be transferred as --no-transfer flag was specified") - - # Check ISPyB (if set up) for ongoing visits - ongoing_visits = [] - if args.visit: - ongoing_visits = [args.visit] - elif server_routing_prefix_found: - for part in Path.cwd().parts: - if "-" in part: - ongoing_visits = [part] - break - if not ongoing_visits: - print("Ongoing visits:") - ongoing_visits = _get_visit_list(murfey_url, instrument_name) - pprint(ongoing_visits) - ongoing_visits = [v.name for v in ongoing_visits] - - _enable_webbrowser_in_cygwin() - - # Set up additional log handlers - log.setLevel(logging.DEBUG) - log_queue = Queue() - input_queue = Queue() - - # Rich-based console handler - rich_handler = DirectableRichHandler(enable_link_path=False) - rich_handler.setLevel(logging.DEBUG if args.debug else logging.INFO) - - # Set up websocket app and handler - client_id_response = capture_get( - base_url=str(murfey_url.geturl()), - router_name="session_control.router", - function_name="new_client_id", - token=token, - ) - if client_id_response.status_code == 401: - exit( - "This instrument is not authorised to run the TUI app; please use the " - "Murfey web UI instead" - ) - elif client_id_response.status_code != 200: - exit( - "Unable to establish connection to Murfey server: \n" - f"{client_id_response.json()}" - ) - client_id: dict = client_id_response.json() - ws = murfey.client.websocket.WSApp( - server=args.server, - id=client_id["new_id"], - ) - ws_handler = CustomHandler(ws.send) - - # Add additional handlers and set logging levels - logging.getLogger().addHandler(rich_handler) - logging.getLogger().addHandler(ws_handler) - logging.getLogger("murfey").setLevel(logging.INFO) - logging.getLogger("websocket").setLevel(logging.WARNING) - - log.info("Starting Websocket connection") - - # Load machine data for subsequent sections - machine_data = capture_get( - base_url=str(murfey_url.geturl()), - router_name="session_control.router", - function_name="machine_info_by_instrument", - token=token, - instrument_name=instrument_name, - ).json() - gain_ref: Path | None = None - - # Set up Murfey environment instance and map it to websocket app - instance_environment = MurfeyInstanceEnvironment( - url=murfey_url, - client_id=ws.id, - instrument_name=instrument_name, - software_versions=machine_data.get("software_versions", {}), - demo=args.demo, - processing_only_mode=server_routing_prefix_found, - rsync_url=( - urlparse(machine_data["rsync_url"]).hostname - if machine_data.get("rsync_url") - else "" - ), - ) - ws.environment = instance_environment - - # Set up and run Murfey TUI app - status_bar = StatusBar() - rich_handler.redirect = True - app = MurfeyTUI( - environment=instance_environment, - visits=ongoing_visits, - queues={"input": input_queue, "logs": log_queue}, - status_bar=status_bar, - dummy_dc=args.fake_dc, - do_transfer=not args.no_transfer, - gain_ref=gain_ref, - redirected_logger=rich_handler, - force_mdoc_metadata=not args.ignore_mdoc_metadata, - processing_enabled=machine_data.get("processing_enabled", True), - skip_existing_processing=args.skip_existing_processing, - ) - app.run() - rich_handler.redirect = False diff --git a/src/murfey/client/tui/progress.py b/src/murfey/client/tui/progress.py deleted file mode 100644 index f23155a49..000000000 --- a/src/murfey/client/tui/progress.py +++ /dev/null @@ -1,159 +0,0 @@ -""" -A progressbar widget that uses block characters rather than a thin -horizontal line. This makes it more visible on Windows. -""" - -from __future__ import annotations - -import math -from functools import lru_cache -from typing import List - -import rich.progress -import rich.progress_bar -from rich.color import Color, blend_rgb -from rich.color_triplet import ColorTriplet -from rich.console import Console, ConsoleOptions, RenderResult -from rich.segment import Segment -from rich.style import Style - -PULSE_SIZE = rich.progress_bar.PULSE_SIZE - - -class BlockProgressBar(rich.progress_bar.ProgressBar): - @lru_cache(maxsize=16) - def _get_pulse_segments( - self, - fore_style: Style, - back_style: Style, - color_system: str, - no_color: bool, - ascii: bool = False, - ) -> List[Segment]: - """Get a list of segments to render a pulse animation. - - Returns: - List[Segment]: A list of segments, one segment per character. - """ - bar = "-" if ascii else "█" - segments: List[Segment] = [] - if color_system not in ("standard", "eight_bit", "truecolor") or no_color: - segments += [Segment(bar, fore_style)] * (PULSE_SIZE // 2) - segments += [Segment(" " if no_color else bar, back_style)] * ( - PULSE_SIZE - (PULSE_SIZE // 2) - ) - return segments - - append = segments.append - fore_color = ( - fore_style.color.get_truecolor() - if fore_style.color - else ColorTriplet(255, 0, 255) - ) - back_color = ( - back_style.color.get_truecolor() - if back_style.color - else ColorTriplet(0, 0, 0) - ) - cos = math.cos - pi = math.pi - _Segment = Segment - _Style = Style - from_triplet = Color.from_triplet - - for index in range(PULSE_SIZE): - position = index / PULSE_SIZE - fade = 0.5 + cos(position * pi * 2) / 2.0 - color = blend_rgb(fore_color, back_color, cross_fade=fade) - append(_Segment(bar, _Style(color=from_triplet(color)))) - return segments - - def __rich_console__( - self, console: Console, options: ConsoleOptions - ) -> RenderResult: - width = min(self.width or options.max_width, options.max_width) - ascii = options.ascii_only - if self.pulse: - yield from self._render_pulse(console, width, ascii=ascii) - return - - completed = min(self.total, max(0, self.completed)) - - bar = "-" if ascii else "█" - half_bar_right = " " if ascii else "▌" - half_bar_left = " " if ascii else "▐" - complete_halves = ( - int(width * 2 * completed / self.total) if self.total else width * 2 - ) - bar_count = complete_halves // 2 - half_bar_count = complete_halves % 2 - style = console.get_style(self.style) - complete_style = console.get_style( - self.complete_style if self.completed < self.total else self.finished_style - ) - _Segment = Segment - if bar_count: - yield _Segment(bar * bar_count, complete_style) - if half_bar_count: - yield _Segment(half_bar_right * half_bar_count, complete_style) - - if not console.no_color: - remaining_bars = width - bar_count - half_bar_count - if remaining_bars and console.color_system is not None: - if not half_bar_count and bar_count: - yield _Segment(half_bar_left, style) - remaining_bars -= 1 - if remaining_bars: - yield _Segment(bar * remaining_bars, style) - - -class BlockBarColumn(rich.progress.BarColumn): - def render(self, task: rich.progress.Task) -> rich.progress_bar.ProgressBar: - """Gets a progress bar widget for a task.""" - return BlockProgressBar( - total=max(0, task.total), - completed=max(0, task.completed), - width=None if self.bar_width is None else max(1, self.bar_width), - pulse=not task.started, - animation_time=task.get_time(), - style=self.style, - complete_style=self.complete_style, - finished_style=self.finished_style, - pulse_style=self.pulse_style, - ) - - -if __name__ == "__main__": # pragma: no cover - from rich.theme import Theme - - custom_theme = Theme( - { - "bar.complete": "rgb(249,38,249)", - "bar.finished": "rgb(31,156,31)", - "bar.pulse": "rgb(249,38,249)", - } - ) - console = Console(theme=custom_theme) - steps = 400 - arguments = { - "width": 50, - } - bars = [ - BlockProgressBar(**arguments, total=steps), - rich.progress_bar.ProgressBar(**arguments, total=steps), - BlockProgressBar(**arguments, pulse=True), - rich.progress_bar.ProgressBar(**arguments, pulse=True), - ] - - import time - - console.show_cursor(False) - for n in range(0, steps + 1, 1): - for bar in bars: - bar.update(n) - console.print(bar) - console.print(" ", end="") - console.file.write("\r") - time.sleep(3 / steps) - console.show_cursor(True) - console.print() diff --git a/src/murfey/client/tui/screens.py b/src/murfey/client/tui/screens.py deleted file mode 100644 index a30261cad..000000000 --- a/src/murfey/client/tui/screens.py +++ /dev/null @@ -1,1260 +0,0 @@ -from __future__ import annotations - -# import contextlib -import logging -import subprocess -from datetime import datetime -from functools import partial -from pathlib import Path -from typing import ( - Any, - Callable, - Dict, - List, - NamedTuple, - Optional, - OrderedDict, - Type, - TypeVar, -) - -from pydantic import BaseModel, ValidationError -from rich.box import SQUARE -from rich.panel import Panel -from textual.app import ScreenStackError -from textual.containers import VerticalScroll -from textual.message import Message -from textual.reactive import reactive -from textual.screen import Screen -from textual.widget import Widget -from textual.widgets import ( - Button, - DataTable, - DirectoryTree, - Footer, - Header, - Input, - Label, - ProgressBar, - RadioButton, - RadioSet, - RichLog, - Static, - Switch, - Tree, -) -from werkzeug.utils import secure_filename - -from murfey.client.contexts.spa import SPAModularContext -from murfey.client.contexts.tomo import TomographyContext -from murfey.client.destinations import determine_default_destination -from murfey.client.gain_ref import determine_gain_ref -from murfey.client.rsync import RSyncer -from murfey.util import posix_path -from murfey.util.client import ( - capture_delete, - capture_get, - capture_post, - get_machine_config_client, - read_config, -) -from murfey.util.models import ProcessingParametersSPA, ProcessingParametersTomo - -log = logging.getLogger("murfey.tui.screens") - -ReactiveType = TypeVar("ReactiveType") - -token = read_config()["Murfey"].get("token", "") -instrument_name = read_config()["Murfey"].get("instrument_name", "") - - -class InputResponse(NamedTuple): - question: str - allowed_responses: List[str] | None = None - default: str = "" - callback: Callable | None = None - key_change_callback: Callable | None = None - kwargs: dict | None = None - form: OrderedDict[str, Any] | None = None - model: BaseModel | None = None - - -class LogBook(RichLog): - class Log(Message): - def __init__(self, log_renderable): - self.renderable = log_renderable - super().__init__() - - -class InfoWidget(Widget): - text: reactive[str] = reactive("") - - def __init__(self, text: str, **kwargs): - super().__init__(**kwargs) - self.text = text - - def render(self) -> Panel: - return Panel(self.text, style=("on dark_magenta"), box=SQUARE) - - def _key_change(self, input_char: str | None): - if input_char is None: - self.text = self.text[:-1] - return - self.text += input_char - - -class QuickPrompt: - def __init__(self, text: str, options: List[str]): - self._text = text - self._options = options - self.warn = False - - def __repr__(self): - return repr(self._text) - - def __str__(self): - return self._text - - def __iter__(self): - return iter(self._options) - - def __bool__(self): - return bool(self._text) - - -def validate_form(form: dict, model: BaseModel) -> bool: - try: - convert = lambda x: None if x == "None" else x - validated = model(**{k: convert(v) for k, v in form.items()}) - log.info(validated.model_dump()) - return True - except (AttributeError, ValidationError) as e: - log.warning(f"Form validation failed: {str(e)}") - return False - - -class _DirectoryTree(DirectoryTree): - valid_selection = reactive(False) - - def __init__(self, *args, data_directories: List[Path] | None = None, **kwargs): - super().__init__(*args, **kwargs) - self._selected_path = self.path - self._data_directories = data_directories or [] - - def on_tree_node_selected(self, event: Tree.NodeSelected) -> None: - event.stop() - dir_entry = event.node.data.path - if dir_entry is None: - return - if dir_entry.is_dir(): - self._selected_path = dir_entry - if not self._data_directories: - self.valid_selection = True - return - for d in self._data_directories: - if Path(self._selected_path).absolute().is_relative_to(d.absolute()): - self.valid_selection = True - break - else: - self.valid_selection = False - else: - self.valid_selection = False - - -class _DirectoryTreeGain(DirectoryTree): - valid_selection = reactive(False) - - def __init__(self, gain_reference: Path, *args, **kwargs): - super().__init__(*args, **kwargs) - self._gain_reference = gain_reference - - def on_tree_node_selected(self, event: Tree.NodeSelected) -> None: - event.stop() - dir_entry = event.node.data.path - if dir_entry is None: - return - if not dir_entry.is_dir(): - self.valid_selection = True - self._gain_reference = dir_entry - else: - self.valid_selection = False - - -class LaunchScreen(Screen): - _launch_btn: Button | None = None - - def __init__( - self, *args, basepath: Path = Path("."), add_basepath: bool = False, **kwargs - ): - super().__init__(*args, **kwargs) - self._selected_dir = basepath - self._add_basepath = add_basepath - self._context: Type[SPAModularContext] | Type[TomographyContext] - self._context = SPAModularContext - - def compose(self): - machine_data = capture_get( - base_url=str(self.app._environment.url.geturl()), - router_name="session_control.router", - function_name="machine_info_by_instrument", - token=token, - instrument_name=instrument_name, - ).json() - self._dir_tree = _DirectoryTree( - str(self._selected_dir), - data_directories=machine_data.get("data_directories", []), - id="dir-select", - ) - - yield self._dir_tree - text_log = RichLog(id="selected-directories") - widgets = [text_log, Button("Clear", id="clear")] - text_log_block = VerticalScroll(*widgets, id="selected-directories-vert") - yield text_log_block - - text_log.write("Selected directories:\n") - btn_disabled = True - for d in machine_data.get("data_directories", []): - if ( - Path(self._dir_tree._selected_path) - .absolute() - .is_relative_to(Path(d).absolute()) - or self.app._environment.processing_only_mode - ): - btn_disabled = False - break - self._launch_btn = Button("Launch", id="launch", disabled=btn_disabled) - self._add_btn = Button("Add directory", id="add", disabled=btn_disabled) - self.watch(self._dir_tree, "valid_selection", self._check_valid_selection) - yield self._add_btn - yield self._launch_btn - yield Button("Quit", id="quit") - - def on_mount(self): - if self._add_basepath: - self._add_directory(str(self._selected_dir)) - - def _check_valid_selection(self, valid: bool): - if self._add_btn: - if valid: - self._add_btn.disabled = False - else: - self._add_btn.disabled = True - - def _add_directory(self, directory: str, add_destination: bool = True): - source = Path(self._dir_tree.path).absolute() / directory - if add_destination: - for s in self.app._environment.sources: - if source.is_relative_to(s): - return - self.app._environment.sources.append(source) - self.app._default_destinations[source] = f"{datetime.now().year}" - if self._launch_btn: - self._launch_btn.disabled = False - self.query_one("#selected-directories").write(str(source) + "\n") - - def on_button_pressed(self, event: Button.Pressed) -> None: - if event.button.id == "quit": - self.app.clean_up_quit() - elif event.button.id == "add": - self._add_directory(self._dir_tree._selected_path) - elif event.button.id == "launch": - text = self.app._visit - visit_path = "" - transfer_routes = {} - for s, defd in self.app._default_destinations.items(): - _default = determine_default_destination( - self.app._visit, - s, - defd, - self.app._environment, - self.app.analysers, - token, - touch=True, - ) - visit_path = defd + f"/{text}" - if self.app._environment.processing_only_mode: - self.app._start_rsyncer( - Path(_default), _default, visit_path=visit_path - ) - transfer_routes[s] = _default - self.app.install_screen( - DestinationSelect(transfer_routes, self._context), - "destination-select-screen", - ) - self.app.pop_screen() - self.app.push_screen("destination-select-screen") - elif event.button.id == "clear": - sel_dir = self.query_one("#selected-directories") - for line in sel_dir.lines[1:]: - source = Path(line.text) - if source in self.app._environment.sources: - self.app._environment.sources.remove(source) - if self.app._default_destinations.get(source): - del self.app._default_destinations[source] - sel_dir.clear() - sel_dir.write("Selected directories:\n") - - -class ConfirmScreen(Screen): - def __init__( - self, - prompt: str, - *args, - params: dict | None = None, - pressed_callback: Callable | None = None, - button_names: dict | None = None, - push: str = "main", - **kwargs, - ): - super().__init__(*args, **kwargs) - self._prompt = prompt - self._params = params or {} - self._callback = pressed_callback - self._button_names = button_names or {} - self._push = push - - def compose(self): - if self._params: - dt = DataTable(id="prompt") - keys = list(self._params.keys()) - dt.add_columns(*keys) - dt.add_rows([[self._params[k] for k in keys]]) - yield dt - else: - yield Static(self._prompt, id="prompt") - yield Button(self._button_names.get("launch") or "Launch", id="launch") - yield Button(self._button_names.get("quit") or "Back", id="quit") - - def on_button_pressed(self, event: Button.Pressed) -> None: - if event.button.id == "quit": - self.app.pop_screen() - self.app.uninstall_screen("confirm") - else: - while True: - try: - if self.app.screen._name == "main": - break - self.app.pop_screen() - except ScreenStackError: - break - if self._push: - log.info(f"Pushing screen {self._push}") - self.app.push_screen(self._push) - self.app.uninstall_screen("confirm") - if self._callback and event.button.id == "launch": - self._callback(params=self._params) - - -class ProcessingForm(Screen): - _form = reactive({}) - _vert = None - - def __init__( - self, - form: dict, - *args, - **kwargs, - ): - super().__init__(*args, **kwargs) - self._form = form - self._inputs: Dict[Input, str] = {} - - def compose(self): - inputs = [] - analyser = list(self.app.analysers.values())[0] - for k in analyser._context.user_params + analyser._context.metadata_params: - t = k.label - inputs.append(Label(t, classes="label")) - if self._form.get(k.name) in ("true", "True", True): - i = Switch(value=True, classes="input", id=f"switch_{k.name}") - elif self._form.get(k.name) in ("false", "False", False): - i = Switch(value=False, classes="input", id=f"switch_{k.name}") - else: - i = Input(placeholder=t, classes="input", id=f"input_{k.name}") - default = self._form.get(k.name, str(k.default)) - i.value = "None" if default is None else default - self._inputs[i] = k.name - inputs.append(i) - confirm_btn = Button("Confirm", id="confirm-btn") - if self._form.get("motion_corr_binning") == "2": - self._vert = VerticalScroll( - *inputs, - Label("Collected in counting mode:"), - Switch(id="superres", value=True, classes="input"), - confirm_btn, - id="input-form", - ) - else: - self._vert = VerticalScroll(*inputs, confirm_btn, id="input-form") - yield self._vert - - def _write_params( - self, - params: dict | None = None, - model: ProcessingParametersTomo | ProcessingParametersSPA | None = None, - ): - if params: - try: - analyser = [a for a in self.app.analysers.values() if a._context][0] - except IndexError: - return - for k in analyser._context.user_params + analyser._context.metadata_params: - self.app.query_one("#info").write(f"{k.label}: {params.get(k.name)}") - self.app._start_dc(params) - if model == ProcessingParametersTomo: - capture_post( - base_url=str(self.app._environment.url.geturl()), - router_name="workflow.tomo_router", - function_name="register_tomo_proc_params", - token=token, - session_id=self.app._environment.murfey_session, - data=params, - ) - elif model == ProcessingParametersSPA: - capture_post( - base_url=str(self.app._environment.url.geturl()), - router_name="workflow.spa_router", - function_name="register_spa_proc_params", - token=token, - session_id=self.app._environment.murfey_session, - data=params, - ) - capture_post( - base_url=str(self.app._environment.url.geturl()), - router_name="workflow.spa_router", - function_name="flush_spa_processing", - token=token, - visit_name=self.app._environment.visit, - session_id=self.app._environment.murfey_session, - ) - - def on_switch_changed(self, event): - if event.switch.id == "superres": - pix_size = self.query_one("#input_pixel_size_on_image") - motion_corr_binning = self.query_one("#input_motion_corr_binning") - if event.value: - pix_size.value = str(float(pix_size.value) / 2) - motion_corr_binning.value = "2" - else: - pix_size.value = str(float(pix_size.value) * 2) - motion_corr_binning.value = "1" - else: - k = self._inputs[event.switch] - self._form[k] = event.value - - def on_input_changed(self, event): - k = self._inputs[event.input] - self._form[k] = event.value - - def on_button_pressed(self, event): - model = None - if self.app.analysers.get(Path(self._form.get("source", ""))): - if model := self.app.analysers[Path(self._form["source"])].parameters_model: - valid = validate_form(self._form, model) - if not valid: - return - if "confirm" not in self.app._installed_screens: - self.app.install_screen( - ConfirmScreen( - "Launch processing?", - params=self._form, - pressed_callback=partial(self._write_params, model=model), - ), - "confirm", - ) - self.app.push_screen("confirm") - - -class SwitchSelection(Screen): - def __init__( - self, - name: str, - elements: List[str], - switch_label: str, - switch_status: bool = True, - *args, - **kwargs, - ): - super().__init__(*args, **kwargs) - self._elements = elements - self._switch_status = switch_status - self._switch_label = switch_label - self._name = name - - def compose(self): - hovers = ( - [ - Button(e, id=f"btn-{self._name}-{e}", classes=f"btn-{self._name}") - for e in self._elements - ] - if self._elements - else [Button("No elements found")] - ) - yield VerticalScroll(*hovers, id=f"select-{self._name}") - yield Static(self._switch_label, id=f"label-{self._name}") - yield Switch(id=f"switch-{self._name}", value=self._switch_status) - - def on_switch_changed(self, event): - self._switch_status = event.value - - -class SessionSelection(Screen): - def __init__( - self, sessions: List[str], sessions_with_client: List[str], *args, **kwargs - ): - super().__init__(*args, **kwargs) - self._sessions = sessions - self._sessions_with_client = sessions_with_client - self._name = "session" - - def compose(self): - hovers = ( - [ - Button(e, id=f"btn-{self._name}-{e}", classes=f"btn-{self._name}") - for e in self._sessions - ] - if self._sessions - else [Button("No elements found")] - ) - deletes = ( - [ - Button( - f"Remove {e}", - id=f"btn-{self._name}-{e}-del", - classes=f"btn-{self._name}", - ) - for e in self._sessions - ] - if self._sessions - else [Button("No elements found")] - ) - yield VerticalScroll( - *[v for pair in zip(hovers, deletes) for v in pair], - id=f"select-{self._name}", - ) - yield Button( - "New session", id=f"btn-{self._name}-new", classes=f"btn-{self._name}" - ) - - def on_button_pressed(self, event: Button.Pressed): - if event.button.id.endswith("new"): - session_id = None - self.app.pop_screen() - elif event.button.id.endswith("del"): - session_id = int( - str(event.button.label.split(":")[0]).replace("Remove ", "") - ) - self.app.pop_screen() - self.app.install_screen( - ConfirmScreen( - ( - f"Remove session {session_id} [WARNING: there are clients already using this session]" - if str(event.button.label) in self._sessions_with_client - else f"Remove session {session_id}" - ), - pressed_callback=partial(self._remove_session, session_id), - button_names={"launch": "Yes"}, - push="visit-select-screen", - ), - "confirm", - ) - self.app.push_screen("confirm") - return - else: - self.app._environment.murfey_session = int( - str(event.button.label.split(":")[0]) - ) - session_id = self.app._environment.murfey_session - self.app.pop_screen() - session_name = "Client connection" - self.app._environment.murfey_session = capture_post( - base_url=str(self.app._environment.url.geturl()), - router_name="session_control.router", - function_name="link_client_to_session", - token=token, - instrument_name=self.app._environment.instrument_name, - client_id=self.app._environment.client_id, - data={"session_id": session_id, "session_name": session_name}, - ).json() - - def _remove_session(self, session_id: int, **kwargs): - capture_delete( - base_url=str(self.app._environment.url.geturl()), - router_name="session_control.router", - function_name="remove_session", - token=token, - session_id=session_id, - ) - exisiting_sessions = capture_get( - base_url=str(self.app._environment.url.geturl()), - router_name="session_control.router", - function_name="get_sessions", - token=token, - ).json() - self.app.uninstall_screen("session-select-screen") - if exisiting_sessions: - self.app.install_screen( - SessionSelection( - [ - f"{s['session']['id']}: {s['session']['name']}" - for s in exisiting_sessions - ], - [ - f"{s['session']['id']}: {s['session']['name']}" - for s in exisiting_sessions - if s["clients"] - ], - ), - "session-select-screen", - ) - self.app.push_screen("session-select-screen") - else: - session_name = "Client connection" - resp = capture_post( - base_url=str(self.app._environment.url.geturl()), - router_name="session_control.router", - function_name="link_client_to_session", - token=token, - instrument_name=self.app._environment.instrument_name, - client_id=self.app._environment.client_id, - data={"session_id": None, "session_name": session_name}, - ) - if resp: - self.app._environment.murfey_session = resp.json() - - -class VisitSelection(SwitchSelection): - def __init__(self, visits: List[str], *args, **kwargs): - super().__init__( - "visit", - visits, - "Create visit directory (suggested)", - *args, - **kwargs, - ) - - def on_button_pressed(self, event: Button.Pressed): - text = str(event.button.label) - self.app._visit = text - self.app._environment.visit = text - response = capture_post( - base_url=str(self.app._environment.url.geturl()), - router_name="session_control.router", - function_name="register_client_to_visit", - token=token, - visit_name=text, - data={"id": self.app._environment.client_id}, - ) - log.info(f"Posted visit registration: {response.status_code}") - machine_data = capture_get( - base_url=str(self.app._environment.url.geturl()), - router_name="session_control.router", - function_name="machine_info_by_instrument", - token=token, - instrument_name=instrument_name, - ).json() - - if self._switch_status: - self.app.install_screen( - DirectorySelection( - [ - path - for path in machine_data.get("data_directories", []) - if Path(path).exists() - ] - ), - "directory-select", - ) - self.app.pop_screen() - - if machine_data.get("gain_reference_directory"): - self.app.install_screen( - GainReference( - determine_gain_ref(Path(machine_data["gain_reference_directory"])), - self._switch_status, - ), - "gain-ref-select", - ) - self.app.push_screen("gain-ref-select") - else: - if self._switch_status: - self.app.push_screen("directory-select") - else: - self.app.install_screen(LaunchScreen(basepath=Path("./")), "launcher") - self.app.push_screen("launcher") - - if machine_data.get("upstream_data_directories"): - upstream_downloads: dict[str, dict[str, Path]] = capture_get( - base_url=str(self.app._environment.url.geturl()), - router_name="session_control.correlative_router", - function_name="find_upstream_visits", - token=token, - session_id=self.app._environment.murfey_session, - ).json() - # Pass flattened dict for backwards compatibility - self.app.install_screen( - UpstreamDownloads( - { - visit_name: visit_dir - for _, upstream_visits in upstream_downloads.items() - for visit_name, visit_dir in upstream_visits.items() - } - ), - "upstream-downloads", - ) - self.app.push_screen("upstream-downloads") - - -class VisitCreation(Screen): - # This allows for the manual creation of a visit name when there is no LIMS system to provide it - # Shares a lot of code with VisitSelection, should be neatened up at some point - visit_name: reactive[str] = reactive("") - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def compose(self): - yield Input(placeholder="Visit name", classes="input-visit-name") - yield Button("Create visit", classes="btn-visit-create") - - def on_input_changed(self, event): - self.visit_name = event.value - - def on_button_pressed(self, event: Button.Pressed): - text = str(self.visit_name) - self.app._visit = text - self.app._environment.visit = text - response = capture_post( - base_url=str(self.app._environment.url.geturl()), - router_name="session_control.router", - function_name="register_client_to_visit", - token=token, - visit_name=text, - data={"id": self.app._environment.client_id}, - ) - log.info(f"Posted visit registration: {response.status_code}") - machine_data = capture_get( - base_url=str(self.app._environment.url.geturl()), - router_name="session_control.router", - function_name="machine_info_by_instrument", - token=token, - instrument_name=instrument_name, - ).json() - - self.app.install_screen( - DirectorySelection( - [ - path - for path in machine_data.get("data_directories", []) - if Path(path).exists() - ] - ), - "directory-select", - ) - self.app.pop_screen() - - if machine_data.get("gain_reference_directory"): - self.app.install_screen( - GainReference( - determine_gain_ref(Path(machine_data["gain_reference_directory"])), - True, - ), - "gain-ref-select", - ) - self.app.push_screen("gain-ref-select") - else: - self.app.push_screen("directory-select") - - if machine_data.get("upstream_data_directories"): - upstream_downloads: dict[str, dict[str, Path]] = capture_get( - base_url=str(self.app._environment.url.geturl()), - router_name="session_control.correlative_router", - function_name="find_upstream_visits", - token=token, - session_id=self.app._environment.murfey_session, - ).json() - # Pass a flattened dict for backwards compatibility - self.app.install_screen( - UpstreamDownloads( - { - visit_name: visit_dir - for _, upstream_visits in upstream_downloads.items() - for visit_name, visit_dir in upstream_visits.items() - } - ), - "upstream-downloads", - ) - self.app.push_screen("upstream-downloads") - - -class UpstreamDownloads(Screen): - def __init__(self, connected_visits: Dict[str, Path], *args, **kwargs): - super().__init__(*args, **kwargs) - self._connected_visits = connected_visits - - def compose(self): - visit_buttons = [ - Button(cv, classes="btn-directory") for cv in self._connected_visits.keys() - ] - yield VerticalScroll(*visit_buttons) - yield Button("Skip", classes="btn-directory") - - def on_button_pressed(self, event: Button.Pressed): - machine_data = capture_get( - base_url=str(self.app._environment.url.geturl()), - router_name="session_control.router", - function_name="machine_info_by_instrument", - token=token, - instrument_name=instrument_name, - ).json() - if machine_data.get("upstream_data_download_directory"): - # Create the directory locally to save files to - download_dir = Path(machine_data["upstream_data_download_directory"]) / str( - event.button.label - ) - download_dir.mkdir(exist_ok=True) - - # Get the paths to the TIFF files generated previously under the same session ID - upstream_tiff_paths_response = capture_get( - base_url=str(self.app._environment.url.geturl()), - router_name="session_control.correlative_router", - function_name="gather_upstream_tiffs", - token=token, - visit_name=event.button.label, - session_id=self.app._environment.murfey_session, - ) - upstream_tiff_paths = upstream_tiff_paths_response.json() or [] - - # Request to download the TIFF files found - for tp in upstream_tiff_paths: - (download_dir / tp).parent.mkdir(exist_ok=True, parents=True) - # Write TIFF to the specified file path - stream_response = capture_get( - base_url=str(self.app._environment.url.geturl()), - router_name="session_control.correlative_router", - function_name="get_tiff_file", - token=token, - visit_name=event.button.label, - session_id=self.app._environment.murfey_session, - tiff_path=tp, - ) - # Write the file chunk-by-chunk to avoid hogging memory - with open(download_dir / tp, "wb") as utiff: - for chunk in stream_response.iter_content(chunk_size=32 * 1024**2): - utiff.write(chunk) - self.app.pop_screen() - - -class GainReference(Screen): - def __init__(self, gain_reference: Path, switch_status: bool, *args, **kwargs): - super().__init__(*args, **kwargs) - self._gain_reference = gain_reference - self._switch_status = switch_status - - def compose(self): - self._dir_tree = _DirectoryTreeGain( - self._gain_reference, - str(self._gain_reference.parent.parent), - id="gain-select", - ) - yield self._dir_tree - self._launch_btn = Button("Launch", id="launch") - self.watch(self._dir_tree, "valid_selection", self._check_valid_selection) - yield Button( - f"Suggested gain reference: {self._gain_reference.parent / self._gain_reference.name}", - id="suggested-gain-ref", - ) - yield self._launch_btn - yield Button("No gain", id="skip-gain") - - def _check_valid_selection(self, valid: bool): - if self._launch_btn: - if valid: - self._launch_btn.disabled = False - else: - self._launch_btn.disabled = True - - def on_button_pressed(self, event): - if event.button.id == "skip-gain": - self.app.pop_screen() - else: - if event.button.id == "suggested-gain-ref": - self._dir_tree._gain_reference = self._gain_reference - visit_path = f"{datetime.now().year}/{self.app._environment.visit}" - # Set up rsync command - rsync_cmd = [ - "rsync", - f"{posix_path(self._dir_tree._gain_reference)!r}", - f"{self.app._environment.rsync_url or self.app._environment.url.hostname}::{self.app._machine_config.get('rsync_module', 'data')}/{visit_path}/processing/{secure_filename(self._dir_tree._gain_reference.name)}", - ] - # Encase in bash shell - cmd = ["bash", "-c", " ".join(rsync_cmd)] - if self.app._environment.demo: - log.info(f"Would perform {' '.join(cmd)}") - else: - # Run rsync subprocess - gain_rsync = subprocess.run(cmd) - if gain_rsync.returncode: - log.warning( - f"Gain reference file {posix_path(self._dir_tree._gain_reference)!r} was not successfully transferred to {visit_path}/processing" - ) - process_gain_response = capture_post( - base_url=str(self.app._environment.url.geturl()), - router_name="file_io_instrument.router", - function_name="process_gain", - token=token, - session_id=self.app._environment.murfey_session, - data={ - "gain_ref": str(self._dir_tree._gain_reference), - "eer": bool( - self.app._machine_config.get("external_executables_eer") - ), - }, - ) - if str(process_gain_response.status_code).startswith("4"): - log.warning( - f"Gain processing failed: status code {process_gain_response.status_code}" - ) - else: - log.info( - f"Gain reference file {process_gain_response.json().get('gain_ref')}" - ) - self.app._environment.gain_ref = process_gain_response.json().get( - "gain_ref" - ) - if self._switch_status: - self.app.push_screen("directory-select") - else: - self.app.install_screen(LaunchScreen(basepath=Path("./")), "launcher") - self.app.push_screen("launcher") - - -class DirectorySelection(SwitchSelection): - def __init__(self, directories: List[str], *args, **kwargs): - super().__init__( - "directory", - directories, - "Automatically transfer and trigger processing for new directories (recommended)", - *args, - **kwargs, - ) - - def on_button_pressed(self, event: Button.Pressed): - self.app._multigrid = self._switch_status - visit_dir = Path(str(event.button.label)).absolute() / self.app._visit - visit_dir.mkdir(exist_ok=True) - self.app._set_default_acquisition_directories(visit_dir) - machine_config = get_machine_config_client( - str(self.app._environment.url.geturl()), - token, - instrument_name=self.app._environment.instrument_name, - demo=self.app._environment.demo, - ) - for dir in machine_config["create_directories"]: - (visit_dir / dir).mkdir(exist_ok=True) - self.app.install_screen( - LaunchScreen(basepath=visit_dir, add_basepath=True), "launcher" - ) - self.app.pop_screen() - self.app.push_screen("launcher") - - -class DestinationSelect(Screen): - def __init__( - self, - transfer_routes: Dict[Path, str], - context: Type[SPAModularContext] | Type[TomographyContext], - *args, - destination_overrides: Optional[Dict[Path, str]] = None, - use_transfer_routes: bool = False, - **kwargs, - ): - super().__init__(*args, **kwargs) - self._transfer_routes = transfer_routes - self._destination_overrides: Dict[Path, str] = destination_overrides or {} - self._user_params: Dict[str, str] = {} - self._inputs: Dict[Input, str] = {} - self._context = context - self._use_transfer_routes = use_transfer_routes - - def compose(self): - bulk = [] - with RadioSet(): - yield RadioButton("SPA", value=self._context is SPAModularContext) - yield RadioButton("Tomography", value=self._context is TomographyContext) - if self.app._multigrid: - machine_config = get_machine_config_client( - str(self.app._environment.url.geturl()), - token, - instrument_name=self.app._environment.instrument_name, - ) - destinations = [] - if self._destination_overrides: - for k, v in self._destination_overrides.items(): - destinations.append(v) - bulk.append(Label(f"Copy the source {k} to:")) - bulk.append( - Input( - value=v, - id=f"destination-{str(k)}", - classes="input-destination", - ) - ) - else: - for s in self._transfer_routes.keys(): - for d in s.glob("*"): - if ( - d.is_dir() - and d.name not in machine_config["create_directories"] - ): - dest = determine_default_destination( - self.app._visit, - s, - f"{datetime.now().year}", - self.app._environment, - self.app.analysers, - token, - touch=True, - ) - if dest and dest in destinations: - dest_path = Path(dest) - name_root = "" - dest_num = 0 - for i, st in enumerate(dest_path.name): - if st.isnumeric(): - dest_num = int(dest_path.name[i:]) - break - name_root += st - if dest_num: - dest = str( - dest_path.parent / f"{name_root}{dest_num + 1}" - ) - else: - dest = str(dest_path.parent / f"{name_root}2") - destinations.append(dest) - bulk.append(Label(f"Copy the source {d} to:")) - bulk.append( - Input( - value=dest, - id=f"destination-{str(d)}", - classes="input-destination", - ) - ) - else: - machine_config = get_machine_config_client( - str(self.app._environment.url.geturl()), - token, - instrument_name=self.app._environment.instrument_name, - ) - for s, d in self._transfer_routes.items(): - if Path(d).name not in machine_config["create_directories"]: - bulk.append(Label(f"Copy the source {s} to:")) - bulk.append( - Input( - value=d, - id=f"destination-{str(s)}", - classes="input-destination", - ) - ) - yield VerticalScroll(*bulk, id="destination-holder") - params_bulk = [] - if self.app._multigrid and self.app._processing_enabled: - for k in self._context.user_params: - params_bulk.append(Label(k.label)) - val = getattr(self.app._environment, k.name, str(k.default)) - self._user_params[k.name] = val - if val in ("true", "True", True): - i = Switch(value=True, id=k.name, classes="input-destination") - elif val in ("false", "False", False): - i = Switch(value=False, id=k.name, classes="input-destination") - else: - i = Input(value=val, id=k.name, classes="input-destination") - params_bulk.append(i) - self._inputs[i] = k.name - machine_config = get_machine_config_client( - str(self.app._environment.url.geturl()), - token, - instrument_name=self.app._environment.instrument_name, - demo=self.app._environment.demo, - ) - if machine_config.get("superres"): - params_bulk.append( - Label("Collected in super resoultion mode unbinned:") - ) - params_bulk.append( - Switch( - value=False, - id="superres-multigrid", - classes="input-destination", - ) - ) - self.app._environment.superres = False - yield VerticalScroll( - *params_bulk, - id="user-params", - ) - yield Button("Confirm", id="destination-btn") - - def on_switch_changed(self, event): - if event.switch.id == "superres-multigrid": - self.app._environment.superres = event.value - else: - for k in self._context.user_params: - if event.switch.id == k.name: - self._user_params[k.name] = event.value - - def on_radio_set_changed(self, event: RadioSet.Changed) -> None: - if event.index == 0: - self._context = SPAModularContext - else: - self._context = TomographyContext - self.app.pop_screen() - self.app.uninstall_screen("destination-select-screen") - self.app.install_screen( - DestinationSelect( - self._transfer_routes, - self._context, - destination_overrides=self._destination_overrides, - use_transfer_routes=True, - ), - "destination-select-screen", - ) - self.app.push_screen("destination-select-screen") - - def on_input_changed(self, event): - if event.input.id.startswith("destination-"): - if not self.app._multigrid: - self._transfer_routes[Path(event.input.id[12:])] = event.value - else: - self._destination_overrides[Path(event.input.id[12:])] = event.value - else: - for k in self._context.user_params: - if event.input.id == k.name: - self._user_params[k.name] = event.value - - def on_button_pressed(self, event): - if self.app._multigrid and self.app._processing_enabled: - if self._context == TomographyContext: - valid = validate_form(self._user_params, ProcessingParametersTomo.Base) - else: - valid = validate_form(self._user_params, ProcessingParametersSPA.Base) - if not valid: - return - for s, d in self._transfer_routes.items(): - self.app._default_destinations[s] = d - self.app._register_dc = True - if self.app._multigrid: - for k, v in self._destination_overrides.items(): - self.app._environment.destination_registry[k.name] = v - self.app._launch_multigrid_watcher( - s, destination_overrides=self._destination_overrides - ) - else: - self.app._start_rsyncer(s, d) - for k, v in self._user_params.items(): - setattr(self.app._environment, k, v) - self.app.pop_screen() - self.app.push_screen("main") - - -class WaitingScreen(Screen): - def __init__( - self, - prompt: str, - num_files: int, - *args, - **kwargs, - ): - super().__init__(*args, **kwargs) - self._num_files = 0 - self._prompt = prompt - self._new_rsyncers: List[RSyncer] = [] - - def compose(self): - yield Static(self._prompt, id="waiting-prompt") - yield ProgressBar(id="progress") - yield Button("Start", id="waiting-launch") - yield Button("Back", id="waiting-quit") - - def on_button_pressed(self, event: Button.Pressed) -> None: - if event.button.id == "waiting-quit": - self.app.pop_screen() - self.app.uninstall_screen("waiting") - else: - sources = [] - if self.app.rsync_processes or self.app._environment.demo: - for k, rp in self.app.rsync_processes.items(): - rp.stop() - if self.app.analysers.get(k): - self.app.analysers[k].stop() - removal_rp = RSyncer.from_rsyncer(rp, remove_files=True) - removal_rp._listeners = [self.file_copied] - self._new_rsyncers.append(removal_rp) - sources.append(k) - log.info( - f"Starting to remove data files {self.app._environment.demo}, {len(self.app.rsync_processes)}" - ) - self.query_one(ProgressBar).update(total=self._num_files) - for k, removal_rp in zip(sources, self._new_rsyncers): - for f in k.absolute().glob("**/*"): - if f.is_file(): - self._num_files += 1 - removal_rp.queue.put(f) - removal_rp.queue.put(None) - self.query_one(ProgressBar).update(total=self._num_files) - for removal_rp in self._new_rsyncers: - removal_rp.start() - - def file_copied(self, *args, **kwargs): - self.query_one(ProgressBar).advance(1) - if self.query_one(ProgressBar).progress == self.query_one(ProgressBar).total: - capture_post( - base_url=str(self.app._environment.url.geturl()), - router_name="session_control.router", - function_name="register_processing_success_in_ispyb", - token=token, - session_id=self.app._environment.murfey_session, - ) - capture_delete( - base_url=str(self.app._environment.url.geturl()), - router_name="session_control.router", - function_name="remove_session", - token=token, - session_id=self.app._environment.murfey_session, - ) - self.app.exit() - - -class MainScreen(Screen): - def compose(self): - self.app.log_book = LogBook(id="log_book", wrap=True, max_lines=200) - if self.app._redirected_logger: - log.info("connecting logger") - self.app._redirected_logger.text_log = self.app.log_book - log.info("logger connected") - self.app.hovers = ( - [Button(v, id="visit-btn") for v in self.app.visits] - if len(self.app.visits) - else [Button("No ongoing visits found")] - ) - self.app.processing_form = ProcessingForm(self.app._form_values) - yield Header() - info_widget = RichLog(id="info", markup=True) - yield info_widget - yield self.app.log_book - info_widget.write( - f"[bold]Welcome to Murfey ({self.app._environment.visit})[/bold]" - ) - yield Button("Visit complete", id="new-visit-btn") - yield Footer() - - def on_mount(self, event): - capture_post( - base_url=str(self.app._environment.url.geturl()), - router_name="prometheus.router", - function_name="change_monitoring_status", - token=token, - visit_name=self.app._environment.visit, - on=1, - ) diff --git a/src/murfey/client/tui/status_bar.py b/src/murfey/client/tui/status_bar.py deleted file mode 100644 index 7e761e8e4..000000000 --- a/src/murfey/client/tui/status_bar.py +++ /dev/null @@ -1,67 +0,0 @@ -from __future__ import annotations - -import functools -import logging -import time -from threading import RLock - -from rich.box import SQUARE -from rich.panel import Panel -from rich.progress import ( - Progress, - SpinnerColumn, - TextColumn, - TimeRemainingColumn, - TransferSpeedColumn, -) -from rich.table import Column -from textual.reactive import Reactive -from textual.widget import Widget - -from murfey.client.tui.progress import BlockBarColumn - -log = logging.getLogger("murfey.client.tui.status_bar") - - -class StatusBar(Widget): - transferred = Reactive([0, 0]) - _current_progress = 0 - lock: RLock = RLock() - - @functools.lru_cache() - def get_progress(self): - text_column = TextColumn("{task.description}", table_column=Column(ratio=1)) - bar_column = BlockBarColumn(bar_width=None, table_column=Column(ratio=3)) - progress = Progress( - text_column, - bar_column, - TransferSpeedColumn(), - SpinnerColumn(), - TimeRemainingColumn(), - expand=True, - ) - - task1 = progress.add_task("[red]Transferring...", total=self.transferred[1]) - task2 = None # progress.add_task("[green]Processing...", total=1000) - task3 = None # progress.add_task("[cyan]Cooking...", total=1000) - return (progress, task1, task2, task3) - - def render(self) -> Panel: - progress, task1, task2, task3 = self.get_progress() - # elapsed = (time.time() - self.start) * 100 - - log.info(f"For transfer {self.transferred[1]}") - advance = self.transferred[0] - self._current_progress - self._current_progress += advance - log.info(f"Advance: {advance}, {self.transferred[0]}") - progress.update(task1, completed=self.transferred[0], total=self.transferred[1]) - # progress.update(task2, completed=max(0, min(1000, elapsed - 1000))) - # progress.update(task3, completed=max(0, min(1000, elapsed - 2000))) - return Panel(progress.make_tasks_table(progress.tasks), height=5, box=SQUARE) - - def on_mount(self): - self.start = time.time() - self.set_interval(0.3, self.tick) - - def tick(self): - self.refresh() diff --git a/src/murfey/client/watchdir.py b/src/murfey/client/watchdir.py index 07bbebcc2..fdb3d363b 100644 --- a/src/murfey/client/watchdir.py +++ b/src/murfey/client/watchdir.py @@ -14,7 +14,6 @@ from pathlib import Path from typing import List, NamedTuple, Optional -from murfey.client.tui.status_bar import StatusBar from murfey.util.client import Observer log = logging.getLogger("murfey.client.watchdir") @@ -32,15 +31,13 @@ def __init__( path: str | os.PathLike, settling_time: float = 60, appearance_time: float | None = None, - substrings_blacklist: dict[str, dict] = {}, + substrings_blacklist: dict[str, list[str]] = {}, transfer_all: bool = True, - status_bar: StatusBar | None = None, ): super().__init__() self._basepath = os.fspath(path) self._lastscan: dict[str, _FileInfo] | None = {} self._file_candidates: dict[str, _FileInfo] = {} - self._statusbar = status_bar self.settling_time = settling_time self._appearance_time = appearance_time self._substrings_blacklist = substrings_blacklist @@ -216,13 +213,6 @@ def _notify_for_transfer(self, file_candidate: str) -> bool: removes it from the file candidates list. """ log.debug(f"File {Path(file_candidate).name!r} is ready to be transferred") - if self._statusbar: - # log.info("Increasing number to be transferred") - with self._statusbar.lock: - self._statusbar.transferred = [ - self._statusbar.transferred[0], - self._statusbar.transferred[1] + 1, - ] # Check that it's not a hidden file, ".", "..", or still downloading transfer_check = not Path(file_candidate).name.startswith(".") and not Path( diff --git a/tests/client/test_watchdir.py b/tests/client/test_watchdir.py index c1ada825e..38c860a74 100644 --- a/tests/client/test_watchdir.py +++ b/tests/client/test_watchdir.py @@ -15,7 +15,6 @@ def test_dirwatcher_initialises(tmp_path: Path): assert watcher._basepath == os.fspath(str(tmp_path)) assert watcher._lastscan == {} assert watcher._file_candidates == {} - assert watcher._statusbar is None assert watcher.settling_time == 60 assert watcher._appearance_time is None assert watcher._substrings_blacklist == {} diff --git a/tests/client/tui/test_main.py b/tests/client/tui/test_main.py index 82b006990..e69de29bb 100644 --- a/tests/client/tui/test_main.py +++ b/tests/client/tui/test_main.py @@ -1,71 +0,0 @@ -from unittest import mock -from unittest.mock import Mock -from urllib.parse import urlparse - -import pytest - -from murfey.client.tui.main import _get_visit_list -from murfey.util.models import Visit - -test_get_visit_list_params_matrix = ( - ("http://0.0.0.0:8000",), - ("http://0.0.0.0:8000/api",), - ("http://murfey_server",), - ("http://murfey_server/api",), - ("http://murfey_server.com",), -) - - -@pytest.mark.parametrize("test_params", test_get_visit_list_params_matrix) -@mock.patch("murfey.client.tui.main.capture_get") -def test_get_visit_list( - mock_request_get, - test_params: tuple[str], - mock_client_configuration, -): - # Unpack test params and set up other params - (server_url,) = test_params - instrument_name = mock_client_configuration["Murfey"]["instrument_name"] - - # Construct the expected request response - example_visits = [ - { - "start": "1999-09-09T09:00:00", - "end": "1999-09-11T09:00:00", - "session_id": 123456789, - "name": "cm12345-0", - "beamline": "murfey", - "proposal_title": "Commissioning Session 1", - }, - { - "start": "1999-09-09T09:00:00", - "end": "1999-09-11T09:00:00", - "session_id": 246913578, - "name": "cm23456-1", - "beamline": "murfey", - "proposal_title": "Cryo-cycle 1999", - }, - ] - mock_response = Mock() - mock_response.status_code = 200 - mock_response.json.return_value = example_visits - mock_request_get.return_value = mock_response - - # read_config() has to be patched using fixture, so has to be done in function - with mock.patch("murfey.util.client.read_config", mock_client_configuration): - visits = _get_visit_list(urlparse(server_url), instrument_name) - - # Check that request was sent with the correct URL - mock_request_get.assert_called_once_with( - base_url=server_url, - router_name="session_control.router", - function_name="get_current_visits", - token=mock.ANY, - instrument_name=instrument_name, - ) - - # Check that expected outputs are correct (order-sensitive) - for v, visit in enumerate(visits): - assert ( - visit.model_dump() == Visit.model_validate(example_visits[v]).model_dump() - ) From 761ff3562cbffe9dc263267f2997abe681d311c3 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 11 Dec 2025 18:37:08 +0000 Subject: [PATCH 02/28] Simplified optional dependency keys used in 'pyproject.toml'; 'instrument-server' as a key is no longer needed --- Dockerfiles/murfey-instrument-server | 2 +- pyproject.toml | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Dockerfiles/murfey-instrument-server b/Dockerfiles/murfey-instrument-server index f90054b56..085c451aa 100644 --- a/Dockerfiles/murfey-instrument-server +++ b/Dockerfiles/murfey-instrument-server @@ -32,7 +32,7 @@ RUN apt-get update && \ pip \ build \ importlib-metadata && \ - /venv/bin/python -m pip install /python-murfey[client,instrument-server] + /venv/bin/python -m pip install /python-murfey[client] # Transfer completed Murfey build to base image diff --git a/pyproject.toml b/pyproject.toml index 5b1c46a82..250a9b734 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,9 +29,12 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies = [ + "aiohttp", "defusedxml", # For safely parsing XML files + "fastapi[standard-no-fastapi-cloud-cli]>=0.116.0", "pydantic>=2", "pydantic-settings", + "python-jose", "requests", "rich", "werkzeug", @@ -42,7 +45,6 @@ cicd = [ "pytest-cov", # Used for generating PyTest coverage reports ] client = [ - "textual==0.42.0", "websocket-client", ] developer = [ @@ -52,15 +54,8 @@ developer = [ "pytest", # Test code functionality "pytest-mock", # Additional mocking tools for unit tests ] -instrument-server = [ - "aiohttp", - "fastapi[standard-no-fastapi-cloud-cli]>=0.116.0", - "python-jose", -] server = [ - "aiohttp", "cryptography", - "fastapi[standard-no-fastapi-cloud-cli]>=0.116.0", "graypy", "ispyb>=10.2.4", # Responsible for setting requirements for SQLAlchemy and mysql-connector-python; "jinja2", @@ -70,7 +65,6 @@ server = [ "passlib", "pillow", "prometheus_client", - "python-jose[cryptography]", "sqlalchemy[postgresql]", # Add as explicit dependency "sqlmodel", "stomp-py>8.1.1", # 8.1.1 (released 2024-04-06) doesn't work with our project From a5b2ee8691f83d4390a8a1dac1388352ea326755 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 11 Dec 2025 18:39:19 +0000 Subject: [PATCH 03/28] Removed 'tui' folder from tests --- tests/client/tui/__init__.py | 0 tests/client/tui/test_main.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/client/tui/__init__.py delete mode 100644 tests/client/tui/test_main.py diff --git a/tests/client/tui/__init__.py b/tests/client/tui/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/client/tui/test_main.py b/tests/client/tui/test_main.py deleted file mode 100644 index e69de29bb..000000000 From ce6d2fd14d25af79ae3208c2c67002c47122252d Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 12 Dec 2025 15:23:11 +0000 Subject: [PATCH 04/28] Added a HTTPS-based log handler to post logs in batches to a specified URL endpoint --- src/murfey/client/customlogging.py | 122 ++++++++++++++++++++++++++++- 1 file changed, 121 insertions(+), 1 deletion(-) diff --git a/src/murfey/client/customlogging.py b/src/murfey/client/customlogging.py index 567cf7b7e..8fb9a63f9 100644 --- a/src/murfey/client/customlogging.py +++ b/src/murfey/client/customlogging.py @@ -2,8 +2,11 @@ import json import logging +import threading +import time +from queue import Empty, Queue -logger = logging.getLogger("murfey.client.customlogging") +import requests class CustomHandler(logging.Handler): @@ -26,3 +29,120 @@ def emit(self, record): self._callback(self.prepare(record)) except Exception: self.handleError(record) + + +class HTTPSHandler(logging.Handler): + """ + A log handler collects log messages and posts them in batches to the backend + FastAPI server using HTTPS POST. + """ + + def __init__( + self, + endpoint_url: str, + min_batch: int = 5, + max_batch: int = 50, + min_interval: float = 0.5, + max_interval: float = 2.0, + max_retry: int = 5, + timeout: int = 3, + token: str = "", + ): + super().__init__() + self.endpoint_url = endpoint_url + self.queue: Queue = Queue() + self._stop_event = threading.Event() + self.min_batch = min_batch + self.max_batch = max_batch + self.min_interval = min_interval + self.max_interval = max_interval + self.max_retry = max_retry + self.timeout = timeout + self.token = token + + self.log_times: list[ + float + ] = [] # Timestamps of recent logs for rate estimation + self.thread = threading.Thread(target=self._worker, daemon=True) + self.thread.start() + + def emit(self, record: logging.LogRecord): + """ + Formats the log and puts it on a queue for submission to the backend server + """ + try: + log_entry = self.format_record(record) + self.queue.put(log_entry) + self.log_times.append(time.time()) + except Exception: + self.handleError(record) + pass + + def format_record(self, record: logging.LogRecord): + """ + Packages the log record as a JSON-formatted string + """ + self.format(record) + log_data = record.__dict__.copy() + log_data["type"] = "log" + return json.dumps(log_data) + + def _worker(self): + """ + Worker function that sends batches of logs to the URL endpoint specified, + with logic to adjust how frequently it should batch and send logs depending + on rate of incoming traffic + """ + + batch: list[str] = [] + last_flush = time.time() + + while not self._stop_event.is_set(): + try: + log_entry = self.queue.get(timeout=0.05) + batch.append(log_entry) + except Empty: + pass + + # Calculate logging rate + now = time.time() + self.log_times = [ + t for t in self.log_times if now - t <= 1.0 + ] # Number of logs in last second + log_rate = len(self.log_times) + + # Adjust batch size and flush interval + batch_size = min(max(self.min_batch, log_rate), self.max_batch) + flush_interval = max( + self.min_interval, min(self.max_interval, 1 / max(log_rate, 1)) + ) + + # Flush if batch is ready + if batch and ( + len(batch) >= batch_size or now - last_flush >= flush_interval + ): + self._send_batch(batch) + batch = [] + last_flush = now + + # Flush remaining logs on shutdown + if batch: + self._send_batch(batch) + + def _send_batch(self, batch: list[str]): + """ + Submits the list of stringified log records to the URL endpoint specified. + """ + for attempt in range(1, self.max_retry + 1): + try: + response = requests.post(self.endpoint_url, json=batch, timeout=5) + if response.status_code == 200: + return + except requests.RequestException: + pass + time.sleep(2**attempt * 0.1) # Exponential backoff + + def close(self): + self._stop_event.set() + self.thread.join() + super().close() From b76c121a81af582bcaebd7861abe87e7fc841cd3 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 12 Dec 2025 15:29:00 +0000 Subject: [PATCH 05/28] Added new backend router for logging-related endpoints --- src/murfey/server/api/logging.py | 35 +++++++++++++++++++++++++++++ src/murfey/server/main.py | 3 +++ src/murfey/util/route_manifest.yaml | 6 +++++ 3 files changed, 44 insertions(+) create mode 100644 src/murfey/server/api/logging.py diff --git a/src/murfey/server/api/logging.py b/src/murfey/server/api/logging.py new file mode 100644 index 000000000..21f54d514 --- /dev/null +++ b/src/murfey/server/api/logging.py @@ -0,0 +1,35 @@ +import json +import logging +from datetime import datetime +from typing import Any + +from fastapi import APIRouter, Request + +logger = logging.getLogger("murfey.server.api.hub") + +router = APIRouter( + prefix="/logging", + tags=["Logging"], +) + + +@router.post("/logs") +async def forward_logs(request: Request): + """ + Receives a list of stringified JSON log records from the instrument server, + unpacks them, and forwards them through the handlers set up on the backend. + """ + + data: list[str] = await request.json() + for line in data: + log_data: dict[str, Any] = json.loads(line) + logger_name = log_data["name"] + log_data.pop("msecs", None) + log_data.pop("relativeCreated", None) + client_timestamp = log_data.pop("created", 0) + if client_timestamp: + log_data["client_time"] = datetime.fromtimestamp( + client_timestamp + ).isoformat() + log_data["client_host"] = request.client.host if request.client else None + logging.getLogger(logger_name).handle(logging.makeLogRecord(log_data)) diff --git a/src/murfey/server/main.py b/src/murfey/server/main.py index a65896aa0..613546bfd 100644 --- a/src/murfey/server/main.py +++ b/src/murfey/server/main.py @@ -18,6 +18,7 @@ import murfey.server.api.file_io_instrument import murfey.server.api.hub import murfey.server.api.instrument +import murfey.server.api.logging import murfey.server.api.mag_table import murfey.server.api.processing_parameters import murfey.server.api.prometheus @@ -77,6 +78,8 @@ class Settings(BaseSettings): app.include_router(murfey.server.api.instrument.router) +app.include_router(murfey.server.api.logging.router) + app.include_router(murfey.server.api.mag_table.router) app.include_router(murfey.server.api.session_control.router) diff --git a/src/murfey/util/route_manifest.yaml b/src/murfey/util/route_manifest.yaml index 2f8cd36a3..cddb8231c 100644 --- a/src/murfey/util/route_manifest.yaml +++ b/src/murfey/util/route_manifest.yaml @@ -692,6 +692,12 @@ murfey.server.api.instrument.router: type: int methods: - GET +murfey.server.api.logging.router: + - path: /logging/logs + function: forward_logs + path_params: [] + methods: + - POST murfey.server.api.mag_table.router: - path: /mag_table/mag_table/ function: get_mag_table From 277cca3b9394ab62924cd6ef0927240f060bbe07 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 12 Dec 2025 15:34:28 +0000 Subject: [PATCH 06/28] Replaced CustomHandler with HTTPSHandler in instrument server --- src/murfey/instrument_server/__init__.py | 28 ++++++++++++------------ tests/instrument_server/test_init.py | 4 ---- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/src/murfey/instrument_server/__init__.py b/src/murfey/instrument_server/__init__.py index 41aca780f..26534122e 100644 --- a/src/murfey/instrument_server/__init__.py +++ b/src/murfey/instrument_server/__init__.py @@ -25,9 +25,10 @@ def start_instrument_server(): from rich.logging import RichHandler import murfey - import murfey.client.websocket - from murfey.client.customlogging import CustomHandler + from murfey.client.customlogging import HTTPSHandler from murfey.util import LogFilter + from murfey.util.api import url_path_for + from murfey.util.client import read_config parser = argparse.ArgumentParser(description="Start the Murfey server") parser.add_argument( @@ -55,22 +56,21 @@ def start_instrument_server(): logging.getLogger("fastapi").addHandler(rich_handler) logging.getLogger("uvicorn").addHandler(rich_handler) - # Create a websocket app to connect to the backend - ws = murfey.client.websocket.WSApp( - server=read_config().get("Murfey", "server", fallback=""), - register_client=False, - ) + # Construct URL for the HTTPS log handler + client_config = dict(read_config()["Murfey"]) + murfey_server_url = client_config["server"].rstrip("/") + logger_url = f"{murfey_server_url}{url_path_for('api.hub.router', 'forward_logs')}" # Forward DEBUG levels logs and above from Murfey to the backend - murfey_ws_handler = CustomHandler(ws.send) - murfey_ws_handler.setLevel(logging.DEBUG) - logging.getLogger("murfey").addHandler(murfey_ws_handler) + murfey_https_handler = HTTPSHandler(endpoint_url=logger_url) + murfey_https_handler.setLevel(logging.DEBUG) + logging.getLogger("murfey").addHandler(murfey_https_handler) # Forward only INFO level logs and above for other packages - other_ws_handler = CustomHandler(ws.send) - other_ws_handler.setLevel(logging.INFO) - logging.getLogger("fastapi").addHandler(other_ws_handler) - logging.getLogger("uvicorn").addHandler(other_ws_handler) + other_https_handler = HTTPSHandler(endpoint_url=logger_url) + other_https_handler.setLevel(logging.INFO) + logging.getLogger("fastapi").addHandler(other_https_handler) + logging.getLogger("uvicorn").addHandler(other_https_handler) logger.info( f"Starting Murfey server version {murfey.__version__}, listening on {args.host}:{args.port}" diff --git a/tests/instrument_server/test_init.py b/tests/instrument_server/test_init.py index e575ecea0..7ae4cb293 100644 --- a/tests/instrument_server/test_init.py +++ b/tests/instrument_server/test_init.py @@ -142,10 +142,6 @@ def test_start_instrument_server( # Disable 'run'; we just want to confirm it's called correctly mock_server.run.return_value = lambda: None - # Patch the websocket instance - mock_wsapp = mocker.patch("murfey.client.websocket.WSApp") - mock_wsapp.return_value = mocker.Mock() # Disable functionality - # Construct the expected Uvicorn Config object and save it as a dict expected_config = vars( uvicorn.Config( From d3effca16df86cefa07b46b8be9297b8ce370d99 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 12 Dec 2025 15:36:25 +0000 Subject: [PATCH 07/28] Removed all websocket components from the client side --- src/murfey/client/multigrid_control.py | 27 ---- src/murfey/client/websocket.py | 191 ------------------------- 2 files changed, 218 deletions(-) delete mode 100644 src/murfey/client/websocket.py diff --git a/src/murfey/client/multigrid_control.py b/src/murfey/client/multigrid_control.py index c4dc5f684..302521cec 100644 --- a/src/murfey/client/multigrid_control.py +++ b/src/murfey/client/multigrid_control.py @@ -1,4 +1,3 @@ -import json import logging import subprocess import threading @@ -10,7 +9,6 @@ from typing import Dict, List, Optional from urllib.parse import urlparse -import murfey.client.websocket from murfey.client.analyser import Analyser from murfey.client.context import ensure_dcg_exists from murfey.client.contexts.spa import SPAModularContext @@ -97,11 +95,6 @@ def __post_init__(self): self.rsync_processes = self.rsync_processes or {} self.analysers = self.analysers or {} - self.ws = murfey.client.websocket.WSApp( - server=self.murfey_url, - register_client=False, - ) - # Calculate the time offset between the client and the server current_time = datetime.now() server_timestamp = capture_get( @@ -182,17 +175,6 @@ def clean_up_once_dormant(self, running_threads: list[threading.Thread]): if not success: log.warning(f"Could not delete database data for {self.session_id}") - # Send message to frontend to trigger a refresh - self.ws.send( - json.dumps( - { - "message": "refresh", - "target": "sessions", - "instrument_name": self.instrument_name, - } - ) - ) - # Mark as dormant self.dormant = True @@ -293,15 +275,6 @@ def _start_rsyncer_multigrid( transfer=machine_data.get("data_transfer_enabled", True), restarted=str(source) in self.rsync_restarts, ) - self.ws.send( - json.dumps( - { - "message": "refresh", - "target": "rsyncer", - "session_id": self.session_id, - } - ) - ) def _rsyncer_stopped(self, source: Path, explicit_stop: bool = False): if explicit_stop: diff --git a/src/murfey/client/websocket.py b/src/murfey/client/websocket.py deleted file mode 100644 index 417120837..000000000 --- a/src/murfey/client/websocket.py +++ /dev/null @@ -1,191 +0,0 @@ -from __future__ import annotations - -import json -import logging -import queue -import threading -import time -import urllib.parse -import uuid -from typing import Optional - -import websocket - -from murfey.client.instance_environment import MurfeyInstanceEnvironment -from murfey.util.api import url_path_for - -log = logging.getLogger("murfey.client.websocket") - - -class WSApp: - environment: MurfeyInstanceEnvironment | None = None - - def __init__( - self, *, server: str, id: int | str | None = None, register_client: bool = True - ): - self.id = str(uuid.uuid4()) if id is None else id - log.info(f"Opening websocket connection for Client {self.id}") - websocket.enableTrace(False) - - # Parse server URL and get proxy path used, if any - url = urllib.parse.urlparse(server)._replace( - scheme="wss" if server.startswith("https") else "ws" - ) - proxy_path = url.path.rstrip("/") - - self._address = url.geturl() - self._alive = True - self._ready = False - self._send_queue: queue.Queue[Optional[str]] = queue.Queue() - self._receive_queue: queue.Queue[Optional[str]] = queue.Queue() - - # Construct the websocket URL - # Prepend the proxy path to the new URL path - # It will evaluate to "" if nothing's there, and starts with "/" if present - ws_url = ( - url._replace( - path=f"{proxy_path}{url_path_for('websocket.ws', 'websocket_endpoint', client_id=self.id)}" - ).geturl() - if register_client - else url._replace( - path=f"{proxy_path}{url_path_for('websocket.ws', 'websocket_connection_endpoint', client_id=self.id)}" - ).geturl() - ) - self._ws = websocket.WebSocketApp( - ws_url, - on_close=self.on_close, - on_message=self.on_message, - on_open=self.on_open, - on_error=self.on_error, - ) - self._ws_thread = threading.Thread( - target=self._run_websocket_event_loop, - daemon=True, - name="websocket-connection", - ) - self._ws_thread.start() - self._feeder_thread = threading.Thread( - target=self._send_queue_feeder, daemon=True, name="websocket-send-queue" - ) - self._feeder_thread.start() - self._receiver_thread = threading.Thread( - target=self._receive_msgs, daemon=True, name="websocket-receive-queue" - ) - self._receiver_thread.start() - log.info("making wsapp") - - def __repr__(self): - if self.alive: - if self._ready: - status = "connected" - else: - status = "connecting" - else: - status = "closed" - return f"" - - @property - def alive(self): - return self._alive and self._ws_thread.is_alive() - - def _run_websocket_event_loop(self): - backoff = 0 - while True: - attempt_start = time.perf_counter() - connection_failure = self._ws.run_forever(ping_interval=30, ping_timeout=10) - if not connection_failure: - break - if (time.perf_counter() - attempt_start) < 5: - # rapid connection cycling - backoff = min(120, backoff * 2 + 1) - else: - backoff = 0 - time.sleep(backoff) - log.info("Websocket connection closed") - self._alive = False - - def _send_queue_feeder(self): - log.debug("Websocket send-queue-feeder thread starting") - while self.alive: - element = self._send_queue.get() - if element is None: - self._send_queue.task_done() - continue - while not self._ready: - time.sleep(0.3) - try: - self._ws.send(element) - except Exception: - log.error("Error sending message through websocket", exc_info=True) - self._send_queue.task_done() - log.debug("Websocket send-queue-feeder thread stopped") - - def _receive_msgs(self): - while self.alive: - element = self._receive_queue.get() - if element is None: - self._send_queue.task_done() - continue - while not self._ready: - time.sleep(0.3) - try: - self._handle_msg(element) - except json.decoder.JSONDecodeError: - pass - self._receive_queue.task_done() - - def close(self): - log.info("Closing websocket connection") - if self._feeder_thread.is_alive(): - self._send_queue.join() - self._alive = False - if self._feeder_thread.is_alive(): - self._send_queue.put(None) - self._feeder_thread.join() - self._receiver_thread.join() - try: - self._ws.close() - except Exception: - log.error("Error closing websocket connection", exc_info=True) - - def on_message(self, ws: websocket.WebSocketApp, message: str): - self._receive_queue.put(message) - - def _handle_msg(self, message: str): - data = json.loads(message) - if data.get("message") == "state-update": - self._register_id(data["attribute"], data["value"]) - elif data.get("message") == "state-update-partial": - self._register_id_partial(data["attribute"], data["value"]) - - def _register_id(self, attribute: str, value): - if self.environment and hasattr(self.environment, attribute): - setattr(self.environment, attribute, value) - - def _register_id_partial(self, attribute: str, value): - if self.environment and hasattr(self.environment, attribute): - if isinstance(value, dict): - new_value = {**getattr(self.environment, attribute), **value} - setattr( - self.environment, - attribute, - new_value, - ) - - def on_error(self, ws: websocket.WebSocketApp, error: websocket.WebSocketException): - log.error(str(error)) - - def on_close(self, ws: websocket.WebSocketApp, close_status_code, close_msg): - self._ready = False - if close_status_code or close_msg: - log.debug(f"Websocket closed (code={close_status_code}, msg={close_msg})") - else: - log.debug("Websocket closed") - - def on_open(self, ws: websocket.WebSocketApp): - log.info("Opened connection") - self._ready = True - - def send(self, message: str): - if self.alive: - self._send_queue.put_nowait(message) From ff1856bd504a34f35a5439d0629087922dc50d1a Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 12 Dec 2025 15:40:27 +0000 Subject: [PATCH 08/28] Removed apparently unused endpoints and functions from websocket router --- src/murfey/server/api/websocket.py | 58 +---------------------------- src/murfey/util/route_manifest.yaml | 15 +------- 2 files changed, 2 insertions(+), 71 deletions(-) diff --git a/src/murfey/server/api/websocket.py b/src/murfey/server/api/websocket.py index b0a012acf..bf27f4003 100644 --- a/src/murfey/server/api/websocket.py +++ b/src/murfey/server/api/websocket.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import json import logging from datetime import datetime @@ -9,7 +8,6 @@ from fastapi import APIRouter, WebSocket, WebSocketDisconnect from sqlmodel import Session, select -import murfey.server.prometheus as prom from murfey.server.murfey_db import get_murfey_db_session from murfey.util import sanitise from murfey.util.db import ClientEnvironment @@ -69,31 +67,6 @@ async def broadcast(self, message: str): manager = ConnectionManager() -@ws.websocket("/test/{client_id}") -async def websocket_endpoint(websocket: WebSocket, client_id: int): - await manager.connect(websocket, client_id) - await manager.broadcast(f"Client {client_id} joined") - try: - while True: - data = await websocket.receive_text() - try: - json_data: dict = json.loads(data) - if json_data["type"] == "log": # and isinstance(json_data, dict) - json_data.pop("type") - await forward_log(json_data, websocket) - except Exception: - await manager.broadcast(f"Client #{client_id} sent message {data}") - except WebSocketDisconnect: - log.info(f"Disconnecting Client {int(sanitise(str(client_id)))}") - murfey_db = next(get_murfey_db_session()) - client_env = murfey_db.exec( - select(ClientEnvironment).where(ClientEnvironment.client_id == client_id) - ).one() - prom.monitoring_switch.labels(visit=client_env.visit).set(0) - manager.disconnect(client_id) - await manager.broadcast(f"Client #{client_id} disconnected") - - @ws.websocket("/connect/{client_id}") async def websocket_connection_endpoint( websocket: WebSocket, @@ -120,17 +93,6 @@ async def websocket_connection_endpoint( await manager.broadcast(f"Client #{client_id} disconnected") -async def check_connections(active_connections: list[WebSocket]): - log.info("Checking connections") - for connection in active_connections: - log.info("Checking response") - try: - await asyncio.wait_for(connection.receive(), timeout=10) - except asyncio.TimeoutError: - log.info(f"Disconnecting Client {connection[0]}") - manager.disconnect(connection[0], connection[1]) - - async def forward_log(logrecord: dict[str, Any], websocket: WebSocket): record_name = logrecord["name"] logrecord.pop("msecs", None) @@ -142,26 +104,8 @@ async def forward_log(logrecord: dict[str, Any], websocket: WebSocket): logging.getLogger(record_name).handle(logging.makeLogRecord(logrecord)) -@ws.delete("/test/{client_id}") -async def close_ws_connection(client_id: int): - murfey_db: Session = next(get_murfey_db_session()) - client_env = murfey_db.exec( - select(ClientEnvironment).where(ClientEnvironment.client_id == client_id) - ).one() - client_env.connected = False - visit_name = client_env.visit - murfey_db.add(client_env) - murfey_db.commit() - murfey_db.close() - client_id_str = str(client_id).replace("\r\n", "").replace("\n", "") - log.info(f"Disconnecting {client_id_str}") - manager.disconnect(client_id) - prom.monitoring_switch.labels(visit=visit_name).set(0) - await manager.broadcast(f"Client #{client_id} disconnected") - - @ws.delete("/connect/{client_id}") -async def close_unrecorded_ws_connection(client_id: int | str): +async def close_websocket_connection(client_id: int | str): client_id_str = str(client_id).replace("\r\n", "").replace("\n", "") log.info(f"Disconnecting {client_id_str}") manager.disconnect(client_id) diff --git a/src/murfey/util/route_manifest.yaml b/src/murfey/util/route_manifest.yaml index cddb8231c..6fdeaf47f 100644 --- a/src/murfey/util/route_manifest.yaml +++ b/src/murfey/util/route_manifest.yaml @@ -1255,27 +1255,14 @@ murfey.server.api.session_info.tomo_router: methods: - GET murfey.server.api.websocket.ws: - - path: /ws/test/{client_id} - function: websocket_endpoint - path_params: - - name: client_id - type: int - methods: [] - path: /ws/connect/{client_id} function: websocket_connection_endpoint path_params: - name: client_id type: int | str methods: [] - - path: /ws/test/{client_id} - function: close_ws_connection - path_params: - - name: client_id - type: int - methods: - - DELETE - path: /ws/connect/{client_id} - function: close_unrecorded_ws_connection + function: close_websocket_connection path_params: - name: client_id type: int | str From 30004b120fec929a23cd7b9fee37c5605e859f36 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 12 Dec 2025 15:44:13 +0000 Subject: [PATCH 09/28] Created 'murfey.util.logging' to store LogFilter and HTTPSHandler classes in; removed CustomHandler --- src/murfey/instrument_server/__init__.py | 3 +- src/murfey/server/run.py | 2 +- src/murfey/util/__init__.py | 36 ------------ .../customlogging.py => util/logging.py} | 57 +++++++++++-------- 4 files changed, 35 insertions(+), 63 deletions(-) rename src/murfey/{client/customlogging.py => util/logging.py} (75%) diff --git a/src/murfey/instrument_server/__init__.py b/src/murfey/instrument_server/__init__.py index 26534122e..7d9527080 100644 --- a/src/murfey/instrument_server/__init__.py +++ b/src/murfey/instrument_server/__init__.py @@ -25,10 +25,9 @@ def start_instrument_server(): from rich.logging import RichHandler import murfey - from murfey.client.customlogging import HTTPSHandler - from murfey.util import LogFilter from murfey.util.api import url_path_for from murfey.util.client import read_config + from murfey.util.logging import HTTPSHandler, LogFilter parser = argparse.ArgumentParser(description="Start the Murfey server") parser.add_argument( diff --git a/src/murfey/server/run.py b/src/murfey/server/run.py index 26eb75bc0..aedbd660f 100644 --- a/src/murfey/server/run.py +++ b/src/murfey/server/run.py @@ -16,8 +16,8 @@ import murfey.server from murfey.server.feedback import feedback_listen from murfey.server.ispyb import TransportManager -from murfey.util import LogFilter from murfey.util.config import get_microscope, get_security_config +from murfey.util.logging import LogFilter logger = logging.getLogger("murfey.server.run") diff --git a/src/murfey/util/__init__.py b/src/murfey/util/__init__.py index 98370e5d0..fc3e878d2 100644 --- a/src/murfey/util/__init__.py +++ b/src/murfey/util/__init__.py @@ -96,42 +96,6 @@ def wait(self): self.thread.join() -class LogFilter(logging.Filter): - """A filter to limit messages going to Graylog""" - - def __repr__(self): - return "" - - def __init__(self): - super().__init__() - self._filter_levels = { - "murfey": logging.DEBUG, - "ispyb": logging.DEBUG, - "zocalo": logging.DEBUG, - "uvicorn": logging.INFO, - "fastapi": logging.INFO, - "starlette": logging.INFO, - "sqlalchemy": logging.INFO, - } - - @staticmethod - def install() -> LogFilter: - logfilter = LogFilter() - root_logger = logging.getLogger() - for handler in root_logger.handlers: - handler.addFilter(logfilter) - return logfilter - - def filter(self, record: logging.LogRecord) -> bool: - logger_name = record.name - while True: - if logger_name in self._filter_levels: - return record.levelno >= self._filter_levels[logger_name] - if "." not in logger_name: - return False - logger_name = logger_name.rsplit(".", maxsplit=1)[0] - - def safe_run( func: Callable, args: list | tuple = [], diff --git a/src/murfey/client/customlogging.py b/src/murfey/util/logging.py similarity index 75% rename from src/murfey/client/customlogging.py rename to src/murfey/util/logging.py index 8fb9a63f9..a590cd5d6 100644 --- a/src/murfey/client/customlogging.py +++ b/src/murfey/util/logging.py @@ -9,26 +9,40 @@ import requests -class CustomHandler(logging.Handler): - def __init__(self, callback): - """Set up a handler instance, record the callback function.""" - super().__init__() - self._callback = callback +class LogFilter(logging.Filter): + """A filter to limit messages going to Graylog""" - def prepare(self, record): - self.format(record) - record_dict = record.__dict__ - record_dict["type"] = "log" - try: - return json.dumps(record_dict) - except TypeError: - return json.dumps({str(k): str(v) for k, v in record_dict.items()}) + def __repr__(self): + return "" - def emit(self, record): - try: - self._callback(self.prepare(record)) - except Exception: - self.handleError(record) + def __init__(self): + super().__init__() + self._filter_levels = { + "murfey": logging.DEBUG, + "ispyb": logging.DEBUG, + "zocalo": logging.DEBUG, + "uvicorn": logging.INFO, + "fastapi": logging.INFO, + "starlette": logging.INFO, + "sqlalchemy": logging.INFO, + } + + @staticmethod + def install() -> LogFilter: + logfilter = LogFilter() + root_logger = logging.getLogger() + for handler in root_logger.handlers: + handler.addFilter(logfilter) + return logfilter + + def filter(self, record: logging.LogRecord) -> bool: + logger_name = record.name + while True: + if logger_name in self._filter_levels: + return record.levelno >= self._filter_levels[logger_name] + if "." not in logger_name: + return False + logger_name = logger_name.rsplit(".", maxsplit=1)[0] class HTTPSHandler(logging.Handler): @@ -89,9 +103,7 @@ def format_record(self, record: logging.LogRecord): def _worker(self): """ - Worker function that sends batches of logs to the URL endpoint specified, - with logic to adjust how frequently it should batch and send logs depending - on rate of incoming traffic + The worker function when the handler is run as a thread. """ batch: list[str] = [] @@ -130,9 +142,6 @@ def _worker(self): self._send_batch(batch) def _send_batch(self, batch: list[str]): - """ - Submits the list of stringified log records to the URL endpoint specified. - """ for attempt in range(1, self.max_retry + 1): try: response = requests.post(self.endpoint_url, json=batch, timeout=5) From 69efbf024bc642c634f94b9b622f1b8c931d8d72 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 12 Dec 2025 15:48:27 +0000 Subject: [PATCH 10/28] Merge recent changes from 'main' branch --- src/murfey/client/context.py | 6 +- src/murfey/server/api/workflow.py | 63 +++++++------ src/murfey/util/config.py | 2 +- tests/server/api/test_workflow.py | 152 +++++++++++++++++++++++++----- 4 files changed, 167 insertions(+), 56 deletions(-) diff --git a/src/murfey/client/context.py b/src/murfey/client/context.py index 57cc0bb3d..11d04a3d6 100644 --- a/src/murfey/client/context.py +++ b/src/murfey/client/context.py @@ -52,7 +52,11 @@ def ensure_dcg_exists( for h in entry_points(group="murfey.hooks"): try: if h.name == "get_epu_session_metadata": - h.load()(session_file, environment=environment) + h.load()( + destination_dir=session_file.parent, + environment=environment, + token=token, + ) except Exception as e: logger.warning(f"Get EPU session hook failed: {e}") else: diff --git a/src/murfey/server/api/workflow.py b/src/murfey/server/api/workflow.py index 220d90e27..33a0b40b0 100644 --- a/src/murfey/server/api/workflow.py +++ b/src/murfey/server/api/workflow.py @@ -124,37 +124,39 @@ def register_dc_group( ): # Either switching atlas for a common (atlas or processing) tag # Or registering a new atlas-type dcg for a sample that is already present - dcg_murfey[0].atlas = dcg_params.atlas or dcg_murfey[0].atlas - dcg_murfey[0].sample = dcg_params.sample or dcg_murfey[0].sample - dcg_murfey[0].atlas_pixel_size = ( - dcg_params.atlas_pixel_size or dcg_murfey[0].atlas_pixel_size - ) + for dcg_instance in dcg_murfey: + # Update all instances in case there are multiple processing runs + dcg_instance.atlas = dcg_params.atlas or dcg_instance.atlas + dcg_instance.sample = dcg_params.sample or dcg_instance.sample + dcg_instance.atlas_pixel_size = ( + dcg_params.atlas_pixel_size or dcg_instance.atlas_pixel_size + ) - if _transport_object: - if dcg_murfey[0].atlas_id is not None: - _transport_object.send( - _transport_object.feedback_queue, - { - "register": "atlas_update", - "atlas_id": dcg_murfey[0].atlas_id, - "atlas": dcg_params.atlas, - "sample": dcg_params.sample, - "atlas_pixel_size": dcg_params.atlas_pixel_size, - "dcgid": dcg_murfey[0].id, - "session_id": session_id, - }, - ) - else: - atlas_id_response = _transport_object.do_insert_atlas( - Atlas( - dataCollectionGroupId=dcg_murfey[0].id, - atlasImage=dcg_params.atlas, - pixelSize=dcg_params.atlas_pixel_size, - cassetteSlot=dcg_params.sample, + if _transport_object: + if dcg_instance.atlas_id is not None: + _transport_object.send( + _transport_object.feedback_queue, + { + "register": "atlas_update", + "atlas_id": dcg_instance.atlas_id, + "atlas": dcg_params.atlas, + "sample": dcg_params.sample, + "atlas_pixel_size": dcg_params.atlas_pixel_size, + "dcgid": dcg_instance.id, + "session_id": session_id, + }, ) - ) - dcg_murfey[0].atlas_id = atlas_id_response["return_value"] - db.add(dcg_murfey[0]) + else: + atlas_id_response = _transport_object.do_insert_atlas( + Atlas( + dataCollectionGroupId=dcg_instance.id, + atlasImage=dcg_params.atlas, + pixelSize=dcg_params.atlas_pixel_size, + cassetteSlot=dcg_params.sample, + ) + ) + dcg_instance.atlas_id = atlas_id_response["return_value"] + db.add(dcg_instance) db.commit() search_maps = db.exec( @@ -172,6 +174,9 @@ def register_dc_group( select(DataCollectionGroup) .where(DataCollectionGroup.session_id == session_id) .where(DataCollectionGroup.sample == dcg_params.sample) + .where( + col(DataCollectionGroup.tag).contains(f"/Sample{dcg_params.sample}/Atlas") + ) ).all(): # Case where we switch from atlas to processing dcg_murfey[0].tag = dcg_params.tag or dcg_murfey[0].tag diff --git a/src/murfey/util/config.py b/src/murfey/util/config.py index f989d5b38..1407a8d7f 100644 --- a/src/murfey/util/config.py +++ b/src/murfey/util/config.py @@ -331,6 +331,6 @@ def get_extended_machine_config( ) if not machine_config: return None - model = entry_points(group="murfey.config", name=extension_name)[0].load() + model = list(entry_points(group="murfey.config", name=extension_name))[0].load() data = getattr(machine_config, extension_name, {}) return model(**data) diff --git a/tests/server/api/test_workflow.py b/tests/server/api/test_workflow.py index 01ebffe6b..afbbbbcdc 100644 --- a/tests/server/api/test_workflow.py +++ b/tests/server/api/test_workflow.py @@ -15,8 +15,8 @@ def test_register_dc_group_new_dcg(mock_transport, murfey_db_session: Session): # Request new dcg registration dcg_params = DCGroupParameters( experiment_type_id=44, - tag="atlas_tag", - atlas="/path/to/Atlas_1.jpg", + tag="/path/to/Sample10/Atlas", + atlas="/path/to/Sample10/Atlas/Atlas_1.jpg", sample=10, atlas_pixel_size=1e-5, ) @@ -34,9 +34,9 @@ def test_register_dc_group_new_dcg(mock_transport, murfey_db_session: Session): "register": "data_collection_group", "start_time": mock.ANY, "experiment_type_id": 44, - "tag": "atlas_tag", + "tag": "/path/to/Sample10/Atlas", "session_id": ExampleVisit.murfey_session_id, - "atlas": "/path/to/Atlas_1.jpg", + "atlas": "/path/to/Sample10/Atlas/Atlas_1.jpg", "sample": 10, "atlas_pixel_size": 1e-5, "microscope": "", @@ -57,15 +57,26 @@ def test_register_dc_group_atlas_to_processing( """ mock_transport.feedback_queue = "mock_feedback_queue" - # Make sure dcg is present - dcg = DataCollectionGroup( + # Add a processing dcg to ensure this is not touched + proc_dcg = DataCollectionGroup( id=1, session_id=ExampleVisit.murfey_session_id, - tag="atlas_tag", + tag="initial_processing_tag", + atlas_id=90, + atlas_pixel_size=1e-5, + sample=10, + atlas="/path/to/Sample10/Atlas/Atlas_1.jpg", + ) + murfey_db_session.add(proc_dcg) + # Make sure dcg is present for update + dcg = DataCollectionGroup( + id=2, + session_id=ExampleVisit.murfey_session_id, + tag="/path/to/Sample10/Atlas", atlas_id=90, atlas_pixel_size=1e-5, sample=10, - atlas="/path/to/Atlas_1.jpg", + atlas="/path/to/Sample10/Atlas/Atlas_1.jpg", ) murfey_db_session.add(dcg) murfey_db_session.commit() @@ -74,7 +85,7 @@ def test_register_dc_group_atlas_to_processing( dcg_params = DCGroupParameters( experiment_type_id=36, tag="processing_tag", - atlas="/path/to/Atlas_1.jpg", + atlas="/path/to/Sample10/Atlas/Atlas_1.jpg", sample=10, atlas_pixel_size=1e-5, ) @@ -91,14 +102,18 @@ def test_register_dc_group_atlas_to_processing( { "register": "experiment_type_update", "experiment_type_id": 36, - "dcgid": 1, + "dcgid": 2, }, ) # Check that the tag of the data collection group was updated - new_dcg = murfey_db_session.exec( + initial_dcg = murfey_db_session.exec( select(DataCollectionGroup).where(DataCollectionGroup.id == 1) ).one() + assert initial_dcg.tag == "initial_processing_tag" + new_dcg = murfey_db_session.exec( + select(DataCollectionGroup).where(DataCollectionGroup.id == 2) + ).one() assert new_dcg.tag == "processing_tag" @@ -120,16 +135,26 @@ def test_register_dc_group_processing_to_atlas( atlas_id=90, atlas_pixel_size=1e-5, sample=10, - atlas="/path/to/Atlas_1.jpg", + atlas="/path/to/Sample10/Atlas/Atlas_1.jpg", ) murfey_db_session.add(dcg) + second_dcg = DataCollectionGroup( + id=2, + session_id=ExampleVisit.murfey_session_id, + tag="second_processing_tag", + atlas_id=90, + atlas_pixel_size=1e-5, + sample=10, + atlas="/path/to/Sample10/Atlas/Atlas_1.jpg", + ) + murfey_db_session.add(second_dcg) murfey_db_session.commit() # Request new dcg registration with atlas experiment type and tag dcg_params = DCGroupParameters( experiment_type_id=44, - tag="atlas_tag", - atlas="/path/to/Atlas_2.jpg", + tag="/path/to/Sample10/Atlas", + atlas="/path/to/Sample10/Atlas/Atlas_2.jpg", sample=10, atlas_pixel_size=1e-4, ) @@ -141,27 +166,104 @@ def test_register_dc_group_processing_to_atlas( ) # Check request to ispyb for updating the experiment type - mock_transport.send.assert_called_once_with( + assert mock_transport.send.call_count == 2 + mock_transport.send.assert_any_call( "mock_feedback_queue", { "register": "atlas_update", "atlas_id": 90, - "atlas": "/path/to/Atlas_2.jpg", + "atlas": "/path/to/Sample10/Atlas/Atlas_2.jpg", "sample": 10, "atlas_pixel_size": 1e-4, "dcgid": 1, "session_id": ExampleVisit.murfey_session_id, }, ) + mock_transport.send.assert_any_call( + "mock_feedback_queue", + { + "register": "atlas_update", + "atlas_id": 90, + "atlas": "/path/to/Sample10/Atlas/Atlas_2.jpg", + "sample": 10, + "atlas_pixel_size": 1e-4, + "dcgid": 2, + "session_id": ExampleVisit.murfey_session_id, + }, + ) # Check the data collection group atlas was updated new_dcg = murfey_db_session.exec( select(DataCollectionGroup).where(DataCollectionGroup.id == 1) ).one() - assert new_dcg.atlas == "/path/to/Atlas_2.jpg" + second_new_dcg = murfey_db_session.exec( + select(DataCollectionGroup).where(DataCollectionGroup.id == 1) + ).one() + assert new_dcg.atlas == "/path/to/Sample10/Atlas/Atlas_2.jpg" assert new_dcg.atlas_pixel_size == 1e-4 + assert second_new_dcg.atlas == "/path/to/Sample10/Atlas/Atlas_2.jpg" + assert second_new_dcg.atlas_pixel_size == 1e-4 # Check the tag of the data collection group was not updated - assert new_dcg.tag != "atlas_tag" + assert new_dcg.tag != "/path/to/Sample10/Atlas" + assert second_new_dcg.tag != "/path/to/Sample10/Atlas" + + +@mock.patch("murfey.server.api.workflow._transport_object") +def test_register_dc_group_new_dcg_old_atlas( + mock_transport, murfey_db_session: Session +): + """ + Test the request to register a new processing type data collection group + in the case where there is already one for that atlas + """ + mock_transport.feedback_queue = "mock_feedback_queue" + + # Make sure dcg is present + dcg = DataCollectionGroup( + id=1, + session_id=ExampleVisit.murfey_session_id, + tag="processing_tag", + atlas_id=90, + atlas_pixel_size=1e-5, + sample=10, + atlas="/path/to/Sample10/Atlas/Atlas_1.jpg", + ) + murfey_db_session.add(dcg) + murfey_db_session.commit() + + # Request new dcg registration with atlas experiment type and new processing tag + dcg_params = DCGroupParameters( + experiment_type_id=37, + tag="second_processing_tag", + atlas="/path/to/Sample10/Atlas/Atlas_1.jpg", + sample=10, + atlas_pixel_size=1e-5, + ) + register_dc_group( + visit_name="cm12345-6", + session_id=ExampleVisit.murfey_session_id, + dcg_params=dcg_params, + db=murfey_db_session, + ) + + # Check request for registering dcg in ispyb and murfey + mock_transport.send.assert_called_once_with( + "mock_feedback_queue", + { + "register": "data_collection_group", + "start_time": mock.ANY, + "experiment_type_id": 37, + "tag": "second_processing_tag", + "session_id": ExampleVisit.murfey_session_id, + "atlas": "/path/to/Sample10/Atlas/Atlas_1.jpg", + "sample": 10, + "atlas_pixel_size": 1e-5, + "microscope": "", + "proposal_code": ExampleVisit.proposal_code, + "proposal_number": str(ExampleVisit.proposal_number), + "visit_number": str(ExampleVisit.visit_number), + }, + ) @mock.patch("murfey.server.api.workflow._transport_object") @@ -186,7 +288,7 @@ def test_register_dc_group_new_atlas(mock_transport, murfey_db_session: Session) dcg_params = DCGroupParameters( experiment_type_id=36, tag="processing_tag", - atlas="/path/to/Atlas_2.jpg", + atlas="/path/to/Sample10/Atlas/Atlas_2.jpg", sample=10, atlas_pixel_size=1e-4, ) @@ -204,7 +306,7 @@ def test_register_dc_group_new_atlas(mock_transport, murfey_db_session: Session) atlas_args = mock_transport.do_insert_atlas.call_args_list assert len(atlas_args) == 1 assert atlas_args[0][0][0].dataCollectionGroupId == 1 - assert atlas_args[0][0][0].atlasImage == "/path/to/Atlas_2.jpg" + assert atlas_args[0][0][0].atlasImage == "/path/to/Sample10/Atlas/Atlas_2.jpg" assert atlas_args[0][0][0].pixelSize == 1e-4 assert atlas_args[0][0][0].cassetteSlot == 10 @@ -212,7 +314,7 @@ def test_register_dc_group_new_atlas(mock_transport, murfey_db_session: Session) new_dcg = murfey_db_session.exec( select(DataCollectionGroup).where(DataCollectionGroup.id == 1) ).one() - assert new_dcg.atlas == "/path/to/Atlas_2.jpg" + assert new_dcg.atlas == "/path/to/Sample10/Atlas/Atlas_2.jpg" assert new_dcg.sample == 10 assert new_dcg.atlas_pixel_size == 1e-4 assert new_dcg.tag == "processing_tag" @@ -238,7 +340,7 @@ def test_register_dc_group_new_atlas_with_searchmaps( atlas_id=90, atlas_pixel_size=1e-5, sample=10, - atlas="/path/to/Atlas_1.jpg", + atlas="/path/to/Sample10/Atlas/Atlas_1.jpg", ) murfey_db_session.add(dcg) murfey_db_session.commit() @@ -271,7 +373,7 @@ def test_register_dc_group_new_atlas_with_searchmaps( dcg_params = DCGroupParameters( experiment_type_id=37, tag="processing_tag", - atlas="/path/to/Atlas_2.jpg", + atlas="/path/to/Sample12/Atlas/Atlas_2.jpg", sample=12, atlas_pixel_size=1e-4, ) @@ -288,7 +390,7 @@ def test_register_dc_group_new_atlas_with_searchmaps( { "register": "atlas_update", "atlas_id": 90, - "atlas": "/path/to/Atlas_2.jpg", + "atlas": "/path/to/Sample12/Atlas/Atlas_2.jpg", "sample": 12, "atlas_pixel_size": 1e-4, "dcgid": 1, @@ -300,7 +402,7 @@ def test_register_dc_group_new_atlas_with_searchmaps( new_dcg = murfey_db_session.exec( select(DataCollectionGroup).where(DataCollectionGroup.id == dcg.id) ).one() - assert new_dcg.atlas == "/path/to/Atlas_2.jpg" + assert new_dcg.atlas == "/path/to/Sample12/Atlas/Atlas_2.jpg" assert new_dcg.sample == 12 assert new_dcg.atlas_pixel_size == 1e-4 assert new_dcg.tag == "processing_tag" From f001b9ca54a133d19eb50246a89157ae261f7cb0 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 12 Dec 2025 15:56:15 +0000 Subject: [PATCH 11/28] Forgot to update router path --- src/murfey/instrument_server/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/murfey/instrument_server/__init__.py b/src/murfey/instrument_server/__init__.py index 7d9527080..f3c0e01d0 100644 --- a/src/murfey/instrument_server/__init__.py +++ b/src/murfey/instrument_server/__init__.py @@ -58,7 +58,9 @@ def start_instrument_server(): # Construct URL for the HTTPS log handler client_config = dict(read_config()["Murfey"]) murfey_server_url = client_config["server"].rstrip("/") - logger_url = f"{murfey_server_url}{url_path_for('api.hub.router', 'forward_logs')}" + logger_url = ( + f"{murfey_server_url}{url_path_for('api.logging.router', 'forward_logs')}" + ) # Forward DEBUG levels logs and above from Murfey to the backend murfey_https_handler = HTTPSHandler(endpoint_url=logger_url) From a7c7debbbd746b7f9acf945aa832b0fa96945f17 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 12 Dec 2025 16:37:26 +0000 Subject: [PATCH 12/28] Fixed broken test --- tests/instrument_server/test_init.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/instrument_server/test_init.py b/tests/instrument_server/test_init.py index 7ae4cb293..50015bae5 100644 --- a/tests/instrument_server/test_init.py +++ b/tests/instrument_server/test_init.py @@ -1,5 +1,7 @@ +import logging import sys from typing import Optional +from unittest.mock import MagicMock from urllib.parse import urlparse import pytest @@ -14,6 +16,7 @@ from murfey.instrument_server import check_for_updates, start_instrument_server from murfey.server.api.bootstrap import pypi as pypi_router, version as version_router from murfey.util.api import url_path_for +from murfey.util.logging import HTTPSHandler # Set up a test router with only the essential endpoints app = FastAPI() @@ -132,11 +135,28 @@ def test_check_for_updates( @pytest.mark.parametrize("test_params", start_instrument_server_test_matrix) def test_start_instrument_server( - mocker: MockerFixture, test_params: tuple[Optional[str], Optional[int]] + mocker: MockerFixture, + mock_client_configuration, + test_params: tuple[Optional[str], Optional[int]], ): # Unpack test params host, port = test_params + # Patch the 'read_config' function + _ = mocker.patch( + "murfey.util.client.read_config", return_value=mock_client_configuration + ) + + # Mock the HTTPSHandler (test it separately in a unit test) + mock_https_handler_instance = MagicMock() + mock_https_handler_instance.level = logging.INFO + mock_https_handler_instance.setLevel.return_value = None + mock_https_handler = mocker.patch( + "murfey.util.logging.HTTPSHandler", + spec=HTTPSHandler, + ) + mock_https_handler.return_value = mock_https_handler_instance + # Patch the Uvicorn Server instance mock_server = mocker.patch("uvicorn.Server") # Disable 'run'; we just want to confirm it's called correctly From 8c2351c497a65f2eb927c622ab60a83c9ae057b7 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 12 Dec 2025 18:00:49 +0000 Subject: [PATCH 13/28] Minor updates to comments and iteration logic --- src/murfey/util/logging.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/murfey/util/logging.py b/src/murfey/util/logging.py index a590cd5d6..b092e11e8 100644 --- a/src/murfey/util/logging.py +++ b/src/murfey/util/logging.py @@ -116,11 +116,9 @@ def _worker(self): except Empty: pass - # Calculate logging rate + # Calculate logging rate based on past second now = time.time() - self.log_times = [ - t for t in self.log_times if now - t <= 1.0 - ] # Number of logs in last second + self.log_times = [t for t in self.log_times if now - t <= 1.0] log_rate = len(self.log_times) # Adjust batch size and flush interval @@ -142,14 +140,14 @@ def _worker(self): self._send_batch(batch) def _send_batch(self, batch: list[str]): - for attempt in range(1, self.max_retry + 1): + for attempt in range(0, self.max_retry): try: response = requests.post(self.endpoint_url, json=batch, timeout=5) if response.status_code == 200: return except requests.RequestException: pass - time.sleep(2**attempt * 0.1) # Exponential backoff + time.sleep(2 ** (attempt + 1) * 0.1) # Exponential backoff def close(self): self._stop_event.set() From d4e7e16a37b420d93d8d458e9bd4a47e21e608ee Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 12 Dec 2025 18:01:12 +0000 Subject: [PATCH 14/28] Added unit test for the HTTPS log handler --- tests/util/test_logging.py | 70 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 tests/util/test_logging.py diff --git a/tests/util/test_logging.py b/tests/util/test_logging.py new file mode 100644 index 000000000..888518be7 --- /dev/null +++ b/tests/util/test_logging.py @@ -0,0 +1,70 @@ +import time +from unittest import mock +from unittest.mock import MagicMock + +import pytest +from fastapi import Response +from pytest_mock import MockerFixture + +from murfey.util.logging import HTTPSHandler + +https_handler_test_matrix = ( + # Num messages | Status code + (10, 200), + (10, 404), +) + + +@pytest.mark.parametrize("test_params", https_handler_test_matrix) +def test_https_handler( + mocker: MockerFixture, + mock_client_configuration, + test_params: tuple[int, int], +): + # Unpack test params + num_messages, status_code = test_params + + # Mock the imported 'requests' module and the HTTPX response + mock_response = MagicMock(spec=Response) + mock_response.status_code = status_code + mock_requests = mocker.patch("murfey.util.logging.requests") + mock_requests.post.return_value = mock_response + + # Import logger and set up a logger object + from logging import getLogger + + # Initialise the logger with URL from mock client config + client_config = dict(mock_client_configuration["Murfey"]) + server_url = client_config["server"] + https_handler = HTTPSHandler( + endpoint_url=server_url, + min_batch=5, + max_batch=10, + min_interval=0.5, + max_interval=1.0, + max_retry=1, + ) + + logger = getLogger("tests.util.test_logging") + logger.setLevel(10) + logger.addHandler(https_handler) + for i in range(num_messages): + # Test all the logging levels + if i % 4 == 0: + logger.debug("This is a debug log") + if i % 4 == 1: + logger.info("This is an info log") + if i % 4 == 2: + logger.warning("This is a warning log") + if i % 4 == 3: + logger.error("This is an error log") + + # Let it run in the background before checking for the expected calls + time.sleep(1) + mock_requests.post.assert_called_with( + server_url, + json=mock.ANY, + timeout=5, + ) + + assert https_handler.close() is None From b35d452e0d07abcb83679d38325486575c726657 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 12 Dec 2025 18:21:49 +0000 Subject: [PATCH 15/28] Don't assert 'https_handler.close()', just run it --- tests/util/test_logging.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/util/test_logging.py b/tests/util/test_logging.py index 888518be7..596ae4118 100644 --- a/tests/util/test_logging.py +++ b/tests/util/test_logging.py @@ -67,4 +67,5 @@ def test_https_handler( timeout=5, ) - assert https_handler.close() is None + # Close the handler thread + https_handler.close() From d75cee794de56e12dd49d7a1f7b1a12117613027 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 12 Dec 2025 18:22:54 +0000 Subject: [PATCH 16/28] Attempt at fixing warnings raised by CodeQL --- src/murfey/util/logging.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/murfey/util/logging.py b/src/murfey/util/logging.py index b092e11e8..acab9c406 100644 --- a/src/murfey/util/logging.py +++ b/src/murfey/util/logging.py @@ -90,7 +90,6 @@ def emit(self, record: logging.LogRecord): self.log_times.append(time.time()) except Exception: self.handleError(record) - pass def format_record(self, record: logging.LogRecord): """ @@ -113,8 +112,10 @@ def _worker(self): try: log_entry = self.queue.get(timeout=0.05) batch.append(log_entry) + # If the queue is empty, check back again except Empty: - pass + time.sleep(1) + continue # Calculate logging rate based on past second now = time.time() @@ -146,8 +147,7 @@ def _send_batch(self, batch: list[str]): if response.status_code == 200: return except requests.RequestException: - pass - time.sleep(2 ** (attempt + 1) * 0.1) # Exponential backoff + time.sleep(2 ** (attempt + 1) * 0.1) # Exponential backoff def close(self): self._stop_event.set() From d7bdd8f9c7d7e676f947d7731157ab51bf4a2b7a Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 12 Dec 2025 18:34:24 +0000 Subject: [PATCH 17/28] Fixed incorrectly named loggers --- src/murfey/server/api/hub.py | 2 +- src/murfey/server/api/instrument.py | 2 +- src/murfey/server/api/logging.py | 2 +- src/murfey/server/api/websocket.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/murfey/server/api/hub.py b/src/murfey/server/api/hub.py index f17db4a1c..c0a74d05a 100644 --- a/src/murfey/server/api/hub.py +++ b/src/murfey/server/api/hub.py @@ -7,7 +7,7 @@ from murfey.util.config import get_machine_config -logger = getLogger("murfey.api.hub") +logger = getLogger("murfey.server.api.hub") config = get_machine_config() diff --git a/src/murfey/server/api/instrument.py b/src/murfey/server/api/instrument.py index 9069535ce..d71e412a2 100644 --- a/src/murfey/server/api/instrument.py +++ b/src/murfey/server/api/instrument.py @@ -36,7 +36,7 @@ tags=["Instrument Server"], ) -log = logging.getLogger("murfey.server.instrument") +log = logging.getLogger("murfey.server.api.instrument") lock = asyncio.Lock() diff --git a/src/murfey/server/api/logging.py b/src/murfey/server/api/logging.py index 21f54d514..fde075d72 100644 --- a/src/murfey/server/api/logging.py +++ b/src/murfey/server/api/logging.py @@ -5,7 +5,7 @@ from fastapi import APIRouter, Request -logger = logging.getLogger("murfey.server.api.hub") +logger = logging.getLogger("murfey.server.api.logging") router = APIRouter( prefix="/logging", diff --git a/src/murfey/server/api/websocket.py b/src/murfey/server/api/websocket.py index bf27f4003..bbabf68b3 100644 --- a/src/murfey/server/api/websocket.py +++ b/src/murfey/server/api/websocket.py @@ -15,7 +15,7 @@ T = TypeVar("T") ws = APIRouter(prefix="/ws", tags=["Websocket"]) -log = logging.getLogger("murfey.server.websocket") +log = logging.getLogger("murfey.server.api.websocket") class ConnectionManager: From bfa7ab3d7b4fc986ebc9cc0bc99e6bbbba6cdf99 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 15 Dec 2025 11:25:44 +0000 Subject: [PATCH 18/28] Added unit test for the 'forward_logs' API endpoint --- pyproject.toml | 1 + tests/server/api/test_logging.py | 89 ++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 tests/server/api/test_logging.py diff --git a/pyproject.toml b/pyproject.toml index 250a9b734..95c7162d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ developer = [ "ipykernel", # Enable interactive coding with VS Code and Jupyter Notebook "pre-commit", # Formatting, linting, type checking, etc. "pytest", # Test code functionality + "pytest-asyncio", # For testing async functions "pytest-mock", # Additional mocking tools for unit tests ] server = [ diff --git a/tests/server/api/test_logging.py b/tests/server/api/test_logging.py new file mode 100644 index 000000000..0d46012cb --- /dev/null +++ b/tests/server/api/test_logging.py @@ -0,0 +1,89 @@ +import json +import logging +import time +from datetime import datetime +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest +from pytest_mock import MockerFixture + +import murfey +from murfey.server.api.logging import forward_logs + + +@pytest.mark.asyncio +async def test_forward_logs( + mocker: MockerFixture, +): + # Create example log messages + message_list = [ + json.dumps( + { + "name": f"murfey.{module_name}", + "msg": "Starting Murfey server version {murfey.__version__}, listening on 0.0.0.0:8000", + "args": [], + "levelname": levelname, + "levelno": levelno, + "pathname": f"{murfey.__file__}/{module_name}/__init__.py", + "filename": "__init__.py", + "module": "__init__", + "exc_info": None, + "exc_text": None, + "stack_info": None, + "lineno": 76, + "funcName": f"start_{module_name}", + "created": time.time(), + "msecs": 930.0, + "relativeCreated": 1379.8329830169678, + "thread": time.time_ns(), + "threadName": "MainThread", + "processName": "MainProcess", + "process": time.time_ns(), + "message": f"Starting Murfey server version {murfey.__version__}, listening on 0.0.0.0:8000", + "type": "log", + } + ) + for module_name, levelname, levelno in ( + ("module_1", "DEBUG", logging.DEBUG), + ("module_2", "INFO", logging.INFO), + ("module_3", "WARNING", logging.WARNING), + ("module_4", "ERROR", logging.ERROR), + ) + ] + + # Create a mock request to pass to the function + mock_request = MagicMock() + mock_request.json = AsyncMock(return_value=message_list) + + # Mock the logging module + mock_logging = mocker.patch("murfey.server.api.logging.logging") + + # Mock the 'getLogger()' and 'handle()' functions + mock_logger = MagicMock() + mock_logger.handle.return_value = None + mock_logging.getLogger.return_value = mock_logger + + # Run the function and check that the results are as expected + await forward_logs(mock_request) + + # Check that the correct logger name was called. + for i, message in enumerate(message_list): + # Process the message as in the actual function + log_data: dict[str, Any] = json.loads(message) + logger_name = log_data["name"] + log_data.pop("msecs", None) + log_data.pop("relativeCreated", None) + client_timestamp = log_data.pop("created", 0) + if client_timestamp: + log_data["client_time"] = datetime.fromtimestamp( + client_timestamp + ).isoformat() + log_data["client_host"] = None # No host, as function is being tested directly + + # Check that messages are unpacked and handled in sequence + mock_logging.getLogger.call_args_list[i][0][0] == logger_name + mock_logger.handle.call_args_list[i][0][0] == logging.makeLogRecord(log_data) + + # Check that 'handle' was called for each message + assert mock_logger.handle.call_count == len(message_list) From ac158c04e5b2e633744e97fa32d91647940a0d33 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 15 Dec 2025 11:56:41 +0000 Subject: [PATCH 19/28] Make test file names more specific --- tests/server/api/{test_logging.py => test_logging_api.py} | 0 tests/util/{test_logging.py => test_logging_util.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/server/api/{test_logging.py => test_logging_api.py} (100%) rename tests/util/{test_logging.py => test_logging_util.py} (100%) diff --git a/tests/server/api/test_logging.py b/tests/server/api/test_logging_api.py similarity index 100% rename from tests/server/api/test_logging.py rename to tests/server/api/test_logging_api.py diff --git a/tests/util/test_logging.py b/tests/util/test_logging_util.py similarity index 100% rename from tests/util/test_logging.py rename to tests/util/test_logging_util.py From 9d364244bc61f6e86816048dcf135944ed146b73 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 15 Dec 2025 12:24:39 +0000 Subject: [PATCH 20/28] Updated code to remove references to the 'client' optional dependency, as it is no longer needed --- Dockerfiles/murfey-instrument-server | 2 +- pyproject.toml | 3 --- src/murfey/bootstrap/__main__.py | 2 +- src/murfey/client/update.py | 2 +- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Dockerfiles/murfey-instrument-server b/Dockerfiles/murfey-instrument-server index 085c451aa..91f33688e 100644 --- a/Dockerfiles/murfey-instrument-server +++ b/Dockerfiles/murfey-instrument-server @@ -32,7 +32,7 @@ RUN apt-get update && \ pip \ build \ importlib-metadata && \ - /venv/bin/python -m pip install /python-murfey[client] + /venv/bin/python -m pip install /python-murfey # Transfer completed Murfey build to base image diff --git a/pyproject.toml b/pyproject.toml index 95c7162d6..10c318a26 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,9 +44,6 @@ dependencies = [ cicd = [ "pytest-cov", # Used for generating PyTest coverage reports ] -client = [ - "websocket-client", -] developer = [ "bump-my-version", # Version control "ipykernel", # Enable interactive coding with VS Code and Jupyter Notebook diff --git a/src/murfey/bootstrap/__main__.py b/src/murfey/bootstrap/__main__.py index 109bd3a9e..393567652 100644 --- a/src/murfey/bootstrap/__main__.py +++ b/src/murfey/bootstrap/__main__.py @@ -144,7 +144,7 @@ def _download_to_file(url: str, outfile: str): murfey_hostname, "-i", f"{murfey_base}{url_path_for('bootstrap.pypi', 'get_pypi_index')}", - "murfey[client]", + "murfey", ] ) if result.returncode: diff --git a/src/murfey/client/update.py b/src/murfey/client/update.py index ce7732488..0f1fe6a7c 100644 --- a/src/murfey/client/update.py +++ b/src/murfey/client/update.py @@ -78,7 +78,7 @@ def install_murfey(api_base: ParseResult, version: str) -> bool: path=f"{proxy_path}{url_path_for('bootstrap.pypi', 'get_pypi_index')}", query="", ).geturl(), - f"murfey[client]=={version}", + f"murfey=={version}", ] ) return result.returncode == 0 From 52e4d508ffb9de34d587c15eeb983442c8fb886a Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 15 Dec 2025 12:37:31 +0000 Subject: [PATCH 21/28] Updated documentation --- README.md | 4 +- src/murfey/templates/bootstrap.html | 57 ++++++----------------------- 2 files changed, 14 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index e3de57436..6bcddff6e 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ then install using the following command. ```text $ git clone git@github.com:DiamondLightSource/python-murfey.git $ cd python-murfey -$ pip install -e .[client,server,developer] +$ pip install -e .[server,developer] ``` The packages included under the `[developer]` installation key contain some helpful tools to aid you with developing Murfey further: @@ -43,7 +43,7 @@ $ murfey.server and connect the client with ```text -$ murfey --server http://127.0.0.1:8000 +$ murfey.instrument_server --port 8000 ``` You can also install a client on a remote machine. This machine only needs to have diff --git a/src/murfey/templates/bootstrap.html b/src/murfey/templates/bootstrap.html index 2dab84d6f..fcb7f05f7 100644 --- a/src/murfey/templates/bootstrap.html +++ b/src/murfey/templates/bootstrap.html @@ -2,7 +2,7 @@ %} {% block content %}

Bootstrapping Instructions

1. Setting Up a POSIX Environment

-

Installing MSYS2

+

A. Installing MSYS2

MSYS2 is a POSIX environment which provides extensive compiler support for the more modern programming languages used by Murfey's package dependencies. @@ -14,9 +14,14 @@

Installing MSYS2

mirror, then run it using the default settings. This will install MSYS2 to - C:\msys64. + + C:\msys64 .

-

Setting Up the MSYS2 Package Manager (If Network-Restricted)

+

B. Setting Up MSYS2

+

i. Setting Up the MSYS2 Package Manager (If Network-Restricted)

By default, MSYS2 comes with preset lists of mirrors and servers that it installs its packages from. On a network-restricted PC, these will need to be @@ -36,7 +41,7 @@

Setting Up the MSYS2 Package Manager (If Network-Restricted)

> C:\msys64\etc\pacman.d -

Installing Dependencies

+

ii. Installing Dependencies

MSYS2 comes with multiple environments, but UCRT64 is the most modern one. In order for the Murfey client to be able to install and run its dependencies @@ -69,7 +74,7 @@

Installing Dependencies

>pacman -Ss <package-name>

-

Configuring the Rust Package Manager (If Network-Restricted)

+

iii. Configuring the Rust Package Manager (If Network-Restricted)

Many newer Python packages now have dependencies written in Rust that allow them to operate more efficiently. MSYS2 supports the compilation and @@ -132,30 +137,6 @@

Configuring the Rust Package Manager (If Network-Restricted)

instead.

-

Running MSYS2 Through Command Prompt

-

- In order to run Murfey via the terminal, MSYS2 will have to be run through - Window's Command Prompt terminal, as there is an ongoing bug with MSYS2's - pre-packaged terminal that prevents mouse interaction with interactive apps in - the terminal. -

-

- To do so, simply right-click on your desktop and navigate to - New > Shortcut. When prompted for the location of the item, enter - the following into the text box: -

-
-    cmd.exe /k "C:\msys64\msys2_shell.cmd -defterm -no-start -ucrt64 -shell bash"
-
-

- After naming the shortcut, click Finish to create the shortcut. This will run - a UCRT64 instance of MSYS2 through the Command Prompt terminal that starts you - off in MSYS2's default home directory. You can proceed to customise the - shortcut icon to taste. -

-

2. Setting Up Python

Once Python and @@ -180,25 +161,11 @@

A. (Optional) Setting Up a Virtual Environment

B. Installing Murfey

You can install Murfey in the Python environment (the base one or a virtual - environment) in either the Cygwin or UCRT64 terminal using the following - commands: + environment) in the UCRT64 terminal using the following commands: 

-    $ pip install murfey[client] --index-url {{ request.url.scheme }}://{{ netloc }}{{ proxy_path }}/pypi/index --trusted-host {{ netloc }}
+    $ pip install murfey --index-url {{ request.url.scheme }}://{{ netloc }}{{ proxy_path }}/pypi/index --trusted-host {{ netloc }}
-

- If you wish to install the client-side dependencies needed to run Murfey via - the web UI, replace - murfey[client] - with - murfey[client,instrument-server]. -

{% endblock %} From c9fe70cebe60f6d0b1cda74273912600a52f50d5 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 15 Dec 2025 12:39:29 +0000 Subject: [PATCH 22/28] Missed a couple of 'pip install' commands in GitHub workflows --- .github/workflows/ci.yml | 2 +- .github/workflows/publish-version.yml | 2 +- .github/workflows/version-bump.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e700d35a0..6ca69bed0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,7 +77,7 @@ jobs: - name: Install Murfey run: | set -eux - pip install --disable-pip-version-check -e "."[cicd,client,server,developer] + pip install --disable-pip-version-check -e "."[cicd,server,developer] - uses: shogo82148/actions-setup-mysql@v1 with: diff --git a/.github/workflows/publish-version.yml b/.github/workflows/publish-version.yml index c054c29a2..9b4e52f02 100644 --- a/.github/workflows/publish-version.yml +++ b/.github/workflows/publish-version.yml @@ -24,7 +24,7 @@ jobs: - name: Check current tag id: checkTag run: | - pip install --disable-pip-version-check -e "."[cicd,client,server,developer] + pip install --disable-pip-version-check -e "."[cicd,server,developer] VERSION=$(python -c "import murfey; print(murfey.__version__)") echo "newVersion=v$VERSION" >> $GITHUB_OUTPUT diff --git a/.github/workflows/version-bump.yml b/.github/workflows/version-bump.yml index b52691360..aeef9c2d9 100644 --- a/.github/workflows/version-bump.yml +++ b/.github/workflows/version-bump.yml @@ -42,7 +42,7 @@ jobs: - name: Install package run: | set -eux - pip install --disable-pip-version-check -e "."[cicd,client,server,developer] + pip install --disable-pip-version-check -e "."[cicd,server,developer] - name: Run bumpversion and push tag env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From d0b40d5f5bc240f13743b912f3f01eafbd886027 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 15 Dec 2025 17:34:38 +0000 Subject: [PATCH 23/28] Added new fields to the Pydantic models for the CLEM results to register thumbnail images for ISPyB with --- src/murfey/workflows/clem/register_align_and_merge_results.py | 3 ++- src/murfey/workflows/clem/register_preprocessing_results.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/murfey/workflows/clem/register_align_and_merge_results.py b/src/murfey/workflows/clem/register_align_and_merge_results.py index fb5f563aa..da97fc015 100644 --- a/src/murfey/workflows/clem/register_align_and_merge_results.py +++ b/src/murfey/workflows/clem/register_align_and_merge_results.py @@ -22,7 +22,8 @@ class AlignAndMergeResult(BaseModel): align_self: Optional[str] = None flatten: Optional[str] = "mean" align_across: Optional[str] = None - composite_image: Path + output_file: Path + thumbnail: Path @field_validator("image_stacks", mode="before") @classmethod diff --git a/src/murfey/workflows/clem/register_preprocessing_results.py b/src/murfey/workflows/clem/register_preprocessing_results.py index 2cb19e0e2..bb38300a1 100644 --- a/src/murfey/workflows/clem/register_preprocessing_results.py +++ b/src/murfey/workflows/clem/register_preprocessing_results.py @@ -38,6 +38,9 @@ class CLEMPreprocessingResult(BaseModel): output_files: dict[ Literal["gray", "red", "green", "blue", "cyan", "magenta", "yellow"], Path ] + thumbnails: dict[ + Literal["gray", "red", "green", "blue", "cyan", "magenta", "yellow"], Path + ] metadata: Path parent_lif: Optional[Path] = None parent_tiffs: dict[ From b9909c3254a9f3a011176af4ff245202a93ab3da Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 19 Dec 2025 10:06:01 +0000 Subject: [PATCH 24/28] Added comments to 'GridSquareParameters' table to clarify what the different fields are used for --- src/murfey/util/models.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/murfey/util/models.py b/src/murfey/util/models.py index c1ae42a56..004cb5f69 100644 --- a/src/murfey/util/models.py +++ b/src/murfey/util/models.py @@ -127,22 +127,34 @@ class Base(BaseModel): class GridSquareParameters(BaseModel): tag: str + image: str = "" + x_location: Optional[float] = None - x_location_scaled: Optional[int] = None y_location: Optional[float] = None + + # Image coordinates when overlaid on atlas (in pixels0) + x_location_scaled: Optional[int] = None y_location_scaled: Optional[int] = None + x_stage_position: Optional[float] = None y_stage_position: Optional[float] = None + + # Size of original image (in pixels) readout_area_x: Optional[int] = None readout_area_y: Optional[int] = None + + # Size of thumbnail used (in pixels) thumbnail_size_x: Optional[int] = None thumbnail_size_y: Optional[int] = None + height: Optional[int] = None - height_scaled: Optional[int] = None width: Optional[int] = None + + # Size of image when overlaid on atlas (in pixels) + height_scaled: Optional[int] = None width_scaled: Optional[int] = None + pixel_size: Optional[float] = None - image: str = "" angle: Optional[float] = None From 7fb882afb4e8359cb2684b65ad1d38d1bb0c1c00 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 19 Dec 2025 10:09:30 +0000 Subject: [PATCH 25/28] Added new columns to the 'CLEMImageSeries' database table to store information about the thumbnails used when registering data to ISPyB --- src/murfey/util/db.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 801ac5e46..193e0ff2a 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -239,7 +239,8 @@ class CLEMImageSeries(SQLModel, table=True): # type: ignore series_name: str = Field( index=True ) # Name of the series, as determined from the metadata - search_string: Optional[str] = Field(default=None) # Path for globbing with + image_search_string: Optional[str] = Field(default=None) + thumbnail_search_string: Optional[str] = Field(default=None) session: Optional["Session"] = Relationship( back_populates="image_series" @@ -295,9 +296,12 @@ class CLEMImageSeries(SQLModel, table=True): # type: ignore number_of_members: Optional[int] = Field(default=None) # Shape and resolution information - pixels_x: Optional[int] = Field(default=None) - pixels_y: Optional[int] = Field(default=None) - pixel_size: Optional[float] = Field(default=None) + image_pixels_x: Optional[int] = Field(default=None) + image_pixels_y: Optional[int] = Field(default=None) + image_pixel_size: Optional[float] = Field(default=None) + thumbnail_pixels_x: Optional[int] = Field(default=None) + thumbnail_pixels_y: Optional[int] = Field(default=None) + thumbnail_pixel_size: Optional[float] = Field(default=None) units: Optional[str] = Field(default=None) # Extent of the imaged area in real space From 4245c8fe177507aa0583ea9f9805fe9a01ed2b53 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 19 Dec 2025 10:13:01 +0000 Subject: [PATCH 26/28] Updated DataCollectionGroup and GridSquare registration logic for CLEM workflow to make use of scaled down thumbnails instead of full-sized TIFFs --- .../clem/register_preprocessing_results.py | 85 ++++++++++++++----- 1 file changed, 63 insertions(+), 22 deletions(-) diff --git a/src/murfey/workflows/clem/register_preprocessing_results.py b/src/murfey/workflows/clem/register_preprocessing_results.py index bb38300a1..a56282682 100644 --- a/src/murfey/workflows/clem/register_preprocessing_results.py +++ b/src/murfey/workflows/clem/register_preprocessing_results.py @@ -40,7 +40,8 @@ class CLEMPreprocessingResult(BaseModel): ] thumbnails: dict[ Literal["gray", "red", "green", "blue", "cyan", "magenta", "yellow"], Path - ] + ] = {} + thumbnail_size: Optional[tuple[int, int]] = None # height, width metadata: Path parent_lif: Optional[Path] = None parent_tiffs: dict[ @@ -57,7 +58,10 @@ class CLEMPreprocessingResult(BaseModel): def _is_clem_atlas(result: CLEMPreprocessingResult): # If an image has a width/height of at least 1.5 mm, it should qualify as an atlas return ( - max(result.pixels_x * result.pixel_size, result.pixels_y * result.pixel_size) + max( + result.pixels_x * result.pixel_size, + result.pixels_y * result.pixel_size, + ) >= processing_params.atlas_threshold ) @@ -152,17 +156,29 @@ def _register_clem_image_series( murfey_db.commit() # Add metadata for this series - clem_img_series.search_string = str(output_file.parent / "*tiff") + clem_img_series.image_search_string = str(output_file.parent / "*tiff") clem_img_series.data_type = "atlas" if _is_clem_atlas(result) else "grid_square" clem_img_series.number_of_members = result.number_of_members - clem_img_series.pixels_x = result.pixels_x - clem_img_series.pixels_y = result.pixels_y - clem_img_series.pixel_size = result.pixel_size + clem_img_series.image_pixels_x = result.pixels_x + clem_img_series.image_pixels_y = result.pixels_y + clem_img_series.image_pixel_size = result.pixel_size clem_img_series.units = result.units clem_img_series.x0 = result.extent[0] clem_img_series.x1 = result.extent[1] clem_img_series.y0 = result.extent[2] clem_img_series.y1 = result.extent[3] + # Register thumbnails if they are present + if result.thumbnails and result.thumbnail_size: + thumbnail = list(result.thumbnails.values())[0] + clem_img_series.thumbnail_search_string = str(thumbnail.parent / "*.png") + + thumbnail_height, thumbnail_width = result.thumbnail_size + scaling_factor = min( + thumbnail_height / result.pixels_y, thumbnail_width / result.pixels_x + ) + clem_img_series.thumbnail_pixel_size = result.pixel_size / scaling_factor + clem_img_series.thumbnail_pixels_x = int(result.pixels_x * scaling_factor) + clem_img_series.thumbnail_pixels_y = int(result.pixels_y * scaling_factor) murfey_db.add(clem_img_series) murfey_db.commit() murfey_db.close() @@ -192,8 +208,23 @@ def _register_dcg_and_atlas( # Determine values for atlas if _is_clem_atlas(result): output_file = list(result.output_files.values())[0] - atlas_name = str(output_file.parent / "*.tiff") - atlas_pixel_size = result.pixel_size + # Register the thumbnail entries if they are provided + if result.thumbnails and result.thumbnail_size is not None: + # Glob path to the thumbnail files + thumbnail = list(result.thumbnails.values())[0] + atlas_name = str(thumbnail.parent / "*.png") + + # Work out the scaling factor used + thumbnail_height, thumbnail_width = result.thumbnail_size + scaling_factor = min( + thumbnail_width / result.pixels_x, + thumbnail_height / result.pixels_y, + ) + atlas_pixel_size = result.pixel_size / scaling_factor + # Otherwise, register the TIFF files themselves + else: + atlas_name = str(output_file.parent / "*.tiff") + atlas_pixel_size = result.pixel_size else: atlas_name = "" atlas_pixel_size = 0.0 @@ -311,8 +342,6 @@ def _register_grid_square( and atlas_entry.x1 is not None and atlas_entry.y0 is not None and atlas_entry.y1 is not None - and atlas_entry.pixels_x is not None - and atlas_entry.pixels_y is not None ): atlas_width_real = atlas_entry.x1 - atlas_entry.x0 atlas_height_real = atlas_entry.y1 - atlas_entry.y0 @@ -321,32 +350,40 @@ def _register_grid_square( return for clem_img_series in clem_img_series_to_register: + # Register datasets using thumbnail sizes and scales if ( clem_img_series.x0 is not None and clem_img_series.x1 is not None and clem_img_series.y0 is not None and clem_img_series.y1 is not None + and clem_img_series.thumbnail_pixels_x is not None + and clem_img_series.thumbnail_pixels_y is not None + and clem_img_series.thumbnail_pixel_size is not None ): # Find pixel corresponding to image midpoint on atlas x_mid_real = ( 0.5 * (clem_img_series.x0 + clem_img_series.x1) - atlas_entry.x0 ) - x_mid_px = int(x_mid_real / atlas_width_real * atlas_entry.pixels_x) + x_mid_px = int( + x_mid_real / atlas_width_real * clem_img_series.thumbnail_pixels_x + ) y_mid_real = ( 0.5 * (clem_img_series.y0 + clem_img_series.y1) - atlas_entry.y0 ) - y_mid_px = int(y_mid_real / atlas_height_real * atlas_entry.pixels_y) + y_mid_px = int( + y_mid_real / atlas_height_real * clem_img_series.thumbnail_pixels_y + ) - # Find the number of pixels in width and height the image corresponds to on the atlas + # Find the size of the image, in pixels, when overlaid the atlas width_scaled = int( (clem_img_series.x1 - clem_img_series.x0) / atlas_width_real - * atlas_entry.pixels_x + * clem_img_series.thumbnail_pixels_x ) height_scaled = int( (clem_img_series.y1 - clem_img_series.y0) / atlas_height_real - * atlas_entry.pixels_y + * clem_img_series.thumbnail_pixels_y ) else: logger.warning( @@ -361,14 +398,18 @@ def _register_grid_square( x_location_scaled=x_mid_px, y_location=clem_img_series.y0, y_location_scaled=y_mid_px, - height=clem_img_series.pixels_x, - height_scaled=height_scaled, - width=clem_img_series.pixels_y, + readout_area_x=clem_img_series.image_pixels_x, + readout_area_y=clem_img_series.image_pixels_y, + thumbnail_size_x=clem_img_series.thumbnail_pixels_x, + thumbnail_size_y=clem_img_series.thumbnail_pixels_y, + width=clem_img_series.image_pixels_x, width_scaled=width_scaled, - x_stage_position=clem_img_series.x0, - y_stage_position=clem_img_series.y0, - pixel_size=clem_img_series.pixel_size, - image=clem_img_series.search_string, + height=clem_img_series.image_pixels_y, + height_scaled=height_scaled, + x_stage_position=0.5 * (clem_img_series.x0 + clem_img_series.x1), + y_stage_position=0.5 * (clem_img_series.y0 + clem_img_series.y1), + pixel_size=clem_img_series.image_pixel_size, + image=clem_img_series.thumbnail_search_string, ) # Register or update the grid square entry as required if grid_square_result := murfey_db.exec( From bab27548bb4ce28fe476dc0a45875db279863f40 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 19 Dec 2025 10:54:46 +0000 Subject: [PATCH 27/28] Updated 'AlignAndMergeResult' Pydantic model --- src/murfey/workflows/clem/register_align_and_merge_results.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/murfey/workflows/clem/register_align_and_merge_results.py b/src/murfey/workflows/clem/register_align_and_merge_results.py index da97fc015..a85ce9da9 100644 --- a/src/murfey/workflows/clem/register_align_and_merge_results.py +++ b/src/murfey/workflows/clem/register_align_and_merge_results.py @@ -23,7 +23,8 @@ class AlignAndMergeResult(BaseModel): flatten: Optional[str] = "mean" align_across: Optional[str] = None output_file: Path - thumbnail: Path + thumbnail: Optional[Path] = None + thumbnail_size: Optional[tuple[int, int]] = None @field_validator("image_stacks", mode="before") @classmethod From 9f0002f8f1725abb25045d0317028b1ef45f4596 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Fri, 19 Dec 2025 11:28:20 +0000 Subject: [PATCH 28/28] Added new message keys to CLEM workflow test --- .../clem/test_register_preprocessing_results.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/workflows/clem/test_register_preprocessing_results.py b/tests/workflows/clem/test_register_preprocessing_results.py index 06f7a5a09..3353be984 100644 --- a/tests/workflows/clem/test_register_preprocessing_results.py +++ b/tests/workflows/clem/test_register_preprocessing_results.py @@ -77,6 +77,14 @@ def generate_preprocessing_messages( output_files = {color: str(series_path / f"{color}.tiff") for color in colors} for output_file in output_files.values(): Path(output_file).touch(exist_ok=True) + thumbnails = { + color: str(series_path / ".thumbnails" / f"{color}.png") for color in colors + } + for v in thumbnails.values(): + if not (thumbnail := Path(v)).parent.exists(): + thumbnail.parent.mkdir(parents=True) + thumbnail.touch(exist_ok=True) + thumbnail_size = (512, 512) is_stack = dataset[1] is_montage = dataset[2] shape = dataset[3] @@ -91,6 +99,8 @@ def generate_preprocessing_messages( "is_stack": is_stack, "is_montage": is_montage, "output_files": output_files, + "thumbnails": thumbnails, + "thumbnail_size": thumbnail_size, "metadata": str(metadata), "parent_lif": None, "parent_tiffs": {},