From a9fa2ad50baf29a6adc98e8d6839b22fb90d8590 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 22 Oct 2025 18:20:05 +0100 Subject: [PATCH 01/10] dev --- cf/__init__.py | 316 ++++++++++++++-------------- cf/aggregate.py | 6 +- cf/constants.py | 116 +++++----- cf/data/array/umarray.py | 6 +- cf/data/collapse/collapse.py | 14 +- cf/data/collapse/collapse_active.py | 10 +- cf/data/collapse/dask_collapse.py | 51 ++++- cf/data/dask_regrid.py | 3 +- cf/data/dask_utils.py | 4 +- cf/data/data.py | 86 +++++++- cf/functions.py | 106 ++++++---- cf/mixin/fielddomain.py | 9 +- cf/read_write/um/umread.py | 34 +-- cf/regrid/regrid.py | 43 ++-- cf/units.py | 8 - 15 files changed, 488 insertions(+), 324 deletions(-) diff --git a/cf/__init__.py b/cf/__init__.py index 7dcf6a90d7..c518dce2da 100644 --- a/cf/__init__.py +++ b/cf/__init__.py @@ -83,167 +83,167 @@ __date__ = "2025-10-16" __version__ = "3.18.2" -_requires = ( - "numpy", - "netCDF4", - "cftime", - "cfunits", - "cfdm", - "psutil", - "dask", - "packaging", - "scipy", - "distributed", -) -x = ", ".join(_requires) -_error0 = f"cf v{__version__} requires the modules {x}. " - -import importlib.util -from platform import python_version - -_found_esmpy = bool(importlib.util.find_spec("esmpy")) - -try: - import packaging - from packaging.version import Version -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - _minimum_vn = "20.0" - if Version(packaging.__version__) < Version(_minimum_vn): - raise RuntimeError( - f"Bad packaging version: cf requires packaging>={_minimum_vn}. " - f"Got {packaging.__version__} at {packaging.__file__}" - ) - -try: - import cfdm -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - # Check the version of cfdm - _minimum_vn = "1.12.3.1" - _maximum_vn = "1.12.4.0" - _cfdm_version = Version(cfdm.__version__) - if _cfdm_version < Version(_minimum_vn) or _cfdm_version >= Version( - _maximum_vn - ): - raise RuntimeError( - "Bad cfdm version: cf requires " - f"{_minimum_vn}<=cfdm<{_maximum_vn}. " - f"Got {_cfdm_version} at {cfdm.__file__}" - ) - +#_requires = ( +# "numpy", +# "netCDF4", +# "cftime", +# "cfunits", +# "cfdm", +# "psutil", +# "dask", +# "packaging", +# "scipy", +# "distributed", +#) +#x = ", ".join(_requires) +#_error0 = f"cf v{__version__} requires the modules {x}. " +# +#import importlib.util +#from platform import python_version +# +#_found_esmpy = bool(importlib.util.find_spec("esmpy")) +# +#try: +# import packaging +# from packaging.version import Version +#except ImportError as error1: +# raise ImportError(_error0 + str(error1)) +#else: +# _minimum_vn = "20.0" +# if Version(packaging.__version__) < Version(_minimum_vn): +# raise RuntimeError( +# f"Bad packaging version: cf requires packaging>={_minimum_vn}. " +# f"Got {packaging.__version__} at {packaging.__file__}" +# ) +# +#try: +import cfdm +#except ImportError as error1: +# raise ImportError(_error0 + str(error1)) +#else: +# # Check the version of cfdm +# _minimum_vn = "1.12.3.1" +# _maximum_vn = "1.12.4.0" +# _cfdm_version = Version(cfdm.__version__) +# if _cfdm_version < Version(_minimum_vn) or _cfdm_version >= Version( +# _maximum_vn +# ): +# raise RuntimeError( +# "Bad cfdm version: cf requires " +# f"{_minimum_vn}<=cfdm<{_maximum_vn}. " +# f"Got {_cfdm_version} at {cfdm.__file__}" +# ) +# __cf_version__ = cfdm.__cf_version__ __Conventions__ = f"CF-{__cf_version__}" -try: - import netCDF4 -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - _set_vn = "1.7.2" - if Version(netCDF4.__version__) != Version(_set_vn): - raise RuntimeError( - "Bad netCDF4 version: cf requires " - f"netCDF4=={_set_vn}. " - f"Got {netCDF4.__version__} at {netCDF4.__file__}" - ) - -try: - import numpy as np -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - _minimum_vn = "2.0.0" - if Version(np.__version__) < Version(_minimum_vn): - raise RuntimeError( - f"Bad numpy version: cf requires numpy>={_minimum_vn} " - f"Got {np.__version__} at {np.__file__}" - ) - -try: - import cftime -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - _minimum_vn = "1.6.4" - if Version(cftime.__version__) < Version(_minimum_vn): - raise RuntimeError( - f"Bad cftime version: cf requires cftime>={_minimum_vn}. " - f"Got {cftime.__version__} at {cftime.__file__}" - ) - -try: - import cfunits -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - _minimum_vn = "3.3.7" - if Version(cfunits.__version__) < Version(_minimum_vn): - raise RuntimeError( - f"Bad cfunits version: cf requires cfunits>={_minimum_vn}. " - f"Got {cfunits.__version__} at {cfunits.__file__}" - ) - -try: - import psutil -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - _minimum_vn = "0.6.0" - if Version(psutil.__version__) < Version(_minimum_vn): - raise RuntimeError( - f"Bad psutil version: cf requires psutil>={_minimum_vn}. " - f"Got {psutil.__version__} at {psutil.__file__}" - ) - -# Check the version of dask -try: - import dask -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - _minimum_vn = "2025.5.1" - if Version(dask.__version__) < Version(_minimum_vn): - raise RuntimeError( - f"Bad dask version: cf requires dask>={_minimum_vn}. " - f"Got {dask.__version__} at {dask.__file__}" - ) - -try: - import distributed -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - _minimum_vn = "2025.5.1" - if Version(distributed.__version__) < Version(_minimum_vn): - raise RuntimeError( - "Bad distributed version: cf requires " - f"distributed>={_minimum_vn}. " - f"Got {distributed.__version__} at {distributed.__file__}" - ) - -try: - import scipy -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - _minimum_vn = "1.10.0" - if Version(scipy.__version__) < Version(_minimum_vn): - raise RuntimeError( - f"Bad scipy version: cf requires scipy>={_minimum_vn}. " - f"Got {scipy.__version__} at {scipy.__file__}" - ) - -_minimum_vn = "3.10.0" -if Version(python_version()) < Version(_minimum_vn): - raise RuntimeError( - f"Bad python version: cf requires python>={_minimum_vn}. " - f"Got {python_version()}" - ) - -del _minimum_vn, _maximum_vn +#try: +# import netCDF4 +#except ImportError as error1: +# raise ImportError(_error0 + str(error1)) +#else: +# _set_vn = "1.7.2" +# if Version(netCDF4.__version__) != Version(_set_vn): +# raise RuntimeError( +# "Bad netCDF4 version: cf requires " +# f"netCDF4=={_set_vn}. " +# f"Got {netCDF4.__version__} at {netCDF4.__file__}" +# ) +# +#try: +# import numpy as np +#except ImportError as error1: +# raise ImportError(_error0 + str(error1)) +#else: +# _minimum_vn = "2.0.0" +# if Version(np.__version__) < Version(_minimum_vn): +# raise RuntimeError( +# f"Bad numpy version: cf requires numpy>={_minimum_vn} " +# f"Got {np.__version__} at {np.__file__}" +# ) +# +#try: +# import cftime +#except ImportError as error1: +# raise ImportError(_error0 + str(error1)) +#else: +# _minimum_vn = "1.6.4" +# if Version(cftime.__version__) < Version(_minimum_vn): +# raise RuntimeError( +# f"Bad cftime version: cf requires cftime>={_minimum_vn}. " +# f"Got {cftime.__version__} at {cftime.__file__}" +# ) +# +#try: +# import cfunits +#except ImportError as error1: +# raise ImportError(_error0 + str(error1)) +#else: +# _minimum_vn = "3.3.7" +# if Version(cfunits.__version__) < Version(_minimum_vn): +# raise RuntimeError( +# f"Bad cfunits version: cf requires cfunits>={_minimum_vn}. " +# f"Got {cfunits.__version__} at {cfunits.__file__}" +# ) +# +#try: +# import psutil +#except ImportError as error1: +# raise ImportError(_error0 + str(error1)) +#else: +# _minimum_vn = "0.6.0" +# if Version(psutil.__version__) < Version(_minimum_vn): +# raise RuntimeError( +# f"Bad psutil version: cf requires psutil>={_minimum_vn}. " +# f"Got {psutil.__version__} at {psutil.__file__}" +# ) +# +## Check the version of dask +#try: +# import dask +#except ImportError as error1: +# raise ImportError(_error0 + str(error1)) +#else: +# _minimum_vn = "2025.5.1" +# if Version(dask.__version__) < Version(_minimum_vn): +# raise RuntimeError( +# f"Bad dask version: cf requires dask>={_minimum_vn}. " +# f"Got {dask.__version__} at {dask.__file__}" +# ) +# +#try: +# import distributed +#except ImportError as error1: +# raise ImportError(_error0 + str(error1)) +#else: +# _minimum_vn = "2025.5.1" +# if Version(distributed.__version__) < Version(_minimum_vn): +# raise RuntimeError( +# "Bad distributed version: cf requires " +# f"distributed>={_minimum_vn}. " +# f"Got {distributed.__version__} at {distributed.__file__}" +# ) +# +#try: +# import scipy +#except ImportError as error1: +# raise ImportError(_error0 + str(error1)) +#else: +# _minimum_vn = "1.10.0" +# if Version(scipy.__version__) < Version(_minimum_vn): +# raise RuntimeError( +# f"Bad scipy version: cf requires scipy>={_minimum_vn}. " +# f"Got {scipy.__version__} at {scipy.__file__}" +# ) +# +#_minimum_vn = "3.10.0" +#if Version(python_version()) < Version(_minimum_vn): +# raise RuntimeError( +# f"Bad python version: cf requires python>={_minimum_vn}. " +# f"Got {python_version()}" +# ) +# +#del _minimum_vn, _maximum_vn from .constructs import Constructs diff --git a/cf/aggregate.py b/cf/aggregate.py index 6b93aa6305..caf3df8438 100644 --- a/cf/aggregate.py +++ b/cf/aggregate.py @@ -6,7 +6,7 @@ import numpy as np from cfdm import is_log_level_debug, is_log_level_detail, is_log_level_info -from dask.base import tokenize +#from dask.base import tokenize from .auxiliarycoordinate import AuxiliaryCoordinate from .data import Data @@ -1986,6 +1986,8 @@ def tokenise_cell_conditions(self, cell_conditions): ('ce9a05dd6ec76c6a6d171b0c055f3127', '8e0216a9a17a20b6620c6502bb45dec9') """ + from dask.base import tokenize + out = [] for x in cell_conditions: if x is None: @@ -4056,6 +4058,8 @@ def _get_hfl( hash_value = d.get_deterministic_name() except ValueError: # Slow + from dask.base import tokenize + hash_value = tokenize(d.compute()) if hash_value in hash_map: diff --git a/cf/constants.py b/cf/constants.py index aa2bfd0fcd..0928b73938 100644 --- a/cf/constants.py +++ b/cf/constants.py @@ -1,72 +1,72 @@ import logging import sys from enum import Enum, auto -from tempfile import gettempdir +#from tempfile import gettempdir import numpy as np -from dask import config -from dask.utils import parse_bytes -from psutil import virtual_memory +#from dask import config +#from dask.utils import parse_bytes +#from psutil import virtual_memory from .units import Units # -------------------------------------------------------------------- # Find the total amount of memory, in bytes # -------------------------------------------------------------------- -_TOTAL_MEMORY = float(virtual_memory().total) - -_CHUNKSIZE = "128 MiB" -config.set({"array.chunk-size": _CHUNKSIZE}) - - -"""A dictionary of useful constants. - -Whilst the dictionary may be modified directly, it is safer to -retrieve and set the values with a function where one is -provided. This is due to interdependencies between some values. - -Note ATOL and RTOL are constants that in essence belong in this dict, -but since they can be read and manipulated directly from cfdm, it is -safest to work with ``cfdm.constants.CONSTANTS['ATOL']`` (and 'RTOL' -equivalent) instead of storing separately and synchronising them here -in cf. - -:Keys: - - TOTAL_MEMORY: `float` - Find the total amount of physical memory (in bytes). - - CHUNKSIZE: `int` - The Dask chunk size (in bytes). See `cf.chunksize`. - - TEMPDIR: `str` - The location to store temporary files. By default it is the - default directory used by the :mod:`tempfile` module. - - REGRID_LOGGING: `bool` - Whether or not to enable `esmpy` logging. If it is logging is - performed after every call to `esmpy`. By default logging is - disabled. - - LOG_LEVEL: `str` - The minimal level of seriousness for which log messages are - shown. See `cf.log_level`. - -""" -CONSTANTS = { - "ATOL": sys.float_info.epsilon, - "RTOL": sys.float_info.epsilon, - "TEMPDIR": gettempdir(), - "TOTAL_MEMORY": _TOTAL_MEMORY, - "REGRID_LOGGING": False, - "RELAXED_IDENTITIES": False, - "LOG_LEVEL": logging.getLevelName(logging.getLogger().level), - "BOUNDS_COMBINATION_MODE": "AND", - "CHUNKSIZE": parse_bytes(_CHUNKSIZE), - "active_storage": False, - "active_storage_url": None, - "active_storage_max_requests": 100, -} +#_TOTAL_MEMORY = float(virtual_memory().total) +# +#_CHUNKSIZE = "128 MiB" +#config.set({"array.chunk-size": _CHUNKSIZE}) +# +# +#"""A dictionary of useful constants. +# +#Whilst the dictionary may be modified directly, it is safer to +#retrieve and set the values with a function where one is +#provided. This is due to interdependencies between some values. +# +#Note ATOL and RTOL are constants that in essence belong in this dict, +#but since they can be read and manipulated directly from cfdm, it is +#safest to work with ``cfdm.constants.CONSTANTS['ATOL']`` (and 'RTOL' +#equivalent) instead of storing separately and synchronising them here +#in cf. +# +#:Keys: +# +# TOTAL_MEMORY: `float` +# Find the total amount of physical memory (in bytes). +# +# CHUNKSIZE: `int` +# The Dask chunk size (in bytes). See `cf.chunksize`. +# +# TEMPDIR: `str` +# The location to store temporary files. By default it is the +# default directory used by the :mod:`tempfile` module. +# +# REGRID_LOGGING: `bool` +# Whether or not to enable `esmpy` logging. If it is logging is +# performed after every call to `esmpy`. By default logging is +# disabled. +# +# LOG_LEVEL: `str` +# The minimal level of seriousness for which log messages are +# shown. See `cf.log_level`. +# +#""" +#CONSTANTS = { +# "ATOL": sys.float_info.epsilon, +# "RTOL": sys.float_info.epsilon, +# "TEMPDIR": gettempdir(), +# "TOTAL_MEMORY": _TOTAL_MEMORY, +# "REGRID_LOGGING": False, +# "RELAXED_IDENTITIES": False, +# "LOG_LEVEL": logging.getLevelName(logging.getLogger().level), +# "BOUNDS_COMBINATION_MODE": "AND", +# "CHUNKSIZE": parse_bytes(_CHUNKSIZE), +# "active_storage": False, +# "active_storage_url": None, +# "active_storage_max_requests": 100, +#} masked = np.ma.masked diff --git a/cf/data/array/umarray.py b/cf/data/array/umarray.py index 6b22ed8e88..60d53e09df 100644 --- a/cf/data/array/umarray.py +++ b/cf/data/array/umarray.py @@ -1,8 +1,8 @@ import cfdm -from ...constants import _stash2standard_name -from ...functions import _DEPRECATION_ERROR_ATTRIBUTE, load_stash2standard_name -from ...umread_lib.umfile import File, Rec +from cf.constants import _stash2standard_name +from cf.functions import _DEPRECATION_ERROR_ATTRIBUTE, load_stash2standard_name +from cf.umread_lib.umfile import File, Rec from .abstract import Array diff --git a/cf/data/collapse/collapse.py b/cf/data/collapse/collapse.py index 550c14959c..b27d16918f 100644 --- a/cf/data/collapse/collapse.py +++ b/cf/data/collapse/collapse.py @@ -2,7 +2,7 @@ import numpy as np from cfdm.core import DocstringRewriteMeta -from dask.array.reductions import reduction +#from dask.array.reductions import reduction from ...docstring import _docstring_substitution_definitions from .collapse_utils import check_input_dtype, double_precision_dtype @@ -129,6 +129,7 @@ def max( The collapsed array. """ + from dask.array.reductions import reduction from .dask_collapse import cf_max_agg, cf_max_chunk, cf_max_combine if chunk_function is None: @@ -244,6 +245,7 @@ def mean( The collapsed array. """ + from dask.array.reductions import reduction from .dask_collapse import cf_mean_agg, cf_mean_chunk, cf_mean_combine if chunk_function is None: @@ -361,6 +363,7 @@ def mid_range( The collapsed array. """ + from dask.array.reductions import reduction from .dask_collapse import ( cf_mid_range_agg, cf_range_chunk, @@ -427,6 +430,7 @@ def min( The collapsed array. """ + from dask.array.reductions import reduction from .dask_collapse import cf_min_agg, cf_min_chunk, cf_min_combine if chunk_function is None: @@ -538,6 +542,7 @@ def range( The collapsed array. """ + from dask.array.reductions import reduction from .dask_collapse import ( cf_range_agg, cf_range_chunk, @@ -608,6 +613,7 @@ def rms( The collapsed array. """ + from dask.array.reductions import reduction from .dask_collapse import cf_mean_combine, cf_rms_agg, cf_rms_chunk if chunk_function is None: @@ -671,6 +677,7 @@ def sample_size( The collapsed array. """ + from dask.array.reductions import reduction from .dask_collapse import ( cf_sample_size_agg, cf_sample_size_chunk, @@ -740,6 +747,7 @@ def sum( The collapsed array. """ + from dask.array.reductions import reduction from .dask_collapse import cf_sum_agg, cf_sum_chunk, cf_sum_combine if chunk_function is None: @@ -809,6 +817,7 @@ def sum_of_weights( The collapsed array. """ + from dask.array.reductions import reduction from .dask_collapse import ( cf_sum_agg, cf_sum_combine, @@ -879,6 +888,7 @@ def sum_of_weights2( The collapsed array. """ + from dask.array.reductions import reduction from .dask_collapse import ( cf_sum_agg, cf_sum_combine, @@ -925,6 +935,7 @@ def unique(self, a, split_every=None, chunk_function=None): The unique values in a 1-d array. """ + from dask.array.reductions import reduction from .dask_collapse import cf_unique_agg, cf_unique_chunk if chunk_function is None: @@ -1005,6 +1016,7 @@ def var( The collapsed array. """ + from dask.array.reductions import reduction from .dask_collapse import cf_var_agg, cf_var_chunk, cf_var_combine if chunk_function is None: diff --git a/cf/data/collapse/collapse_active.py b/cf/data/collapse/collapse_active.py index 51ac197076..5fffa43a57 100644 --- a/cf/data/collapse/collapse_active.py +++ b/cf/data/collapse/collapse_active.py @@ -4,10 +4,10 @@ from functools import wraps from numbers import Integral -try: - from activestorage import Active -except ModuleNotFoundError: - pass +#try: +# from activestorage import Active +#except ModuleNotFoundError: +# pass from ...functions import ( active_storage, @@ -184,6 +184,8 @@ def active_chunk_function(method, *args, **kwargs): # reason, then this will trigger (inside `actify`) a local # reduction being carried out instead. # ---------------------------------------------------------------- + from activestorage import Active + filename = x.get_filename() address = x.get_address() max_requests = active_storage_max_requests() diff --git a/cf/data/collapse/dask_collapse.py b/cf/data/collapse/dask_collapse.py index 1238860a32..7ea741f956 100644 --- a/cf/data/collapse/dask_collapse.py +++ b/cf/data/collapse/dask_collapse.py @@ -11,11 +11,11 @@ import numpy as np from cfdm.data.dask_utils import cfdm_to_memory -from dask.array import chunk -from dask.array.core import _concatenate2 -from dask.array.reductions import divide, numel -from dask.core import flatten -from dask.utils import deepmap +#from dask.array import chunk +#from dask.array.core import _concatenate2 +#from dask.array.reductions import divide #, numel +#from dask.core import flatten +#from dask.utils import deepmap from .collapse_active import actify from .collapse_utils import double_precision_dtype @@ -145,6 +145,8 @@ def sum_weights_chunk( if np.ma.is_masked(x): weights = np.ma.masked_where(x.mask, weights) + from dask.array import chunk + return chunk.sum(weights, dtype=dtype, **kwargs) @@ -167,6 +169,9 @@ def combine_arrays( `numpy.ndarray` """ + from dask.utils import deepmap + from dask.array.core import _concatenate2 + x = deepmap(lambda pair: pair[key], pairs) if not computing_meta else pairs if dtype: @@ -182,6 +187,8 @@ def sum_arrays(pairs, key, axis, dtype, computing_meta=False, **kwargs): .. versionadded:: 3.14.0 """ + from dask.array import chunk + return combine_arrays( pairs, key, chunk.sum, axis, dtype, computing_meta, **kwargs ) @@ -193,6 +200,8 @@ def max_arrays(pairs, key, axis, dtype, computing_meta=False, **kwargs): .. versionadded:: 3.14.0 """ + from dask.array import chunk + return combine_arrays( pairs, key, chunk.max, axis, dtype, computing_meta, **kwargs ) @@ -204,6 +213,8 @@ def min_arrays(pairs, key, axis, dtype, computing_meta=False, **kwargs): .. versionadded:: 3.14.0 """ + from dask.array import chunk + return combine_arrays( pairs, key, chunk.min, axis, dtype, computing_meta, **kwargs ) @@ -216,6 +227,8 @@ def sum_sample_sizes(pairs, axis, computing_meta=False, **kwargs): .. versionadded:: 3.14.0 """ + from dask.array import chunk + return combine_arrays( pairs, "N", @@ -310,6 +323,8 @@ def cf_mean_combine( As for `cf_mean_chunk`. """ + from dask.core import flatten + if not isinstance(pairs, list): pairs = [pairs] @@ -368,6 +383,8 @@ def cf_mean_agg( if computing_meta: return d + from dask.array.reductions import divide + x = divide(d["sum"], d["V1"], dtype=dtype) x = mask_small_sample_size(x, d["N"], axis, mtol, original_shape) return x @@ -403,6 +420,8 @@ def cf_max_chunk(x, dtype=None, computing_meta=False, **kwargs): x = cfdm_to_memory(x) + from dask.array import chunk + return { "max": chunk.max(x, **kwargs), "N": cf_sample_size_chunk(x, **kwargs)["N"], @@ -522,6 +541,8 @@ def cf_mid_range_agg( return d # Calculate the mid-range + from dask.array.reductions import divide + x = divide(d["max"] + d["min"], 2.0, dtype=dtype) x = mask_small_sample_size(x, d["N"], axis, mtol, original_shape) return x @@ -557,6 +578,8 @@ def cf_min_chunk(x, dtype=None, computing_meta=False, **kwargs): x = cfdm_to_memory(x) + from dask.array import chunk + return { "min": chunk.min(x, **kwargs), "N": cf_sample_size_chunk(x, **kwargs)["N"], @@ -667,6 +690,8 @@ def cf_range_chunk(x, dtype=None, computing_meta=False, **kwargs): # N, max d = cf_max_chunk(x, **kwargs) + from dask.array import chunk + d["min"] = chunk.min(x, **kwargs) return d @@ -867,9 +892,13 @@ def cf_sample_size_chunk(x, dtype="i8", computing_meta=False, **kwargs): # dtype=ndtype)". See # https://github.com/numpy/numpy/issues/28255 for more # details. + from dask.array import chunk + x = np.ma.array(np.ones((x.shape), dtype=x.dtype), mask=x.mask) N = chunk.sum(x, **kwargs) else: + from dask.array.reductions import numel + if dtype: kwargs["dtype"] = dtype @@ -1009,6 +1038,9 @@ def cf_sum_chunk( x = np.multiply(x, weights, dtype=dtype) d = cf_sample_size_chunk(x, **kwargs) + + from dask.array import chunk + d["sum"] = chunk.sum(x, dtype=dtype, **kwargs) return d @@ -1226,6 +1258,8 @@ def cf_unique_agg(pairs, axis=None, computing_meta=False, **kwargs): The unique values. """ + from dask.utils import deepmap + x = ( deepmap(lambda pair: pair["unique"], pairs) if not computing_meta @@ -1234,6 +1268,8 @@ def cf_unique_agg(pairs, axis=None, computing_meta=False, **kwargs): if computing_meta: return x + from dask.array.core import _concatenate2 + x = _concatenate2(x, axes=[0]) return np.unique(x) @@ -1306,6 +1342,9 @@ def cf_var_chunk( if computing_meta: return x + from dask.array.reductions import divide + from dask.array import chunk + x = cfdm_to_memory(x) weighted = weights is not None @@ -1361,6 +1400,8 @@ def cf_var_combine( As for `cf_var_chunk`. """ + from dask.core import flatten + if not isinstance(pairs, list): pairs = [pairs] diff --git a/cf/data/dask_regrid.py b/cf/data/dask_regrid.py index 2536c4c606..66461bc4f1 100644 --- a/cf/data/dask_regrid.py +++ b/cf/data/dask_regrid.py @@ -651,7 +651,6 @@ def regrid_weights(operator, dst_dtype=None): mask of all `False`. """ - from math import prod operator.tosparse() @@ -662,6 +661,8 @@ def regrid_weights(operator, dst_dtype=None): dst_mask = operator.dst_mask if dst_mask is not None: + from math import prod + # Convert dst_mask to a 1-d array dst_mask = dst_mask.reshape((prod(operator.dst_shape),)) diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index 4c6923541d..ab64fae473 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -9,7 +9,7 @@ import numpy as np from cfdm.data.dask_utils import cfdm_to_memory -from scipy.ndimage import convolve1d +#from scipy.ndimage import convolve1d from ..cfdatetime import dt, dt2rt, rt2dt from ..units import Units @@ -73,6 +73,8 @@ def cf_convolve1d(a, window=None, axis=-1, origin=0): Convolved float array with same shape as input. """ + from scipy.ndimage import convolve1d + a = cfdm_to_memory(a) # Cast to float to ensure that NaNs can be stored diff --git a/cf/data/data.py b/cf/data/data.py index 1bbc5db614..4549be0a89 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -6,14 +6,14 @@ import cfdm import cftime -import dask.array as da +#import dask.array as da import numpy as np from cfdm.data.dask_utils import cfdm_where from cfdm.data.utils import new_axis_identifier -from dask import compute, delayed # noqa: F401 -from dask.array.core import normalize_chunks -from dask.base import is_dask_collection, tokenize -from dask.highlevelgraph import HighLevelGraph +#from dask import compute, delayed # noqa: F401 +#from dask.array.core import normalize_chunks +#from dask.base import is_dask_collection, tokenize +#from dask.highlevelgraph import HighLevelGraph from ..cfdatetime import dt as cf_dt from ..constants import masked @@ -186,6 +186,9 @@ def __contains__(self, value): False """ + import dask.array as da + from dask.base import is_dask_collection + # Check that value is scalar by seeing if its shape is () shape = getattr(value, "shape", None) if shape is None: @@ -549,6 +552,8 @@ def diff(self, axis=-1, n=1, inplace=False): [[0.0 1.0 -- 0.0]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = self.to_dask_array() @@ -726,6 +731,8 @@ def digitize( [ 1 1 1 --]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) org_units = d.Units @@ -1192,7 +1199,11 @@ def percentile( [2 2 3 3]] """ + import dask.array as da from dask.core import flatten + from dask.base import is_dask_collection, tokenize + from dask.array.core import normalize_chunks + from dask.highlevelgraph import HighLevelGraph # TODODASKAPI: interpolation -> method if interpolation is not None: @@ -1336,6 +1347,8 @@ def ceil(self, inplace=False, i=False): [-1. -1. -1. -1. 0. 1. 2. 2. 2.] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() dx = da.ceil(dx) @@ -2283,6 +2296,7 @@ def _regrid( The regridded data. """ + from dask import delayed from .dask_regrid import regrid, regrid_weights shape = self.shape @@ -2357,6 +2371,8 @@ def _regrid( src_mask = operator.src_mask if src_mask is not None: + import dask.array as da + src_mask = da.asanyarray(src_mask) weights_dst_mask = delayed(regrid_weights, pure=True)( @@ -2781,6 +2797,8 @@ def is_masked(self): True """ + import dask.array as da + # 'cf_is_masked' has its own call to 'cfdm_to_memory', so we # can set '_force_to_memory=False'. dx = self.to_dask_array(_force_to_memory=False) @@ -2946,6 +2964,8 @@ def arctan(self, inplace=False): 0.5404195002705842 --] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() @@ -3102,6 +3122,8 @@ def arcsinh(self, inplace=False): --] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() @@ -3350,6 +3372,8 @@ def argmax(self, axis=None, unravel=False): if unravel and (axis is None or self.ndim <= 1): # Return a multidimensional index tuple + import dask.array as da + return tuple(np.array(da.unravel_index(a, self.shape))) return type(self)(a) @@ -3434,6 +3458,8 @@ def argmin(self, axis=None, unravel=False): if unravel and (axis is None or self.ndim <= 1): # Return a multidimensional index tuple + import dask.array as da + return tuple(np.array(da.unravel_index(a, self.shape))) return type(self)(a) @@ -4349,6 +4375,8 @@ def clip(self, a_min, a_max, units=None, inplace=False, i=False): [8. 9. 9. 9.]] """ + import dask.array as da + if units is not None: # Convert the limits to the same units as the data array units = self._Units_class(units) @@ -4436,6 +4464,7 @@ def arctan2(cls, x1, x2): [90.0 -90.0] """ + import dask.array as da x1 = conform_units(x1, x2.Units) @@ -4502,6 +4531,8 @@ def compressed(self, inplace=False): [9] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() @@ -4565,6 +4596,8 @@ def cos(self, inplace=False, i=False): [[0.540302305868 -0.416146836547 -0.9899924966 --]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) if d.Units.equivalent(_units_radians): @@ -4629,6 +4662,8 @@ def count(self, axis=None, keepdims=True, split_every=None): 8 """ + import dask.array as da + d = self.copy(array=False) dx = self.to_dask_array() dx = da.ma.count( @@ -5033,6 +5068,8 @@ def exp(self, inplace=False, i=False): **Examples** """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) units = self.Units @@ -5474,6 +5511,8 @@ def floor(self, inplace=False, i=False): [-2. -2. -2. -1. 0. 1. 1. 1. 1.] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() d._set_dask(da.floor(dx)) @@ -5534,6 +5573,8 @@ def outerproduct(self, a, inplace=False, i=False): [18 21 24 27]]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) shape = d.shape @@ -5925,6 +5966,8 @@ def masked_invalid(self, inplace=False): """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = self.to_dask_array() dx = da.ma.masked_invalid(dx) @@ -6182,6 +6225,8 @@ def isclose(self, y, rtol=None, atol=None): [ True True True] """ + import dask.array as da + a = np.empty((), dtype=self.dtype) b = np.empty((), dtype=da.asanyarray(y).dtype) try: @@ -6313,6 +6358,8 @@ def rint(self, inplace=False, i=False): [-2. -2. -1. -1. 0. 1. 1. 2. 2.] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() d._set_dask(da.rint(dx)) @@ -6436,6 +6483,8 @@ def round(self, decimals=0, inplace=False, i=False): [-0., -0., -0., -0., 0., 0., 0., 0., 0.] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() d._set_dask(da.round(dx, decimals=decimals)) @@ -6612,6 +6661,8 @@ def stats( 'sample_size': } """ + from dask import compute,delayed + no_weights = ( "minimum", "median", @@ -6698,6 +6749,8 @@ def swapaxes(self, axis0, axis1, inplace=False, i=False): """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = self.to_dask_array() dx = da.swapaxes(dx, axis0, axis1) @@ -7024,6 +7077,8 @@ def where( x, y = xy # Apply the where operation + import dask.array as da + dx = da.core.elemwise(cfdm_where, dx, condition, x, y, d.hardmask) d._set_dask(dx) @@ -7079,6 +7134,8 @@ def sin(self, inplace=False, i=False): [[0.841470984808 0.909297426826 0.14112000806 --]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) if d.Units.equivalent(_units_radians): @@ -7140,6 +7197,8 @@ def sinh(self, inplace=False): [[1.1752011936438014 3.626860407847019 10.017874927409903 --]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) if d.Units.equivalent(_units_radians): @@ -7199,6 +7258,8 @@ def cosh(self, inplace=False): [[1.5430806348152437 3.7621956910836314 10.067661995777765 --]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) if d.Units.equivalent(_units_radians): @@ -7261,6 +7322,8 @@ def tanh(self, inplace=False): [[0.7615941559557649 0.9640275800758169 0.9950547536867305 --]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) if d.Units.equivalent(_units_radians): @@ -7292,6 +7355,8 @@ def log(self, base=None, inplace=False, i=False): `Data` or `None` """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() @@ -7361,6 +7426,8 @@ def tan(self, inplace=False, i=False): [[1.55740772465 -2.18503986326 -0.142546543074 --]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) if d.Units.equivalent(_units_radians): @@ -7468,6 +7535,8 @@ def trunc(self, inplace=False, i=False): [-1. -1. -1. -1. 0. 1. 1. 1. 1.] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() dx = da.trunc(dx) @@ -7543,6 +7612,8 @@ def func( dtype=float64) """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() @@ -7722,6 +7793,7 @@ def roll(self, axis, shift, inplace=False, i=False): # "shift,axis=", and the default axis behaviour # of a flattened roll followed by shape # restore + import dask.array as da d = _inplace_enabled_define_and_cleanup(self) @@ -8388,6 +8460,8 @@ def square(self, dtype=None, inplace=False): [[0.0 1.0 6.25 -- 16.0]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() dx = da.square(dx, dtype=dtype) @@ -8457,6 +8531,8 @@ def sqrt(self, dtype=None, inplace=False): [-- 1.0 --] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() dx = da.sqrt(dx, dtype=dtype) diff --git a/cf/functions.py b/cf/functions.py index 3bfdcb3a60..056024634c 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -6,6 +6,7 @@ import re import warnings from collections.abc import Iterable +from functools import partial from itertools import product from math import isnan from os import mkdir @@ -14,17 +15,18 @@ from os.path import expandvars as _os_path_expandvars from os.path import join as _os_path_join from os.path import relpath as _os_path_relpath +from tempfile import gettempdir from urllib.parse import urljoin, urlparse import cfdm import netCDF4 import numpy as np -from dask.base import is_dask_collection +#from dask.base import is_dask_collection from psutil import virtual_memory from . import __file__, __version__ from .constants import ( - CONSTANTS, +# CONSTANTS, OperandBoundsCombination, _stash2standard_name, ) @@ -444,15 +446,6 @@ def _configuration(_Configuration, **kwargs): values are specified. """ - old = {name.lower(): val for name, val in CONSTANTS.items()} - - old.pop("total_memory", None) - - # Filter out 'None' kwargs from configuration() defaults. Note that this - # does not filter out '0' or 'True' values, which is important as the user - # might be trying to set those, as opposed to None emerging as default. - kwargs = {name: val for name, val in kwargs.items() if val is not None} - # Note values are the functions not the keyword arguments of same name: reset_mapping = { "new_atol": atol, @@ -468,6 +461,21 @@ def _configuration(_Configuration, **kwargs): "active_storage_max_requests": active_storage_max_requests, } + # Make sure that the constants dictionary is fully populated + for func in reset_mapping.values(): + func() + + old = ConstantAccess.constants(copy=True) + +# old = {name.lower(): val for name, val in CONSTANTS.items()} +# +# old.pop("total_memory", None) + + # Filter out 'None' kwargs from configuration() defaults. Note that this + # does not filter out '0' or 'True' values, which is important as the user + # might be trying to set those, as opposed to None emerging as default. + kwargs = {name: val for name, val in kwargs.items() if val is not None} + old_values = {} try: @@ -540,7 +548,7 @@ def FREE_MEMORY(): _disable_logging = cfdm._disable_logging # We can inherit the generic logic for the cf-python log_level() # function as contained in _log_level, but can't inherit the -# user-facing log_level() from cfdm as it operates on cfdm's CONSTANTS +# user-facing log_level() from cfdm as it operates on cfdm's constants # dict. Define cf-python's own. This also means the log_level # dostrings are independent which is important for providing # module-specific documentation links and directives, etc. @@ -552,15 +560,12 @@ def FREE_MEMORY(): # Functions inherited from cfdm # -------------------------------------------------------------------- class ConstantAccess(cfdm.ConstantAccess): - _CONSTANTS = CONSTANTS + _constants = {} _Constant = Constant def __docstring_substitutions__(self): return _docstring_substitution_definitions - def __docstring_package_depth__(self): - return 0 - class atol(ConstantAccess, cfdm.atol): pass @@ -614,7 +619,8 @@ class regrid_logging(ConstantAccess): """ - _name = "REGRID_LOGGING" + _name = "regrid_logging" + _default = False def _parse(cls, arg): """Parse a new constant value. @@ -631,8 +637,8 @@ def _parse(cls, arg): :Returns: - A version of the new constant value suitable for insertion - into the `CONSTANTS` dictionary. + A version of the new constant value suitable for + insertion into the `_constants` dictionary. """ return bool(arg) @@ -681,7 +687,7 @@ class collapse_parallel_mode(ConstantAccess): """ - _name = "COLLAPSE_PARALLEL_MODE" + _name = "collapse_parallel_mode" def _parse(cls, arg): """Parse a new constant value. @@ -700,8 +706,8 @@ def _parse(cls, arg): :Returns: - A version of the new constant value suitable for insertion - into the `CONSTANTS` dictionary. + A version of the new constant value suitable for + insertion into the `_constants` dictionary. """ # TODODASKAPI @@ -751,7 +757,8 @@ class relaxed_identities(ConstantAccess): """ - _name = "RELAXED_IDENTITIES" + _name = "relaxed_identities" + _default = False def _parse(cls, arg): """Parse a new constant value. @@ -768,8 +775,8 @@ def _parse(cls, arg): :Returns: - A version of the new constant value suitable for insertion - into the `CONSTANTS` dictionary. + A version of the new constant value suitable for + insertion into the `_constants` dictionary. """ return bool(arg) @@ -817,7 +824,8 @@ class tempdir(ConstantAccess): """ - _name = "TEMPDIR" + _name = "tempdir" + _default = gettempdir() def _parse(cls, arg): """Parse a new constant value. @@ -834,8 +842,8 @@ def _parse(cls, arg): :Returns: - A version of the new constant value suitable for insertion - into the `CONSTANTS` dictionary. + A version of the new constant value suitable for + insertion into the `_constants` dictionary. """ arg = _os_path_expanduser(_os_path_expandvars(arg)) @@ -898,7 +906,7 @@ class of_fraction(ConstantAccess): """ - _name = "OF_FRACTION" + _name = "of_fraction" def _parse(cls, arg): """Parse a new constant value. @@ -917,8 +925,8 @@ def _parse(cls, arg): :Returns: - A version of the new constant value suitable for insertion - into the `CONSTANTS` dictionary. + A version of the new constant value suitable for + insertion into the `_constants` dictionary. """ # TODODASKAPI @@ -951,7 +959,7 @@ class free_memory_factor(ConstantAccess): """ - _name = "FREE_MEMORY_FACTOR" + _name = "free_memory_factor" def _parse(cls, arg): """Parse a new constant value. @@ -970,8 +978,8 @@ def _parse(cls, arg): :Returns: - A version of the new constant value suitable for insertion - into the `CONSTANTS` dictionary. + A version of the new constant value suitable for + insertion into the `_constants` dictionary. """ # TODODASKAPI @@ -1089,7 +1097,8 @@ class bounds_combination_mode(ConstantAccess): """ - _name = "BOUNDS_COMBINATION_MODE" + _name = "bounds_combination_mode" + _default = "AND" def _parse(cls, arg): """Parse a new constant value. @@ -1106,8 +1115,8 @@ def _parse(cls, arg): :Returns: - A version of the new constant value suitable for insertion - into the `CONSTANTS` dictionary. + A version of the new constant value suitable for + insertion into the `_constants` dictionary. """ try: @@ -1164,6 +1173,7 @@ class active_storage(ConstantAccess): """ _name = "active_storage" + _default = False def _parse(cls, arg): """Parse a new constant value. @@ -1181,16 +1191,19 @@ def _parse(cls, arg): :Returns: A version of the new constant value suitable for - insertion into the `CONSTANTS` dictionary. + insertion into the `_constants` dictionary. """ try: - from activestorage import Active # noqa: F401 + import activestorage # noqa: F401 except ModuleNotFoundError as error: if arg: - raise ModuleNotFoundError( - f"Can't enable active storage operations: {error}" + error.msg += ( + ". Install the 'activestorage' package " + "(https://pypi.org/project/PyActiveStorage) to enable " + "active storage reductions" ) + raise return bool(arg) @@ -1250,7 +1263,7 @@ def _parse(cls, arg): :Returns: A version of the new constant value suitable for - insertion into the `CONSTANTS` dictionary. + insertion into the `_constants` dictionary. """ if arg is None: @@ -1315,6 +1328,7 @@ class active_storage_max_requests(ConstantAccess): """ _name = "active_storage_max_requests" + _default = 100 def _parse(cls, arg): """Parse a new constant value. @@ -1332,7 +1346,7 @@ def _parse(cls, arg): :Returns: A version of the new constant value suitable for - insertion into the `CONSTANTS` dictionary. + insertion into the `_constants` dictionary. """ return int(arg) @@ -1479,7 +1493,7 @@ def min_total_memory(): def total_memory(): """The total amount of physical memory (in bytes).""" - return CONSTANTS["TOTAL_MEMORY"] + return float(virtual_memory().total) def is_log_level_info(logger): @@ -2067,6 +2081,8 @@ def indices_shape(indices, full_shape, keepdims=True): [] """ + from dask.base import is_dask_collection + shape = [] for index, full_size in zip(indices, full_shape): if isinstance(index, slice): @@ -2751,8 +2767,6 @@ def dirname(path, normalise=False, uri=None, isdir=False, sep=False): dirname.__doc__ = cfdm.dirname.__doc__.replace("cfdm.", "cf.") -from functools import partial - dirname2 = partial(cfdm.dirname) dirname2.__doc__ = cfdm.dirname.__doc__.replace("cfdm.", "cf.") diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 56f55b8755..caf5d38b10 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -3,8 +3,8 @@ import numpy as np from cfdm import is_log_level_debug, is_log_level_info -from dask.array.slicing import normalize_index -from dask.base import is_dask_collection +#from dask.array.slicing import normalize_index +#from dask.base import is_dask_collection from ..data import Data from ..decorators import ( @@ -252,6 +252,9 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): empty if the *ancillary_mask* parameter is False. """ + from dask.array.slicing import normalize_index + from dask.base import is_dask_collection + debug = is_log_level_debug(logger) # Parse mode and halo @@ -824,7 +827,7 @@ def _point_not_in_cell(nodes_x, nodes_y, point): # Note: We're about to make 'indices' inconsistent # with 'ind', but that's OK because we're not # going to use 'ind' again as 'create_mask' is - # False. + # False. reduced_halo = False for axis in item_axes: index = indices[axis] diff --git a/cf/read_write/um/umread.py b/cf/read_write/um/umread.py index 3289ea88b4..7e74e3fa1a 100644 --- a/cf/read_write/um/umread.py +++ b/cf/read_write/um/umread.py @@ -6,28 +6,28 @@ import cfdm import cftime -import dask.array as da +#import dask.array as da import numpy as np from cfdm import Constructs, is_log_level_info from cfdm.read_write.exceptions import DatasetTypeError -from dask.array.core import getter, normalize_chunks -from dask.base import tokenize +#from dask.array.core import getter, normalize_chunks +#from dask.base import tokenize from netCDF4 import date2num as netCDF4_date2num -from ... import __Conventions__, __version__ -from ...constants import _stash2standard_name -from ...data import Data -from ...data.array import UMArray -from ...decorators import ( +from cf import __Conventions__, __version__ +from cf.constants import _stash2standard_name +from cf.data import Data +from cf.data.array import UMArray +from cf.decorators import ( _manage_log_level_via_verbose_attr, _manage_log_level_via_verbosity, ) -from ...functions import abspath -from ...functions import atol as cf_atol -from ...functions import load_stash2standard_name -from ...functions import rtol as cf_rtol -from ...umread_lib.umfile import File -from ...units import Units +from cf.functions import abspath +from cf.functions import atol as cf_atol +from cf.functions import load_stash2standard_name +from cf.functions import rtol as cf_rtol +from cf.umread_lib.umfile import File +from cf.units import Units logger = logging.getLogger(__name__) @@ -2005,6 +2005,10 @@ def create_data(self): `Data` """ + import dask.array as da + from dask.base import tokenize + from dask.array.core import getter, normalize_chunks + if self.info: logger.info("Creating data:") # pragma: no cover @@ -3243,6 +3247,8 @@ def get_data(self, array, units, fill_value=None, bounds=False): An independent copy of the new data. """ + from dask.base import tokenize + token = tokenize(array, units) data = _cached_data.get(token) if data is None: diff --git a/cf/regrid/regrid.py b/cf/regrid/regrid.py index a2cc34dc64..e4373caeae 100644 --- a/cf/regrid/regrid.py +++ b/cf/regrid/regrid.py @@ -5,7 +5,7 @@ from datetime import datetime from typing import Any -import dask.array as da +#import dask.array as da import numpy as np from cfdm import is_log_level_debug @@ -13,26 +13,26 @@ from ..units import Units from .regridoperator import RegridOperator -esmpy_imported = True -try: - import esmpy -except ImportError: - esmpy_imported = False +#esmpy_imported = True +#try: +# import esmpy +#except ImportError: +# esmpy_imported = False logger = logging.getLogger(__name__) # Mapping of regrid method strings to esmpy method codes. The values -# get replaced with `esmpy.RegridMethod` constants the first time +# get created with `esmpy.RegridMethod` constants the first time # `esmpy_initialise` is run. esmpy_methods = { - "linear": None, - "bilinear": None, - "conservative": None, - "conservative_1st": None, - "conservative_2nd": None, - "nearest_dtos": None, - "nearest_stod": None, - "patch": None, +# "linear": None, +# "bilinear": None, +# "conservative": None, +# "conservative_1st": None, +# "conservative_2nd": None, +# "nearest_dtos": None, +# "nearest_stod": None, +# "patch": None, } @@ -1909,7 +1909,9 @@ def esmpy_initialise(): The `esmpy` manager. """ - if not esmpy_imported: + try: + import esmpy + except ImportError: raise RuntimeError( "Regridding will not work unless the esmpy library is installed" ) @@ -1962,6 +1964,8 @@ def create_esmpy_grid(grid, mask=None, grid_partitions=1): grid partition. """ + import esmpy + debug = is_log_level_debug(logger) if mask is not None: @@ -2247,6 +2251,8 @@ def create_esmpy_mesh(grid, mask=None, grid_partitions=1): grid partition. """ + import esmpy + debug = is_log_level_debug(logger) if grid.mesh_location != "face": @@ -2385,6 +2391,8 @@ def create_esmpy_locstream(grid, mask=None, grid_partitions=1): each grid partition. """ + import esmpy + debug = is_log_level_debug(logger) if mask is not None: @@ -2563,6 +2571,7 @@ def create_esmpy_weights( written to, a file. Otherwise `False`. """ + import esmpy from cfdm import integer_dtype debug = is_log_level_debug(logger) @@ -3073,6 +3082,8 @@ def get_mask(f, grid): The Boolean mask. """ + import dask.array as da + regrid_axes = grid.axis_indices index = [slice(None) if i in regrid_axes else 0 for i in range(f.ndim)] diff --git a/cf/units.py b/cf/units.py index 49486800a8..021b68cd5c 100644 --- a/cf/units.py +++ b/cf/units.py @@ -1,13 +1,5 @@ -from ctypes.util import find_library - from cfunits import Units as cfUnits -_libpath = find_library("udunits2") -if _libpath is None: - raise FileNotFoundError( - "cf requires UNIDATA UDUNITS-2. Can't find the 'udunits2' library." - ) - class Units: """Store, combine and compare physical units and convert numeric From c181de29b4ecedd041e45571e1cb72c20f091255 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 23 Oct 2025 16:52:43 +0100 Subject: [PATCH 02/10] dev --- cf/__init__.py | 177 +++----------------------- cf/aggregate.py | 5 +- cf/constants.py | 64 ---------- cf/data/array/umarray.py | 1 + cf/data/collapse/collapse.py | 13 +- cf/data/collapse/collapse_active.py | 13 +- cf/data/collapse/dask_collapse.py | 17 +-- cf/data/dask_regrid.py | 2 +- cf/data/dask_utils.py | 3 +- cf/data/data.py | 14 +- cf/field.py | 4 +- cf/functions.py | 18 ++- cf/mixin/fielddomain.py | 6 +- cf/read_write/read.py | 2 +- cf/read_write/um/umread.py | 5 +- cf/regrid/regrid.py | 31 ++--- docs/source/conf.py | 3 +- docs/source/recipes/plot_08_recipe.py | 3 +- docs/source/recipes/plot_12_recipe.py | 2 +- docs/source/recipes/plot_13_recipe.py | 4 +- docs/source/recipes/plot_17_recipe.py | 2 +- docs/source/recipes/plot_18_recipe.py | 4 +- docs/source/recipes/plot_19_recipe.py | 8 +- docs/source/recipes/plot_22_recipe.py | 5 +- docs/source/recipes/plot_23_recipe.py | 8 +- 25 files changed, 100 insertions(+), 314 deletions(-) diff --git a/cf/__init__.py b/cf/__init__.py index c518dce2da..deaac4a813 100644 --- a/cf/__init__.py +++ b/cf/__init__.py @@ -80,170 +80,29 @@ """ +import cfdm + +from packaging.version import Version + + __date__ = "2025-10-16" __version__ = "3.18.2" - -#_requires = ( -# "numpy", -# "netCDF4", -# "cftime", -# "cfunits", -# "cfdm", -# "psutil", -# "dask", -# "packaging", -# "scipy", -# "distributed", -#) -#x = ", ".join(_requires) -#_error0 = f"cf v{__version__} requires the modules {x}. " -# -#import importlib.util -#from platform import python_version -# -#_found_esmpy = bool(importlib.util.find_spec("esmpy")) -# -#try: -# import packaging -# from packaging.version import Version -#except ImportError as error1: -# raise ImportError(_error0 + str(error1)) -#else: -# _minimum_vn = "20.0" -# if Version(packaging.__version__) < Version(_minimum_vn): -# raise RuntimeError( -# f"Bad packaging version: cf requires packaging>={_minimum_vn}. " -# f"Got {packaging.__version__} at {packaging.__file__}" -# ) -# -#try: -import cfdm -#except ImportError as error1: -# raise ImportError(_error0 + str(error1)) -#else: -# # Check the version of cfdm -# _minimum_vn = "1.12.3.1" -# _maximum_vn = "1.12.4.0" -# _cfdm_version = Version(cfdm.__version__) -# if _cfdm_version < Version(_minimum_vn) or _cfdm_version >= Version( -# _maximum_vn -# ): -# raise RuntimeError( -# "Bad cfdm version: cf requires " -# f"{_minimum_vn}<=cfdm<{_maximum_vn}. " -# f"Got {_cfdm_version} at {cfdm.__file__}" -# ) -# __cf_version__ = cfdm.__cf_version__ __Conventions__ = f"CF-{__cf_version__}" -#try: -# import netCDF4 -#except ImportError as error1: -# raise ImportError(_error0 + str(error1)) -#else: -# _set_vn = "1.7.2" -# if Version(netCDF4.__version__) != Version(_set_vn): -# raise RuntimeError( -# "Bad netCDF4 version: cf requires " -# f"netCDF4=={_set_vn}. " -# f"Got {netCDF4.__version__} at {netCDF4.__file__}" -# ) -# -#try: -# import numpy as np -#except ImportError as error1: -# raise ImportError(_error0 + str(error1)) -#else: -# _minimum_vn = "2.0.0" -# if Version(np.__version__) < Version(_minimum_vn): -# raise RuntimeError( -# f"Bad numpy version: cf requires numpy>={_minimum_vn} " -# f"Got {np.__version__} at {np.__file__}" -# ) -# -#try: -# import cftime -#except ImportError as error1: -# raise ImportError(_error0 + str(error1)) -#else: -# _minimum_vn = "1.6.4" -# if Version(cftime.__version__) < Version(_minimum_vn): -# raise RuntimeError( -# f"Bad cftime version: cf requires cftime>={_minimum_vn}. " -# f"Got {cftime.__version__} at {cftime.__file__}" -# ) -# -#try: -# import cfunits -#except ImportError as error1: -# raise ImportError(_error0 + str(error1)) -#else: -# _minimum_vn = "3.3.7" -# if Version(cfunits.__version__) < Version(_minimum_vn): -# raise RuntimeError( -# f"Bad cfunits version: cf requires cfunits>={_minimum_vn}. " -# f"Got {cfunits.__version__} at {cfunits.__file__}" -# ) -# -#try: -# import psutil -#except ImportError as error1: -# raise ImportError(_error0 + str(error1)) -#else: -# _minimum_vn = "0.6.0" -# if Version(psutil.__version__) < Version(_minimum_vn): -# raise RuntimeError( -# f"Bad psutil version: cf requires psutil>={_minimum_vn}. " -# f"Got {psutil.__version__} at {psutil.__file__}" -# ) -# -## Check the version of dask -#try: -# import dask -#except ImportError as error1: -# raise ImportError(_error0 + str(error1)) -#else: -# _minimum_vn = "2025.5.1" -# if Version(dask.__version__) < Version(_minimum_vn): -# raise RuntimeError( -# f"Bad dask version: cf requires dask>={_minimum_vn}. " -# f"Got {dask.__version__} at {dask.__file__}" -# ) -# -#try: -# import distributed -#except ImportError as error1: -# raise ImportError(_error0 + str(error1)) -#else: -# _minimum_vn = "2025.5.1" -# if Version(distributed.__version__) < Version(_minimum_vn): -# raise RuntimeError( -# "Bad distributed version: cf requires " -# f"distributed>={_minimum_vn}. " -# f"Got {distributed.__version__} at {distributed.__file__}" -# ) -# -#try: -# import scipy -#except ImportError as error1: -# raise ImportError(_error0 + str(error1)) -#else: -# _minimum_vn = "1.10.0" -# if Version(scipy.__version__) < Version(_minimum_vn): -# raise RuntimeError( -# f"Bad scipy version: cf requires scipy>={_minimum_vn}. " -# f"Got {scipy.__version__} at {scipy.__file__}" -# ) -# -#_minimum_vn = "3.10.0" -#if Version(python_version()) < Version(_minimum_vn): -# raise RuntimeError( -# f"Bad python version: cf requires python>={_minimum_vn}. " -# f"Got {python_version()}" -# ) -# -#del _minimum_vn, _maximum_vn +# Check the version of cfdm (this is worth doing because of the very +# tight coupling between cf and cfdm, and the risk of bad things +# happening at run time if the versions are mismatched). +_minimum_vn = "1.12.3.1" +_maximum_vn = "1.12.4.0" +_cfdm_vn = Version(cfdm.__version__) +if _cfdm_vn < Version(_minimum_vn) or _cfdm_vn >= Version(_maximum_vn): + raise RuntimeError( + f"cf v{__version__} requires {_minimum_vn}<=cfdm<{_maximum_vn}. " + f"Got {_cfdm_vn} at {cfdm.__file__}" + ) + +del _minimum_vn, _maximum_vn from .constructs import Constructs diff --git a/cf/aggregate.py b/cf/aggregate.py index caf3df8438..ff53f31aa3 100644 --- a/cf/aggregate.py +++ b/cf/aggregate.py @@ -6,7 +6,6 @@ import numpy as np from cfdm import is_log_level_debug, is_log_level_detail, is_log_level_info -#from dask.base import tokenize from .auxiliarycoordinate import AuxiliaryCoordinate from .data import Data @@ -1987,7 +1986,7 @@ def tokenise_cell_conditions(self, cell_conditions): """ from dask.base import tokenize - + out = [] for x in cell_conditions: if x is None: @@ -4059,7 +4058,7 @@ def _get_hfl( except ValueError: # Slow from dask.base import tokenize - + hash_value = tokenize(d.compute()) if hash_value in hash_map: diff --git a/cf/constants.py b/cf/constants.py index 0928b73938..685bbddb83 100644 --- a/cf/constants.py +++ b/cf/constants.py @@ -1,73 +1,9 @@ -import logging -import sys from enum import Enum, auto -#from tempfile import gettempdir import numpy as np -#from dask import config -#from dask.utils import parse_bytes -#from psutil import virtual_memory from .units import Units -# -------------------------------------------------------------------- -# Find the total amount of memory, in bytes -# -------------------------------------------------------------------- -#_TOTAL_MEMORY = float(virtual_memory().total) -# -#_CHUNKSIZE = "128 MiB" -#config.set({"array.chunk-size": _CHUNKSIZE}) -# -# -#"""A dictionary of useful constants. -# -#Whilst the dictionary may be modified directly, it is safer to -#retrieve and set the values with a function where one is -#provided. This is due to interdependencies between some values. -# -#Note ATOL and RTOL are constants that in essence belong in this dict, -#but since they can be read and manipulated directly from cfdm, it is -#safest to work with ``cfdm.constants.CONSTANTS['ATOL']`` (and 'RTOL' -#equivalent) instead of storing separately and synchronising them here -#in cf. -# -#:Keys: -# -# TOTAL_MEMORY: `float` -# Find the total amount of physical memory (in bytes). -# -# CHUNKSIZE: `int` -# The Dask chunk size (in bytes). See `cf.chunksize`. -# -# TEMPDIR: `str` -# The location to store temporary files. By default it is the -# default directory used by the :mod:`tempfile` module. -# -# REGRID_LOGGING: `bool` -# Whether or not to enable `esmpy` logging. If it is logging is -# performed after every call to `esmpy`. By default logging is -# disabled. -# -# LOG_LEVEL: `str` -# The minimal level of seriousness for which log messages are -# shown. See `cf.log_level`. -# -#""" -#CONSTANTS = { -# "ATOL": sys.float_info.epsilon, -# "RTOL": sys.float_info.epsilon, -# "TEMPDIR": gettempdir(), -# "TOTAL_MEMORY": _TOTAL_MEMORY, -# "REGRID_LOGGING": False, -# "RELAXED_IDENTITIES": False, -# "LOG_LEVEL": logging.getLevelName(logging.getLogger().level), -# "BOUNDS_COMBINATION_MODE": "AND", -# "CHUNKSIZE": parse_bytes(_CHUNKSIZE), -# "active_storage": False, -# "active_storage_url": None, -# "active_storage_max_requests": 100, -#} - masked = np.ma.masked repr_prefix = "CF " diff --git a/cf/data/array/umarray.py b/cf/data/array/umarray.py index 60d53e09df..29016180c6 100644 --- a/cf/data/array/umarray.py +++ b/cf/data/array/umarray.py @@ -3,6 +3,7 @@ from cf.constants import _stash2standard_name from cf.functions import _DEPRECATION_ERROR_ATTRIBUTE, load_stash2standard_name from cf.umread_lib.umfile import File, Rec + from .abstract import Array diff --git a/cf/data/collapse/collapse.py b/cf/data/collapse/collapse.py index b27d16918f..346dbaae08 100644 --- a/cf/data/collapse/collapse.py +++ b/cf/data/collapse/collapse.py @@ -2,7 +2,6 @@ import numpy as np from cfdm.core import DocstringRewriteMeta -#from dask.array.reductions import reduction from ...docstring import _docstring_substitution_definitions from .collapse_utils import check_input_dtype, double_precision_dtype @@ -130,6 +129,7 @@ def max( """ from dask.array.reductions import reduction + from .dask_collapse import cf_max_agg, cf_max_chunk, cf_max_combine if chunk_function is None: @@ -246,6 +246,7 @@ def mean( """ from dask.array.reductions import reduction + from .dask_collapse import cf_mean_agg, cf_mean_chunk, cf_mean_combine if chunk_function is None: @@ -364,6 +365,7 @@ def mid_range( """ from dask.array.reductions import reduction + from .dask_collapse import ( cf_mid_range_agg, cf_range_chunk, @@ -431,6 +433,7 @@ def min( """ from dask.array.reductions import reduction + from .dask_collapse import cf_min_agg, cf_min_chunk, cf_min_combine if chunk_function is None: @@ -543,6 +546,7 @@ def range( """ from dask.array.reductions import reduction + from .dask_collapse import ( cf_range_agg, cf_range_chunk, @@ -614,6 +618,7 @@ def rms( """ from dask.array.reductions import reduction + from .dask_collapse import cf_mean_combine, cf_rms_agg, cf_rms_chunk if chunk_function is None: @@ -678,6 +683,7 @@ def sample_size( """ from dask.array.reductions import reduction + from .dask_collapse import ( cf_sample_size_agg, cf_sample_size_chunk, @@ -748,6 +754,7 @@ def sum( """ from dask.array.reductions import reduction + from .dask_collapse import cf_sum_agg, cf_sum_chunk, cf_sum_combine if chunk_function is None: @@ -818,6 +825,7 @@ def sum_of_weights( """ from dask.array.reductions import reduction + from .dask_collapse import ( cf_sum_agg, cf_sum_combine, @@ -889,6 +897,7 @@ def sum_of_weights2( """ from dask.array.reductions import reduction + from .dask_collapse import ( cf_sum_agg, cf_sum_combine, @@ -936,6 +945,7 @@ def unique(self, a, split_every=None, chunk_function=None): """ from dask.array.reductions import reduction + from .dask_collapse import cf_unique_agg, cf_unique_chunk if chunk_function is None: @@ -1017,6 +1027,7 @@ def var( """ from dask.array.reductions import reduction + from .dask_collapse import cf_var_agg, cf_var_chunk, cf_var_combine if chunk_function is None: diff --git a/cf/data/collapse/collapse_active.py b/cf/data/collapse/collapse_active.py index 5fffa43a57..5360cac197 100644 --- a/cf/data/collapse/collapse_active.py +++ b/cf/data/collapse/collapse_active.py @@ -4,11 +4,6 @@ from functools import wraps from numbers import Integral -#try: -# from activestorage import Active -#except ModuleNotFoundError: -# pass - from ...functions import ( active_storage, active_storage_max_requests, @@ -16,6 +11,12 @@ is_log_level_info, ) +# try: +# from activestorage import Active +# except ModuleNotFoundError: +# pass + + logger = logging.getLogger(__name__) # -------------------------------------------------------------------- @@ -185,7 +186,7 @@ def active_chunk_function(method, *args, **kwargs): # reduction being carried out instead. # ---------------------------------------------------------------- from activestorage import Active - + filename = x.get_filename() address = x.get_address() max_requests = active_storage_max_requests() diff --git a/cf/data/collapse/dask_collapse.py b/cf/data/collapse/dask_collapse.py index 7ea741f956..46c672bb0a 100644 --- a/cf/data/collapse/dask_collapse.py +++ b/cf/data/collapse/dask_collapse.py @@ -11,11 +11,6 @@ import numpy as np from cfdm.data.dask_utils import cfdm_to_memory -#from dask.array import chunk -#from dask.array.core import _concatenate2 -#from dask.array.reductions import divide #, numel -#from dask.core import flatten -#from dask.utils import deepmap from .collapse_active import actify from .collapse_utils import double_precision_dtype @@ -169,8 +164,8 @@ def combine_arrays( `numpy.ndarray` """ - from dask.utils import deepmap from dask.array.core import _concatenate2 + from dask.utils import deepmap x = deepmap(lambda pair: pair[key], pairs) if not computing_meta else pairs @@ -542,7 +537,7 @@ def cf_mid_range_agg( # Calculate the mid-range from dask.array.reductions import divide - + x = divide(d["max"] + d["min"], 2.0, dtype=dtype) x = mask_small_sample_size(x, d["N"], axis, mtol, original_shape) return x @@ -898,7 +893,7 @@ def cf_sample_size_chunk(x, dtype="i8", computing_meta=False, **kwargs): N = chunk.sum(x, **kwargs) else: from dask.array.reductions import numel - + if dtype: kwargs["dtype"] = dtype @@ -1259,7 +1254,7 @@ def cf_unique_agg(pairs, axis=None, computing_meta=False, **kwargs): """ from dask.utils import deepmap - + x = ( deepmap(lambda pair: pair["unique"], pairs) if not computing_meta @@ -1342,8 +1337,8 @@ def cf_var_chunk( if computing_meta: return x - from dask.array.reductions import divide from dask.array import chunk + from dask.array.reductions import divide x = cfdm_to_memory(x) @@ -1401,7 +1396,7 @@ def cf_var_combine( """ from dask.core import flatten - + if not isinstance(pairs, list): pairs = [pairs] diff --git a/cf/data/dask_regrid.py b/cf/data/dask_regrid.py index 66461bc4f1..160fe7b209 100644 --- a/cf/data/dask_regrid.py +++ b/cf/data/dask_regrid.py @@ -662,7 +662,7 @@ def regrid_weights(operator, dst_dtype=None): dst_mask = operator.dst_mask if dst_mask is not None: from math import prod - + # Convert dst_mask to a 1-d array dst_mask = dst_mask.reshape((prod(operator.dst_shape),)) diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index ab64fae473..819c294009 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -9,11 +9,12 @@ import numpy as np from cfdm.data.dask_utils import cfdm_to_memory -#from scipy.ndimage import convolve1d from ..cfdatetime import dt, dt2rt, rt2dt from ..units import Units +# from scipy.ndimage import convolve1d + def cf_contains(a, value): """Whether or not an array contains a value. diff --git a/cf/data/data.py b/cf/data/data.py index 4549be0a89..859d99995a 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -6,14 +6,9 @@ import cfdm import cftime -#import dask.array as da import numpy as np from cfdm.data.dask_utils import cfdm_where from cfdm.data.utils import new_axis_identifier -#from dask import compute, delayed # noqa: F401 -#from dask.array.core import normalize_chunks -#from dask.base import is_dask_collection, tokenize -#from dask.highlevelgraph import HighLevelGraph from ..cfdatetime import dt as cf_dt from ..constants import masked @@ -1200,9 +1195,9 @@ def percentile( """ import dask.array as da + from dask.array.core import normalize_chunks + from dask.base import is_dask_collection, tokenize from dask.core import flatten - from dask.base import is_dask_collection, tokenize - from dask.array.core import normalize_chunks from dask.highlevelgraph import HighLevelGraph # TODODASKAPI: interpolation -> method @@ -2297,6 +2292,7 @@ def _regrid( """ from dask import delayed + from .dask_regrid import regrid, regrid_weights shape = self.shape @@ -6661,8 +6657,8 @@ def stats( 'sample_size': } """ - from dask import compute,delayed - + from dask import compute, delayed + no_weights = ( "minimum", "median", diff --git a/cf/field.py b/cf/field.py index 04a9683aaa..2b206c1006 100644 --- a/cf/field.py +++ b/cf/field.py @@ -8784,14 +8784,14 @@ def indices(self, *config, **kwargs): # Check that there are no invalid indices for size 1 axes not # spanned by the data if len(axis_indices) > len(data_axes): + import dask.array as da + for axis, index in axis_indices.items(): if axis in data_axes or ( isinstance(index, slice) and index == slice(None) ): continue - import dask.array as da - shape = da.from_array([0])[index].compute_chunk_sizes().shape if 0 in shape: raise IndexError( diff --git a/cf/functions.py b/cf/functions.py index 056024634c..72f43382b4 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -21,15 +21,10 @@ import cfdm import netCDF4 import numpy as np -#from dask.base import is_dask_collection from psutil import virtual_memory from . import __file__, __version__ -from .constants import ( -# CONSTANTS, - OperandBoundsCombination, - _stash2standard_name, -) +from .constants import OperandBoundsCombination, _stash2standard_name from .docstring import _docstring_substitution_definitions @@ -467,9 +462,9 @@ def _configuration(_Configuration, **kwargs): old = ConstantAccess.constants(copy=True) -# old = {name.lower(): val for name, val in CONSTANTS.items()} -# -# old.pop("total_memory", None) + # old = {name.lower(): val for name, val in CONSTANTS.items()} + # + # old.pop("total_memory", None) # Filter out 'None' kwargs from configuration() defaults. Note that this # does not filter out '0' or 'True' values, which is important as the user @@ -2082,7 +2077,7 @@ def indices_shape(indices, full_shape, keepdims=True): """ from dask.base import is_dask_collection - + shape = [] for index, full_size in zip(indices, full_shape): if isinstance(index, slice): @@ -3222,6 +3217,8 @@ def environment(display=True, paths=True): cf: 3.18.0 """ + # regridding = _get_module_info("esmpy", alternative_name="ESMF", try_except=True) + # Get cfdm env out = cfdm.environment(display=False, paths=paths) @@ -3232,6 +3229,7 @@ def environment(display=True, paths=True): ), "psutil": _get_module_info("psutil"), "matplotlib": _get_module_info("matplotlib", try_except=True), + "activestorage": _get_module_info("activestorage", try_except=True), "cfplot": _get_module_info("cfplot", try_except=True), "cf": (__version__, _os_path_abspath(__file__)), } diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index caf5d38b10..9aa43cfcaa 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -3,8 +3,6 @@ import numpy as np from cfdm import is_log_level_debug, is_log_level_info -#from dask.array.slicing import normalize_index -#from dask.base import is_dask_collection from ..data import Data from ..decorators import ( @@ -254,7 +252,7 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): """ from dask.array.slicing import normalize_index from dask.base import is_dask_collection - + debug = is_log_level_debug(logger) # Parse mode and halo @@ -827,7 +825,7 @@ def _point_not_in_cell(nodes_x, nodes_y, point): # Note: We're about to make 'indices' inconsistent # with 'ind', but that's OK because we're not # going to use 'ind' again as 'create_mask' is - # False. + # False. reduced_halo = False for axis in item_axes: index = indices[axis] diff --git a/cf/read_write/read.py b/cf/read_write/read.py index 082b0dde95..7fd4639afe 100644 --- a/cf/read_write/read.py +++ b/cf/read_write/read.py @@ -725,7 +725,7 @@ def _read(self, dataset): # ------------------------------------------------------------ # Try to read as a GRIB dataset # - # Not yet availabl. When (if!) the time comes, the framework + # Not yet available. When (if!) the time comes, the framework # will be: # ------------------------------------------------------------ # diff --git a/cf/read_write/um/umread.py b/cf/read_write/um/umread.py index 7e74e3fa1a..919fac02c8 100644 --- a/cf/read_write/um/umread.py +++ b/cf/read_write/um/umread.py @@ -6,12 +6,9 @@ import cfdm import cftime -#import dask.array as da import numpy as np from cfdm import Constructs, is_log_level_info from cfdm.read_write.exceptions import DatasetTypeError -#from dask.array.core import getter, normalize_chunks -#from dask.base import tokenize from netCDF4 import date2num as netCDF4_date2num from cf import __Conventions__, __version__ @@ -2006,8 +2003,8 @@ def create_data(self): """ import dask.array as da - from dask.base import tokenize from dask.array.core import getter, normalize_chunks + from dask.base import tokenize if self.info: logger.info("Creating data:") # pragma: no cover diff --git a/cf/regrid/regrid.py b/cf/regrid/regrid.py index e4373caeae..59ce6484bc 100644 --- a/cf/regrid/regrid.py +++ b/cf/regrid/regrid.py @@ -5,7 +5,6 @@ from datetime import datetime from typing import Any -#import dask.array as da import numpy as np from cfdm import is_log_level_debug @@ -13,26 +12,20 @@ from ..units import Units from .regridoperator import RegridOperator -#esmpy_imported = True -#try: -# import esmpy -#except ImportError: -# esmpy_imported = False - logger = logging.getLogger(__name__) # Mapping of regrid method strings to esmpy method codes. The values # get created with `esmpy.RegridMethod` constants the first time # `esmpy_initialise` is run. esmpy_methods = { -# "linear": None, -# "bilinear": None, -# "conservative": None, -# "conservative_1st": None, -# "conservative_2nd": None, -# "nearest_dtos": None, -# "nearest_stod": None, -# "patch": None, + "linear": None, + "bilinear": None, + "conservative": None, + "conservative_1st": None, + "conservative_2nd": None, + "nearest_dtos": None, + "nearest_stod": None, + "patch": None, } @@ -1965,7 +1958,7 @@ def create_esmpy_grid(grid, mask=None, grid_partitions=1): """ import esmpy - + debug = is_log_level_debug(logger) if mask is not None: @@ -2252,7 +2245,7 @@ def create_esmpy_mesh(grid, mask=None, grid_partitions=1): """ import esmpy - + debug = is_log_level_debug(logger) if grid.mesh_location != "face": @@ -2392,7 +2385,7 @@ def create_esmpy_locstream(grid, mask=None, grid_partitions=1): """ import esmpy - + debug = is_log_level_debug(logger) if mask is not None: @@ -3083,7 +3076,7 @@ def get_mask(f, grid): """ import dask.array as da - + regrid_axes = grid.axis_indices index = [slice(None) if i in regrid_axes else 0 for i in range(f.ndim)] diff --git a/docs/source/conf.py b/docs/source/conf.py index 239aa988fa..4f45911a65 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -389,7 +389,7 @@ def _get_date(): "run_stale_examples": False, # Below setting can be buggy: see: # https://github.com/sphinx-gallery/sphinx-gallery/issues/967 - #"reference_url": {"cf": None}, + # "reference_url": {"cf": None}, "backreferences_dir": "gen_modules/backreferences", "doc_module": ("cf",), "inspect_global_variables": True, @@ -476,7 +476,6 @@ def _get_date(): import cf - def linkcode_resolve(domain, info): # ================================================================= # Must delete all .doctrees directories in build for changes to be diff --git a/docs/source/recipes/plot_08_recipe.py b/docs/source/recipes/plot_08_recipe.py index 63427f62a7..6045f51448 100644 --- a/docs/source/recipes/plot_08_recipe.py +++ b/docs/source/recipes/plot_08_recipe.py @@ -9,11 +9,10 @@ # 1. Import cf-python, cf-plot, numpy and scipy.stats: import cfplot as cfp -import cf - import numpy as np import scipy.stats as stats +import cf # %% # 2. Three functions are defined: diff --git a/docs/source/recipes/plot_12_recipe.py b/docs/source/recipes/plot_12_recipe.py index b09db0b29f..5304194b19 100644 --- a/docs/source/recipes/plot_12_recipe.py +++ b/docs/source/recipes/plot_12_recipe.py @@ -13,8 +13,8 @@ # %% # 1. Import cf-python, cf-plot and matplotlib.pyplot: -import matplotlib.pyplot as plt import cfplot as cfp +import matplotlib.pyplot as plt import cf diff --git a/docs/source/recipes/plot_13_recipe.py b/docs/source/recipes/plot_13_recipe.py index bf0398713e..9b658597d8 100644 --- a/docs/source/recipes/plot_13_recipe.py +++ b/docs/source/recipes/plot_13_recipe.py @@ -18,13 +18,11 @@ # in next steps. import cartopy.crs as ccrs -import matplotlib.patches as mpatches - import cfplot as cfp +import matplotlib.patches as mpatches import cf - # %% # 2. Read and select the SST by index and look at its contents: sst = cf.read("~/recipes/ERA5_monthly_averaged_SST.nc")[0] diff --git a/docs/source/recipes/plot_17_recipe.py b/docs/source/recipes/plot_17_recipe.py index c94769e2ba..a66c90b518 100644 --- a/docs/source/recipes/plot_17_recipe.py +++ b/docs/source/recipes/plot_17_recipe.py @@ -11,8 +11,8 @@ # %% # 1. Import cf-python and cf-plot: -import matplotlib.pyplot as plt import cfplot as cfp +import matplotlib.pyplot as plt import cf diff --git a/docs/source/recipes/plot_18_recipe.py b/docs/source/recipes/plot_18_recipe.py index f0eae36e35..3beb9d0db9 100644 --- a/docs/source/recipes/plot_18_recipe.py +++ b/docs/source/recipes/plot_18_recipe.py @@ -10,15 +10,15 @@ """ +import cfplot as cfp + # %% # 1. Import cf-python, cf-plot and other required packages: import matplotlib.pyplot as plt import scipy.stats.mstats as mstats -import cfplot as cfp import cf - # %% # 2. Read the data in and unpack the Fields from FieldLists using indexing. # In our example We are investigating the influence of the land height on diff --git a/docs/source/recipes/plot_19_recipe.py b/docs/source/recipes/plot_19_recipe.py index dcc0926fbd..ceb9db1c5c 100644 --- a/docs/source/recipes/plot_19_recipe.py +++ b/docs/source/recipes/plot_19_recipe.py @@ -9,10 +9,11 @@ maxima. """ +import cfplot as cfp + # %% # 1. Import cf-python, cf-plot and other required packages: import matplotlib.pyplot as plt -import cfplot as cfp import cf @@ -55,7 +56,10 @@ # of the maxima, we loop through the season query mapping and do a # "T: mean" collapse setting the season as the grouping: cfp.gopen( - rows=2, columns=1, bottom=0.1, top=0.85, + rows=2, + columns=1, + bottom=0.1, + top=0.85, ) cfp.gpos(1) cfp.gset(xmin="1980-01-01", xmax="2022-12-01", ymin=304, ymax=312) diff --git a/docs/source/recipes/plot_22_recipe.py b/docs/source/recipes/plot_22_recipe.py index 377313c899..fe329cda9d 100644 --- a/docs/source/recipes/plot_22_recipe.py +++ b/docs/source/recipes/plot_22_recipe.py @@ -11,10 +11,11 @@ # %% # 1. Import cf-python, Dask.array, NumPy, and Matplotlib: -import cf import dask.array as da -import numpy as np import matplotlib.pyplot as plt +import numpy as np + +import cf # %% # 2. Read the field constructs and load the wind speed component fields: diff --git a/docs/source/recipes/plot_23_recipe.py b/docs/source/recipes/plot_23_recipe.py index 2499b0d875..29537803af 100644 --- a/docs/source/recipes/plot_23_recipe.py +++ b/docs/source/recipes/plot_23_recipe.py @@ -18,12 +18,12 @@ # sphinx_gallery_thumbnail_number = 2 # sphinx_gallery_end_ignore -import matplotlib.pyplot as plt import cfplot as cfp -import cf - -import numpy as np import dask.array as da +import matplotlib.pyplot as plt +import numpy as np + +import cf # %% # 2. Read example data field constructs, and set region for our plots: From 445eb74c66639b277a92104f1f0d697bc5bb644d Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 24 Oct 2025 11:22:11 +0100 Subject: [PATCH 03/10] dev --- cf/cfdatetime.py | 52 +++++++++++++++++++---------- cf/data/collapse/collapse_active.py | 16 ++++----- cf/data/dask_utils.py | 2 -- cf/data/data.py | 3 +- cf/field.py | 10 +++--- cf/functions.py | 28 ++++++++-------- cf/mixin/coordinate.py | 2 -- cf/mixin/fielddomain.py | 6 ++-- cf/read_write/um/umread.py | 10 ++++-- cf/regrid/regrid.py | 6 ++-- 10 files changed, 77 insertions(+), 58 deletions(-) diff --git a/cf/cfdatetime.py b/cf/cfdatetime.py index f7bf73cdf8..1c647018a6 100644 --- a/cf/cfdatetime.py +++ b/cf/cfdatetime.py @@ -1,7 +1,6 @@ import datetime from functools import partial -import cftime import numpy as np from .functions import _DEPRECATION_ERROR_CLASS @@ -10,17 +9,10 @@ default_calendar = "gregorian" # -------------------------------------------------------------------- -# Mapping of CF calendars to cftime date-time objects +# Mapping of CF calendars to cftime date-time objects (that gets +# populated in the `dt` function). # -------------------------------------------------------------------- -_datetime_object = { - ("",): partial(cftime.datetime, calendar=""), - (None, "gregorian", "standard", "none"): cftime.DatetimeGregorian, - ("proleptic_gregorian",): cftime.DatetimeProlepticGregorian, - ("360_day",): cftime.Datetime360Day, - ("noleap", "365_day"): cftime.DatetimeNoLeap, - ("all_leap", "366_day"): cftime.DatetimeAllLeap, - ("julian",): cftime.DatetimeJulian, -} +_datetime_object = {} canonical_calendar = { None: "standard", @@ -40,7 +32,7 @@ _calendar_map = {None: "gregorian"} -class Datetime(cftime.datetime): +class Datetime: """A date-time object which supports CF calendars. Deprecated at version 3.0.0. Use function 'cf.dt' to create date- @@ -134,6 +126,26 @@ def dt( (2003, 4, 5, 12, 30, 15) """ + import cftime + + if not _datetime_object: + _datetime_object.update( + { + ("",): partial(cftime.datetime, calendar=""), + ( + None, + "gregorian", + "standard", + "none", + ): cftime.DatetimeGregorian, + ("proleptic_gregorian",): cftime.DatetimeProlepticGregorian, + ("360_day",): cftime.Datetime360Day, + ("noleap", "365_day"): cftime.DatetimeNoLeap, + ("all_leap", "366_day"): cftime.DatetimeAllLeap, + ("julian",): cftime.DatetimeJulian, + } + ) + if isinstance(arg, str): (year, month, day, hour, minute, second, microsecond) = st2elements( arg @@ -161,11 +173,6 @@ def dt( else: year = arg - # calendar=_calendar_map.get(calendar, calendar) - # - # return cftime.datetime(year, month, day, hour, minute, second, - # microsecond, calendar=calendar) - for calendars, datetime_cls in _datetime_object.items(): if calendar in calendars: return datetime_cls( @@ -354,6 +361,8 @@ def st2datetime(date_string, calendar=None): `cftime.datetime` """ + import cftime + if date_string.count("-") != 2: raise ValueError( "Input date-time string must contain at least a year, a month " @@ -388,6 +397,8 @@ def st2elements(date_string): `tuple` """ + import cftime + if date_string.count("-") != 2: raise ValueError( "Input date-time string must contain at least a year, a month " @@ -450,6 +461,8 @@ def rt2dt(array, units_in, units_out=None, dummy1=None): # mask return np.ma.masked_all((), dtype=object) + import cftime + units = units_in.units calendar = getattr(units_in, "calendar", "standard") @@ -510,6 +523,8 @@ def dt2rt(array, units_in, units_out, dummy1=None): [-- 685.5] """ + import cftime + isscalar = not np.ndim(array) array = cftime.date2num( @@ -545,8 +560,9 @@ def st2rt(array, units_in, units_out, dummy1=None): An array of floats with the same shape as *array*. """ + import cftime + array = st2dt(array, units_in) - # array = units_out._utime.date2num(array) array = cftime.date2num( array, units=units_out.units, calendar=units_out._utime.calendar ) diff --git a/cf/data/collapse/collapse_active.py b/cf/data/collapse/collapse_active.py index 5360cac197..2b42e34aa4 100644 --- a/cf/data/collapse/collapse_active.py +++ b/cf/data/collapse/collapse_active.py @@ -1,22 +1,14 @@ -import datetime import logging -import time from functools import wraps from numbers import Integral -from ...functions import ( +from cf.functions import ( active_storage, active_storage_max_requests, active_storage_url, is_log_level_info, ) -# try: -# from activestorage import Active -# except ModuleNotFoundError: -# pass - - logger = logging.getLogger(__name__) # -------------------------------------------------------------------- @@ -185,6 +177,12 @@ def active_chunk_function(method, *args, **kwargs): # reason, then this will trigger (inside `actify`) a local # reduction being carried out instead. # ---------------------------------------------------------------- + import datetime + import time + + # Note: We know that the optional `activestorage` pacakge exists + # because this was checked when active storage was enabled + # with `cf.active_storage(True)` from activestorage import Active filename = x.get_filename() diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index 819c294009..20751482df 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -13,8 +13,6 @@ from ..cfdatetime import dt, dt2rt, rt2dt from ..units import Units -# from scipy.ndimage import convolve1d - def cf_contains(a, value): """Whether or not an array contains a value. diff --git a/cf/data/data.py b/cf/data/data.py index 859d99995a..dd6d31173d 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -5,7 +5,6 @@ from operator import mul import cfdm -import cftime import numpy as np from cfdm.data.dask_utils import cfdm_where from cfdm.data.utils import new_axis_identifier @@ -2198,6 +2197,8 @@ def _binary_operation(cls, data, other, method): # so that combination with cf.Query objects works. # ------------------------------------------------------------ if not isinstance(other, cls): + import cftime + if ( isinstance(other, cftime.datetime) and other.calendar == "" diff --git a/cf/field.py b/cf/field.py index 2b206c1006..2a2cbcbb4a 100644 --- a/cf/field.py +++ b/cf/field.py @@ -9949,12 +9949,12 @@ def convolution_filter( An unweighted 5-point moving average can be computed with ``window=[0.2, 0.2, 0.2, 0.2, 0.2]`` - Note that the `scipy.signal.windows` package has suite - of window functions for creating window weights for - filtering (see the examples for details). + .. note:: The `scipy.signal.windows` package has a + suite of window functions for creating + window weights for filtering (see the + examples for details). - .. versionadded:: 3.3.0 (replaces the old weights - parameter) + .. versionadded:: 3.3.0 axis: Select the domain axis over which the filter is to be diff --git a/cf/functions.py b/cf/functions.py index 72f43382b4..46b479057e 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -7,6 +7,7 @@ import warnings from collections.abc import Iterable from functools import partial +from importlib.util import find_spec from itertools import product from math import isnan from os import mkdir @@ -19,9 +20,7 @@ from urllib.parse import urljoin, urlparse import cfdm -import netCDF4 import numpy as np -from psutil import virtual_memory from . import __file__, __version__ from .constants import OperandBoundsCombination, _stash2standard_name @@ -150,6 +149,8 @@ def _free_memory(): 96496240.0 """ + from psutil import virtual_memory + return float(virtual_memory().available) @@ -1189,18 +1190,15 @@ def _parse(cls, arg): insertion into the `_constants` dictionary. """ - try: - import activestorage # noqa: F401 - except ModuleNotFoundError as error: - if arg: - error.msg += ( - ". Install the 'activestorage' package " - "(https://pypi.org/project/PyActiveStorage) to enable " - "active storage reductions" - ) - raise + arg = bool(arg) + if arg and not find_spec("activestorage"): + raise ModuleNotFoundError( + "Must install the 'activestorage' package " + "(https://pypi.org/project/PyActiveStorage) to enable " + "active storage reductions" + ) - return bool(arg) + return arg class active_storage_url(ConstantAccess): @@ -1488,6 +1486,8 @@ def min_total_memory(): def total_memory(): """The total amount of physical memory (in bytes).""" + from psutil import virtual_memory + return float(virtual_memory().total) @@ -3275,6 +3275,8 @@ def default_netCDF_fillvals(): 'f8': 9.969209968386869e+36} """ + import netCDF4 + return netCDF4.default_fillvals diff --git a/cf/mixin/coordinate.py b/cf/mixin/coordinate.py index 0b6adb05bb..3bc6e8cf5c 100644 --- a/cf/mixin/coordinate.py +++ b/cf/mixin/coordinate.py @@ -1,5 +1,3 @@ -# from itertools import chain - from ..data.data import Data from ..decorators import ( _deprecated_kwarg_check, diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 9aa43cfcaa..ca244d77f3 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -726,9 +726,9 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): for i, p in zip(identities, points) ] ) - raise ImportError( - "Must install matplotlib to create indices " - f"for {self!r} from: {x}" + raise ModuleNotFoundError( + "Must install the 'matplotlib' package to " + f"create indices for {self!r} from: {x}" ) def _point_not_in_cell(nodes_x, nodes_y, point): diff --git a/cf/read_write/um/umread.py b/cf/read_write/um/umread.py index 919fac02c8..8f21984094 100644 --- a/cf/read_write/um/umread.py +++ b/cf/read_write/um/umread.py @@ -5,11 +5,9 @@ from uuid import uuid4 import cfdm -import cftime import numpy as np from cfdm import Constructs, is_log_level_info from cfdm.read_write.exceptions import DatasetTypeError -from netCDF4 import date2num as netCDF4_date2num from cf import __Conventions__, __version__ from cf.constants import _stash2standard_name @@ -1866,6 +1864,8 @@ def coord_positive(self, c, axiscode, domain_axis_key): def ctime(self, rec): """Return elapsed time since the clock time of the given record.""" + import cftime + reftime = self.refUnits LBVTIME = tuple(self.header_vtime(rec)) LBDTIME = tuple(self.header_dtime(rec)) @@ -2287,6 +2287,8 @@ def dtime(self, rec): key = (LBDTIME, units, calendar) time = _cached_date2num.get(key, None) if time is None: + from netCDF4 import date2num as netCDF4_date2num + # It is important to use the same time_units as vtime try: if self.calendar == "gregorian": @@ -2294,6 +2296,8 @@ def dtime(self, rec): datetime(*LBDTIME), units, calendar ) else: + import cftime + time = netCDF4_date2num( cftime.datetime(*LBDTIME, calendar=self.calendar), units, @@ -2949,6 +2953,8 @@ def vtime(self, rec): time = _cached_date2num.get(key, None) if time is None: + import cftime + # It is important to use the same time_units as dtime try: time = cftime.date2num( diff --git a/cf/regrid/regrid.py b/cf/regrid/regrid.py index 59ce6484bc..b67db47f8e 100644 --- a/cf/regrid/regrid.py +++ b/cf/regrid/regrid.py @@ -1904,9 +1904,9 @@ def esmpy_initialise(): """ try: import esmpy - except ImportError: - raise RuntimeError( - "Regridding will not work unless the esmpy library is installed" + except ModuleNotFoundError: + raise ModuleNotFoundError( + "Must install the 'esmpy' package to enable regridding" ) # Update the global 'esmpy_methods' dictionary From b8ed67c81b4d2cf2deefa1ca6402a28f59a1f128 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 24 Oct 2025 13:12:46 +0100 Subject: [PATCH 04/10] dev --- cf/field.py | 4 ++-- cf/timeduration.py | 32 ++++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cf/field.py b/cf/field.py index 2a2cbcbb4a..53b889259d 100644 --- a/cf/field.py +++ b/cf/field.py @@ -178,7 +178,7 @@ # -------------------------------------------------------------------- _collapse_ddof_methods = set(("sd", "var")) -_earth_radius = Data(6371229.0, "m") +_earth_radius = 6371229.0 _relational_methods = ( "__eq__", @@ -2701,7 +2701,7 @@ def radius(self, default=None): "or None" ) - return _earth_radius.copy() + return Data(_earth_radius, "m") r = Data.asdata(default).squeeze() else: diff --git a/cf/timeduration.py b/cf/timeduration.py index 1cf79eb15c..573ed820d1 100644 --- a/cf/timeduration.py +++ b/cf/timeduration.py @@ -25,12 +25,8 @@ _minutes = Units("minutes") _seconds = Units("seconds") -# Define some useful constants -_one_year = Data(1, "calendar_years") -_one_day = Data(1, "day") -_one_hour = Data(1, "hour") -_one_minute = Data(1, "minute") -_one_second = Data(1, "second") +# Define some data values +_data = {} # Default month lengths in days _default_month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] @@ -340,27 +336,31 @@ def __init__( f"Can't create {self.__class__.__name__} of {self.duration}" ) + if not _data: + _data.update( + { + "one_year": Data(1, "calendar_year"), + "one_day": Data(1, "day"), + "one_hour": Data(1, "hour"), + "one_minute": Data(1, "minute"), + } + ) + duration = self.duration offset = [None, month, day, hour, minute, second, 0] if units.equivalent(_calendar_years): - if duration < _one_year: + if duration < _data["one_year"]: offset[1] = None else: offset[1] = None offset[2] = None - if duration < _one_day: + if duration < _data["one_day"]: offset[3] = None - if duration < _one_hour: + if duration < _data["one_hour"]: offset[4] = None - if duration < _one_minute: + if duration < _data["one_minute"]: offset[5] = None - # if units <= _hours and duration < _one_day: - # offset[3] = None - # if units <= _minutes and duration < _one_hour: - # offset[4] = None - # if units <= _seconds and duration < _one_minute: - # offset[5] = None self.offset = Offset(*offset) From 660d48f0a19e047338302ca1f162a8fd7b3f0a98 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 24 Oct 2025 13:42:04 +0100 Subject: [PATCH 05/10] dev --- Changelog.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Changelog.rst b/Changelog.rst index 09f90ba8eb..a1f70ccb45 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -1,3 +1,13 @@ +Version NEXTVERSION +------------------- + +**2025-12-??** + +* Reduce the time taken to import `cf` + (https://github.com/NCAS-CMS/cfdm/issues/902) + +---- + Version 3.18.2 -------------- From 7640ef0723fb754c2752d0f8104c1c2f44ab2de7 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 27 Oct 2025 11:58:48 +0000 Subject: [PATCH 06/10] dev --- cf/cellmethod.py | 3 ++- cf/data/collapse/collapse_active.py | 3 ++- cf/functions.py | 20 ++++++++++++++------ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/cf/cellmethod.py b/cf/cellmethod.py index dda1e343b0..0d33a1c2a0 100644 --- a/cf/cellmethod.py +++ b/cf/cellmethod.py @@ -1,5 +1,4 @@ import logging -import re from ast import literal_eval as ast_literal_eval import cfdm @@ -89,6 +88,8 @@ def create(cls, cell_methods_string=None): >>> c = CellMethod.create('lat: mean (interval: 1 hour)') """ + import re + incorrect_interval = "Cell method interval is incorrectly formatted" out = [] diff --git a/cf/data/collapse/collapse_active.py b/cf/data/collapse/collapse_active.py index 2b42e34aa4..425f2769e6 100644 --- a/cf/data/collapse/collapse_active.py +++ b/cf/data/collapse/collapse_active.py @@ -2,11 +2,12 @@ from functools import wraps from numbers import Integral +from cfdm import is_log_level_info + from cf.functions import ( active_storage, active_storage_max_requests, active_storage_url, - is_log_level_info, ) logger = logging.getLogger(__name__) diff --git a/cf/functions.py b/cf/functions.py index 46b479057e..5543d4b524 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -1,15 +1,11 @@ import atexit -import csv -import logging import os import platform -import re import warnings from collections.abc import Iterable from functools import partial from importlib.util import find_spec from itertools import product -from math import isnan from os import mkdir from os.path import abspath as _os_path_abspath from os.path import expanduser as _os_path_expanduser @@ -1494,6 +1490,9 @@ def total_memory(): def is_log_level_info(logger): """Return True if and only if log level is at least as verbose as INFO. + Deprecated at version NEXTVERSION and is no longer available. Use + `cfdm.is_log_level_info` instead. + .. versionadded:: 3.16.3 .. seealso:: `log_level` @@ -1509,7 +1508,12 @@ def is_log_level_info(logger): Whether or not the log level is at least INFO. """ - return logger.parent.level <= logging.INFO + _DEPRECATION_ERROR_FUNCTION( + "is_log_level_info", + message="Use cfdm.is_log_level_info instead", + version="NEXTVERSION", + removed_at="5.0.0", + ) # pragma: no cover # -------------------------------------------------------------------- @@ -2076,6 +2080,8 @@ def indices_shape(indices, full_shape, keepdims=True): [] """ + from math import isnan + from dask.base import is_dask_collection shape = [] @@ -2498,6 +2504,9 @@ def load_stash2standard_name(table=None, delimiter="!", merge=True): >>> cf.load_stash2standard_name('my_table4.txt', merge=False) """ + import csv + import re + # 0 Model # 1 STASH code # 2 STASH name @@ -2507,7 +2516,6 @@ def load_stash2standard_name(table=None, delimiter="!", merge=True): # 6 standard_name # 7 CF extra info # 8 PP extra info - # Number matching regular expression number_regex = r"([-+]?\d*\.?\d+(e[-+]?\d+)?)" From 973f2fa059c704a6fba1a6b8666651153e19871f Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 12 Nov 2025 09:36:34 +0000 Subject: [PATCH 07/10] typo Co-authored-by: Sadie L. Bartholomew --- Changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index a1f70ccb45..64934e39ac 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -4,7 +4,7 @@ Version NEXTVERSION **2025-12-??** * Reduce the time taken to import `cf` - (https://github.com/NCAS-CMS/cfdm/issues/902) + (https://github.com/NCAS-CMS/cf-python/issues/902) ---- From 6f0a1e9de29fd5b93f980492d9266e98b9951aaf Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 12 Nov 2025 09:36:55 +0000 Subject: [PATCH 08/10] Remove dead code Co-authored-by: Sadie L. Bartholomew --- cf/functions.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cf/functions.py b/cf/functions.py index 5543d4b524..f63b037305 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -3225,8 +3225,6 @@ def environment(display=True, paths=True): cf: 3.18.0 """ - # regridding = _get_module_info("esmpy", alternative_name="ESMF", try_except=True) - # Get cfdm env out = cfdm.environment(display=False, paths=paths) From 9a076bd4a281897ae7de7ba1955e463a5ec1a9f4 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 12 Nov 2025 09:44:02 +0000 Subject: [PATCH 09/10] rename time duration dict --- cf/timeduration.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cf/timeduration.py b/cf/timeduration.py index 573ed820d1..21e466394d 100644 --- a/cf/timeduration.py +++ b/cf/timeduration.py @@ -25,8 +25,8 @@ _minutes = Units("minutes") _seconds = Units("seconds") -# Define some data values -_data = {} +# Define some duration data values +_durations = {} # Default month lengths in days _default_month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] @@ -336,8 +336,8 @@ def __init__( f"Can't create {self.__class__.__name__} of {self.duration}" ) - if not _data: - _data.update( + if not _durations: + _durations.update( { "one_year": Data(1, "calendar_year"), "one_day": Data(1, "day"), @@ -350,16 +350,16 @@ def __init__( offset = [None, month, day, hour, minute, second, 0] if units.equivalent(_calendar_years): - if duration < _data["one_year"]: + if duration < _durations["one_year"]: offset[1] = None else: offset[1] = None offset[2] = None - if duration < _data["one_day"]: + if duration < _durations["one_day"]: offset[3] = None - if duration < _data["one_hour"]: + if duration < _durations["one_hour"]: offset[4] = None - if duration < _data["one_minute"]: + if duration < _durations["one_minute"]: offset[5] = None self.offset = Offset(*offset) From 727bb79d5974e57da0a3fbf660308ec6b98f291a Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 12 Nov 2025 09:58:04 +0000 Subject: [PATCH 10/10] revert isort change on recipe import orders --- docs/source/recipes/plot_08_recipe.py | 3 ++- docs/source/recipes/plot_12_recipe.py | 2 +- docs/source/recipes/plot_13_recipe.py | 4 +++- docs/source/recipes/plot_17_recipe.py | 2 +- docs/source/recipes/plot_18_recipe.py | 4 ++-- docs/source/recipes/plot_19_recipe.py | 3 +-- docs/source/recipes/plot_22_recipe.py | 5 ++--- docs/source/recipes/plot_23_recipe.py | 8 ++++---- 8 files changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/source/recipes/plot_08_recipe.py b/docs/source/recipes/plot_08_recipe.py index 6045f51448..63427f62a7 100644 --- a/docs/source/recipes/plot_08_recipe.py +++ b/docs/source/recipes/plot_08_recipe.py @@ -9,10 +9,11 @@ # 1. Import cf-python, cf-plot, numpy and scipy.stats: import cfplot as cfp +import cf + import numpy as np import scipy.stats as stats -import cf # %% # 2. Three functions are defined: diff --git a/docs/source/recipes/plot_12_recipe.py b/docs/source/recipes/plot_12_recipe.py index 5304194b19..b09db0b29f 100644 --- a/docs/source/recipes/plot_12_recipe.py +++ b/docs/source/recipes/plot_12_recipe.py @@ -13,8 +13,8 @@ # %% # 1. Import cf-python, cf-plot and matplotlib.pyplot: -import cfplot as cfp import matplotlib.pyplot as plt +import cfplot as cfp import cf diff --git a/docs/source/recipes/plot_13_recipe.py b/docs/source/recipes/plot_13_recipe.py index 9b658597d8..bf0398713e 100644 --- a/docs/source/recipes/plot_13_recipe.py +++ b/docs/source/recipes/plot_13_recipe.py @@ -18,11 +18,13 @@ # in next steps. import cartopy.crs as ccrs -import cfplot as cfp import matplotlib.patches as mpatches +import cfplot as cfp + import cf + # %% # 2. Read and select the SST by index and look at its contents: sst = cf.read("~/recipes/ERA5_monthly_averaged_SST.nc")[0] diff --git a/docs/source/recipes/plot_17_recipe.py b/docs/source/recipes/plot_17_recipe.py index a66c90b518..c94769e2ba 100644 --- a/docs/source/recipes/plot_17_recipe.py +++ b/docs/source/recipes/plot_17_recipe.py @@ -11,8 +11,8 @@ # %% # 1. Import cf-python and cf-plot: -import cfplot as cfp import matplotlib.pyplot as plt +import cfplot as cfp import cf diff --git a/docs/source/recipes/plot_18_recipe.py b/docs/source/recipes/plot_18_recipe.py index 3beb9d0db9..f0eae36e35 100644 --- a/docs/source/recipes/plot_18_recipe.py +++ b/docs/source/recipes/plot_18_recipe.py @@ -10,15 +10,15 @@ """ -import cfplot as cfp - # %% # 1. Import cf-python, cf-plot and other required packages: import matplotlib.pyplot as plt import scipy.stats.mstats as mstats +import cfplot as cfp import cf + # %% # 2. Read the data in and unpack the Fields from FieldLists using indexing. # In our example We are investigating the influence of the land height on diff --git a/docs/source/recipes/plot_19_recipe.py b/docs/source/recipes/plot_19_recipe.py index ceb9db1c5c..02d493dc21 100644 --- a/docs/source/recipes/plot_19_recipe.py +++ b/docs/source/recipes/plot_19_recipe.py @@ -9,11 +9,10 @@ maxima. """ -import cfplot as cfp - # %% # 1. Import cf-python, cf-plot and other required packages: import matplotlib.pyplot as plt +import cfplot as cfp import cf diff --git a/docs/source/recipes/plot_22_recipe.py b/docs/source/recipes/plot_22_recipe.py index fe329cda9d..377313c899 100644 --- a/docs/source/recipes/plot_22_recipe.py +++ b/docs/source/recipes/plot_22_recipe.py @@ -11,11 +11,10 @@ # %% # 1. Import cf-python, Dask.array, NumPy, and Matplotlib: +import cf import dask.array as da -import matplotlib.pyplot as plt import numpy as np - -import cf +import matplotlib.pyplot as plt # %% # 2. Read the field constructs and load the wind speed component fields: diff --git a/docs/source/recipes/plot_23_recipe.py b/docs/source/recipes/plot_23_recipe.py index 29537803af..2499b0d875 100644 --- a/docs/source/recipes/plot_23_recipe.py +++ b/docs/source/recipes/plot_23_recipe.py @@ -18,13 +18,13 @@ # sphinx_gallery_thumbnail_number = 2 # sphinx_gallery_end_ignore -import cfplot as cfp -import dask.array as da import matplotlib.pyplot as plt -import numpy as np - +import cfplot as cfp import cf +import numpy as np +import dask.array as da + # %% # 2. Read example data field constructs, and set region for our plots: