diff --git a/Changelog.rst b/Changelog.rst index 79b6898037..1bbaf8583d 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -1,3 +1,13 @@ +Version NEXTVERSION +------------------- + +**2025-12-??** + +* Reduce the time taken to import `cf` + (https://github.com/NCAS-CMS/cf-python/issues/902) + +---- + Version 3.18.2 -------------- diff --git a/cf/__init__.py b/cf/__init__.py index 7dcf6a90d7..deaac4a813 100644 --- a/cf/__init__.py +++ b/cf/__init__.py @@ -80,167 +80,26 @@ """ -__date__ = "2025-10-16" -__version__ = "3.18.2" +import cfdm -_requires = ( - "numpy", - "netCDF4", - "cftime", - "cfunits", - "cfdm", - "psutil", - "dask", - "packaging", - "scipy", - "distributed", -) -x = ", ".join(_requires) -_error0 = f"cf v{__version__} requires the modules {x}. " - -import importlib.util -from platform import python_version - -_found_esmpy = bool(importlib.util.find_spec("esmpy")) - -try: - import packaging - from packaging.version import Version -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - _minimum_vn = "20.0" - if Version(packaging.__version__) < Version(_minimum_vn): - raise RuntimeError( - f"Bad packaging version: cf requires packaging>={_minimum_vn}. " - f"Got {packaging.__version__} at {packaging.__file__}" - ) - -try: - import cfdm -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - # Check the version of cfdm - _minimum_vn = "1.12.3.1" - _maximum_vn = "1.12.4.0" - _cfdm_version = Version(cfdm.__version__) - if _cfdm_version < Version(_minimum_vn) or _cfdm_version >= Version( - _maximum_vn - ): - raise RuntimeError( - "Bad cfdm version: cf requires " - f"{_minimum_vn}<=cfdm<{_maximum_vn}. " - f"Got {_cfdm_version} at {cfdm.__file__}" - ) +from packaging.version import Version + +__date__ = "2025-10-16" +__version__ = "3.18.2" __cf_version__ = cfdm.__cf_version__ __Conventions__ = f"CF-{__cf_version__}" -try: - import netCDF4 -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - _set_vn = "1.7.2" - if Version(netCDF4.__version__) != Version(_set_vn): - raise RuntimeError( - "Bad netCDF4 version: cf requires " - f"netCDF4=={_set_vn}. " - f"Got {netCDF4.__version__} at {netCDF4.__file__}" - ) - -try: - import numpy as np -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - _minimum_vn = "2.0.0" - if Version(np.__version__) < Version(_minimum_vn): - raise RuntimeError( - f"Bad numpy version: cf requires numpy>={_minimum_vn} " - f"Got {np.__version__} at {np.__file__}" - ) - -try: - import cftime -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - _minimum_vn = "1.6.4" - if Version(cftime.__version__) < Version(_minimum_vn): - raise RuntimeError( - f"Bad cftime version: cf requires cftime>={_minimum_vn}. " - f"Got {cftime.__version__} at {cftime.__file__}" - ) - -try: - import cfunits -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - _minimum_vn = "3.3.7" - if Version(cfunits.__version__) < Version(_minimum_vn): - raise RuntimeError( - f"Bad cfunits version: cf requires cfunits>={_minimum_vn}. " - f"Got {cfunits.__version__} at {cfunits.__file__}" - ) - -try: - import psutil -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - _minimum_vn = "0.6.0" - if Version(psutil.__version__) < Version(_minimum_vn): - raise RuntimeError( - f"Bad psutil version: cf requires psutil>={_minimum_vn}. " - f"Got {psutil.__version__} at {psutil.__file__}" - ) - -# Check the version of dask -try: - import dask -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - _minimum_vn = "2025.5.1" - if Version(dask.__version__) < Version(_minimum_vn): - raise RuntimeError( - f"Bad dask version: cf requires dask>={_minimum_vn}. " - f"Got {dask.__version__} at {dask.__file__}" - ) - -try: - import distributed -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - _minimum_vn = "2025.5.1" - if Version(distributed.__version__) < Version(_minimum_vn): - raise RuntimeError( - "Bad distributed version: cf requires " - f"distributed>={_minimum_vn}. " - f"Got {distributed.__version__} at {distributed.__file__}" - ) - -try: - import scipy -except ImportError as error1: - raise ImportError(_error0 + str(error1)) -else: - _minimum_vn = "1.10.0" - if Version(scipy.__version__) < Version(_minimum_vn): - raise RuntimeError( - f"Bad scipy version: cf requires scipy>={_minimum_vn}. " - f"Got {scipy.__version__} at {scipy.__file__}" - ) - -_minimum_vn = "3.10.0" -if Version(python_version()) < Version(_minimum_vn): +# Check the version of cfdm (this is worth doing because of the very +# tight coupling between cf and cfdm, and the risk of bad things +# happening at run time if the versions are mismatched). +_minimum_vn = "1.12.3.1" +_maximum_vn = "1.12.4.0" +_cfdm_vn = Version(cfdm.__version__) +if _cfdm_vn < Version(_minimum_vn) or _cfdm_vn >= Version(_maximum_vn): raise RuntimeError( - f"Bad python version: cf requires python>={_minimum_vn}. " - f"Got {python_version()}" + f"cf v{__version__} requires {_minimum_vn}<=cfdm<{_maximum_vn}. " + f"Got {_cfdm_vn} at {cfdm.__file__}" ) del _minimum_vn, _maximum_vn diff --git a/cf/aggregate.py b/cf/aggregate.py index 6b93aa6305..ff53f31aa3 100644 --- a/cf/aggregate.py +++ b/cf/aggregate.py @@ -6,7 +6,6 @@ import numpy as np from cfdm import is_log_level_debug, is_log_level_detail, is_log_level_info -from dask.base import tokenize from .auxiliarycoordinate import AuxiliaryCoordinate from .data import Data @@ -1986,6 +1985,8 @@ def tokenise_cell_conditions(self, cell_conditions): ('ce9a05dd6ec76c6a6d171b0c055f3127', '8e0216a9a17a20b6620c6502bb45dec9') """ + from dask.base import tokenize + out = [] for x in cell_conditions: if x is None: @@ -4056,6 +4057,8 @@ def _get_hfl( hash_value = d.get_deterministic_name() except ValueError: # Slow + from dask.base import tokenize + hash_value = tokenize(d.compute()) if hash_value in hash_map: diff --git a/cf/cellmethod.py b/cf/cellmethod.py index dda1e343b0..0d33a1c2a0 100644 --- a/cf/cellmethod.py +++ b/cf/cellmethod.py @@ -1,5 +1,4 @@ import logging -import re from ast import literal_eval as ast_literal_eval import cfdm @@ -89,6 +88,8 @@ def create(cls, cell_methods_string=None): >>> c = CellMethod.create('lat: mean (interval: 1 hour)') """ + import re + incorrect_interval = "Cell method interval is incorrectly formatted" out = [] diff --git a/cf/cfdatetime.py b/cf/cfdatetime.py index f7bf73cdf8..1c647018a6 100644 --- a/cf/cfdatetime.py +++ b/cf/cfdatetime.py @@ -1,7 +1,6 @@ import datetime from functools import partial -import cftime import numpy as np from .functions import _DEPRECATION_ERROR_CLASS @@ -10,17 +9,10 @@ default_calendar = "gregorian" # -------------------------------------------------------------------- -# Mapping of CF calendars to cftime date-time objects +# Mapping of CF calendars to cftime date-time objects (that gets +# populated in the `dt` function). # -------------------------------------------------------------------- -_datetime_object = { - ("",): partial(cftime.datetime, calendar=""), - (None, "gregorian", "standard", "none"): cftime.DatetimeGregorian, - ("proleptic_gregorian",): cftime.DatetimeProlepticGregorian, - ("360_day",): cftime.Datetime360Day, - ("noleap", "365_day"): cftime.DatetimeNoLeap, - ("all_leap", "366_day"): cftime.DatetimeAllLeap, - ("julian",): cftime.DatetimeJulian, -} +_datetime_object = {} canonical_calendar = { None: "standard", @@ -40,7 +32,7 @@ _calendar_map = {None: "gregorian"} -class Datetime(cftime.datetime): +class Datetime: """A date-time object which supports CF calendars. Deprecated at version 3.0.0. Use function 'cf.dt' to create date- @@ -134,6 +126,26 @@ def dt( (2003, 4, 5, 12, 30, 15) """ + import cftime + + if not _datetime_object: + _datetime_object.update( + { + ("",): partial(cftime.datetime, calendar=""), + ( + None, + "gregorian", + "standard", + "none", + ): cftime.DatetimeGregorian, + ("proleptic_gregorian",): cftime.DatetimeProlepticGregorian, + ("360_day",): cftime.Datetime360Day, + ("noleap", "365_day"): cftime.DatetimeNoLeap, + ("all_leap", "366_day"): cftime.DatetimeAllLeap, + ("julian",): cftime.DatetimeJulian, + } + ) + if isinstance(arg, str): (year, month, day, hour, minute, second, microsecond) = st2elements( arg @@ -161,11 +173,6 @@ def dt( else: year = arg - # calendar=_calendar_map.get(calendar, calendar) - # - # return cftime.datetime(year, month, day, hour, minute, second, - # microsecond, calendar=calendar) - for calendars, datetime_cls in _datetime_object.items(): if calendar in calendars: return datetime_cls( @@ -354,6 +361,8 @@ def st2datetime(date_string, calendar=None): `cftime.datetime` """ + import cftime + if date_string.count("-") != 2: raise ValueError( "Input date-time string must contain at least a year, a month " @@ -388,6 +397,8 @@ def st2elements(date_string): `tuple` """ + import cftime + if date_string.count("-") != 2: raise ValueError( "Input date-time string must contain at least a year, a month " @@ -450,6 +461,8 @@ def rt2dt(array, units_in, units_out=None, dummy1=None): # mask return np.ma.masked_all((), dtype=object) + import cftime + units = units_in.units calendar = getattr(units_in, "calendar", "standard") @@ -510,6 +523,8 @@ def dt2rt(array, units_in, units_out, dummy1=None): [-- 685.5] """ + import cftime + isscalar = not np.ndim(array) array = cftime.date2num( @@ -545,8 +560,9 @@ def st2rt(array, units_in, units_out, dummy1=None): An array of floats with the same shape as *array*. """ + import cftime + array = st2dt(array, units_in) - # array = units_out._utime.date2num(array) array = cftime.date2num( array, units=units_out.units, calendar=units_out._utime.calendar ) diff --git a/cf/constants.py b/cf/constants.py index aa2bfd0fcd..685bbddb83 100644 --- a/cf/constants.py +++ b/cf/constants.py @@ -1,73 +1,9 @@ -import logging -import sys from enum import Enum, auto -from tempfile import gettempdir import numpy as np -from dask import config -from dask.utils import parse_bytes -from psutil import virtual_memory from .units import Units -# -------------------------------------------------------------------- -# Find the total amount of memory, in bytes -# -------------------------------------------------------------------- -_TOTAL_MEMORY = float(virtual_memory().total) - -_CHUNKSIZE = "128 MiB" -config.set({"array.chunk-size": _CHUNKSIZE}) - - -"""A dictionary of useful constants. - -Whilst the dictionary may be modified directly, it is safer to -retrieve and set the values with a function where one is -provided. This is due to interdependencies between some values. - -Note ATOL and RTOL are constants that in essence belong in this dict, -but since they can be read and manipulated directly from cfdm, it is -safest to work with ``cfdm.constants.CONSTANTS['ATOL']`` (and 'RTOL' -equivalent) instead of storing separately and synchronising them here -in cf. - -:Keys: - - TOTAL_MEMORY: `float` - Find the total amount of physical memory (in bytes). - - CHUNKSIZE: `int` - The Dask chunk size (in bytes). See `cf.chunksize`. - - TEMPDIR: `str` - The location to store temporary files. By default it is the - default directory used by the :mod:`tempfile` module. - - REGRID_LOGGING: `bool` - Whether or not to enable `esmpy` logging. If it is logging is - performed after every call to `esmpy`. By default logging is - disabled. - - LOG_LEVEL: `str` - The minimal level of seriousness for which log messages are - shown. See `cf.log_level`. - -""" -CONSTANTS = { - "ATOL": sys.float_info.epsilon, - "RTOL": sys.float_info.epsilon, - "TEMPDIR": gettempdir(), - "TOTAL_MEMORY": _TOTAL_MEMORY, - "REGRID_LOGGING": False, - "RELAXED_IDENTITIES": False, - "LOG_LEVEL": logging.getLevelName(logging.getLogger().level), - "BOUNDS_COMBINATION_MODE": "AND", - "CHUNKSIZE": parse_bytes(_CHUNKSIZE), - "active_storage": False, - "active_storage_url": None, - "active_storage_max_requests": 100, -} - masked = np.ma.masked repr_prefix = "CF " diff --git a/cf/data/array/umarray.py b/cf/data/array/umarray.py index 6b22ed8e88..29016180c6 100644 --- a/cf/data/array/umarray.py +++ b/cf/data/array/umarray.py @@ -1,8 +1,9 @@ import cfdm -from ...constants import _stash2standard_name -from ...functions import _DEPRECATION_ERROR_ATTRIBUTE, load_stash2standard_name -from ...umread_lib.umfile import File, Rec +from cf.constants import _stash2standard_name +from cf.functions import _DEPRECATION_ERROR_ATTRIBUTE, load_stash2standard_name +from cf.umread_lib.umfile import File, Rec + from .abstract import Array diff --git a/cf/data/collapse/collapse.py b/cf/data/collapse/collapse.py index 550c14959c..346dbaae08 100644 --- a/cf/data/collapse/collapse.py +++ b/cf/data/collapse/collapse.py @@ -2,7 +2,6 @@ import numpy as np from cfdm.core import DocstringRewriteMeta -from dask.array.reductions import reduction from ...docstring import _docstring_substitution_definitions from .collapse_utils import check_input_dtype, double_precision_dtype @@ -129,6 +128,8 @@ def max( The collapsed array. """ + from dask.array.reductions import reduction + from .dask_collapse import cf_max_agg, cf_max_chunk, cf_max_combine if chunk_function is None: @@ -244,6 +245,8 @@ def mean( The collapsed array. """ + from dask.array.reductions import reduction + from .dask_collapse import cf_mean_agg, cf_mean_chunk, cf_mean_combine if chunk_function is None: @@ -361,6 +364,8 @@ def mid_range( The collapsed array. """ + from dask.array.reductions import reduction + from .dask_collapse import ( cf_mid_range_agg, cf_range_chunk, @@ -427,6 +432,8 @@ def min( The collapsed array. """ + from dask.array.reductions import reduction + from .dask_collapse import cf_min_agg, cf_min_chunk, cf_min_combine if chunk_function is None: @@ -538,6 +545,8 @@ def range( The collapsed array. """ + from dask.array.reductions import reduction + from .dask_collapse import ( cf_range_agg, cf_range_chunk, @@ -608,6 +617,8 @@ def rms( The collapsed array. """ + from dask.array.reductions import reduction + from .dask_collapse import cf_mean_combine, cf_rms_agg, cf_rms_chunk if chunk_function is None: @@ -671,6 +682,8 @@ def sample_size( The collapsed array. """ + from dask.array.reductions import reduction + from .dask_collapse import ( cf_sample_size_agg, cf_sample_size_chunk, @@ -740,6 +753,8 @@ def sum( The collapsed array. """ + from dask.array.reductions import reduction + from .dask_collapse import cf_sum_agg, cf_sum_chunk, cf_sum_combine if chunk_function is None: @@ -809,6 +824,8 @@ def sum_of_weights( The collapsed array. """ + from dask.array.reductions import reduction + from .dask_collapse import ( cf_sum_agg, cf_sum_combine, @@ -879,6 +896,8 @@ def sum_of_weights2( The collapsed array. """ + from dask.array.reductions import reduction + from .dask_collapse import ( cf_sum_agg, cf_sum_combine, @@ -925,6 +944,8 @@ def unique(self, a, split_every=None, chunk_function=None): The unique values in a 1-d array. """ + from dask.array.reductions import reduction + from .dask_collapse import cf_unique_agg, cf_unique_chunk if chunk_function is None: @@ -1005,6 +1026,8 @@ def var( The collapsed array. """ + from dask.array.reductions import reduction + from .dask_collapse import cf_var_agg, cf_var_chunk, cf_var_combine if chunk_function is None: diff --git a/cf/data/collapse/collapse_active.py b/cf/data/collapse/collapse_active.py index 51ac197076..425f2769e6 100644 --- a/cf/data/collapse/collapse_active.py +++ b/cf/data/collapse/collapse_active.py @@ -1,19 +1,13 @@ -import datetime import logging -import time from functools import wraps from numbers import Integral -try: - from activestorage import Active -except ModuleNotFoundError: - pass +from cfdm import is_log_level_info -from ...functions import ( +from cf.functions import ( active_storage, active_storage_max_requests, active_storage_url, - is_log_level_info, ) logger = logging.getLogger(__name__) @@ -184,6 +178,14 @@ def active_chunk_function(method, *args, **kwargs): # reason, then this will trigger (inside `actify`) a local # reduction being carried out instead. # ---------------------------------------------------------------- + import datetime + import time + + # Note: We know that the optional `activestorage` pacakge exists + # because this was checked when active storage was enabled + # with `cf.active_storage(True)` + from activestorage import Active + filename = x.get_filename() address = x.get_address() max_requests = active_storage_max_requests() diff --git a/cf/data/collapse/dask_collapse.py b/cf/data/collapse/dask_collapse.py index 1238860a32..46c672bb0a 100644 --- a/cf/data/collapse/dask_collapse.py +++ b/cf/data/collapse/dask_collapse.py @@ -11,11 +11,6 @@ import numpy as np from cfdm.data.dask_utils import cfdm_to_memory -from dask.array import chunk -from dask.array.core import _concatenate2 -from dask.array.reductions import divide, numel -from dask.core import flatten -from dask.utils import deepmap from .collapse_active import actify from .collapse_utils import double_precision_dtype @@ -145,6 +140,8 @@ def sum_weights_chunk( if np.ma.is_masked(x): weights = np.ma.masked_where(x.mask, weights) + from dask.array import chunk + return chunk.sum(weights, dtype=dtype, **kwargs) @@ -167,6 +164,9 @@ def combine_arrays( `numpy.ndarray` """ + from dask.array.core import _concatenate2 + from dask.utils import deepmap + x = deepmap(lambda pair: pair[key], pairs) if not computing_meta else pairs if dtype: @@ -182,6 +182,8 @@ def sum_arrays(pairs, key, axis, dtype, computing_meta=False, **kwargs): .. versionadded:: 3.14.0 """ + from dask.array import chunk + return combine_arrays( pairs, key, chunk.sum, axis, dtype, computing_meta, **kwargs ) @@ -193,6 +195,8 @@ def max_arrays(pairs, key, axis, dtype, computing_meta=False, **kwargs): .. versionadded:: 3.14.0 """ + from dask.array import chunk + return combine_arrays( pairs, key, chunk.max, axis, dtype, computing_meta, **kwargs ) @@ -204,6 +208,8 @@ def min_arrays(pairs, key, axis, dtype, computing_meta=False, **kwargs): .. versionadded:: 3.14.0 """ + from dask.array import chunk + return combine_arrays( pairs, key, chunk.min, axis, dtype, computing_meta, **kwargs ) @@ -216,6 +222,8 @@ def sum_sample_sizes(pairs, axis, computing_meta=False, **kwargs): .. versionadded:: 3.14.0 """ + from dask.array import chunk + return combine_arrays( pairs, "N", @@ -310,6 +318,8 @@ def cf_mean_combine( As for `cf_mean_chunk`. """ + from dask.core import flatten + if not isinstance(pairs, list): pairs = [pairs] @@ -368,6 +378,8 @@ def cf_mean_agg( if computing_meta: return d + from dask.array.reductions import divide + x = divide(d["sum"], d["V1"], dtype=dtype) x = mask_small_sample_size(x, d["N"], axis, mtol, original_shape) return x @@ -403,6 +415,8 @@ def cf_max_chunk(x, dtype=None, computing_meta=False, **kwargs): x = cfdm_to_memory(x) + from dask.array import chunk + return { "max": chunk.max(x, **kwargs), "N": cf_sample_size_chunk(x, **kwargs)["N"], @@ -522,6 +536,8 @@ def cf_mid_range_agg( return d # Calculate the mid-range + from dask.array.reductions import divide + x = divide(d["max"] + d["min"], 2.0, dtype=dtype) x = mask_small_sample_size(x, d["N"], axis, mtol, original_shape) return x @@ -557,6 +573,8 @@ def cf_min_chunk(x, dtype=None, computing_meta=False, **kwargs): x = cfdm_to_memory(x) + from dask.array import chunk + return { "min": chunk.min(x, **kwargs), "N": cf_sample_size_chunk(x, **kwargs)["N"], @@ -667,6 +685,8 @@ def cf_range_chunk(x, dtype=None, computing_meta=False, **kwargs): # N, max d = cf_max_chunk(x, **kwargs) + from dask.array import chunk + d["min"] = chunk.min(x, **kwargs) return d @@ -867,9 +887,13 @@ def cf_sample_size_chunk(x, dtype="i8", computing_meta=False, **kwargs): # dtype=ndtype)". See # https://github.com/numpy/numpy/issues/28255 for more # details. + from dask.array import chunk + x = np.ma.array(np.ones((x.shape), dtype=x.dtype), mask=x.mask) N = chunk.sum(x, **kwargs) else: + from dask.array.reductions import numel + if dtype: kwargs["dtype"] = dtype @@ -1009,6 +1033,9 @@ def cf_sum_chunk( x = np.multiply(x, weights, dtype=dtype) d = cf_sample_size_chunk(x, **kwargs) + + from dask.array import chunk + d["sum"] = chunk.sum(x, dtype=dtype, **kwargs) return d @@ -1226,6 +1253,8 @@ def cf_unique_agg(pairs, axis=None, computing_meta=False, **kwargs): The unique values. """ + from dask.utils import deepmap + x = ( deepmap(lambda pair: pair["unique"], pairs) if not computing_meta @@ -1234,6 +1263,8 @@ def cf_unique_agg(pairs, axis=None, computing_meta=False, **kwargs): if computing_meta: return x + from dask.array.core import _concatenate2 + x = _concatenate2(x, axes=[0]) return np.unique(x) @@ -1306,6 +1337,9 @@ def cf_var_chunk( if computing_meta: return x + from dask.array import chunk + from dask.array.reductions import divide + x = cfdm_to_memory(x) weighted = weights is not None @@ -1361,6 +1395,8 @@ def cf_var_combine( As for `cf_var_chunk`. """ + from dask.core import flatten + if not isinstance(pairs, list): pairs = [pairs] diff --git a/cf/data/dask_regrid.py b/cf/data/dask_regrid.py index 2536c4c606..160fe7b209 100644 --- a/cf/data/dask_regrid.py +++ b/cf/data/dask_regrid.py @@ -651,7 +651,6 @@ def regrid_weights(operator, dst_dtype=None): mask of all `False`. """ - from math import prod operator.tosparse() @@ -662,6 +661,8 @@ def regrid_weights(operator, dst_dtype=None): dst_mask = operator.dst_mask if dst_mask is not None: + from math import prod + # Convert dst_mask to a 1-d array dst_mask = dst_mask.reshape((prod(operator.dst_shape),)) diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index 4c6923541d..20751482df 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -9,7 +9,6 @@ import numpy as np from cfdm.data.dask_utils import cfdm_to_memory -from scipy.ndimage import convolve1d from ..cfdatetime import dt, dt2rt, rt2dt from ..units import Units @@ -73,6 +72,8 @@ def cf_convolve1d(a, window=None, axis=-1, origin=0): Convolved float array with same shape as input. """ + from scipy.ndimage import convolve1d + a = cfdm_to_memory(a) # Cast to float to ensure that NaNs can be stored diff --git a/cf/data/data.py b/cf/data/data.py index 1bbc5db614..dd6d31173d 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -5,15 +5,9 @@ from operator import mul import cfdm -import cftime -import dask.array as da import numpy as np from cfdm.data.dask_utils import cfdm_where from cfdm.data.utils import new_axis_identifier -from dask import compute, delayed # noqa: F401 -from dask.array.core import normalize_chunks -from dask.base import is_dask_collection, tokenize -from dask.highlevelgraph import HighLevelGraph from ..cfdatetime import dt as cf_dt from ..constants import masked @@ -186,6 +180,9 @@ def __contains__(self, value): False """ + import dask.array as da + from dask.base import is_dask_collection + # Check that value is scalar by seeing if its shape is () shape = getattr(value, "shape", None) if shape is None: @@ -549,6 +546,8 @@ def diff(self, axis=-1, n=1, inplace=False): [[0.0 1.0 -- 0.0]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = self.to_dask_array() @@ -726,6 +725,8 @@ def digitize( [ 1 1 1 --]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) org_units = d.Units @@ -1192,7 +1193,11 @@ def percentile( [2 2 3 3]] """ + import dask.array as da + from dask.array.core import normalize_chunks + from dask.base import is_dask_collection, tokenize from dask.core import flatten + from dask.highlevelgraph import HighLevelGraph # TODODASKAPI: interpolation -> method if interpolation is not None: @@ -1336,6 +1341,8 @@ def ceil(self, inplace=False, i=False): [-1. -1. -1. -1. 0. 1. 2. 2. 2.] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() dx = da.ceil(dx) @@ -2190,6 +2197,8 @@ def _binary_operation(cls, data, other, method): # so that combination with cf.Query objects works. # ------------------------------------------------------------ if not isinstance(other, cls): + import cftime + if ( isinstance(other, cftime.datetime) and other.calendar == "" @@ -2283,6 +2292,8 @@ def _regrid( The regridded data. """ + from dask import delayed + from .dask_regrid import regrid, regrid_weights shape = self.shape @@ -2357,6 +2368,8 @@ def _regrid( src_mask = operator.src_mask if src_mask is not None: + import dask.array as da + src_mask = da.asanyarray(src_mask) weights_dst_mask = delayed(regrid_weights, pure=True)( @@ -2781,6 +2794,8 @@ def is_masked(self): True """ + import dask.array as da + # 'cf_is_masked' has its own call to 'cfdm_to_memory', so we # can set '_force_to_memory=False'. dx = self.to_dask_array(_force_to_memory=False) @@ -2946,6 +2961,8 @@ def arctan(self, inplace=False): 0.5404195002705842 --] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() @@ -3102,6 +3119,8 @@ def arcsinh(self, inplace=False): --] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() @@ -3350,6 +3369,8 @@ def argmax(self, axis=None, unravel=False): if unravel and (axis is None or self.ndim <= 1): # Return a multidimensional index tuple + import dask.array as da + return tuple(np.array(da.unravel_index(a, self.shape))) return type(self)(a) @@ -3434,6 +3455,8 @@ def argmin(self, axis=None, unravel=False): if unravel and (axis is None or self.ndim <= 1): # Return a multidimensional index tuple + import dask.array as da + return tuple(np.array(da.unravel_index(a, self.shape))) return type(self)(a) @@ -4349,6 +4372,8 @@ def clip(self, a_min, a_max, units=None, inplace=False, i=False): [8. 9. 9. 9.]] """ + import dask.array as da + if units is not None: # Convert the limits to the same units as the data array units = self._Units_class(units) @@ -4436,6 +4461,7 @@ def arctan2(cls, x1, x2): [90.0 -90.0] """ + import dask.array as da x1 = conform_units(x1, x2.Units) @@ -4502,6 +4528,8 @@ def compressed(self, inplace=False): [9] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() @@ -4565,6 +4593,8 @@ def cos(self, inplace=False, i=False): [[0.540302305868 -0.416146836547 -0.9899924966 --]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) if d.Units.equivalent(_units_radians): @@ -4629,6 +4659,8 @@ def count(self, axis=None, keepdims=True, split_every=None): 8 """ + import dask.array as da + d = self.copy(array=False) dx = self.to_dask_array() dx = da.ma.count( @@ -5033,6 +5065,8 @@ def exp(self, inplace=False, i=False): **Examples** """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) units = self.Units @@ -5474,6 +5508,8 @@ def floor(self, inplace=False, i=False): [-2. -2. -2. -1. 0. 1. 1. 1. 1.] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() d._set_dask(da.floor(dx)) @@ -5534,6 +5570,8 @@ def outerproduct(self, a, inplace=False, i=False): [18 21 24 27]]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) shape = d.shape @@ -5925,6 +5963,8 @@ def masked_invalid(self, inplace=False): """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = self.to_dask_array() dx = da.ma.masked_invalid(dx) @@ -6182,6 +6222,8 @@ def isclose(self, y, rtol=None, atol=None): [ True True True] """ + import dask.array as da + a = np.empty((), dtype=self.dtype) b = np.empty((), dtype=da.asanyarray(y).dtype) try: @@ -6313,6 +6355,8 @@ def rint(self, inplace=False, i=False): [-2. -2. -1. -1. 0. 1. 1. 2. 2.] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() d._set_dask(da.rint(dx)) @@ -6436,6 +6480,8 @@ def round(self, decimals=0, inplace=False, i=False): [-0., -0., -0., -0., 0., 0., 0., 0., 0.] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() d._set_dask(da.round(dx, decimals=decimals)) @@ -6612,6 +6658,8 @@ def stats( 'sample_size': } """ + from dask import compute, delayed + no_weights = ( "minimum", "median", @@ -6698,6 +6746,8 @@ def swapaxes(self, axis0, axis1, inplace=False, i=False): """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = self.to_dask_array() dx = da.swapaxes(dx, axis0, axis1) @@ -7024,6 +7074,8 @@ def where( x, y = xy # Apply the where operation + import dask.array as da + dx = da.core.elemwise(cfdm_where, dx, condition, x, y, d.hardmask) d._set_dask(dx) @@ -7079,6 +7131,8 @@ def sin(self, inplace=False, i=False): [[0.841470984808 0.909297426826 0.14112000806 --]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) if d.Units.equivalent(_units_radians): @@ -7140,6 +7194,8 @@ def sinh(self, inplace=False): [[1.1752011936438014 3.626860407847019 10.017874927409903 --]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) if d.Units.equivalent(_units_radians): @@ -7199,6 +7255,8 @@ def cosh(self, inplace=False): [[1.5430806348152437 3.7621956910836314 10.067661995777765 --]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) if d.Units.equivalent(_units_radians): @@ -7261,6 +7319,8 @@ def tanh(self, inplace=False): [[0.7615941559557649 0.9640275800758169 0.9950547536867305 --]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) if d.Units.equivalent(_units_radians): @@ -7292,6 +7352,8 @@ def log(self, base=None, inplace=False, i=False): `Data` or `None` """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() @@ -7361,6 +7423,8 @@ def tan(self, inplace=False, i=False): [[1.55740772465 -2.18503986326 -0.142546543074 --]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) if d.Units.equivalent(_units_radians): @@ -7468,6 +7532,8 @@ def trunc(self, inplace=False, i=False): [-1. -1. -1. -1. 0. 1. 1. 1. 1.] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() dx = da.trunc(dx) @@ -7543,6 +7609,8 @@ def func( dtype=float64) """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() @@ -7722,6 +7790,7 @@ def roll(self, axis, shift, inplace=False, i=False): # "shift,axis=", and the default axis behaviour # of a flattened roll followed by shape # restore + import dask.array as da d = _inplace_enabled_define_and_cleanup(self) @@ -8388,6 +8457,8 @@ def square(self, dtype=None, inplace=False): [[0.0 1.0 6.25 -- 16.0]] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() dx = da.square(dx, dtype=dtype) @@ -8457,6 +8528,8 @@ def sqrt(self, dtype=None, inplace=False): [-- 1.0 --] """ + import dask.array as da + d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() dx = da.sqrt(dx, dtype=dtype) diff --git a/cf/field.py b/cf/field.py index 04a9683aaa..53b889259d 100644 --- a/cf/field.py +++ b/cf/field.py @@ -178,7 +178,7 @@ # -------------------------------------------------------------------- _collapse_ddof_methods = set(("sd", "var")) -_earth_radius = Data(6371229.0, "m") +_earth_radius = 6371229.0 _relational_methods = ( "__eq__", @@ -2701,7 +2701,7 @@ def radius(self, default=None): "or None" ) - return _earth_radius.copy() + return Data(_earth_radius, "m") r = Data.asdata(default).squeeze() else: @@ -8784,14 +8784,14 @@ def indices(self, *config, **kwargs): # Check that there are no invalid indices for size 1 axes not # spanned by the data if len(axis_indices) > len(data_axes): + import dask.array as da + for axis, index in axis_indices.items(): if axis in data_axes or ( isinstance(index, slice) and index == slice(None) ): continue - import dask.array as da - shape = da.from_array([0])[index].compute_chunk_sizes().shape if 0 in shape: raise IndexError( @@ -9949,12 +9949,12 @@ def convolution_filter( An unweighted 5-point moving average can be computed with ``window=[0.2, 0.2, 0.2, 0.2, 0.2]`` - Note that the `scipy.signal.windows` package has suite - of window functions for creating window weights for - filtering (see the examples for details). + .. note:: The `scipy.signal.windows` package has a + suite of window functions for creating + window weights for filtering (see the + examples for details). - .. versionadded:: 3.3.0 (replaces the old weights - parameter) + .. versionadded:: 3.3.0 axis: Select the domain axis over which the filter is to be diff --git a/cf/functions.py b/cf/functions.py index 3bfdcb3a60..f63b037305 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -1,33 +1,25 @@ import atexit -import csv -import logging import os import platform -import re import warnings from collections.abc import Iterable +from functools import partial +from importlib.util import find_spec from itertools import product -from math import isnan from os import mkdir from os.path import abspath as _os_path_abspath from os.path import expanduser as _os_path_expanduser from os.path import expandvars as _os_path_expandvars from os.path import join as _os_path_join from os.path import relpath as _os_path_relpath +from tempfile import gettempdir from urllib.parse import urljoin, urlparse import cfdm -import netCDF4 import numpy as np -from dask.base import is_dask_collection -from psutil import virtual_memory from . import __file__, __version__ -from .constants import ( - CONSTANTS, - OperandBoundsCombination, - _stash2standard_name, -) +from .constants import OperandBoundsCombination, _stash2standard_name from .docstring import _docstring_substitution_definitions @@ -153,6 +145,8 @@ def _free_memory(): 96496240.0 """ + from psutil import virtual_memory + return float(virtual_memory().available) @@ -444,15 +438,6 @@ def _configuration(_Configuration, **kwargs): values are specified. """ - old = {name.lower(): val for name, val in CONSTANTS.items()} - - old.pop("total_memory", None) - - # Filter out 'None' kwargs from configuration() defaults. Note that this - # does not filter out '0' or 'True' values, which is important as the user - # might be trying to set those, as opposed to None emerging as default. - kwargs = {name: val for name, val in kwargs.items() if val is not None} - # Note values are the functions not the keyword arguments of same name: reset_mapping = { "new_atol": atol, @@ -468,6 +453,21 @@ def _configuration(_Configuration, **kwargs): "active_storage_max_requests": active_storage_max_requests, } + # Make sure that the constants dictionary is fully populated + for func in reset_mapping.values(): + func() + + old = ConstantAccess.constants(copy=True) + + # old = {name.lower(): val for name, val in CONSTANTS.items()} + # + # old.pop("total_memory", None) + + # Filter out 'None' kwargs from configuration() defaults. Note that this + # does not filter out '0' or 'True' values, which is important as the user + # might be trying to set those, as opposed to None emerging as default. + kwargs = {name: val for name, val in kwargs.items() if val is not None} + old_values = {} try: @@ -540,7 +540,7 @@ def FREE_MEMORY(): _disable_logging = cfdm._disable_logging # We can inherit the generic logic for the cf-python log_level() # function as contained in _log_level, but can't inherit the -# user-facing log_level() from cfdm as it operates on cfdm's CONSTANTS +# user-facing log_level() from cfdm as it operates on cfdm's constants # dict. Define cf-python's own. This also means the log_level # dostrings are independent which is important for providing # module-specific documentation links and directives, etc. @@ -552,15 +552,12 @@ def FREE_MEMORY(): # Functions inherited from cfdm # -------------------------------------------------------------------- class ConstantAccess(cfdm.ConstantAccess): - _CONSTANTS = CONSTANTS + _constants = {} _Constant = Constant def __docstring_substitutions__(self): return _docstring_substitution_definitions - def __docstring_package_depth__(self): - return 0 - class atol(ConstantAccess, cfdm.atol): pass @@ -614,7 +611,8 @@ class regrid_logging(ConstantAccess): """ - _name = "REGRID_LOGGING" + _name = "regrid_logging" + _default = False def _parse(cls, arg): """Parse a new constant value. @@ -631,8 +629,8 @@ def _parse(cls, arg): :Returns: - A version of the new constant value suitable for insertion - into the `CONSTANTS` dictionary. + A version of the new constant value suitable for + insertion into the `_constants` dictionary. """ return bool(arg) @@ -681,7 +679,7 @@ class collapse_parallel_mode(ConstantAccess): """ - _name = "COLLAPSE_PARALLEL_MODE" + _name = "collapse_parallel_mode" def _parse(cls, arg): """Parse a new constant value. @@ -700,8 +698,8 @@ def _parse(cls, arg): :Returns: - A version of the new constant value suitable for insertion - into the `CONSTANTS` dictionary. + A version of the new constant value suitable for + insertion into the `_constants` dictionary. """ # TODODASKAPI @@ -751,7 +749,8 @@ class relaxed_identities(ConstantAccess): """ - _name = "RELAXED_IDENTITIES" + _name = "relaxed_identities" + _default = False def _parse(cls, arg): """Parse a new constant value. @@ -768,8 +767,8 @@ def _parse(cls, arg): :Returns: - A version of the new constant value suitable for insertion - into the `CONSTANTS` dictionary. + A version of the new constant value suitable for + insertion into the `_constants` dictionary. """ return bool(arg) @@ -817,7 +816,8 @@ class tempdir(ConstantAccess): """ - _name = "TEMPDIR" + _name = "tempdir" + _default = gettempdir() def _parse(cls, arg): """Parse a new constant value. @@ -834,8 +834,8 @@ def _parse(cls, arg): :Returns: - A version of the new constant value suitable for insertion - into the `CONSTANTS` dictionary. + A version of the new constant value suitable for + insertion into the `_constants` dictionary. """ arg = _os_path_expanduser(_os_path_expandvars(arg)) @@ -898,7 +898,7 @@ class of_fraction(ConstantAccess): """ - _name = "OF_FRACTION" + _name = "of_fraction" def _parse(cls, arg): """Parse a new constant value. @@ -917,8 +917,8 @@ def _parse(cls, arg): :Returns: - A version of the new constant value suitable for insertion - into the `CONSTANTS` dictionary. + A version of the new constant value suitable for + insertion into the `_constants` dictionary. """ # TODODASKAPI @@ -951,7 +951,7 @@ class free_memory_factor(ConstantAccess): """ - _name = "FREE_MEMORY_FACTOR" + _name = "free_memory_factor" def _parse(cls, arg): """Parse a new constant value. @@ -970,8 +970,8 @@ def _parse(cls, arg): :Returns: - A version of the new constant value suitable for insertion - into the `CONSTANTS` dictionary. + A version of the new constant value suitable for + insertion into the `_constants` dictionary. """ # TODODASKAPI @@ -1089,7 +1089,8 @@ class bounds_combination_mode(ConstantAccess): """ - _name = "BOUNDS_COMBINATION_MODE" + _name = "bounds_combination_mode" + _default = "AND" def _parse(cls, arg): """Parse a new constant value. @@ -1106,8 +1107,8 @@ def _parse(cls, arg): :Returns: - A version of the new constant value suitable for insertion - into the `CONSTANTS` dictionary. + A version of the new constant value suitable for + insertion into the `_constants` dictionary. """ try: @@ -1164,6 +1165,7 @@ class active_storage(ConstantAccess): """ _name = "active_storage" + _default = False def _parse(cls, arg): """Parse a new constant value. @@ -1181,18 +1183,18 @@ def _parse(cls, arg): :Returns: A version of the new constant value suitable for - insertion into the `CONSTANTS` dictionary. + insertion into the `_constants` dictionary. """ - try: - from activestorage import Active # noqa: F401 - except ModuleNotFoundError as error: - if arg: - raise ModuleNotFoundError( - f"Can't enable active storage operations: {error}" - ) + arg = bool(arg) + if arg and not find_spec("activestorage"): + raise ModuleNotFoundError( + "Must install the 'activestorage' package " + "(https://pypi.org/project/PyActiveStorage) to enable " + "active storage reductions" + ) - return bool(arg) + return arg class active_storage_url(ConstantAccess): @@ -1250,7 +1252,7 @@ def _parse(cls, arg): :Returns: A version of the new constant value suitable for - insertion into the `CONSTANTS` dictionary. + insertion into the `_constants` dictionary. """ if arg is None: @@ -1315,6 +1317,7 @@ class active_storage_max_requests(ConstantAccess): """ _name = "active_storage_max_requests" + _default = 100 def _parse(cls, arg): """Parse a new constant value. @@ -1332,7 +1335,7 @@ def _parse(cls, arg): :Returns: A version of the new constant value suitable for - insertion into the `CONSTANTS` dictionary. + insertion into the `_constants` dictionary. """ return int(arg) @@ -1479,12 +1482,17 @@ def min_total_memory(): def total_memory(): """The total amount of physical memory (in bytes).""" - return CONSTANTS["TOTAL_MEMORY"] + from psutil import virtual_memory + + return float(virtual_memory().total) def is_log_level_info(logger): """Return True if and only if log level is at least as verbose as INFO. + Deprecated at version NEXTVERSION and is no longer available. Use + `cfdm.is_log_level_info` instead. + .. versionadded:: 3.16.3 .. seealso:: `log_level` @@ -1500,7 +1508,12 @@ def is_log_level_info(logger): Whether or not the log level is at least INFO. """ - return logger.parent.level <= logging.INFO + _DEPRECATION_ERROR_FUNCTION( + "is_log_level_info", + message="Use cfdm.is_log_level_info instead", + version="NEXTVERSION", + removed_at="5.0.0", + ) # pragma: no cover # -------------------------------------------------------------------- @@ -2067,6 +2080,10 @@ def indices_shape(indices, full_shape, keepdims=True): [] """ + from math import isnan + + from dask.base import is_dask_collection + shape = [] for index, full_size in zip(indices, full_shape): if isinstance(index, slice): @@ -2487,6 +2504,9 @@ def load_stash2standard_name(table=None, delimiter="!", merge=True): >>> cf.load_stash2standard_name('my_table4.txt', merge=False) """ + import csv + import re + # 0 Model # 1 STASH code # 2 STASH name @@ -2496,7 +2516,6 @@ def load_stash2standard_name(table=None, delimiter="!", merge=True): # 6 standard_name # 7 CF extra info # 8 PP extra info - # Number matching regular expression number_regex = r"([-+]?\d*\.?\d+(e[-+]?\d+)?)" @@ -2751,8 +2770,6 @@ def dirname(path, normalise=False, uri=None, isdir=False, sep=False): dirname.__doc__ = cfdm.dirname.__doc__.replace("cfdm.", "cf.") -from functools import partial - dirname2 = partial(cfdm.dirname) dirname2.__doc__ = cfdm.dirname.__doc__.replace("cfdm.", "cf.") @@ -3218,6 +3235,7 @@ def environment(display=True, paths=True): ), "psutil": _get_module_info("psutil"), "matplotlib": _get_module_info("matplotlib", try_except=True), + "activestorage": _get_module_info("activestorage", try_except=True), "cfplot": _get_module_info("cfplot", try_except=True), "cf": (__version__, _os_path_abspath(__file__)), } @@ -3263,6 +3281,8 @@ def default_netCDF_fillvals(): 'f8': 9.969209968386869e+36} """ + import netCDF4 + return netCDF4.default_fillvals diff --git a/cf/mixin/coordinate.py b/cf/mixin/coordinate.py index 0b6adb05bb..3bc6e8cf5c 100644 --- a/cf/mixin/coordinate.py +++ b/cf/mixin/coordinate.py @@ -1,5 +1,3 @@ -# from itertools import chain - from ..data.data import Data from ..decorators import ( _deprecated_kwarg_check, diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 56f55b8755..ca244d77f3 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -3,8 +3,6 @@ import numpy as np from cfdm import is_log_level_debug, is_log_level_info -from dask.array.slicing import normalize_index -from dask.base import is_dask_collection from ..data import Data from ..decorators import ( @@ -252,6 +250,9 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): empty if the *ancillary_mask* parameter is False. """ + from dask.array.slicing import normalize_index + from dask.base import is_dask_collection + debug = is_log_level_debug(logger) # Parse mode and halo @@ -725,9 +726,9 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): for i, p in zip(identities, points) ] ) - raise ImportError( - "Must install matplotlib to create indices " - f"for {self!r} from: {x}" + raise ModuleNotFoundError( + "Must install the 'matplotlib' package to " + f"create indices for {self!r} from: {x}" ) def _point_not_in_cell(nodes_x, nodes_y, point): diff --git a/cf/read_write/read.py b/cf/read_write/read.py index 082b0dde95..7fd4639afe 100644 --- a/cf/read_write/read.py +++ b/cf/read_write/read.py @@ -725,7 +725,7 @@ def _read(self, dataset): # ------------------------------------------------------------ # Try to read as a GRIB dataset # - # Not yet availabl. When (if!) the time comes, the framework + # Not yet available. When (if!) the time comes, the framework # will be: # ------------------------------------------------------------ # diff --git a/cf/read_write/um/umread.py b/cf/read_write/um/umread.py index 3289ea88b4..8f21984094 100644 --- a/cf/read_write/um/umread.py +++ b/cf/read_write/um/umread.py @@ -5,29 +5,24 @@ from uuid import uuid4 import cfdm -import cftime -import dask.array as da import numpy as np from cfdm import Constructs, is_log_level_info from cfdm.read_write.exceptions import DatasetTypeError -from dask.array.core import getter, normalize_chunks -from dask.base import tokenize -from netCDF4 import date2num as netCDF4_date2num - -from ... import __Conventions__, __version__ -from ...constants import _stash2standard_name -from ...data import Data -from ...data.array import UMArray -from ...decorators import ( + +from cf import __Conventions__, __version__ +from cf.constants import _stash2standard_name +from cf.data import Data +from cf.data.array import UMArray +from cf.decorators import ( _manage_log_level_via_verbose_attr, _manage_log_level_via_verbosity, ) -from ...functions import abspath -from ...functions import atol as cf_atol -from ...functions import load_stash2standard_name -from ...functions import rtol as cf_rtol -from ...umread_lib.umfile import File -from ...units import Units +from cf.functions import abspath +from cf.functions import atol as cf_atol +from cf.functions import load_stash2standard_name +from cf.functions import rtol as cf_rtol +from cf.umread_lib.umfile import File +from cf.units import Units logger = logging.getLogger(__name__) @@ -1869,6 +1864,8 @@ def coord_positive(self, c, axiscode, domain_axis_key): def ctime(self, rec): """Return elapsed time since the clock time of the given record.""" + import cftime + reftime = self.refUnits LBVTIME = tuple(self.header_vtime(rec)) LBDTIME = tuple(self.header_dtime(rec)) @@ -2005,6 +2002,10 @@ def create_data(self): `Data` """ + import dask.array as da + from dask.array.core import getter, normalize_chunks + from dask.base import tokenize + if self.info: logger.info("Creating data:") # pragma: no cover @@ -2286,6 +2287,8 @@ def dtime(self, rec): key = (LBDTIME, units, calendar) time = _cached_date2num.get(key, None) if time is None: + from netCDF4 import date2num as netCDF4_date2num + # It is important to use the same time_units as vtime try: if self.calendar == "gregorian": @@ -2293,6 +2296,8 @@ def dtime(self, rec): datetime(*LBDTIME), units, calendar ) else: + import cftime + time = netCDF4_date2num( cftime.datetime(*LBDTIME, calendar=self.calendar), units, @@ -2948,6 +2953,8 @@ def vtime(self, rec): time = _cached_date2num.get(key, None) if time is None: + import cftime + # It is important to use the same time_units as dtime try: time = cftime.date2num( @@ -3243,6 +3250,8 @@ def get_data(self, array, units, fill_value=None, bounds=False): An independent copy of the new data. """ + from dask.base import tokenize + token = tokenize(array, units) data = _cached_data.get(token) if data is None: diff --git a/cf/regrid/regrid.py b/cf/regrid/regrid.py index a2cc34dc64..b67db47f8e 100644 --- a/cf/regrid/regrid.py +++ b/cf/regrid/regrid.py @@ -5,7 +5,6 @@ from datetime import datetime from typing import Any -import dask.array as da import numpy as np from cfdm import is_log_level_debug @@ -13,16 +12,10 @@ from ..units import Units from .regridoperator import RegridOperator -esmpy_imported = True -try: - import esmpy -except ImportError: - esmpy_imported = False - logger = logging.getLogger(__name__) # Mapping of regrid method strings to esmpy method codes. The values -# get replaced with `esmpy.RegridMethod` constants the first time +# get created with `esmpy.RegridMethod` constants the first time # `esmpy_initialise` is run. esmpy_methods = { "linear": None, @@ -1909,9 +1902,11 @@ def esmpy_initialise(): The `esmpy` manager. """ - if not esmpy_imported: - raise RuntimeError( - "Regridding will not work unless the esmpy library is installed" + try: + import esmpy + except ModuleNotFoundError: + raise ModuleNotFoundError( + "Must install the 'esmpy' package to enable regridding" ) # Update the global 'esmpy_methods' dictionary @@ -1962,6 +1957,8 @@ def create_esmpy_grid(grid, mask=None, grid_partitions=1): grid partition. """ + import esmpy + debug = is_log_level_debug(logger) if mask is not None: @@ -2247,6 +2244,8 @@ def create_esmpy_mesh(grid, mask=None, grid_partitions=1): grid partition. """ + import esmpy + debug = is_log_level_debug(logger) if grid.mesh_location != "face": @@ -2385,6 +2384,8 @@ def create_esmpy_locstream(grid, mask=None, grid_partitions=1): each grid partition. """ + import esmpy + debug = is_log_level_debug(logger) if mask is not None: @@ -2563,6 +2564,7 @@ def create_esmpy_weights( written to, a file. Otherwise `False`. """ + import esmpy from cfdm import integer_dtype debug = is_log_level_debug(logger) @@ -3073,6 +3075,8 @@ def get_mask(f, grid): The Boolean mask. """ + import dask.array as da + regrid_axes = grid.axis_indices index = [slice(None) if i in regrid_axes else 0 for i in range(f.ndim)] diff --git a/cf/timeduration.py b/cf/timeduration.py index 1cf79eb15c..21e466394d 100644 --- a/cf/timeduration.py +++ b/cf/timeduration.py @@ -25,12 +25,8 @@ _minutes = Units("minutes") _seconds = Units("seconds") -# Define some useful constants -_one_year = Data(1, "calendar_years") -_one_day = Data(1, "day") -_one_hour = Data(1, "hour") -_one_minute = Data(1, "minute") -_one_second = Data(1, "second") +# Define some duration data values +_durations = {} # Default month lengths in days _default_month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] @@ -340,27 +336,31 @@ def __init__( f"Can't create {self.__class__.__name__} of {self.duration}" ) + if not _durations: + _durations.update( + { + "one_year": Data(1, "calendar_year"), + "one_day": Data(1, "day"), + "one_hour": Data(1, "hour"), + "one_minute": Data(1, "minute"), + } + ) + duration = self.duration offset = [None, month, day, hour, minute, second, 0] if units.equivalent(_calendar_years): - if duration < _one_year: + if duration < _durations["one_year"]: offset[1] = None else: offset[1] = None offset[2] = None - if duration < _one_day: + if duration < _durations["one_day"]: offset[3] = None - if duration < _one_hour: + if duration < _durations["one_hour"]: offset[4] = None - if duration < _one_minute: + if duration < _durations["one_minute"]: offset[5] = None - # if units <= _hours and duration < _one_day: - # offset[3] = None - # if units <= _minutes and duration < _one_hour: - # offset[4] = None - # if units <= _seconds and duration < _one_minute: - # offset[5] = None self.offset = Offset(*offset) diff --git a/cf/units.py b/cf/units.py index 49486800a8..021b68cd5c 100644 --- a/cf/units.py +++ b/cf/units.py @@ -1,13 +1,5 @@ -from ctypes.util import find_library - from cfunits import Units as cfUnits -_libpath = find_library("udunits2") -if _libpath is None: - raise FileNotFoundError( - "cf requires UNIDATA UDUNITS-2. Can't find the 'udunits2' library." - ) - class Units: """Store, combine and compare physical units and convert numeric diff --git a/docs/source/conf.py b/docs/source/conf.py index 239aa988fa..4f45911a65 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -389,7 +389,7 @@ def _get_date(): "run_stale_examples": False, # Below setting can be buggy: see: # https://github.com/sphinx-gallery/sphinx-gallery/issues/967 - #"reference_url": {"cf": None}, + # "reference_url": {"cf": None}, "backreferences_dir": "gen_modules/backreferences", "doc_module": ("cf",), "inspect_global_variables": True, @@ -476,7 +476,6 @@ def _get_date(): import cf - def linkcode_resolve(domain, info): # ================================================================= # Must delete all .doctrees directories in build for changes to be diff --git a/docs/source/recipes/plot_19_recipe.py b/docs/source/recipes/plot_19_recipe.py index dcc0926fbd..02d493dc21 100644 --- a/docs/source/recipes/plot_19_recipe.py +++ b/docs/source/recipes/plot_19_recipe.py @@ -55,7 +55,10 @@ # of the maxima, we loop through the season query mapping and do a # "T: mean" collapse setting the season as the grouping: cfp.gopen( - rows=2, columns=1, bottom=0.1, top=0.85, + rows=2, + columns=1, + bottom=0.1, + top=0.85, ) cfp.gpos(1) cfp.gset(xmin="1980-01-01", xmax="2022-12-01", ymin=304, ymax=312)