From 01c5473721e022de3ecc01c1c1cf92ed1066f46d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 30 May 2023 14:37:48 +0200 Subject: [PATCH] Custom hierarchies (squashed commit) JSON backend: Fail when trying to open non-existing groups Insert CustomHierarchy class to Iteration Help older compilers deal with this Add vector variants of meshes/particlesPath Move meshes and particles over to CustomHierarchies class Move dirtyRecursive to CustomHierarchy Move Iteration reading logic to CustomHierarchy Move Iteration flushing logic to CustomHierarchy class Support for custom datasets Treat "meshes"/"particles" as normal subgroups Introduction of iteration["meshes"].asContainerOf() as a more explicit variant for iteration.meshes. Regex-based list of meshes/particlesPaths More extended testing Fix Python bindings without adding new functionality yet Overload resolution Add simple Python bindings and an example Replace Regexes with Globbing TODO: Since meshes/particles can no longer be directly addressed with this, maybe adapt the class hierarchy to disallow mixed groups that contain meshes, particles, groups and datasets at the same time. Only maybe though.. Move .meshes and .particles back to Iteration class The have their own meaning now and are no longer just carefully maintained for backwards compatibility. Instead, they are supposed to serve as a shortcut to all openPMD data found further down the hierarchy. Some fixes in read error handling More symmetric design for container types Don't write unitSI in custom datasets Discouraged support for custom datasets inside the particlesPath Fix after rebase: dirtyRecursive Fixes to the dirty/dirtyRecursive logic [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Some cleanup in CustomHierarchies class Use polymorphism for meshes/particlesPath in Python Remove hasMeshes / hasParticles logic Sort dirty files This is a workaround only, only one file should be dirty in this test. Formatting [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Fixes after rebase --- CMakeLists.txt | 3 + examples/14_custom_hierarchy.py | 48 ++ include/openPMD/CustomHierarchy.hpp | 256 +++++++ include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 3 +- include/openPMD/Iteration.hpp | 61 +- include/openPMD/Mesh.hpp | 1 + include/openPMD/ParticleSpecies.hpp | 1 + include/openPMD/RecordComponent.hpp | 8 +- include/openPMD/Series.hpp | 7 +- include/openPMD/backend/Attributable.hpp | 17 + include/openPMD/backend/Attribute.hpp | 19 + include/openPMD/backend/BaseRecord.hpp | 1 + include/openPMD/backend/Container.hpp | 20 +- include/openPMD/backend/Writable.hpp | 1 + include/openPMD/binding/python/Common.hpp | 3 + src/CustomHierarchy.cpp | 672 ++++++++++++++++++ src/IO/ADIOS/ADIOS2IOHandler.cpp | 15 +- src/IO/JSON/JSONIOHandlerImpl.cpp | 53 +- src/Iteration.cpp | 374 ++++------ src/ParticlePatches.cpp | 2 +- src/Record.cpp | 23 +- src/RecordComponent.cpp | 14 +- src/Series.cpp | 115 ++- src/backend/Attributable.cpp | 8 +- src/backend/Container.cpp | 2 + src/backend/MeshRecordComponent.cpp | 4 +- src/backend/PatchRecord.cpp | 7 +- src/binding/python/Container.cpp | 71 ++ src/binding/python/CustomHierarchy.cpp | 52 ++ src/binding/python/Iteration.cpp | 8 +- src/binding/python/Series.cpp | 72 +- src/binding/python/openPMD.cpp | 2 + test/CoreTest.cpp | 255 ++++++- test/SerialIOTest.cpp | 24 +- 34 files changed, 1886 insertions(+), 336 deletions(-) create mode 100755 examples/14_custom_hierarchy.py create mode 100644 include/openPMD/CustomHierarchy.hpp create mode 100644 src/CustomHierarchy.cpp create mode 100644 src/binding/python/Container.cpp create mode 100644 src/binding/python/CustomHierarchy.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index dba77d38be..7c8aef7f75 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -394,6 +394,7 @@ include(${openPMD_SOURCE_DIR}/cmake/dependencies/pybind11.cmake) set(CORE_SOURCE src/config.cpp src/ChunkInfo.cpp + src/CustomHierarchy.cpp src/Dataset.cpp src/Datatype.cpp src/Error.cpp @@ -578,6 +579,7 @@ if(openPMD_HAVE_PYTHON) src/binding/python/Attributable.cpp src/binding/python/BaseRecordComponent.cpp src/binding/python/ChunkInfo.cpp + src/binding/python/CustomHierarchy.cpp src/binding/python/Dataset.cpp src/binding/python/Datatype.cpp src/binding/python/Error.cpp @@ -746,6 +748,7 @@ set(openPMD_PYTHON_EXAMPLE_NAMES 11_particle_dataframe 12_span_write 13_write_dynamic_configuration + 14_custom_hierarchy 15_compression ) diff --git a/examples/14_custom_hierarchy.py b/examples/14_custom_hierarchy.py new file mode 100755 index 0000000000..b3eff208a9 --- /dev/null +++ b/examples/14_custom_hierarchy.py @@ -0,0 +1,48 @@ +import numpy as np +import openpmd_api as io + + +def main(): + if "bp" in io.file_extensions: + filename = "../samples/custom_hierarchy.bp" + else: + filename = "../samples/custom_hierarchy.json" + s = io.Series(filename, io.Access.create) + it = s.write_iterations()[100] + + # write openPMD part + temp = it.meshes["temperature"] + temp.axis_labels = ["x", "y"] + temp.unit_dimension = {io.Unit_Dimension.T: 1} + temp.position = [0.5, 0.5] + temp.grid_spacing = [1, 1] + temp.grid_global_offset = [0, 0] + temp.reset_dataset(io.Dataset(np.dtype("double"), [5, 5])) + temp[()] = np.zeros((5, 5)) + + # write NeXus part + nxentry = it["Scan"] + nxentry.set_attribute("NX_class", "NXentry") + nxentry.set_attribute("default", "data") + + data = nxentry["data"] + data.set_attribute("NX_class", "NXdata") + data.set_attribute("signal", "counts") + data.set_attribute("axes", ["two_theta"]) + data.set_attribute("two_theta_indices", [0]) + + counts = data.as_container_of_datasets()["counts"] + counts.set_attribute("units", "counts") + counts.set_attribute("long_name", "photodiode counts") + counts.reset_dataset(io.Dataset(np.dtype("int"), [15])) + counts[()] = np.zeros(15, dtype=np.dtype("int")) + + two_theta = data.as_container_of_datasets()["two_theta"] + two_theta.set_attribute("units", "degrees") + two_theta.set_attribute("long_name", "two_theta (degrees)") + two_theta.reset_dataset(io.Dataset(np.dtype("double"), [15])) + two_theta[()] = np.zeros(15) + + +if __name__ == "__main__": + main() diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp new file mode 100644 index 0000000000..dc23a97e64 --- /dev/null +++ b/include/openPMD/CustomHierarchy.hpp @@ -0,0 +1,256 @@ +/* Copyright 2023 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/RecordComponent.hpp" +#include "openPMD/backend/Container.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace openPMD +{ +class CustomHierarchy; +namespace internal +{ + enum class ContainedType + { + Group, + Mesh, + Particle + }; + struct MeshesParticlesPath + { + std::regex meshRegex; + std::set collectNewMeshesPaths; + std::regex particleRegex; + std::set collectNewParticlesPaths; + + /* + * These values decide which path will be returned upon use of the + * shorthand notation s.iterations[0].meshes or .particles. + * + */ + std::string m_defaultMeshesPath = "meshes"; + std::string m_defaultParticlesPath = "particles"; + + explicit MeshesParticlesPath() = default; + MeshesParticlesPath( + std::vector const &meshes, + std::vector const &particles); + MeshesParticlesPath(Series const &); + + [[nodiscard]] ContainedType + determineType(std::vector const &path) const; + [[nodiscard]] bool + isParticleContainer(std::vector const &path) const; + [[nodiscard]] bool + isMeshContainer(std::vector const &path) const; + }; + + struct CustomHierarchyData + : ContainerData + , ContainerData + , ContainerData + , ContainerData + { + explicit CustomHierarchyData(); + + void syncAttributables(); + +#if 0 + inline Container customHierarchiesWrapped() + { + Container res; + res.setData( + {static_cast *>(this), + [](auto const *) {}}); + return res; + } +#endif + inline Container embeddedDatasetsWrapped() + { + Container res; + res.setData( + {static_cast *>(this), + [](auto const *) {}}); + return res; + } + inline Container embeddedMeshesWrapped() + { + Container res; + res.setData( + {static_cast *>(this), + [](auto const *) {}}); + return res; + } + + inline Container embeddedParticlesWrapped() + { + Container res; + res.setData( + {static_cast *>(this), + [](auto const *) {}}); + return res; + } + +#if 0 + inline Container::InternalContainer & + customHierarchiesInternal() + { + return static_cast *>(this) + ->m_container; + } +#endif + inline Container::InternalContainer & + embeddedDatasetsInternal() + { + return static_cast *>(this) + ->m_container; + } + inline Container::InternalContainer &embeddedMeshesInternal() + { + return static_cast *>(this)->m_container; + } + + inline Container::InternalContainer & + embeddedParticlesInternal() + { + return static_cast *>(this) + ->m_container; + } + }; +} // namespace internal + +template +class ConversibleContainer : public Container +{ + template + friend class ConversibleContainer; + +protected: + using Container_t = Container; + using Data_t = internal::CustomHierarchyData; + static_assert( + std::is_base_of_v); + + ConversibleContainer(Attributable::NoInit) + : Container_t(Attributable::NoInit{}) + {} + + std::shared_ptr m_customHierarchyData; + + [[nodiscard]] Data_t &get() + { + return *m_customHierarchyData; + } + [[nodiscard]] Data_t const &get() const + { + return *m_customHierarchyData; + } + + inline void setData(std::shared_ptr data) + { + m_customHierarchyData = data; + Container_t::setData(std::move(data)); + } + +public: + template + auto asContainerOf() -> ConversibleContainer + { + if constexpr ( + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) + { + ConversibleContainer res(Attributable::NoInit{}); + res.setData(m_customHierarchyData); + return res; + } + else + { + static_assert( + auxiliary::dependent_false_v, + "[CustomHierarchy::asContainerOf] Type parameter must be " + "one of: CustomHierarchy, RecordComponent, Mesh, " + "ParticleSpecies."); + } + } +}; + +class CustomHierarchy : public ConversibleContainer +{ + friend class Iteration; + friend class Container; + +private: + using Container_t = Container; + using Parent_t = ConversibleContainer; + using Data_t = typename Parent_t::Data_t; + + using EraseStaleMeshes = internal::EraseStaleEntries>; + using EraseStaleParticles = + internal::EraseStaleEntries>; + void readNonscalarMesh(EraseStaleMeshes &map, std::string const &name); + void readScalarMesh(EraseStaleMeshes &map, std::string const &name); + void readParticleSpecies(EraseStaleParticles &map, std::string const &name); + +protected: + CustomHierarchy(); + CustomHierarchy(NoInit); + + void read(internal::MeshesParticlesPath const &); + void read( + internal::MeshesParticlesPath const &, + std::vector ¤tPath); + + void flush_internal( + internal::FlushParams const &, + internal::MeshesParticlesPath &, + std::vector currentPath); + void flush(std::string const &path, internal::FlushParams const &) override; + + /** + * @brief Link with parent. + * + * @param w The Writable representing the parent. + */ + void linkHierarchy(Writable &w) override; + +public: + CustomHierarchy(CustomHierarchy const &other) = default; + CustomHierarchy(CustomHierarchy &&other) = default; + + CustomHierarchy &operator=(CustomHierarchy const &) = default; + CustomHierarchy &operator=(CustomHierarchy &&) = default; +}; +} // namespace openPMD diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 6df0c60ced..2be9f8c30d 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -399,7 +399,8 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // make sure that the given path exists in proper form in // the passed json value - static void ensurePath(nlohmann::json *json, std::string const &path); + static void + ensurePath(nlohmann::json *json, std::string const &path, Access); // In order not to insert the same file name into the data structures // with a new pointer (e.g. when reopening), search for a possibly diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index c66199c46f..3d1bd251d0 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -20,6 +20,7 @@ */ #pragma once +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/Mesh.hpp" #include "openPMD/ParticleSpecies.hpp" @@ -98,7 +99,7 @@ namespace internal BeginStep beginStep = BeginStepTypes::DontBeginStep{}; }; - class IterationData : public AttributableData + class IterationData : public CustomHierarchyData { public: /* @@ -142,10 +143,22 @@ namespace internal * @see * https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md#required-attributes-for-the-basepath */ -class Iteration : public Attributable +class Iteration : public CustomHierarchy { - template - friend class Container; +public: + using IterationIndex_t = uint64_t; + + /* + * Some old compilers have trouble with befriending the entire Container + * template here, so we restrict it + * to Container, more is not needed anyway. + * + * E.g. on gcc-7: + * > error: specialization of 'openPMD::Container' + * > after instantiation + * > friend class Container; + */ + friend class Container; friend class Series; friend class internal::AttributableData; template @@ -154,13 +167,17 @@ class Iteration : public Attributable friend class StatefulIterator; friend class StatefulSnapshotsContainer; -public: Iteration(Iteration const &) = default; Iteration(Iteration &&) = default; Iteration &operator=(Iteration const &) = default; Iteration &operator=(Iteration &&) = default; - using IterationIndex_t = uint64_t; + // These use the openPMD Container class mainly for consistency. + // But they are in fact only aliases that don't actually exist + // in the backend. + // Hence meshes.written() and particles.written() will always be false. + Container meshes{}; + Container particles{}; /** * @tparam T Floating point type of user-selected precision (e.g. float, @@ -268,9 +285,6 @@ class Iteration : public Attributable [[deprecated("This attribute is no longer set by the openPMD-api.")]] bool closedByWriter() const; - Container meshes{}; - Container particles{}; // particleSpecies? - virtual ~Iteration() = default; private: @@ -297,14 +311,25 @@ class Iteration : public Attributable inline void setData(std::shared_ptr data) { m_iterationData = std::move(data); - Attributable::setData(m_iterationData); + CustomHierarchy::setData(m_iterationData); } void flushFileBased( std::string const &, IterationIndex_t, internal::FlushParams const &); void flushGroupBased(IterationIndex_t, internal::FlushParams const &); void flushVariableBased(IterationIndex_t, internal::FlushParams const &); - void flush(internal::FlushParams const &); + /* + * Named flushIteration instead of flush to avoid naming + * conflicts with overridden virtual flush from CustomHierarchy + * class. + */ + void flushIteration(internal::FlushParams const &); + + void sync_meshes_and_particles_from_alias_to_subgroups( + internal::MeshesParticlesPath const &); + void sync_meshes_and_particles_from_subgroups_to_alias( + internal::MeshesParticlesPath const &); + void deferParseAccess(internal::DeferredParseAccess); /* * Control flow for runDeferredParseAccess(), readFileBased(), @@ -334,8 +359,6 @@ class Iteration : public Attributable void readGorVBased( std::string const &groupPath, internal::BeginStep const &beginStep); void read_impl(std::string const &groupPath); - void readMeshes(std::string const &meshesPath); - void readParticles(std::string const &particlesPath); /** * Status after beginning an IO step. Currently includes: @@ -416,12 +439,22 @@ class Iteration : public Attributable */ void setStepStatus(StepStatus); + /* + * @brief Check recursively whether this Iteration is dirty. + * It is dirty if any attribute or dataset is read from or written to + * the backend. + * + * @return true If dirty. + * @return false Otherwise. + */ + bool dirtyRecursive() const; + /** * @brief Link with parent. * * @param w The Writable representing the parent. */ - virtual void linkHierarchy(Writable &w); + void linkHierarchy(Writable &w); /** * @brief Access an iteration in read mode that has potentially not been diff --git a/include/openPMD/Mesh.hpp b/include/openPMD/Mesh.hpp index d0bf81ddef..513aaa2fee 100644 --- a/include/openPMD/Mesh.hpp +++ b/include/openPMD/Mesh.hpp @@ -41,6 +41,7 @@ class Mesh : public BaseRecord { friend class Container; friend class Iteration; + friend class CustomHierarchy; public: Mesh(Mesh const &) = default; diff --git a/include/openPMD/ParticleSpecies.hpp b/include/openPMD/ParticleSpecies.hpp index 4f309c0f2a..f7b8448a75 100644 --- a/include/openPMD/ParticleSpecies.hpp +++ b/include/openPMD/ParticleSpecies.hpp @@ -37,6 +37,7 @@ class ParticleSpecies : public Container friend class Iteration; template friend T &internal::makeOwning(T &self, Series); + friend class CustomHierarchy; public: ParticlePatches particlePatches; diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index d06b4213f4..4ca3afec4a 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -128,6 +128,7 @@ class RecordComponent : public BaseRecordComponent friend class MeshRecordComponent; template friend T &internal::makeOwning(T &self, Series); + friend class CustomHierarchy; public: enum class Allocation @@ -485,8 +486,9 @@ class RecordComponent : public BaseRecordComponent static constexpr char const *const SCALAR = "\vScalar"; protected: - void flush(std::string const &, internal::FlushParams const &); - void read(bool require_unit_si); + void flush( + std::string const &, internal::FlushParams const &, bool set_defaults); + void read(bool read_defaults); private: /** @@ -534,7 +536,7 @@ OPENPMD_protected BaseRecordComponent::setData(m_recordComponentData); } - void readBase(bool require_unit_si); + void readBase(bool read_defaults); template void verifyChunk(Offset const &, Extent const &) const; diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 603e540c2b..5ec8902baf 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -294,6 +294,7 @@ class Series : public Attributable friend class internal::SeriesData; friend class internal::AttributableData; friend class StatefulSnapshotsContainer; + friend class CustomHierarchy; public: explicit Series(); @@ -447,6 +448,7 @@ class Series : public Attributable * basePath. */ std::string meshesPath() const; + std::vector meshesPaths() const; /** Set the path to mesh * records, relative(!) to basePath. @@ -457,6 +459,7 @@ class Series : public Attributable * @return Reference to modified series. */ Series &setMeshesPath(std::string const &meshesPath); + Series &setMeshesPath(std::vector const &meshesPath); /** * @return True if there is a rankTable dataset defined for this Series. @@ -502,6 +505,7 @@ class Series : public Attributable * basePath. */ std::string particlesPath() const; + std::vector particlesPaths() const; /** Set the path to groups for each particle * species, relative(!) to basePath. @@ -512,6 +516,7 @@ class Series : public Attributable * @return Reference to modified series. */ Series &setParticlesPath(std::string const &particlesPath); + Series &setParticlesPath(std::vector const &particlesPath); /** * @throw no_such_attribute_error If optional attribute is not present. @@ -898,8 +903,6 @@ OPENPMD_private iterations_iterator end, internal::FlushParams const &flushParams, bool flushIOHandler = true); - void flushMeshesPath(); - void flushParticlesPath(); void flushRankTable(); /* Parameter `read_only_this_single_iteration` used for reopening an * Iteration after closing it. diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 1c1d3db4a5..20d3279f71 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -51,6 +51,7 @@ class AbstractFilePosition; class Attributable; class Iteration; class Series; +class CustomHierarchy; namespace internal { @@ -61,6 +62,7 @@ namespace internal class SharedAttributableData { friend class openPMD::Attributable; + friend class openPMD::CustomHierarchy; public: SharedAttributableData(AttributableData *); @@ -106,6 +108,7 @@ namespace internal class AttributableData : public std::shared_ptr { friend class openPMD::Attributable; + friend class openPMD::CustomHierarchy; using SharedData_t = std::shared_ptr; using A_MAP = SharedData_t::element_type::A_MAP; @@ -117,6 +120,17 @@ namespace internal AttributableData(AttributableData &&) = delete; virtual ~AttributableData() = default; + inline std::shared_ptr & + asSharedPtrOfAttributable() + { + return *this; + } + inline std::shared_ptr const & + asSharedPtrOfAttributable() const + { + return *this; + } + AttributableData &operator=(AttributableData const &) = delete; AttributableData &operator=(AttributableData &&) = delete; @@ -191,6 +205,7 @@ namespace internal class BaseRecordData; class RecordComponentData; + struct CustomHierarchyData; /* * Internal function to turn a handle into an owning handle that will keep @@ -244,6 +259,8 @@ class Attributable friend class internal::AttributableData; friend class Snapshots; friend struct internal::HomogenizeExtents; + friend class CustomHierarchy; + friend struct internal::CustomHierarchyData; protected: // tag for internal constructor diff --git a/include/openPMD/backend/Attribute.hpp b/include/openPMD/backend/Attribute.hpp index 446b4a7375..986e3418de 100644 --- a/include/openPMD/backend/Attribute.hpp +++ b/include/openPMD/backend/Attribute.hpp @@ -310,6 +310,25 @@ namespace detail } } } + // conversion cast: turn a 1-element vector into a single value + else if constexpr (auxiliary::IsVector_v) + { + if constexpr (std::is_convertible_v) + { + if (pv->size() != 1) + { + return {std::runtime_error( + "getCast: vector to scalar conversion requires " + "single-element vectors")}; + } + return {U(*pv->begin())}; + } + else + { + return {std::runtime_error( + "getCast: no vector to scalar conversion possible.")}; + } + } else { return {std::runtime_error("getCast: no cast possible.")}; diff --git a/include/openPMD/backend/BaseRecord.hpp b/include/openPMD/backend/BaseRecord.hpp index f75536e3bd..89c622b6bb 100644 --- a/include/openPMD/backend/BaseRecord.hpp +++ b/include/openPMD/backend/BaseRecord.hpp @@ -186,6 +186,7 @@ class BaseRecord private: using T_Self = BaseRecord; + friend class CustomHierarchy; friend class Iteration; friend class ParticleSpecies; friend class PatchRecord; diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 963fd34802..d4f38bd0d6 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -57,11 +57,26 @@ namespace traits }; } // namespace traits +class CustomHierarchy; + namespace internal { + template + constexpr inline bool isDerivedFromAttributable = + std::is_base_of_v; + + /* + * Opt out from this check due to the recursive definition of + * class CustomHierarchy : public Container{ ... }; + * Cannot check this while CustomHierarchy is still an incomplete type. + */ + template <> + constexpr inline bool isDerivedFromAttributable = true; + class SeriesData; template class EraseStaleEntries; + struct CustomHierarchyData; template < typename T, @@ -103,7 +118,7 @@ template < class Container : virtual public Attributable { static_assert( - std::is_base_of::value, + internal::isDerivedFromAttributable, "Type of container element must be derived from Writable"); friend class Iteration; @@ -114,6 +129,9 @@ class Container : virtual public Attributable template friend class internal::EraseStaleEntries; friend class StatefulIterator; + friend class SeriesIterator; + friend struct internal::CustomHierarchyData; + friend class CustomHierarchy; protected: using ContainerData = internal::ContainerData; diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index 0396aa1f93..397886b85a 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -106,6 +106,7 @@ class Writable final friend void debug::printDirty(Series const &); friend struct Parameter; friend struct Parameter; + friend class CustomHierarchy; private: Writable(internal::AttributableData *); diff --git a/include/openPMD/binding/python/Common.hpp b/include/openPMD/binding/python/Common.hpp index c32e6b5fef..3cada7d1bd 100644 --- a/include/openPMD/binding/python/Common.hpp +++ b/include/openPMD/binding/python/Common.hpp @@ -9,6 +9,7 @@ #pragma once #include "openPMD/ChunkInfo.hpp" +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Mesh.hpp" #include "openPMD/ParticlePatches.hpp" @@ -60,6 +61,7 @@ using PyPatchRecordComponentContainer = Container; using PyBaseRecordRecordComponent = BaseRecord; using PyBaseRecordMeshRecordComponent = BaseRecord; using PyBaseRecordPatchRecordComponent = BaseRecord; +using PyCustomHierarchyContainer = Container; PYBIND11_MAKE_OPAQUE(PyIterationContainer) PYBIND11_MAKE_OPAQUE(PyMeshContainer) PYBIND11_MAKE_OPAQUE(PyPartContainer) @@ -71,3 +73,4 @@ PYBIND11_MAKE_OPAQUE(PyMeshRecordComponentContainer) PYBIND11_MAKE_OPAQUE(PyPatchRecordComponentContainer) PYBIND11_MAKE_OPAQUE(PyBaseRecordRecordComponent) PYBIND11_MAKE_OPAQUE(PyBaseRecordPatchRecordComponent) +PYBIND11_MAKE_OPAQUE(PyCustomHierarchyContainer) diff --git a/src/CustomHierarchy.cpp b/src/CustomHierarchy.cpp new file mode 100644 index 0000000000..0dfb8da0f7 --- /dev/null +++ b/src/CustomHierarchy.cpp @@ -0,0 +1,672 @@ +/* Copyright 2023 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/CustomHierarchy.hpp" + +#include "openPMD/Dataset.hpp" +#include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/Access.hpp" +#include "openPMD/IO/IOTask.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/RecordComponent.hpp" +#include "openPMD/Series.hpp" +#include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/MeshRecordComponent.hpp" +#include "openPMD/backend/Writable.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// @todo add handselected choice of [:punct:] characters to this +// using a macro here to make string interpolation simpler +#define OPENPMD_LEGAL_IDENTIFIER_CHARS "[:alnum:]_" +#define OPENPMD_SINGLE_GLOBBING_CHAR "%" +#define OPENPMD_DOUBLE_GLOBBING_CHAR "%%" + +namespace +{ +template +std::string +concatWithSep(Iterator &&begin, Iterator const &end, std::string const &sep) +{ + if (begin == end) + { + return ""; + } + std::stringstream res; + res << *(begin++); + for (; begin != end; ++begin) + { + res << sep << *begin; + } + return res.str(); +} + +std::string +concatWithSep(std::vector const &v, std::string const &sep) +{ + return concatWithSep(v.begin(), v.end(), sep); +} + +// Not specifying std::regex_constants::optimize here, only using it where +// it makes sense to. +constexpr std::regex_constants::syntax_option_type regex_flags = + std::regex_constants::egrep; + +template +void setDefaultMeshesParticlesPath( + std::vector const &meshes, + std::vector const &particles, + OutParam &writeTarget) +{ + std::regex is_default_path_specification( + "[" OPENPMD_LEGAL_IDENTIFIER_CHARS "]+/", + regex_flags | std::regex_constants::optimize); + constexpr char const *default_default_mesh = "meshes"; + constexpr char const *default_default_particle = "particles"; + for (auto [vec, defaultPath, default_default] : + {std::make_tuple( + &meshes, &writeTarget.m_defaultMeshesPath, default_default_mesh), + std::make_tuple( + &particles, + &writeTarget.m_defaultParticlesPath, + default_default_particle)}) + { + bool set_default = true; + /* + * The first eligible path in meshesPath/particlesPath is used as + * the default, "meshes"/"particles" otherwise. + */ + for (auto const &path : *vec) + { + if (std::regex_match(path, is_default_path_specification)) + { + *defaultPath = openPMD::auxiliary::replace_last(path, "/", ""); + set_default = false; + break; + } + } + if (set_default) + { + *defaultPath = default_default; + } + } +} + +bool anyPathRegexMatches( + std::regex const ®ex, std::vector const &path) +{ + std::string pathToMatch = '/' + concatWithSep(path, "/") + '/'; + return std::regex_match(pathToMatch, regex); +} +} // namespace + +namespace openPMD +{ +namespace internal +{ + namespace + { + std::string globToRegexLongForm(std::string const &glob) + { + return auxiliary::replace_all( + auxiliary::replace_all( + glob, + OPENPMD_DOUBLE_GLOBBING_CHAR, + "([" OPENPMD_LEGAL_IDENTIFIER_CHARS "/]*)"), + OPENPMD_SINGLE_GLOBBING_CHAR, + "([" OPENPMD_LEGAL_IDENTIFIER_CHARS "]*)"); + } + + std::string globToRegexShortForm(std::string const &glob) + { + return "[" OPENPMD_LEGAL_IDENTIFIER_CHARS "/]*/" + glob; + } + } // namespace + + MeshesParticlesPath::MeshesParticlesPath( + std::vector const &meshes, + std::vector const &particles) + { + /* + * /group/meshes/E is a mesh if the meshes path contains: + * + * 1) '/group/meshes/' (absolute path to mesh container) + * 2) 'meshes/' (relative path to mesh container) + * + * All this analogously for particles path. + */ + + // regex for detecting option 1) + // e.g. '/path/to/meshes/': The path to the meshes. Mandatory slashes at + // beginning and end, possibly slashes in + // between. Mandatory slash at beginning might + // be replaced with '%%' to enable paths like + // '%%/path/to/meshes'. + // resolves to: `(/|%%)[[:alnum:]_%/]+/` + std::regex is_legal_long_path_specification( + "(/|" OPENPMD_DOUBLE_GLOBBING_CHAR + ")[" OPENPMD_LEGAL_IDENTIFIER_CHARS OPENPMD_SINGLE_GLOBBING_CHAR + "/]+/", + regex_flags | std::regex_constants::optimize); + + // Regex for detecting option 2) + // e.g. 'meshes/': The name without path. One single mandatory slash + // at the end, no slashes otherwise. + // resolves to `[[:alnum:]_]+/` + std::regex is_legal_short_path_specification( + "[" OPENPMD_LEGAL_IDENTIFIER_CHARS "]+/", + regex_flags | std::regex_constants::optimize); + + for (auto [target_regex, vec] : + {std::make_tuple(&this->meshRegex, &meshes), + std::make_tuple(&this->particleRegex, &particles)}) + { + std::stringstream build_regex; + // neutral element: empty language, regex doesn't match anything + build_regex << "(a^)"; + for (auto const &entry : *vec) + { + if (std::regex_match(entry, is_legal_short_path_specification)) + { + build_regex << "|(" << globToRegexShortForm(entry) << ')'; + } + else if (std::regex_match( + entry, is_legal_long_path_specification)) + { + build_regex << "|(" << globToRegexLongForm(entry) << ')'; + } + else + { + std::cerr + << "[WARNING] Not a legal meshes-/particles-path: '" + << entry << "'. Will skip." << std::endl; + } + } + auto regex_string = build_regex.str(); + // std::cout << "Using regex string: " << regex_string << std::endl; + *target_regex = std::regex( + regex_string, regex_flags | std::regex_constants::optimize); + } + setDefaultMeshesParticlesPath(meshes, particles, *this); + } + + ContainedType MeshesParticlesPath::determineType( + std::vector const &path) const + { + if (isMeshContainer(path)) + { + return ContainedType::Mesh; + } + else if (isParticleContainer(path)) + { + return ContainedType::Particle; + } + else + { + return ContainedType::Group; + } + } + + bool MeshesParticlesPath::isParticleContainer( + std::vector const &path) const + { + return anyPathRegexMatches(particleRegex, path); + } + bool MeshesParticlesPath::isMeshContainer( + std::vector const &path) const + { + return anyPathRegexMatches(meshRegex, path); + } + + CustomHierarchyData::CustomHierarchyData() + { + syncAttributables(); + } + + void CustomHierarchyData::syncAttributables() + { + /* + * m_embeddeddatasets and its friends should point to the same instance + * of Attributable. + * Not strictly necessary to do this explicitly due to virtual + * inheritance (all Attributable instances are the same anyway), + * but let's be explicit about this. + */ + for (auto p : std::initializer_list{ + static_cast *>(this), + static_cast *>(this), + static_cast *>(this), + static_cast *>(this)}) + { + p->asSharedPtrOfAttributable() = this->asSharedPtrOfAttributable(); + } + } +} // namespace internal + +// template +// class ConversibleContainer; + +CustomHierarchy::CustomHierarchy() : ConversibleContainer(NoInit{}) +{ + setData(std::make_shared()); +} +CustomHierarchy::CustomHierarchy(NoInit) : ConversibleContainer(NoInit{}) +{} + +void CustomHierarchy::readNonscalarMesh( + EraseStaleMeshes &map, std::string const &mesh_name) +{ + Parameter pOpen; + Parameter aList; + + Mesh &m = map[mesh_name]; + + pOpen.path = mesh_name; + aList.attributes->clear(); + IOHandler()->enqueue(IOTask(&m, pOpen)); + IOHandler()->enqueue(IOTask(&m, aList)); + IOHandler()->flush(internal::defaultFlushParams); + + // Find constant scalar meshes. shape generally required for meshes, + // shape also required for scalars. + // https://github.com/openPMD/openPMD-standard/pull/289 + auto att_begin = aList.attributes->begin(); + auto att_end = aList.attributes->end(); + auto value = std::find(att_begin, att_end, "value"); + auto shape = std::find(att_begin, att_end, "shape"); + if (value != att_end && shape != att_end) + { + MeshRecordComponent &mrc = m; + IOHandler()->enqueue(IOTask(&mrc, pOpen)); + IOHandler()->flush(internal::defaultFlushParams); + mrc.get().m_isConstant = true; + } + try + { + m.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh with name '" << mesh_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(mesh_name); + } +} + +void CustomHierarchy::readScalarMesh( + EraseStaleMeshes &map, std::string const &mesh_name) +{ + Parameter pOpen; + Parameter pList; + + Parameter dOpen; + Mesh &m = map[mesh_name]; + dOpen.name = mesh_name; + MeshRecordComponent &mrc = m; + IOHandler()->enqueue(IOTask(&mrc, dOpen)); + IOHandler()->flush(internal::defaultFlushParams); + mrc.setWritten(false, Attributable::EnqueueAsynchronously::No); + mrc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); + mrc.setWritten(true, Attributable::EnqueueAsynchronously::No); + try + { + m.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh with name '" << mesh_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(mesh_name); + } +} + +void CustomHierarchy::readParticleSpecies( + EraseStaleParticles &map, std::string const &species_name) +{ + Parameter pOpen; + Parameter pList; + + ParticleSpecies &p = map[species_name]; + pOpen.path = species_name; + IOHandler()->enqueue(IOTask(&p, pOpen)); + IOHandler()->flush(internal::defaultFlushParams); + try + { + p.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read particle species with name '" << species_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(species_name); + } +} + +void CustomHierarchy::read(internal::MeshesParticlesPath const &mpp) +{ + std::vector currentPath; + read(mpp, currentPath); +} + +void CustomHierarchy::read( + internal::MeshesParticlesPath const &mpp, + std::vector ¤tPath) +{ + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + Parameter pList; + IOHandler()->enqueue(IOTask(this, pList)); + + Attributable::readAttributes(ReadMode::FullyReread); + Parameter dList; + IOHandler()->enqueue(IOTask(this, dList)); + IOHandler()->flush(internal::defaultFlushParams); + + std::deque constantComponentsPushback; + auto &data = get(); + auto embeddedMeshes = data.embeddedMeshesWrapped(); + auto embeddedParticles = data.embeddedParticlesWrapped(); + EraseStaleMeshes meshesMap(embeddedMeshes); + EraseStaleParticles particlesMap(embeddedParticles); + for (auto const &path : *pList.paths) + { + switch (mpp.determineType(currentPath)) + { + case internal::ContainedType::Group: { + Parameter pOpen; + pOpen.path = path; + auto &subpath = this->operator[](path); + IOHandler()->enqueue(IOTask(&subpath, pOpen)); + currentPath.emplace_back(path); + try + { + subpath.read(mpp, currentPath); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read subgroup '" << path << "' at path '" + << myPath().openPMDPath() + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + container().erase(path); + } + currentPath.pop_back(); + if (subpath.size() == 0 && subpath.containsAttribute("shape") && + subpath.containsAttribute("value")) + { + // This is not a group, but a constant record component + // Writable::~Writable() will deal with removing this from the + // backend again. + constantComponentsPushback.push_back(path); + container().erase(path); + } + break; + } + case internal::ContainedType::Mesh: { + try + { + readNonscalarMesh(meshesMap, path); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + meshesMap.forget(path); + } + break; + } + case internal::ContainedType::Particle: { + try + { + readParticleSpecies(particlesMap, path); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read particle species at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + particlesMap.forget(path); + } + break; + } + } + } + for (auto const &path : *dList.datasets) + { + switch (mpp.determineType(currentPath)) + { + + case internal::ContainedType::Particle: + std::cerr << "[Warning] Dataset found at '" + << (concatWithSep(currentPath, "/") + "/" + path) + << "' inside the particles path. A particle species is " + "always a group, never a dataset. Will parse as a " + "custom dataset. Storing custom datasets inside the " + "particles path is discouraged." + << std::endl; + [[fallthrough]]; + // Group is a bit of an internal misnomer here, it just means that + // it matches neither meshes nor particles path + case internal::ContainedType::Group: { + auto embeddedDatasets = data.embeddedDatasetsWrapped(); + auto &rc = embeddedDatasets[path]; + Parameter dOpen; + dOpen.name = path; + IOHandler()->enqueue(IOTask(&rc, dOpen)); + try + { + IOHandler()->flush(internal::defaultFlushParams); + rc.setWritten(false, Attributable::EnqueueAsynchronously::No); + rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); + rc.setWritten(true, Attributable::EnqueueAsynchronously::No); + rc.read(/* read_defaults = */ false); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read contained custom dataset '" << path + << "' at path '" << myPath().openPMDPath() + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + embeddedDatasets.erase(path); + } + break; + } + case internal::ContainedType::Mesh: + try + { + readScalarMesh(meshesMap, path); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read scalar mesh at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + meshesMap.forget(path); + } + break; + } + } + + for (auto const &path : constantComponentsPushback) + { + auto embeddedDatasets = data.embeddedDatasetsWrapped(); + auto &rc = embeddedDatasets[path]; + try + { + Parameter pOpen; + pOpen.path = path; + IOHandler()->enqueue(IOTask(&rc, pOpen)); + rc.get().m_isConstant = true; + rc.read(/* read_defaults = */ false); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read dataset at location '" + << myPath().openPMDPath() << "/" << path + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + embeddedDatasets.erase(path); + } + } + setDirty(false); +} + +void CustomHierarchy::flush_internal( + internal::FlushParams const &flushParams, + internal::MeshesParticlesPath &mpp, + std::vector currentPath) +{ + if (!dirtyRecursive()) + { + return; + } + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + // No need to do anything in access::readOnly since meshes and particles + // are initialized as aliases for subgroups at parsing time + auto &data = get(); + if (access::write(IOHandler()->m_frontendAccess)) + { + flushAttributes(flushParams); + } + + Parameter pCreate; + for (auto &[name, subpath] : *this) + { + if (!subpath.written()) + { + pCreate.path = name; + IOHandler()->enqueue(IOTask(&subpath, pCreate)); + } + currentPath.emplace_back(name); + subpath.flush_internal(flushParams, mpp, currentPath); + currentPath.pop_back(); + } + for (auto &[name, mesh] : data.embeddedMeshesInternal()) + { + if (!mpp.isMeshContainer(currentPath)) + { + std::string extend_meshes_path; + // Check if this can be covered by shorthand notation + // (e.g. meshesPath == "meshes/") + if (!currentPath.empty() && + *currentPath.rbegin() == mpp.m_defaultMeshesPath) + { + extend_meshes_path = *currentPath.rbegin() + "/"; + } + else + { + // Otherwise use full path + extend_meshes_path = "/" + + (currentPath.empty() + ? "" + : concatWithSep(currentPath, "/") + "/"); + } + mpp.collectNewMeshesPaths.emplace(std::move(extend_meshes_path)); + } + mesh.flush(name, flushParams); + } + for (auto &[name, particleSpecies] : data.embeddedParticlesInternal()) + { + if (!mpp.isParticleContainer(currentPath)) + { + std::string extend_particles_path; + if (!currentPath.empty() && + *currentPath.rbegin() == mpp.m_defaultParticlesPath) + { + // Check if this can be covered by shorthand notation + // (e.g. particlesPath == "particles/") + extend_particles_path = *currentPath.rbegin() + "/"; + } + else + { + // Otherwise use full path + extend_particles_path = "/" + + (currentPath.empty() + ? "" + : concatWithSep(currentPath, "/") + "/"); + ; + } + mpp.collectNewParticlesPaths.emplace( + std::move(extend_particles_path)); + } + particleSpecies.flush(name, flushParams); + } + for (auto &[name, dataset] : get().embeddedDatasetsInternal()) + { + dataset.flush(name, flushParams, /* set_defaults = */ false); + } + + if (flushParams.flushLevel != FlushLevel::SkeletonOnly && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) + { + setDirty(false); + } +} + +void CustomHierarchy::flush( + std::string const & /* path */, internal::FlushParams const &) +{ + throw std::runtime_error( + "[CustomHierarchy::flush()] Don't use this method. Flushing should be " + "triggered via Iteration class."); +} + +void CustomHierarchy::linkHierarchy(Writable &w) +{ + Attributable::linkHierarchy(w); +} +} // namespace openPMD + +#undef OPENPMD_LEGAL_IDENTIFIER_CHARS +#undef OPENPMD_SINGLE_GLOBBING_CHAR +#undef OPENPMD_DOUBLE_GLOBBING_CHAR diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 0ee10a77c4..36a1d21835 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -629,13 +629,24 @@ ADIOS2IOHandlerImpl::flush(internal::ParsedFlushParams &flushParams) } } + std::vector sorted; + sorted.reserve(m_dirty.size()); for (auto const &file : m_dirty) { - auto file_data = m_fileData.find(file); + sorted.emplace_back(&file); + } + std::sort( + sorted.begin(), sorted.end(), [](auto const &left, auto const &right) { + return **left <= **right; + }); + + for (auto file : sorted) + { + auto file_data = m_fileData.find(*file); if (file_data == m_fileData.end()) { throw error::Internal( - "[ADIOS2 backend] No associated data found for file'" + *file + + "[ADIOS2 backend] No associated data found for file'" + **file + "'."); } file_data->second->flush( diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 59541c1e30..1dd6d10061 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -24,6 +24,7 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" +#include "openPMD/IO/Access.hpp" #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSONMatcher.hpp" @@ -569,13 +570,13 @@ void JSONIOHandlerImpl::createPath( auto filepos = setAndGetFilePosition(writable, false); jsonVal = &(*jsonVal)[filepos->id]; - ensurePath(jsonVal, path); + ensurePath(jsonVal, path, m_handler->m_backendAccess); path = filepos->id.to_string() + "/" + path; } else { - ensurePath(jsonVal, path); + ensurePath(jsonVal, path, m_handler->m_backendAccess); } m_dirty.emplace(file); @@ -948,7 +949,10 @@ void JSONIOHandlerImpl::openPath( std::make_shared(json::json_pointer(path)); } - ensurePath(j, removeSlashes(parameters.path)); + ensurePath( + j, + removeSlashes(parameters.path), + /* Must not modify j */ Access::READ_ONLY); writable->written = true; } @@ -1850,18 +1854,45 @@ bool JSONIOHandlerImpl::hasKey(nlohmann::json const &j, KeyT &&key) } void JSONIOHandlerImpl::ensurePath( - nlohmann::json *jsonp, std::string const &path) + nlohmann::json *jsonp, std::string const &path, Access access) { auto groups = auxiliary::split(path, "/"); - for (std::string &group : groups) + if (access::readOnly(access)) { - // Enforce a JSON object - // the library will automatically create a list if the first - // key added to it is parseable as an int - jsonp = &(*jsonp)[group]; - if (jsonp->is_null()) + for (std::string const &group : groups) { - *jsonp = nlohmann::json::object(); + if (!jsonp->contains(group)) + { + throw error::ReadError( + error::AffectedObject::Group, + error::Reason::NotFound, + "JSON", + "Required group '" + path + "' not present."); + } + jsonp = &(*jsonp).at(group); + if (!jsonp->is_object()) + { + throw error::ReadError( + error::AffectedObject::Group, + error::Reason::UnexpectedContent, + "JSON", + "Required group '" + path + + "' is present, but not a JSON object."); + } + } + } + else + { + for (std::string const &group : groups) + { + // Enforce a JSON object + // the library will automatically create a list if the first + // key added to it is parseable as an int + jsonp = &(*jsonp)[group]; + if (jsonp->is_null()) + { + *jsonp = nlohmann::json::object(); + } } } } diff --git a/src/Iteration.cpp b/src/Iteration.cpp index ab35c1e3ae..1aa13ecd46 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -19,6 +19,7 @@ * If not, see . */ #include "openPMD/Iteration.hpp" +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/Dataset.hpp" #include "openPMD/Datatype.hpp" #include "openPMD/Error.hpp" @@ -36,12 +37,11 @@ #include "openPMD/backend/Writable.hpp" #include -#include -#include #include #include #include -#include +#include +#include #include namespace openPMD @@ -49,7 +49,7 @@ namespace openPMD using internal::CloseStatus; using internal::DeferredParseAccess; -Iteration::Iteration() : Attributable(NoInit()) +Iteration::Iteration() : CustomHierarchy(NoInit()) { setData(std::make_shared()); setTime(static_cast(0)); @@ -275,7 +275,7 @@ void Iteration::flushFileBased( case FlushLevel::SkeletonOnly: case FlushLevel::InternalFlush: case FlushLevel::UserFlush: - flush(flushParams); + flushIteration(flushParams); break; } } @@ -298,7 +298,7 @@ void Iteration::flushGroupBased( case FlushLevel::SkeletonOnly: case FlushLevel::InternalFlush: case FlushLevel::UserFlush: - flush(flushParams); + flushIteration(flushParams); break; } } @@ -321,7 +321,7 @@ void Iteration::flushVariableBased( case FlushLevel::SkeletonOnly: case FlushLevel::InternalFlush: case FlushLevel::UserFlush: - flush(flushParams); + flushIteration(flushParams); break; } @@ -347,74 +347,130 @@ void Iteration::flushVariableBased( } } -void Iteration::flush(internal::FlushParams const &flushParams) +void Iteration::flushIteration(internal::FlushParams const &flushParams) { Parameter touch; IOHandler()->enqueue(IOTask(&writable(), touch)); - if (access::readOnly(IOHandler()->m_frontendAccess)) + if (flushParams.flushLevel == FlushLevel::CreateOrOpenFiles) { - for (auto &m : meshes) - m.second.flush(m.first, flushParams); - for (auto &species : particles) - species.second.flush(species.first, flushParams); - setDirty(false); + return; } - else + + /* + * Convention for CustomHierarchy::flush and CustomHierarchy::read: + * Path is created/opened already at entry point of method, method needs + * to create/open path for contained subpaths. + */ + + Series s = retrieveSeries(); + std::vector meshesPaths = s.meshesPaths(), + particlesPaths = s.particlesPaths(); + internal::MeshesParticlesPath mpp(meshesPaths, particlesPaths); + + sync_meshes_and_particles_from_alias_to_subgroups(mpp); + + std::vector currentPath; + CustomHierarchy::flush_internal(flushParams, mpp, currentPath); + + sync_meshes_and_particles_from_subgroups_to_alias(mpp); + + if (!mpp.collectNewMeshesPaths.empty() || + !mpp.collectNewParticlesPaths.empty()) { - /* Find the root point [Series] of this file, - * meshesPath and particlesPath are stored there */ - Series s = retrieveSeries(); + for (auto [newly_added_paths, vec] : + {std::make_pair(&mpp.collectNewMeshesPaths, &meshesPaths), + std::make_pair(&mpp.collectNewParticlesPaths, &particlesPaths)}) + { + std::transform( + newly_added_paths->begin(), + newly_added_paths->end(), + std::back_inserter(*vec), + [](auto const &pair) { return pair; }); + } + s.setMeshesPath(meshesPaths); + s.setParticlesPath(particlesPaths); + } - if (!meshes.empty() || s.containsAttribute("meshesPath")) + if (flushParams.flushLevel != FlushLevel::SkeletonOnly && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles) + { + if (access::write(IOHandler()->m_frontendAccess)) { - if (!s.containsAttribute("meshesPath")) + flushAttributes(flushParams); + } + setDirty(false); + meshes.setDirty(false); + particles.setDirty(false); + } +} + +void Iteration::sync_meshes_and_particles_from_alias_to_subgroups( + internal::MeshesParticlesPath const &mpp) +{ + auto sync_meshes_and_particles = + [this](auto &m_or_p, std::string const &defaultPath) { + using type = + typename std::remove_reference_t::mapped_type; + + if (m_or_p.empty()) { - s.setMeshesPath("meshes/"); - s.flushMeshesPath(); + return; } - if (meshes.dirtyRecursive()) + auto container = (*this)[defaultPath].asContainerOf(); + + for (auto &[name, entry] : m_or_p) { - meshes.flush(s.meshesPath(), flushParams); - for (auto &m : meshes) + if (auxiliary::contains(name, '/')) + { + throw std::runtime_error( + "Unimplemented: Multi-level paths in " + "Iteration::meshes/Iteration::particles"); + } + if (auto it = container.find(name); it != container.end()) { - m.second.flush(m.first, flushParams); + if (it->second.m_attri->asSharedPtrOfAttributable() == + entry.m_attri->asSharedPtrOfAttributable()) + { + continue; // has been emplaced previously + } + else + { + throw std::runtime_error("asdfasdfasdfasd"); + } + } + else + { + container.emplace(name, entry); + entry.linkHierarchy(container.writable()); } } - } - else - { - meshes.setDirty(false); - } + }; - if (!particles.empty() || s.containsAttribute("particlesPath")) - { - if (!s.containsAttribute("particlesPath")) + sync_meshes_and_particles(meshes, mpp.m_defaultMeshesPath); + sync_meshes_and_particles(particles, mpp.m_defaultParticlesPath); +} + +void Iteration::sync_meshes_and_particles_from_subgroups_to_alias( + internal::MeshesParticlesPath const &mpp) +{ + auto sync_meshes_and_particles = + [this](auto &m_or_p, std::string const &defaultPath) { + using type = + typename std::remove_reference_t::mapped_type; + auto it = this->find(defaultPath); + if (it == this->end()) { - s.setParticlesPath("particles/"); - s.flushParticlesPath(); + return; } - if (particles.dirtyRecursive()) + auto container = it->second.asContainerOf(); + for (auto &[name, entry] : container) { - particles.flush(s.particlesPath(), flushParams); - for (auto &species : particles) - { - species.second.flush(species.first, flushParams); - } + m_or_p.emplace(name, entry); } - } - else - { - particles.setDirty(false); - } + }; - flushAttributes(flushParams); - } - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - setDirty(false); - meshes.setDirty(false); - particles.setDirty(false); - } + sync_meshes_and_particles(meshes, mpp.m_defaultMeshesPath); + sync_meshes_and_particles(particles, mpp.m_defaultParticlesPath); } void Iteration::deferParseAccess(DeferredParseAccess dr) @@ -572,68 +628,19 @@ void Iteration::read_impl(std::string const &groupPath) Attribute(Attribute::from_any, *aRead.m_resource).dtype) + ")"); - /* Find the root point [Series] of this file, - * meshesPath and particlesPath are stored there */ Series s = retrieveSeries(); Parameter pList; - auto version = IOHandler()->m_standard; - bool hasMeshes = false; - bool hasParticles = false; - if (version <= OpenpmdStandard::v_1_0_1) - { - IOHandler()->enqueue(IOTask(this, pList)); - IOHandler()->flush(internal::defaultFlushParams); - hasMeshes = std::count( - pList.paths->begin(), - pList.paths->end(), - auxiliary::replace_last(s.meshesPath(), "/", "")) == 1; - hasParticles = - std::count( - pList.paths->begin(), - pList.paths->end(), - auxiliary::replace_last(s.particlesPath(), "/", "")) == 1; - pList.paths->clear(); - } - else - { - hasMeshes = s.containsAttribute("meshesPath"); - hasParticles = s.containsAttribute("particlesPath"); - } + IOHandler()->enqueue(IOTask(this, pList)); - if (hasMeshes) - { - try - { - readMeshes(s.meshesPath()); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read meshes in iteration " << groupPath - << " and will skip them due to read error:\n" - << err.what() << std::endl; - meshes.container().clear(); - } - } - meshes.setDirty(false); + // @todo restore compatibility with openPMD 1.0.*: + // hasMeshes <-> meshesPath is defined - if (hasParticles) - { - try - { - readParticles(s.particlesPath()); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read particles in iteration " << groupPath - << " and will skip them due to read error:\n" - << err.what() << std::endl; - particles.container().clear(); - } - } - particles.setDirty(false); + internal::MeshesParticlesPath mpp(s.meshesPaths(), s.particlesPaths()); + CustomHierarchy::read(mpp); + + sync_meshes_and_particles_from_subgroups_to_alias(mpp); - readAttributes(ReadMode::FullyReread); #ifdef openPMD_USE_INVASIVE_TESTS if (containsAttribute("__openPMD_internal_fail")) { @@ -650,128 +657,6 @@ void Iteration::read_impl(std::string const &groupPath) #endif } -void Iteration::readMeshes(std::string const &meshesPath) -{ - Parameter pOpen; - Parameter pList; - - pOpen.path = meshesPath; - IOHandler()->enqueue(IOTask(&meshes, pOpen)); - - meshes.readAttributes(ReadMode::FullyReread); - - internal::EraseStaleEntries map{meshes}; - - /* obtain all non-scalar meshes */ - IOHandler()->enqueue(IOTask(&meshes, pList)); - IOHandler()->flush(internal::defaultFlushParams); - - Parameter aList; - for (auto const &mesh_name : *pList.paths) - { - Mesh &m = map[mesh_name]; - pOpen.path = mesh_name; - aList.attributes->clear(); - IOHandler()->enqueue(IOTask(&m, pOpen)); - IOHandler()->enqueue(IOTask(&m, aList)); - IOHandler()->flush(internal::defaultFlushParams); - - // Find constant scalar meshes. shape generally required for meshes, - // shape also required for scalars. - // https://github.com/openPMD/openPMD-standard/pull/289 - auto att_begin = aList.attributes->begin(); - auto att_end = aList.attributes->end(); - auto value = std::find(att_begin, att_end, "value"); - auto shape = std::find(att_begin, att_end, "shape"); - if (value != att_end && shape != att_end) - { - MeshRecordComponent &mrc = m; - IOHandler()->enqueue(IOTask(&mrc, pOpen)); - IOHandler()->flush(internal::defaultFlushParams); - mrc.get().m_isConstant = true; - } - try - { - m.read(); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read mesh with name '" << mesh_name - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - map.forget(mesh_name); - } - } - - /* obtain all scalar meshes */ - Parameter dList; - IOHandler()->enqueue(IOTask(&meshes, dList)); - IOHandler()->flush(internal::defaultFlushParams); - - Parameter dOpen; - for (auto const &mesh_name : *dList.datasets) - { - Mesh &m = map[mesh_name]; - dOpen.name = mesh_name; - IOHandler()->enqueue(IOTask(&m, dOpen)); - IOHandler()->flush(internal::defaultFlushParams); - MeshRecordComponent &mrc = m; - IOHandler()->enqueue(IOTask(&mrc, dOpen)); - IOHandler()->flush(internal::defaultFlushParams); - mrc.setWritten(false, Attributable::EnqueueAsynchronously::No); - mrc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); - mrc.setWritten(true, Attributable::EnqueueAsynchronously::No); - try - { - m.read(); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read mesh with name '" << mesh_name - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - map.forget(mesh_name); - } - } -} - -void Iteration::readParticles(std::string const &particlesPath) -{ - Parameter pOpen; - Parameter pList; - - pOpen.path = particlesPath; - IOHandler()->enqueue(IOTask(&particles, pOpen)); - - particles.readAttributes(ReadMode::FullyReread); - - /* obtain all particle species */ - pList.paths->clear(); - IOHandler()->enqueue(IOTask(&particles, pList)); - IOHandler()->flush(internal::defaultFlushParams); - - internal::EraseStaleEntries map{particles}; - for (auto const &species_name : *pList.paths) - { - ParticleSpecies &p = map[species_name]; - pOpen.path = species_name; - IOHandler()->enqueue(IOTask(&p, pOpen)); - IOHandler()->flush(internal::defaultFlushParams); - try - { - p.read(); - } - catch (error::ReadError const &err) - { - std::cerr << "Cannot read particle species with name '" - << species_name - << "' and will skip it due to read error:\n" - << err.what() << std::endl; - map.forget(species_name); - } - } -} - auto Iteration::beginStep(bool reread) -> BeginStepStatus { BeginStepStatus res; @@ -948,6 +833,29 @@ void Iteration::setStepStatus(StepStatus status) } } +bool Iteration::dirtyRecursive() const +{ + if (dirty() || CustomHierarchy::dirtyRecursive()) + { + return true; + } + for (auto const &pair : particles) + { + if (!pair.second.written()) + { + return true; + } + } + for (auto const &pair : meshes) + { + if (!pair.second.written()) + { + return true; + } + } + return false; +} + void Iteration::linkHierarchy(Writable &w) { Attributable::linkHierarchy(w); diff --git a/src/ParticlePatches.cpp b/src/ParticlePatches.cpp index 789559a67f..df87896896 100644 --- a/src/ParticlePatches.cpp +++ b/src/ParticlePatches.cpp @@ -100,7 +100,7 @@ void ParticlePatches::read() pr.setDirty(false); try { - prc.PatchRecordComponent::read(/* require_unit_si = */ false); + prc.PatchRecordComponent::read(/* read_defaults = */ false); } catch (error::ReadError const &err) { diff --git a/src/Record.cpp b/src/Record.cpp index 8056718edd..d7f124d697 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -62,12 +62,14 @@ void Record::flush_impl( { if (scalar()) { - T_RecordComponent::flush(SCALAR, flushParams); + T_RecordComponent::flush( + SCALAR, flushParams, /* set_defaults = */ true); } else { for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* set_defaults = */ true); } } else @@ -77,7 +79,7 @@ void Record::flush_impl( if (scalar()) { RecordComponent &rc = *this; - rc.flush(name, flushParams); + rc.flush(name, flushParams, /* set_defaults = */ true); } else { @@ -87,7 +89,8 @@ void Record::flush_impl( for (auto &comp : *this) { comp.second.parent() = getWritable(this); - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* set_defaults = */ true); } } } @@ -96,12 +99,14 @@ void Record::flush_impl( if (scalar()) { - T_RecordComponent::flush(name, flushParams); + T_RecordComponent::flush( + name, flushParams, /* set_defaults = */ true); } else { for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* set_defaults = */ true); } } @@ -121,7 +126,7 @@ auto Record::read() -> internal::HomogenizeExtents /* using operator[] will incorrectly update parent */ try { - T_RecordComponent::read(/* require_unit_si = */ true); + T_RecordComponent::read(/* read_defaults = */ true); } catch (error::ReadError const &err) { @@ -149,7 +154,7 @@ auto Record::read() -> internal::HomogenizeExtents rc.get().m_isConstant = true; try { - rc.read(/* require_unit_si = */ true); + rc.read(/* read_defaults = */ true); } catch (error::ReadError const &err) { @@ -178,7 +183,7 @@ auto Record::read() -> internal::HomogenizeExtents rc.setWritten(true, Attributable::EnqueueAsynchronously::No); try { - rc.read(/* require_unit_si = */ true); + rc.read(/* read_defaults = */ true); } catch (error::ReadError const &err) { diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index eb41dddd19..b83c685412 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -372,7 +372,9 @@ bool RecordComponent::empty() const } void RecordComponent::flush( - std::string const &name, internal::FlushParams const &flushParams) + std::string const &name, + internal::FlushParams const &flushParams, + bool set_defaults) { if (!dirtyRecursive()) { @@ -415,7 +417,7 @@ void RecordComponent::flush( "RecordComponent::resetDataset())."); } } - if (!containsAttribute("unitSI")) + if (set_defaults && !containsAttribute("unitSI")) { setUnitSI(1); } @@ -516,9 +518,9 @@ void RecordComponent::flush( } } -void RecordComponent::read(bool require_unit_si) +void RecordComponent::read(bool read_defaults) { - readBase(require_unit_si); + readBase(read_defaults); } namespace @@ -543,7 +545,7 @@ namespace }; } // namespace -void RecordComponent::readBase(bool require_unit_si) +void RecordComponent::readBase(bool read_defaults) { using DT = Datatype; auto &rc = get(); @@ -596,7 +598,7 @@ void RecordComponent::readBase(bool require_unit_si) read_constant(); } - if (require_unit_si) + if (read_defaults) { if (!containsAttribute("unitSI")) { diff --git a/src/Series.cpp b/src/Series.cpp index 6d13de73c3..b20a63249f 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -269,7 +269,27 @@ Series &Series::setBasePath(std::string const &bp) std::string Series::meshesPath() const { - return getAttribute("meshesPath").get(); + auto res = meshesPaths(); + if (res.empty()) + { + throw no_such_attribute_error("meshesPath"); + } + /* + * @todo: Verify that meshesPath has canonical form + */ + return res.at(0); +} + +std::vector Series::meshesPaths() const +{ + if (containsAttribute("meshesPath")) + { + return getAttribute("meshesPath").get>(); + } + else + { + return {}; + } } Series &Series::setMeshesPath(std::string const &mp) @@ -292,6 +312,23 @@ Series &Series::setMeshesPath(std::string const &mp) setDirty(true); return *this; } +Series &Series::setMeshesPath(std::vector const &mp) +{ + // @todo if already written, then append + switch (mp.size()) + { + case 0: + return *this; + case 1: + setAttribute("meshesPath", *mp.begin()); + break; + default: + setAttribute("meshesPath", mp); + break; + } + setDirty(true); + return *this; +} std::vector Series::availableDatasets() { @@ -588,7 +625,27 @@ void Series::flushRankTable() std::string Series::particlesPath() const { - return getAttribute("particlesPath").get(); + auto res = particlesPaths(); + if (res.empty()) + { + throw no_such_attribute_error("particlesPath"); + } + /* + * @todo: Verify that particlesPath has canonical form + */ + return res.at(0); +} + +std::vector Series::particlesPaths() const +{ + if (containsAttribute("particlesPath")) + { + return getAttribute("particlesPath").get>(); + } + else + { + return {}; + } } Series &Series::setParticlesPath(std::string const &pp) @@ -611,6 +668,23 @@ Series &Series::setParticlesPath(std::string const &pp) setDirty(true); return *this; } +Series &Series::setParticlesPath(std::vector const &pp) +{ + // @todo if already written, then append + switch (pp.size()) + { + case 0: + return *this; + case 1: + setAttribute("particlesPath", *pp.begin()); + break; + default: + setAttribute("particlesPath", pp); + break; + } + setDirty(true); + return *this; +} std::string Series::author() const { @@ -1463,7 +1537,7 @@ void Series::flushFileBased( break; case IO::HasBeenOpened: // continue below - it->second.flush(flushParams); + it->second.flushIteration(flushParams); break; } @@ -1491,6 +1565,10 @@ void Series::flushFileBased( bool allDirty = dirty(); for (auto it = begin; it != end; ++it) { + /* reset the dirty bit for every iteration (i.e. file) + * otherwise only the first iteration will have updates attributes + */ + setDirty(allDirty); // Phase 1 switch (openIterationIfDirty(it->first, it->second)) { @@ -1536,12 +1614,7 @@ void Series::flushFileBased( IOHandler()->enqueue(IOTask(&it->second, std::move(fClose))); it->second.get().m_closed = internal::CloseStatus::Closed; } - /* reset the dirty bit for every iteration (i.e. file) - * otherwise only the first iteration will have updates attributes - */ - setDirty(allDirty); } - setDirty(false); // Phase 3 if (flushIOHandler) @@ -1582,7 +1655,7 @@ void Series::flushGorVBased( series.m_snapshotToStep.at(it->first)}; IOHandler()->enqueue(IOTask(this, std::move(param))); } - it->second.flush(flushParams); + it->second.flushIteration(flushParams); break; } @@ -1683,26 +1756,6 @@ void Series::flushGorVBased( } } -void Series::flushMeshesPath() -{ - Parameter aWrite; - aWrite.name = "meshesPath"; - Attribute a = getAttribute("meshesPath"); - aWrite.m_resource = a.getAny(); - aWrite.dtype = a.dtype; - IOHandler()->enqueue(IOTask(this, aWrite)); -} - -void Series::flushParticlesPath() -{ - Parameter aWrite; - aWrite.name = "particlesPath"; - Attribute a = getAttribute("particlesPath"); - aWrite.m_resource = a.getAny(); - aWrite.dtype = a.dtype; - IOHandler()->enqueue(IOTask(this, aWrite)); -} - void Series::readFileBased( std::optional read_only_this_single_iteration) { @@ -2495,7 +2548,7 @@ void Series::readBase() IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); if (auto val = Attribute(Attribute::from_any, *aRead.m_resource) - .getOptional(); + .getOptional>(); val.has_value()) { /* allow setting the meshes path after completed IO */ @@ -2537,7 +2590,7 @@ void Series::readBase() IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); if (auto val = Attribute(Attribute::from_any, *aRead.m_resource) - .getOptional(); + .getOptional>(); val.has_value()) { /* allow setting the meshes path after completed IO */ diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index ce1c2936cb..7048c87f79 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -304,10 +304,10 @@ void Attributable::flushAttributes(internal::FlushParams const &flushParams) } } // Do this outside the if branch to also setDirty to dirtyRecursive - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - setDirty(false); - } + assert( + flushParams.flushLevel != FlushLevel::SkeletonOnly && + flushParams.flushLevel != FlushLevel::CreateOrOpenFiles); + setDirty(false); } void Attributable::readAttributes(ReadMode mode) diff --git a/src/backend/Container.cpp b/src/backend/Container.cpp index 7d421ac1a9..d8fe8bd5dd 100644 --- a/src/backend/Container.cpp +++ b/src/backend/Container.cpp @@ -21,6 +21,7 @@ #include "openPMD/backend/ContainerImpl.tpp" +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/Mesh.hpp" #include "openPMD/ParticlePatches.hpp" @@ -33,6 +34,7 @@ namespace openPMD #define OPENPMD_COMMA , #define OPENPMD_INSTANTIATE(type) template class Container; +OPENPMD_INSTANTIATE(CustomHierarchy) OPENPMD_INSTANTIATE(Mesh) OPENPMD_INSTANTIATE(MeshRecordComponent) OPENPMD_INSTANTIATE(ParticlePatches) diff --git a/src/backend/MeshRecordComponent.cpp b/src/backend/MeshRecordComponent.cpp index 8298a8abbd..7a1dc84343 100644 --- a/src/backend/MeshRecordComponent.cpp +++ b/src/backend/MeshRecordComponent.cpp @@ -66,7 +66,7 @@ void MeshRecordComponent::read() Attribute(Attribute::from_any, *aRead.m_resource).dtype) + ")"); - readBase(/* require_unit_si = */ true); + readBase(/* read_defaults = */ true); } void MeshRecordComponent::flush( @@ -81,7 +81,7 @@ void MeshRecordComponent::flush( { setPosition(std::vector{0}); } - RecordComponent::flush(name, params); + RecordComponent::flush(name, params, /* set_defaults = */ true); } template diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index 3080523d24..f14acf1758 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -52,10 +52,11 @@ void PatchRecord::flush_impl( path, flushParams); // warning (clang-tidy-10): // bugprone-parent-virtual-call for (auto &comp : *this) - comp.second.flush(comp.first, flushParams); + comp.second.flush( + comp.first, flushParams, /* set_defaults = */ true); } else - T_RecordComponent::flush(path, flushParams); + T_RecordComponent::flush(path, flushParams, /* set_defaults = */ true); if (flushParams.flushLevel != FlushLevel::SkeletonOnly) { setDirty(false); @@ -101,7 +102,7 @@ void PatchRecord::read() prc.setWritten(true, Attributable::EnqueueAsynchronously::No); try { - prc.read(/* require_unit_si = */ false); + prc.read(/* read_defaults = */ false); } catch (error::ReadError const &err) { diff --git a/src/binding/python/Container.cpp b/src/binding/python/Container.cpp new file mode 100644 index 0000000000..d49185e57d --- /dev/null +++ b/src/binding/python/Container.cpp @@ -0,0 +1,71 @@ +/* Copyright 2018-2021 Axel Huebl + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + * + * The function `bind_container` is based on std_bind.h in pybind11 + * Copyright (c) 2016 Sergey Lyskov and Wenzel Jakob + * + * BSD-style license, see pybind11 LICENSE file. + */ + +#include + +#include "openPMD/Iteration.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticlePatches.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/Record.hpp" +#include "openPMD/Series.hpp" +#include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/BaseRecordComponent.hpp" +#include "openPMD/backend/Container.hpp" +#include "openPMD/backend/MeshRecordComponent.hpp" +#include "openPMD/backend/PatchRecord.hpp" +#include "openPMD/backend/PatchRecordComponent.hpp" +#include "openPMD/binding/python/Container.hpp" + +#include "openPMD/binding/python/Common.hpp" + +void init_Container(py::module &m) +{ + ::detail::create_and_bind_container( + m, "Iteration_Container"); + ::detail::create_and_bind_container( + m, "Mesh_Container"); + ::detail::create_and_bind_container( + m, "Particle_Container"); + ::detail::create_and_bind_container( + m, "Particle_Patches_Container"); + ::detail::create_and_bind_container( + m, "Record_Container"); + ::detail::create_and_bind_container( + m, "Patch_Record_Container"); + ::detail:: + create_and_bind_container( + m, "Record_Component_Container"); + ::detail:: + create_and_bind_container( + m, "Mesh_Record_Component_Container"); + ::detail::create_and_bind_container< + PyPatchRecordComponentContainer, + Attributable>(m, "Patch_Record_Component_Container"); + ::detail:: + create_and_bind_container( + m, "Custom_Hierarchy_Container"); +} diff --git a/src/binding/python/CustomHierarchy.cpp b/src/binding/python/CustomHierarchy.cpp new file mode 100644 index 0000000000..b893d6a174 --- /dev/null +++ b/src/binding/python/CustomHierarchy.cpp @@ -0,0 +1,52 @@ + + +#include "openPMD/CustomHierarchy.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/RecordComponent.hpp" +#include "openPMD/backend/Attributable.hpp" +#include "openPMD/binding/python/Common.hpp" +#include "openPMD/binding/python/Container.H" +#include + +namespace py = pybind11; +using namespace openPMD; + +template +void define_conversible_container(py::module &m, std::string const &name) +{ + using CC = ConversibleContainer; + py::class_, Attributable>(m, name.c_str()) + .def( + "as_container_of_datasets", + &CC::template asContainerOf) + .def("as_container_of_meshes", &CC::template asContainerOf) + .def( + "as_container_of_particles", + &CC::template asContainerOf) + .def( + "as_container_of_custom_hierarchy", + &CC::template asContainerOf); +} + +void init_CustomHierarchy(py::module &m) +{ + auto py_ch_cont = + declare_container( + m, "Container_CustomHierarchy"); + + define_conversible_container( + m, "ConversibleContainer_CustomHierarchy"); + define_conversible_container( + m, "ConversibleContainer_ParticleSpecies"); + define_conversible_container( + m, "ConversibleContainer_RecordComponent"); + define_conversible_container(m, "ConversibleContainer_Mesh"); + + [[maybe_unused]] py::class_< + CustomHierarchy, + ConversibleContainer, + Container, + Attributable> custom_hierarchy(m, "CustomHierarchy"); + + finalize_container(py_ch_cont); +} diff --git a/src/binding/python/Iteration.cpp b/src/binding/python/Iteration.cpp index 846861d262..ad3feaa757 100644 --- a/src/binding/python/Iteration.cpp +++ b/src/binding/python/Iteration.cpp @@ -19,6 +19,7 @@ * If not, see . */ #include "openPMD/Iteration.hpp" +#include "openPMD/CustomHierarchy.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/binding/python/Common.hpp" @@ -40,8 +41,11 @@ void init_Iteration(py::module &m) #define OPENPMD_AVOID_CLANG_FORMAT auto cl = OPENPMD_AVOID_CLANG_FORMAT #undef OPENPMD_AVOID_CLANG_FORMAT - - py::class_(m, "Iteration") + py::class_< + Iteration, + CustomHierarchy, + PyCustomHierarchyContainer, + Attributable>(m, "Iteration") .def(py::init()) .def( diff --git a/src/binding/python/Series.cpp b/src/binding/python/Series.cpp index 384462403d..e3e1971541 100644 --- a/src/binding/python/Series.cpp +++ b/src/binding/python/Series.cpp @@ -24,6 +24,7 @@ #include "openPMD/Iteration.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/auxiliary/JSON.hpp" +#include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Container.H" @@ -37,6 +38,7 @@ #include #include #include +#include #if openPMD_USE_FILESYSTEM_HEADER #include @@ -440,13 +442,61 @@ this method. &Series::openPMDextension, &Series::setOpenPMDextension) .def_property("base_path", &Series::basePath, &Series::setBasePath) - .def_property( - "meshes_path", &Series::meshesPath, &Series::setMeshesPath) .def_property_readonly("has_rank_table_read", &Series::hasRankTableRead) .def("get_rank_table", &Series::rankTable, py::arg("collective")) .def("set_rank_table", &Series::setRankTable, py::arg("my_rank_info")) .def_property( - "particles_path", &Series::particlesPath, &Series::setParticlesPath) + "meshes_path", + [](Series &self) + -> std::variant> { + using res_t = + std::variant>; + auto res = self.meshesPaths(); + if (res.size() == 1) + { + return res_t{std::move(res[0])}; + } + else + { + return res_t{std::move(res)}; + } + }, + [](Series &self, + std::variant> const &arg) + -> Series & { + std::visit( + [&](auto const &arg_resolved) { + self.setMeshesPath(arg_resolved); + }, + arg); + return self; + }) + .def_property( + "particles_path", + [](Series &self) + -> std::variant> { + using res_t = + std::variant>; + auto res = self.particlesPaths(); + if (res.size() == 1) + { + return res_t{std::move(res[0])}; + } + else + { + return res_t{std::move(res)}; + } + }, + [](Series &self, + std::variant> const &arg) + -> Series & { + std::visit( + [&](auto const &arg_resolved) { + self.setParticlesPath(arg_resolved); + }, + arg); + return self; + }) .def_property("author", &Series::author, &Series::setAuthor) .def_property( "machine", @@ -489,8 +539,20 @@ this method. .def("set_openPMD", &Series::setOpenPMD) .def("set_openPMD_extension", &Series::setOpenPMDextension) .def("set_base_path", &Series::setBasePath) - .def("set_meshes_path", &Series::setMeshesPath) - .def("set_particles_path", &Series::setParticlesPath) + .def( + "set_meshes_path", + py::overload_cast(&Series::setMeshesPath)) + .def( + "set_meshes_path", + py::overload_cast const &>( + &Series::setMeshesPath)) + .def( + "set_particles_path", + py::overload_cast const &>( + &Series::setParticlesPath)) + .def( + "set_particles_path", + py::overload_cast(&Series::setParticlesPath)) .def("set_author", &Series::setAuthor) .def("set_date", &Series::setDate) .def("set_iteration_encoding", &Series::setIterationEncoding) diff --git a/src/binding/python/openPMD.cpp b/src/binding/python/openPMD.cpp index dcf01d6c5f..01907205b8 100644 --- a/src/binding/python/openPMD.cpp +++ b/src/binding/python/openPMD.cpp @@ -37,6 +37,7 @@ void init_Dataset(py::module &); void init_Datatype(py::module &); void init_Error(py::module &); void init_Helper(py::module &); +void init_CustomHierarchy(py::module &); void init_Iteration(py::module &); void init_IterationEncoding(py::module &); void init_Mesh(py::module &); @@ -101,6 +102,7 @@ PYBIND11_MODULE(openpmd_api_cxx, m) init_ParticleSpecies(m); init_Mesh(m); + init_CustomHierarchy(m); init_Iteration(m); init_IterationEncoding(m); init_Series(m); diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index ad22ae1cf8..4a44a30ca1 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -399,6 +399,252 @@ TEST_CASE("attribute_dtype_test", "[core]") } } +TEST_CASE("custom_hierarchies", "[core]") +{ + std::string filePath = "../samples/custom_hierarchies.json"; + Series write(filePath, Access::CREATE); + write.iterations[0]; + write.close(); + + Series read(filePath, Access::READ_ONLY); + REQUIRE(read.iterations[0].size() == 0); + read.close(); + + write = Series(filePath, Access::READ_WRITE); + write.iterations[0]["custom"]["hierarchy"]; + write.iterations[0]["custom"].setAttribute("string", "attribute"); + write.iterations[0]["custom"]["hierarchy"].setAttribute("number", 3); + write.iterations[0]["no_attributes"]; + write.close(); + + read = Series(filePath, Access::READ_ONLY); + REQUIRE(read.iterations[0].size() == 2); + REQUIRE(read.iterations[0].count("custom") == 1); + REQUIRE(read.iterations[0].count("no_attributes") == 1); + REQUIRE(read.iterations[0]["custom"].size() == 1); + REQUIRE(read.iterations[0]["custom"].count("hierarchy") == 1); + REQUIRE(read.iterations[0]["custom"]["hierarchy"].size() == 0); + REQUIRE(read.iterations[0]["no_attributes"].size() == 0); + REQUIRE( + read.iterations[0]["custom"] + .getAttribute("string") + .get() == "attribute"); + REQUIRE( + read.iterations[0]["custom"]["hierarchy"] + .getAttribute("number") + .get() == 3); + read.close(); + + write = Series(filePath, Access::READ_WRITE); + { + write.iterations[0]["custom"]["hierarchy"]; + write.iterations[0]["custom"] + .asContainerOf()["emptyDataset"] + .makeEmpty(Datatype::FLOAT, 3); + write.iterations[0]["custom"]["hierarchy"].setAttribute("number", 3); + write.iterations[0]["no_attributes"]; + auto iteration_level_ds = + write.iterations[0] + .asContainerOf()["iteration_level_dataset"]; + iteration_level_ds.resetDataset({Datatype::INT, {10}}); + std::vector data(10, 5); + iteration_level_ds.storeChunk(data); + + auto meshesViaAlias = write.iterations[0].meshes; + meshesViaAlias["E"]["x"].makeEmpty(2); + write.setMeshesPath(std::vector{"fields/", "%%/meshes/"}); + auto meshesManually = + write.iterations[0]["fields"].asContainerOf(); + REQUIRE(meshesManually.size() == 0); + write.flush(); // Synchronized upon flushing + REQUIRE(meshesManually.contains("E")); + REQUIRE(meshesManually.size() == 1); + meshesManually["B"]["x"].makeEmpty(2); + REQUIRE(meshesViaAlias.size() == 1); + write.flush(); + REQUIRE(meshesViaAlias.contains("B")); + REQUIRE(meshesViaAlias.size() == 2); + + write.setParticlesPath("species"); + auto particlesManually = + write.iterations[0]["species"].asContainerOf(); + particlesManually["e"]["position"]["x"].makeEmpty(1); + auto particlesViaAlias = write.iterations[0].particles; + particlesViaAlias["i"]["position"]["x"].makeEmpty(1); + + write.close(); + } + + read = Series(filePath, Access::READ_ONLY); + { + REQUIRE(read.iterations[0].size() == 4); + REQUIRE(read.iterations[0].count("custom") == 1); + REQUIRE(read.iterations[0].count("no_attributes") == 1); + REQUIRE(read.iterations[0].count("fields") == 1); + REQUIRE(read.iterations[0].count("species") == 1); + REQUIRE(read.iterations[0]["custom"].size() == 1); + REQUIRE(read.iterations[0]["custom"].count("hierarchy") == 1); + REQUIRE(read.iterations[0]["custom"]["hierarchy"].size() == 0); + REQUIRE(read.iterations[0]["no_attributes"].size() == 0); + REQUIRE(read.iterations[0]["fields"].asContainerOf().size() == 2); + REQUIRE( + read.iterations[0]["fields"].asContainerOf().contains("E")); + REQUIRE( + read.iterations[0]["fields"].asContainerOf().contains("B")); + REQUIRE(read.iterations[0].meshes.size() == 2); + REQUIRE(read.iterations[0].meshes.contains("E")); + REQUIRE(read.iterations[0].meshes.contains("B")); + REQUIRE( + read.iterations[0]["species"] + .asContainerOf() + .size() == 2); + REQUIRE(read.iterations[0]["species"] + .asContainerOf() + .contains("e")); + REQUIRE(read.iterations[0]["species"] + .asContainerOf() + .contains("i")); + REQUIRE(read.iterations[0].particles.size() == 2); + REQUIRE(read.iterations[0].particles.contains("e")); + REQUIRE(read.iterations[0].particles.contains("i")); + + REQUIRE( + read.iterations[0].asContainerOf().size() == 1); + REQUIRE( + read.iterations[0]["custom"] + .asContainerOf() + .size() == 1); + REQUIRE( + read.iterations[0]["custom"]["hierarchy"] + .asContainerOf() + .size() == 0); + REQUIRE( + read.iterations[0]["no_attributes"] + .asContainerOf() + .size() == 0); + + auto iteration_level_ds = + read.iterations[0] + .asContainerOf()["iteration_level_dataset"]; + REQUIRE(iteration_level_ds.getDatatype() == Datatype::INT); + REQUIRE(iteration_level_ds.getExtent() == Extent{10}); + auto loaded_chunk = iteration_level_ds.loadChunk(); + iteration_level_ds.seriesFlush(); + for (size_t i = 0; i < 10; ++i) + { + REQUIRE(loaded_chunk.get()[i] == 5); + } + + auto constant_dataset = + read.iterations[0]["custom"] + .asContainerOf()["emptyDataset"]; + REQUIRE(constant_dataset.getDatatype() == Datatype::FLOAT); + REQUIRE(constant_dataset.getExtent() == Extent{0, 0, 0}); + } + read.close(); + + write = Series(filePath, Access::READ_WRITE); + { + std::vector data(10, 3); + + auto E_x = write.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf()["E"]["x"]; + E_x.resetDataset({Datatype::INT, {10}}); + E_x.storeChunk(data, {0}, {10}); + + auto e_pos_x = + write.iterations[0]["custom_particles"]["particles"] + .asContainerOf()["e"]["position"]["x"]; + e_pos_x.resetDataset({Datatype::INT, {10}}); + e_pos_x.storeChunk(data, {0}, {10}); + write.close(); + } + + read = Series(filePath, Access::READ_ONLY); + { + auto it0 = read.iterations[0]; + auto custom_meshes = it0["custom_meshes"]; + REQUIRE(custom_meshes["meshes"].asContainerOf().size() == 1); + REQUIRE( + read.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf() + .count("E") == 1); + auto E_x_loaded = read.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf()["E"]["x"] + .loadChunk(); + REQUIRE( + read.iterations[0]["custom_particles"]["particles"] + .asContainerOf() + .size() == 1); + REQUIRE( + read.iterations[0]["custom_particles"]["particles"] + .asContainerOf() + .count("e") == 1); + auto e_pos_x_loaded = + read.iterations[0]["custom_particles"]["particles"] + .asContainerOf()["e"]["position"]["x"] + .loadChunk(); + read.flush(); + + for (size_t i = 0; i < 10; ++i) + { + REQUIRE(E_x_loaded.get()[i] == 3); + REQUIRE(e_pos_x_loaded.get()[i] == 3); + } + } +} + +TEST_CASE("custom_hierarchies_no_rw", "[core]") +{ + std::string filePath = "../samples/custom_hierarchies_no_rw.json"; + Series write(filePath, Access::CREATE); + write.setMeshesPath(std::vector{"%%/meshes/"}); + write.iterations[0]["custom"]["hierarchy"]; + write.iterations[0]["custom"].setAttribute("string", "attribute"); + write.iterations[0]["custom"]["hierarchy"].setAttribute("number", 3); + write.iterations[0]["no_attributes"]; + + { + write.iterations[0]["custom"]["hierarchy"]; + write.iterations[0]["custom"] + .asContainerOf()["emptyDataset"] + .makeEmpty(Datatype::FLOAT, 3); + write.iterations[0]["custom"]["hierarchy"].setAttribute("number", 3); + write.iterations[0]["no_attributes"]; + auto iteration_level_ds = + write.iterations[0] + .asContainerOf()["iteration_level_dataset"]; + iteration_level_ds.resetDataset({Datatype::INT, {10}}); + std::vector data(10, 5); + iteration_level_ds.storeChunk(data); + write.flush(); + } + + { + std::vector data(10, 3); + + auto E_x = write.iterations[0]["custom_meshes"]["meshes"] + .asContainerOf()["E"]["x"]; + E_x.resetDataset({Datatype::INT, {10}}); + E_x.storeChunk(data, {0}, {10}); + + auto e_pos_x = + write.iterations[0]["custom_particles"]["particles"] + .asContainerOf()["e"]["position"]["x"]; + e_pos_x.resetDataset({Datatype::INT, {10}}); + e_pos_x.storeChunk(data, {0}, {10}); + + auto gnihihi = write.iterations[0]["custom_particles"]["particles"] + .asContainerOf(); + auto dataset = gnihihi["custom_dataset"]; + dataset.resetDataset({Datatype::INT, {10}}); + dataset.storeChunk(std::unique_ptr(new int[10]{}), {0}, {10}); + write.close(); + } + + Series read(filePath, Access::READ_ONLY); +} + TEST_CASE("myPath", "[core]") { #if openPMD_USE_INVASIVE_TESTS @@ -429,7 +675,8 @@ TEST_CASE("myPath", "[core]") recordComponent.template makeConstant(5678); }; - REQUIRE(pathOf(iteration.meshes) == vec_t{"data", "1234", "meshes"}); + // iteration.meshes is only an alias without a path of its own + // REQUIRE(pathOf(iteration.meshes) == vec_t{"data", "1234", "meshes"}); auto scalarMesh = iteration.meshes["e_chargeDensity"]; REQUIRE( @@ -450,7 +697,11 @@ TEST_CASE("myPath", "[core]") pathOf(vectorMeshComponent) == vec_t{"data", "1234", "meshes", "E", "x"}); - REQUIRE(pathOf(iteration.particles) == vec_t{"data", "1234", "particles"}); + // iteration.particles is only an alias without a path of its own + // REQUIRE(pathOf(iteration.particles) == vec_t{"data", "1234", + // "particles"}); REQUIRE( + // pathOf(iteration.particles) == + // vec_t{"iterations", "1234", "particles"}); auto speciesE = iteration.particles["e"]; REQUIRE(pathOf(speciesE) == vec_t{"data", "1234", "particles", "e"}); diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 7fd13822f3..c8de327464 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -2848,9 +2848,17 @@ TEST_CASE("git_hdf5_sample_structure_test", "[serial][hdf5]") REQUIRE( o.iterations[100].meshes.parent() == getWritable(&o.iterations[100])); + REQUIRE( + getWritable( + &o.iterations[100]["fields"].asContainerOf()["E"]) == + getWritable(&o.iterations[100].meshes["E"])); + REQUIRE( + o.iterations[100]["fields"].asContainerOf()["E"].parent() == + &o.iterations[100]["fields"].asContainerOf().writable()); REQUIRE( o.iterations[100].meshes["E"].parent() == - getWritable(&o.iterations[100].meshes)); + // Iteration::meshes is only an alias, this is the actual parent + &o.iterations[100]["fields"].asContainerOf().writable()); REQUIRE( o.iterations[100].meshes["E"]["x"].parent() == getWritable(&o.iterations[100].meshes["E"])); @@ -2860,13 +2868,17 @@ TEST_CASE("git_hdf5_sample_structure_test", "[serial][hdf5]") REQUIRE( o.iterations[100].meshes["E"]["z"].parent() == getWritable(&o.iterations[100].meshes["E"])); + REQUIRE( + getWritable(&o.iterations[100].meshes["rho"]) == + getWritable( + &o.iterations[100]["fields"].asContainerOf()["rho"])); REQUIRE( o.iterations[100].meshes["rho"].parent() == - getWritable(&o.iterations[100].meshes)); + getWritable(&o.iterations[100]["fields"])); REQUIRE( o.iterations[100] .meshes["rho"][MeshRecordComponent::SCALAR] - .parent() == getWritable(&o.iterations[100].meshes)); + .parent() == getWritable(&o.iterations[100]["fields"])); REQUIRE_THROWS_AS( o.iterations[100].meshes["cherries"], std::out_of_range); REQUIRE( @@ -2874,7 +2886,11 @@ TEST_CASE("git_hdf5_sample_structure_test", "[serial][hdf5]") getWritable(&o.iterations[100])); REQUIRE( o.iterations[100].particles["electrons"].parent() == - getWritable(&o.iterations[100].particles)); + getWritable(&o.iterations[100]["particles"])); + REQUIRE( + getWritable(&o.iterations[100].particles["electrons"]) == + getWritable(&o.iterations[100]["particles"] + .asContainerOf()["electrons"])); REQUIRE( o.iterations[100].particles["electrons"]["charge"].parent() == getWritable(&o.iterations[100].particles["electrons"]));