Fix #80, support an include mechanism for PDI config

.github/pull_request_template.md

@@ -7,7 +7,7 @@ Before merging your code, please check the following:
 * [ ] you have added a line describing your changes to the Changelog;
 * [ ] you have added unit tests for any new or improved feature;
 * [ ] in case you updated dependencies, you have checked pdi/docs/CheckList.md;
-* [ ] in case of a change in pdi.h, this same change must be reflected in no-pdi/include/pdi.h;
+* [ ] in case of a change in pdi.h, this same change must be reflected in mock_pdi/pdi.h;
 * you have checked your code format:
   - [ ] you have checked that you respect all conventions specified in CONTRIBUTING.md;
   - [ ] you have checked that the indentation and formatting conforms to the `.clang-format`;

CHANGELOG.md

@@ -161,6 +161,8 @@ and this project adheres to
 
 #### Added
 * Added tests for using an installed PDI
+* Support an include mechanism for PDI config,
+  to include other YAML files into root YAML document.
 
 #### Fixed
 * Support multiple consecutive calls to `find_package(PDI)`

CMakeLists.txt

@@ -529,6 +529,14 @@ sbuild_add_module(PDI_COMBINATION_TESTS
 	SUBSTEPS test
 )
 
+sbuild_add_module(PDI_TEST_INCLUDE_YAML
+	ENABLE_BUILD_FLAG BUILD_TESTING
+	SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tests/include_yaml_tests"
+	DEPENDS PDI
+	INSTALL_COMMAND ""
+	SUBSTEPS test
+)
+
 sbuild_add_module(PDI_API_TESTS
 	ENABLE_BUILD_FLAG BUILD_TESTING
 	SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tests/api_tests"

pdi/CMakeLists.txt

@@ -94,6 +94,7 @@ if("${BUILD_FORTRAN}")
 	list(APPEND PARACONF_COMPONENTS "f90")
 endif()
 set(PDI_REQUIRED_PARACONF_VERSION 1.0) # used in PDIConfig too
+# set(PDI_REQUIRED_PARACONF_VERSION 1.1) # used in PDIConfig too
 find_package(paraconf "${PDI_REQUIRED_PARACONF_VERSION}" REQUIRED COMPONENTS ${PARACONF_COMPONENTS}) # must match PDIConfig.cmake.in
 set(PDI_REQUIRED_SPDLOG_VERSION 1.9) # used in PDIConfig too
 find_package(spdlog "${PDI_REQUIRED_SPDLOG_VERSION}" REQUIRED) # must match PDIConfig.cmake.in

pdi/docs/Specification_tree_ref.md

@@ -9,28 +9,34 @@ The *specification tree root* is a **mapping** that contains the following keys:
 
 |key|value|
 |:--|:----|
-|`"types"` (*optional*)|a \ref types_map_node|
-|`"data"` (*optional*)|a \ref data_map_node|
+|`"include"` (*optional*)|a \ref include_node|
+|`"logging"` (*optional*)|a \ref logging_node|
 |`"metadata"` (*optional*)|a \ref data_map_node|
+|`"data"` (*optional*)|a \ref data_map_node|
 |`"plugins"` (*optional*)|a \ref plugin_map_node|
-|`"logging"` (*optional*)|a \ref logging_node|
 |`"plugin_path"` (*optional*)|a \ref plugin_path_map_node|
+|`"types"` (*optional*)|a \ref types_map_node|
 |`".*"` (*optional*)| *anything* |
 
-* the `types` section specifies user-defined datatypes
-* the `data` and `metadata` sections specify the type of the data in buffers
+* the `include` section specify other YAML configuration part of a same
+  Paraconf tree
+* the `logging` section specify logger properties
+* the `metadata` and `data` sections specify the type of the data in buffers
   exposed by the application; for `metadata`, %PDI keeps a copy while it only
   keeps references for `data`,
 * the `plugins` section specifies the list of plugins to load and their
   configuration,
 * the `plugin_path` section specifies the path to a directory where %PDI should
   search for plugins
-* the `logging` section specify logger properties
+* the `types` section specifies user-defined datatypes
 * additional sections are ignored.
 
 ### Example:
 
 ```{.python}
+include:
+  - my_other_configuration_file.yml
+logging: trace
 metadata:
   my_metadata: int
 data:
@@ -43,6 +49,116 @@ plugins:
   mpi: #...
 ```
 
+## include {#include_node}
+
+An *include* is a sequence of scalars, each scalar representing a path to
+a \subpage YAML configuration file. 
+
+An *include* is fully supported in both the root YAML configuration and
+further included YAML configuration.
+
+Examples:
+
+```yaml
+include: /<full>/<path>/<to>/subfile.yml
+```
+Paraconf 1.1+ also enables relative path :
+```yaml
+include: subfile.yml
+```
+
+* entirely optional, without default value
+
+
+## logging {#logging_node}
+
+A *logging* can be **any of**:
+* a \ref logging_map_node,
+* a \ref logging_level_node,
+
+A *logging* is fully supported in \ref root_node and
+any \ref plugin_map_node "plugin_map_node".
+
+
+## logging_level {#logging_level_node}
+
+A *logging_level* is a scalar which determines verbosity level. It can be set to 
+  (from the most to the least verbose): 
+* `"debug"` - shows a log when a normal situation of the execution might be
+  useful to understand the behavior of the library,
+* `"info"` - shows a log when a normal situation of the execution is likely
+  useful to understand the behavior of the library,
+* `"warn"` - shows a log when a very likely invalid situation has been detected
+  by the library (user input that is technically valid, but very unusual for
+  example),
+* `"error"` - shows a log when an invalid situation has been detected by the
+  library (invalid user input, invalid hardware behaviour, etc.),
+* `"off"` - logs are disabled.
+
+Example:
+
+```yaml
+logging: "debug"
+```
+
+* by default `level` is set to `info`
+
+
+## logging_map {#logging_map_node}
+
+A *logging_map* is a **mapping** that contains the following keys:
+
+|key|value|
+|:--|:----|
+|`"level"`  (*optional*)|a \ref logging_level_node|
+|`"pattern"` (*optional*)|a logger prefix pattern of spdlog|
+|`"output"` (*optional*)|a \ref logging_output_map_node|
+
+* spdlog pattern is a string that is parsed by spdlog library,
+  (see more: https://github.com/gabime/spdlog/wiki/3.-Custom-formatting)
+* %PDI introduces new special flag `%{<EXPR>}`, where `<EXPR>` represents a
+  \ref expression_node "string-valued $-expression"|a $-expression that will be
+  evaluated just after all plugins have been initialized,
+* *pattern* by default is set to (where %n is `PDI` or a plugin name):
+  ```
+  [%T][%n] *** %^%l%$: %v
+  ```
+  for serial execution and:
+  ```
+  [%T][%{MPI_COMM_WORLD.rank:06d}][%n] *** %^%l%$: %v
+  ```
+  when running application with MPI/
+
+Example:
+
+```yaml
+logging:
+  level: "debug"
+  pattern: "[%{MPI_COMM_WORLD.rank:04d}][%n][%l]"
+```
+
+
+## logging_output_map {#logging_output_map_node}
+
+A *logging_output_map* is a **mapping** that contains the following keys:
+
+|key|value|
+|:--|:----|
+|`"file"`  (*optional*)|a path of the file where to write logs|
+|`"console"` (*optional*)|`on` or `off`|
+
+* by default when `file` is defined, `console` is set to `off`.
+
+Example:
+
+```yaml
+logging:
+  level: "debug"
+  output:
+    file: "test.log"
+    console: "on"
+```
+
 
 ## array_type {#array_type_node}
 
@@ -774,115 +890,6 @@ type: intptr
 ```
 
 
-## logging {#logging_node}
-
-A *logging* can be **any of**:
-* a \ref logging_map_node,
-* a \ref logging_level_node,
-
-A *logging* is fully supported in \ref root_node and any \ref plugin_map_node .
-
-
-## logging_level {#logging_level_node}
-
-A *logging_level* is a scalar which determines verbosity level. It can be set to 
-  (from the most to the least verbose): 
-* `"debug"` - shows a log when a normal situation of the execution might be
-  useful to understand the behavior of the library,
-* `"info"` - shows a log when a normal situation of the execution is likely
-  useful to understand the behavior of the library,
-* `"warn"` - shows a log when a very likely invalid situation has been detected
-  by the library (user input that is technically valid, but very unusual for
-  example),
-* `"error"` - shows a log when an invalid situation has been detected by the
-  library (invalid user input, invalid hardware behaviour, etc.),
-* `"off"` - logs are disabled.
-
-Examples:
-
-```yaml
-logging: "debug"
-```
-
-* by default `level` is set to `info`
-
-
-## logging_map {#logging_map_node}
-
-A *logging_map* is a **mapping** that contains the following keys:
-
-|key|value|
-|:--|:----|
-|`"level"`  (*optional*)|a \ref logging_level_node|
-|`"pattern"` (*optional*)|a logger prefix pattern of spdlog|
-|`"output"` (*optional*)|a \ref logging_output_map_node|
-
-* spdlog pattern is a string that is parsed by spdlog library,
-  (see more: https://github.com/gabime/spdlog/wiki/3.-Custom-formatting)
-* %PDI introduces new special flag `%{<EXPR>}`, where `<EXPR>` represents a
-  \ref expression_node "string-valued $-expression"|a $-expression that will be
-  evaluated just after all plugins have been initialized,
-* *pattern* by default is set to (where %n is `PDI` or a plugin name):
-  ```
-  [%T][%n] *** %^%l%$: %v
-  ```
-  for serial execution and:
-  ```
-  [%T][%{MPI_COMM_WORLD.rank:06d}][%n] *** %^%l%$: %v
-  ```
-  when running application with MPI/
-
-Example:
-
-```yaml
-logging:
-  level: "debug"
-  pattern: "[%{MPI_COMM_WORLD.rank:04d}][%n][%l]"
-```
-
-
-## logging_output_map {#logging_output_map_node}
-
-A *logging_output_map* is a **mapping** that contains the following keys:
-
-|key|value|
-|:--|:----|
-|`"file"`  (*optional*)|a path of the file where to write logs|
-|`"console"` (*optional*)|`on` or `off`|
-
-* by default when `file` is defined, `console` is set to `off`.
-
-Example:
-
-```yaml
-logging:
-  level: "debug"
-  output:
-    file: "test.log"
-    console: "on"
-```
-
-### Example:
-
-```{.python}
-type: struct
-members:
-  - my_char: char
-```
-
-```{.python}
-type: struct
-members:
-  - my_long: int64
-  - my_array:
-      type: array
-      subtype: int64
-      size: [10, 10]
-```
-
-See \ref struct_type_node for more examples.
-
-
 ## logical_type {#logical_type_node}
 
 A *logical_type* is a **mapping** that contains the following keys:

pdi/docs/Yaml.md

@@ -104,9 +104,9 @@ complete syntax).
 
 ## YAML Parsing with Paraconf
 
-The PDI_init function gets as parameter a tree with `logging`, `data`,
-`metadata` and `plugins` maps defined in its root. User can define its own
-values in yaml and pass to %PDI only the subtree:
+The PDI_init function gets as parameter a tree with `logging`, `include`,
+`data`, `metadata` and `plugins` maps defined in its root. User can define its
+own values in yaml and pass to %PDI only the subtree:
 
 ```yaml
 duration: 0.75

pdi/include/pdi/context.h

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (C) 2015-2024 Commissariat a l'energie atomique et aux energies alternatives (CEA)
+ * Copyright (C) 2015-2026 Commissariat a l'energie atomique et aux energies alternatives (CEA)
  * Copyright (C) 2021 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC)
  * All rights reserved.
  *

pdi/include/pdi/context_proxy.h

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (C) 2021-2024 Commissariat a l'energie atomique et aux energies alternatives (CEA)
+ * Copyright (C) 2021-2026 Commissariat a l'energie atomique et aux energies alternatives (CEA)
  * Copyright (C) 2019-2021 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC)
  * All rights reserved.
  *

pdi/src/data_descriptor_impl.cxx

@@ -76,6 +76,14 @@ Data_descriptor_impl::Data_descriptor_impl(Global_context& ctx, const char* name
 	, m_metadata{false}
 {}
 
+Data_descriptor_impl::Data_descriptor_impl(Global_context& ctx, const char* name, PC_tree_t source_node)
+	: m_context{ctx}
+	, m_type{UNDEF_TYPE}
+	, m_name{name}
+	, m_metadata{false}
+	, m_source_node{source_node}
+{}
+
 Data_descriptor_impl::Data_descriptor_impl(Data_descriptor_impl&&) = default;
 
 Data_descriptor_impl::~Data_descriptor_impl()

pdi/src/data_descriptor_impl.h

@@ -59,11 +59,14 @@ class PDI_EXPORT Data_descriptor_impl: public Data_descriptor
 
 	bool m_metadata;
 
+	PC_tree_t m_source_node; ///< YAML key node of the first definition, for duplicate error reporting
 
 	/** Create an empty descriptor
 	 */
 	Data_descriptor_impl(Global_context& ctx, const char* name);
 
+	Data_descriptor_impl(Global_context& ctx, const char* name, PC_tree_t source_node);
+
 	Data_descriptor_impl(const Data_descriptor_impl&) = delete;
 
 	Data_descriptor_impl& operator= (const Data_descriptor_impl&) = delete;