tools/mkconstfs2: Make templates configurable.

dist/tools/mkconstfs/README.md

@@ -19,3 +19,8 @@ structures that can be mounted using constfs.
 
 This is an alternative tool that takes a list of files instead of a whole
 directory.
+
+File generation can be customized via a INI file containing template
+fragments. Note that the INI parser has comments disabled (to allow for
+literal ";" and "#" characters). Also, leading whitespace is removed,
+requiring the use of escape sequences.

dist/tools/xxd-plus/default.ini

@@ -0,0 +1,54 @@
+-- To avoid colliding with ; and # in C, this INI file uses the nonstandard
+-- "--" for comments
+
+[Main]
+
+-- File header. This is placed before any other content.
+header= /* This file was automatically generated by xxd-plus.
+ \40* !!!! DO NOT EDIT !!!!!
+ \40*/
+
+ #include <stdint.h>
+ #include "fs/constfs.h"
+ \n
+
+-- File footer. This is placed at the end.
+-- mount_point: mount point (as given by the command line's "-m" argument)
+-- constfs_name: structure name (also provided in the command line)
+footer= \nstatic const constfs_t _fs_data = {{
+ \40   .files = _files,
+ \40   .nfiles = sizeof(_files) / sizeof(_files[0]),
+ }};
+
+ vfs_mount_t {constfs_name} = {{
+ \40   .fs = &constfs_file_system,
+ \40   .mount_point = "{mount_point}",
+ \40   .private_data = (void *)&_fs_data,
+ }};\n
+
+[Files]
+
+-- These chunks are used once, after all blobs have been printed
+header= \nstatic const constfs_file_t _files[] = {\n
+footer= };\n
+
+-- between the header and the footer, for each file the following template is
+-- executed:
+-- target_name: is the target filename (i.e the filename relative to the mount point)
+-- target_name_addroot: like target_name but it always starts with a "/".
+-- buff_name: the value of {varname} for the Blob corresponding to this file.
+template = \40{{
+ \40 .path = "{target_name_addroot}",
+ \40 .data = {buff_name},
+ \40 .size = sizeof({buff_name})
+ \40}},\n
+
+[Blob]
+-- These chunks are placed at the beginning and end of each "textified"
+-- file.
+-- fname is the target filename (i.e the filename relative to the mount point)
+-- varname is a unique identifiear assigned by the tool to this variable.
+
+header = \n/** {fname} **/
+ static const uint8_t {varname}[] = {{\n
+footer = };\n

dist/tools/mkconstfs/mkconstfs2.py → dist/tools/xxd-plus/xxd-plus.py

@@ -9,77 +9,51 @@
 import itertools
 import mmap
 import shutil
+import configparser
+import codecs
+from functools import partial
 from binascii import hexlify
 
-C_HEADER = """/* This file was automatically generated by mkconstfs2.
- * !!!! DO NOT EDIT !!!!!
- */
-
-#include <stdint.h>
-#include "fs/constfs.h"
-
-"""
-
-FILE_TEMPLATE = """ {{
-  .path = "{target_name}",
-  .data = {buff_name},
-  .size = sizeof({buff_name})
- }},
-"""
-
-C_FOOTER = """
-static const constfs_t _fs_data = {{
-    .files = _files,
-    .nfiles = sizeof(_files) / sizeof(_files[0]),
-}};
-
-vfs_mount_t {constfs_name} = {{
-    .fs = &constfs_file_system,
-    .mount_point = "{mount_pount}",
-    .private_data = (void *)&_fs_data,
-}};
-"""
-
-FILES_DECL = """
-static const constfs_file_t _files[] = {
-"""
-
-BLOB_DECL = """
-/** {fname} **/
-static const uint8_t {varname}[] = {{
-"""
-
 
 def _relpath_p(path, start):
     return posixpath.relpath(pathlib.Path(os.path.abspath(path)).as_posix(),
                              pathlib.Path(os.path.abspath(start)).as_posix())
 
 
-def mkconstfs(files, root_path, mount_point, constfs_name):
+def xxd(templates, files, root_path, mount_point, constfs_name):
     """Generate a C file containing a constant file system
 
     Return
     ------
 
     chunks: Iterator yielding fragments of the of the output file.
     """
+    def _d(x):
+        return codecs.decode(x, "unicode-escape")
+
+    FILE_TEMPLATE = _d(templates['Files']['template'])
+
+    _print_file_data = partial(print_file_data, _d(templates['Blob']['header']),
+                               _d(templates['Blob']['footer']))
 
     filemap = {f: (_mkident(i), _relpath_p(f, root_path))
                for i, f in enumerate(files)}
 
-    yield C_HEADER
+    yield _d(templates['Main']['header'])
     yield from itertools.chain.from_iterable(
-                print_file_data(local_f, *f_data) for local_f, f_data in filemap.items())
+                _print_file_data(local_f, *f_data) for local_f, f_data in filemap.items())
 
-    yield FILES_DECL
+    yield _d(templates['Files']['header'])
 
-    yield from (FILE_TEMPLATE.format(target_name=_addroot(relp),
+    yield from (FILE_TEMPLATE.format(target_name=relp,
+                                     target_name_addroot=_addroot(relp),
                                      buff_name=ident)
                 for ident, relp in sorted(filemap.values()))
 
-    yield "};\n"
+    yield _d(templates['Files']['footer'])
 
-    yield C_FOOTER.format(constfs_name=constfs_name, mount_pount=mount_point)
+    yield _d(templates['Main']['footer']).format(constfs_name=constfs_name,
+                                                 mount_point=mount_point)
 
 
 def _addroot(fname):
@@ -90,7 +64,7 @@ def _mkident(k):
     return "_file{:02X}".format(k)
 
 
-def print_file_data(local_fname, varname, target_fname=""):
+def print_file_data(header, footer, local_fname, varname, target_fname=""):
     """Convert a file into a static C array:
 
     Parameters
@@ -106,7 +80,7 @@ def print_file_data(local_fname, varname, target_fname=""):
     chunks: Iterator yielding fragments of the of the output text.
     """
 
-    yield BLOB_DECL.format(fname=target_fname, varname=varname)
+    yield header.format(fname=target_fname, varname=varname)
 
     def byte2s(b):
         return "0x{},".format(hexlify(b).decode('utf-8'))
@@ -127,36 +101,51 @@ def chunk(iterable, blocksize):
                         )
                        )
 
-    yield "};\n"
+    yield footer
 
 
 def main():
+    this_script_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
+    default_templates = os.path.join(this_script_dir, 'default.ini')
+
     parser = argparse.ArgumentParser(
-            description="Embed files into a constant file system")
+            description="Embed files into a C arrays and structures. "
+            "By default this generates a vfs_mount_t structure for a constant file "
+            "system (constfs), but that can be changed via templates. "
+            "See %s to learn how to customize the generator." % default_templates)
 
     parser.add_argument("-m", '--mount', metavar="mountpoint",
                         help="Where to mount the resulting fs", default="/")
 
+    parser.add_argument("-t", '--template', metavar="ini_file",
+                        help="Configuration file containing template chunks. "
+                             "The default template will generate a constfs-compatible structure.",
+                        type=argparse.FileType(),
+                        default=default_templates)
+
     parser.add_argument("-o", '--output', metavar="output_file",
                         help="Write the output to a file instead of stdout. "
                              "The file is only written if the command is successful "
                              "(i.e. there is no partial output")
 
     parser.add_argument("-r", '--root', metavar="root_base_path",
                         type=pathlib.Path,
-                        help="Paths on the constf will be generated for the real "
+                        help="Paths on the C code will be generated from the real "
                         "path of the files by considering this path to be the root "
                         "By default the current directory (.) is used",
                         default=pathlib.Path())
 
-    parser.add_argument("name", help="Name for the vfs_mount_t structure")
+    parser.add_argument("name", help="Name for the main data structure")
 
     parser.add_argument("files", nargs="+", type=pathlib.Path,
                         help="Files to be included.")
 
     ns = parser.parse_args()
 
-    f_chunks = mkconstfs(ns.files, ns.root, ns.mount, ns.name)
+    template = configparser.ConfigParser(comment_prefixes=("--",))
+    template.read_file(ns.template)
+
+    f_chunks = xxd(template, ns.files, ns.root, ns.mount, ns.name)
 
     if ns.output:
         tmp_out = io.StringIO()