From b8d6d4746a7e93c3310af09a700f1a16280e2130 Mon Sep 17 00:00:00 2001 From: Juan Carrano Date: Thu, 12 Jul 2018 18:23:01 +0200 Subject: [PATCH 1/2] tools/mkconstfs2: Make templates configurable. The template strings to build the file can now be provided in an INI files. This allows reusing the script for other purposes other than doing constfs filesystems. --- dist/tools/mkconstfs/README.md | 5 ++ dist/tools/mkconstfs/default.ini | 37 +++++++++++++ dist/tools/mkconstfs/mkconstfs2.py | 86 +++++++++++++----------------- 3 files changed, 78 insertions(+), 50 deletions(-) create mode 100644 dist/tools/mkconstfs/default.ini diff --git a/dist/tools/mkconstfs/README.md b/dist/tools/mkconstfs/README.md index c5fc8bf08dd9..792f8fd737e2 100644 --- a/dist/tools/mkconstfs/README.md +++ b/dist/tools/mkconstfs/README.md @@ -19,3 +19,8 @@ structures that can be mounted using constfs. This is an alternative tool that takes a list of files instead of a whole directory. + +File generation can be customized via a INI file containing template +fragments. Note that the INI parser has comments disabled (to allow for +literal ";" and "#" characters). Also, leading whitespace is removed, +requiring the use of escape sequences. diff --git a/dist/tools/mkconstfs/default.ini b/dist/tools/mkconstfs/default.ini new file mode 100644 index 000000000000..2196d84321a3 --- /dev/null +++ b/dist/tools/mkconstfs/default.ini @@ -0,0 +1,37 @@ +[Main] + +header= /* This file was automatically generated by mkconstfs2. + \40* !!!! DO NOT EDIT !!!!! + \40*/ + + #include + #include "fs/constfs.h" + \n + +footer= \nstatic const constfs_t _fs_data = {{ + \40 .files = _files, + \40 .nfiles = sizeof(_files) / sizeof(_files[0]), + }}; + + vfs_mount_t {constfs_name} = {{ + \40 .fs = &constfs_file_system, + \40 .mount_point = "{mount_point}", + \40 .private_data = (void *)&_fs_data, + }};\n + +[Files] + +header= \nstatic const constfs_file_t _files[] = {\n +footer= };\n + +template = \40{{ + \40 .path = "{target_name_addroot}", + \40 .data = {buff_name}, + \40 .size = sizeof({buff_name}) + \40}},\n + +[Blob] + +header = \n/** {fname} **/ + static const uint8_t {varname}[] = {{\n +footer = };\n diff --git a/dist/tools/mkconstfs/mkconstfs2.py b/dist/tools/mkconstfs/mkconstfs2.py index a552baefefe3..6ac459e8242e 100755 --- a/dist/tools/mkconstfs/mkconstfs2.py +++ b/dist/tools/mkconstfs/mkconstfs2.py @@ -9,53 +9,18 @@ import itertools import mmap import shutil +import configparser +import codecs +from functools import partial from binascii import hexlify -C_HEADER = """/* This file was automatically generated by mkconstfs2. - * !!!! DO NOT EDIT !!!!! - */ - -#include -#include "fs/constfs.h" - -""" - -FILE_TEMPLATE = """ {{ - .path = "{target_name}", - .data = {buff_name}, - .size = sizeof({buff_name}) - }}, -""" - -C_FOOTER = """ -static const constfs_t _fs_data = {{ - .files = _files, - .nfiles = sizeof(_files) / sizeof(_files[0]), -}}; - -vfs_mount_t {constfs_name} = {{ - .fs = &constfs_file_system, - .mount_point = "{mount_pount}", - .private_data = (void *)&_fs_data, -}}; -""" - -FILES_DECL = """ -static const constfs_file_t _files[] = { -""" - -BLOB_DECL = """ -/** {fname} **/ -static const uint8_t {varname}[] = {{ -""" - def _relpath_p(path, start): return posixpath.relpath(pathlib.Path(os.path.abspath(path)).as_posix(), pathlib.Path(os.path.abspath(start)).as_posix()) -def mkconstfs(files, root_path, mount_point, constfs_name): +def mkconstfs(templates, files, root_path, mount_point, constfs_name): """Generate a C file containing a constant file system Return @@ -63,23 +28,32 @@ def mkconstfs(files, root_path, mount_point, constfs_name): chunks: Iterator yielding fragments of the of the output file. """ + def _d(x): + return codecs.decode(x, "unicode-escape") + + FILE_TEMPLATE = _d(templates['Files']['template']) + + _print_file_data = partial(print_file_data, _d(templates['Blob']['header']), + _d(templates['Blob']['footer'])) filemap = {f: (_mkident(i), _relpath_p(f, root_path)) for i, f in enumerate(files)} - yield C_HEADER + yield _d(templates['Main']['header']) yield from itertools.chain.from_iterable( - print_file_data(local_f, *f_data) for local_f, f_data in filemap.items()) + _print_file_data(local_f, *f_data) for local_f, f_data in filemap.items()) - yield FILES_DECL + yield _d(templates['Files']['header']) - yield from (FILE_TEMPLATE.format(target_name=_addroot(relp), + yield from (FILE_TEMPLATE.format(target_name=relp, + target_name_addroot=_addroot(relp), buff_name=ident) for ident, relp in sorted(filemap.values())) - yield "};\n" + yield _d(templates['Files']['footer']) - yield C_FOOTER.format(constfs_name=constfs_name, mount_pount=mount_point) + yield _d(templates['Main']['footer']).format(constfs_name=constfs_name, + mount_point=mount_point) def _addroot(fname): @@ -90,7 +64,7 @@ def _mkident(k): return "_file{:02X}".format(k) -def print_file_data(local_fname, varname, target_fname=""): +def print_file_data(header, footer, local_fname, varname, target_fname=""): """Convert a file into a static C array: Parameters @@ -106,7 +80,7 @@ def print_file_data(local_fname, varname, target_fname=""): chunks: Iterator yielding fragments of the of the output text. """ - yield BLOB_DECL.format(fname=target_fname, varname=varname) + yield header.format(fname=target_fname, varname=varname) def byte2s(b): return "0x{},".format(hexlify(b).decode('utf-8')) @@ -127,16 +101,25 @@ def chunk(iterable, blocksize): ) ) - yield "};\n" + yield footer def main(): + this_script_dir = os.path.dirname(os.path.realpath(sys.argv[0])) + default_templates = os.path.join(this_script_dir, 'default.ini') + parser = argparse.ArgumentParser( - description="Embed files into a constant file system") + description="Embed files into a constant file system. " + "See %s to learn how to customize the generator." % default_templates) parser.add_argument("-m", '--mount', metavar="mountpoint", help="Where to mount the resulting fs", default="/") + parser.add_argument("-t", '--template', metavar="ini_file", + help="Configuration file containing template chunks", + type=argparse.FileType(), + default=default_templates) + parser.add_argument("-o", '--output', metavar="output_file", help="Write the output to a file instead of stdout. " "The file is only written if the command is successful " @@ -156,7 +139,10 @@ def main(): ns = parser.parse_args() - f_chunks = mkconstfs(ns.files, ns.root, ns.mount, ns.name) + template = configparser.ConfigParser(comment_prefixes=()) + template.read_file(ns.template) + + f_chunks = mkconstfs(template, ns.files, ns.root, ns.mount, ns.name) if ns.output: tmp_out = io.StringIO() From ac4dd603296977d55361782e1ba5d5f1e056bb93 Mon Sep 17 00:00:00 2001 From: Juan Carrano Date: Thu, 1 Nov 2018 15:24:41 +0100 Subject: [PATCH 2/2] tools/xxd-plus: rename from mkconstfs2. Changed the name of the tool and document the template. --- dist/tools/mkconstfs/default.ini | 37 ------------- dist/tools/xxd-plus/default.ini | 54 +++++++++++++++++++ .../mkconstfs2.py => xxd-plus/xxd-plus.py} | 17 +++--- 3 files changed, 64 insertions(+), 44 deletions(-) delete mode 100644 dist/tools/mkconstfs/default.ini create mode 100644 dist/tools/xxd-plus/default.ini rename dist/tools/{mkconstfs/mkconstfs2.py => xxd-plus/xxd-plus.py} (87%) diff --git a/dist/tools/mkconstfs/default.ini b/dist/tools/mkconstfs/default.ini deleted file mode 100644 index 2196d84321a3..000000000000 --- a/dist/tools/mkconstfs/default.ini +++ /dev/null @@ -1,37 +0,0 @@ -[Main] - -header= /* This file was automatically generated by mkconstfs2. - \40* !!!! DO NOT EDIT !!!!! - \40*/ - - #include - #include "fs/constfs.h" - \n - -footer= \nstatic const constfs_t _fs_data = {{ - \40 .files = _files, - \40 .nfiles = sizeof(_files) / sizeof(_files[0]), - }}; - - vfs_mount_t {constfs_name} = {{ - \40 .fs = &constfs_file_system, - \40 .mount_point = "{mount_point}", - \40 .private_data = (void *)&_fs_data, - }};\n - -[Files] - -header= \nstatic const constfs_file_t _files[] = {\n -footer= };\n - -template = \40{{ - \40 .path = "{target_name_addroot}", - \40 .data = {buff_name}, - \40 .size = sizeof({buff_name}) - \40}},\n - -[Blob] - -header = \n/** {fname} **/ - static const uint8_t {varname}[] = {{\n -footer = };\n diff --git a/dist/tools/xxd-plus/default.ini b/dist/tools/xxd-plus/default.ini new file mode 100644 index 000000000000..fbe096a5b4a9 --- /dev/null +++ b/dist/tools/xxd-plus/default.ini @@ -0,0 +1,54 @@ +-- To avoid colliding with ; and # in C, this INI file uses the nonstandard +-- "--" for comments + +[Main] + +-- File header. This is placed before any other content. +header= /* This file was automatically generated by xxd-plus. + \40* !!!! DO NOT EDIT !!!!! + \40*/ + + #include + #include "fs/constfs.h" + \n + +-- File footer. This is placed at the end. +-- mount_point: mount point (as given by the command line's "-m" argument) +-- constfs_name: structure name (also provided in the command line) +footer= \nstatic const constfs_t _fs_data = {{ + \40 .files = _files, + \40 .nfiles = sizeof(_files) / sizeof(_files[0]), + }}; + + vfs_mount_t {constfs_name} = {{ + \40 .fs = &constfs_file_system, + \40 .mount_point = "{mount_point}", + \40 .private_data = (void *)&_fs_data, + }};\n + +[Files] + +-- These chunks are used once, after all blobs have been printed +header= \nstatic const constfs_file_t _files[] = {\n +footer= };\n + +-- between the header and the footer, for each file the following template is +-- executed: +-- target_name: is the target filename (i.e the filename relative to the mount point) +-- target_name_addroot: like target_name but it always starts with a "/". +-- buff_name: the value of {varname} for the Blob corresponding to this file. +template = \40{{ + \40 .path = "{target_name_addroot}", + \40 .data = {buff_name}, + \40 .size = sizeof({buff_name}) + \40}},\n + +[Blob] +-- These chunks are placed at the beginning and end of each "textified" +-- file. +-- fname is the target filename (i.e the filename relative to the mount point) +-- varname is a unique identifiear assigned by the tool to this variable. + +header = \n/** {fname} **/ + static const uint8_t {varname}[] = {{\n +footer = };\n diff --git a/dist/tools/mkconstfs/mkconstfs2.py b/dist/tools/xxd-plus/xxd-plus.py similarity index 87% rename from dist/tools/mkconstfs/mkconstfs2.py rename to dist/tools/xxd-plus/xxd-plus.py index 6ac459e8242e..df2f805d1ec3 100755 --- a/dist/tools/mkconstfs/mkconstfs2.py +++ b/dist/tools/xxd-plus/xxd-plus.py @@ -20,7 +20,7 @@ def _relpath_p(path, start): pathlib.Path(os.path.abspath(start)).as_posix()) -def mkconstfs(templates, files, root_path, mount_point, constfs_name): +def xxd(templates, files, root_path, mount_point, constfs_name): """Generate a C file containing a constant file system Return @@ -109,14 +109,17 @@ def main(): default_templates = os.path.join(this_script_dir, 'default.ini') parser = argparse.ArgumentParser( - description="Embed files into a constant file system. " + description="Embed files into a C arrays and structures. " + "By default this generates a vfs_mount_t structure for a constant file " + "system (constfs), but that can be changed via templates. " "See %s to learn how to customize the generator." % default_templates) parser.add_argument("-m", '--mount', metavar="mountpoint", help="Where to mount the resulting fs", default="/") parser.add_argument("-t", '--template', metavar="ini_file", - help="Configuration file containing template chunks", + help="Configuration file containing template chunks. " + "The default template will generate a constfs-compatible structure.", type=argparse.FileType(), default=default_templates) @@ -127,22 +130,22 @@ def main(): parser.add_argument("-r", '--root', metavar="root_base_path", type=pathlib.Path, - help="Paths on the constf will be generated for the real " + help="Paths on the C code will be generated from the real " "path of the files by considering this path to be the root " "By default the current directory (.) is used", default=pathlib.Path()) - parser.add_argument("name", help="Name for the vfs_mount_t structure") + parser.add_argument("name", help="Name for the main data structure") parser.add_argument("files", nargs="+", type=pathlib.Path, help="Files to be included.") ns = parser.parse_args() - template = configparser.ConfigParser(comment_prefixes=()) + template = configparser.ConfigParser(comment_prefixes=("--",)) template.read_file(ns.template) - f_chunks = mkconstfs(template, ns.files, ns.root, ns.mount, ns.name) + f_chunks = xxd(template, ns.files, ns.root, ns.mount, ns.name) if ns.output: tmp_out = io.StringIO()