From d59a7d75bf65a4438a21e74297ea44530e61a2e4 Mon Sep 17 00:00:00 2001 From: David Schneller Date: Tue, 29 Apr 2025 15:46:40 +0200 Subject: [PATCH 1/9] Add meta code generation (templating) --- yateto/generator.py | 8 ++++- yateto/metagen.py | 82 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 yateto/metagen.py diff --git a/yateto/generator.py b/yateto/generator.py index ce65cd9..6ac9eaf 100644 --- a/yateto/generator.py +++ b/yateto/generator.py @@ -455,7 +455,13 @@ def unit_test_body(cpp, testFramework): cpp.include(fInit.hName) with cpp.Namespace(namespace): initGen.generateInitCpp(cpp) - + + prefixnsp = lambda a: a.name if a.namespace == '' else f'{a.namespace}::{a.name}' + return { + 'namespace': namespace, + 'tensors': set(tensor.baseNameWithNamespace() for tensor in tensors.values()) | set(scalar.baseNameWithNamespace() for scalar in scalars), + 'kernels': set(prefixnsp(kernel) for kernel in self._kernels) | set(prefixnsp(family) for family in self._kernelFamilies.values()) + } class NamespacedGenerator(object): def __init__(self, generator, namespace): diff --git a/yateto/metagen.py b/yateto/metagen.py new file mode 100644 index 0000000..46dfb38 --- /dev/null +++ b/yateto/metagen.py @@ -0,0 +1,82 @@ +from .codegen.code import Cpp +import os + +class MetaGenerator: + def __init__(self, templateType): + self.templateType = templateType + self.generators = [] + + def add_generator(self, template, generator, *args, **kwargs): + assert len(self.templateType) == len(template) + self.generators += [{ + 'template': template, + 'generator': generator, + 'args': args, + 'kwargs': kwargs + }] + + def generate(self, outputDir='', namespace='yateto'): + tensors = {} + kernels = {} + + namespacepfx = 'yatetometagen' + for i, gendata in enumerate(self.generators): + subnamespace = f'{namespace}::{namespacepfx}{i}' + outdir = os.path.join(outputDir, f'metagen{i}') + os.makedirs(outdir, exist_ok=True) + + generator = gendata['generator'] + template = gendata['template'] + args = gendata['args'] + kwargs = gendata['kwargs'] + result = generator.generate(*args, **kwargs, namespace=subnamespace, outputDir=outdir) + for tensor in result['tensors']: + if tensor not in tensors: + tensors[tensor] = [] + tensors[tensor] += [(subnamespace, template)] + for tensor in result['kernels']: + if tensor not in kernels: + kernels[tensor] = [] + kernels[tensor] += [(subnamespace, template)] + + # TODO: open meta header + with Cpp(os.path.join(outputDir, 'tensor.h')) as header: + with header.HeaderGuard('METAGEN_TENSOR_H_'): + for i, gendata in enumerate(self.generators): + header.include(f'metagen{i}/tensor.h') + with header.Namespace(namespace): + for tensor in tensors: + self.template(header, tensor, tensors[tensor], 'tensor') + + with Cpp(os.path.join(outputDir, 'init.h')) as header: + with header.HeaderGuard('METAGEN_INIT_H_'): + for i, gendata in enumerate(self.generators): + header.include(f'metagen{i}/init.h') + with header.Namespace(namespace): + for tensor in tensors: + self.template(header, tensor, tensors[tensor], 'init') + + with Cpp(os.path.join(outputDir, 'kernel.h')) as header: + with header.HeaderGuard('METAGEN_KERNEL_H_'): + for i, gendata in enumerate(self.generators): + header.include(f'metagen{i}/kernel.h') + with header.Namespace(namespace): + for kernel in kernels: + self.template(header, kernel, kernels[kernel], 'kernel') + + def template(self, header, prename, foundin, subnsp): + splitname = prename.split('::') + + name = '::'.join(splitname[:-1] + [subnsp, splitname[-1]]) + + escname = name.replace(':', '_') + internalName = f'Internal_{escname}' + + templatetypes = ', '.join(f'{typ} Arg{i}' for i, typ in enumerate(self.templateType)) + templateargs = ', '.join(f'Arg{i}' for i, _ in enumerate(self.templateType)) + + header(f'template<{templatetypes}> struct internal::{internalName};') + for gnsp, spec in foundin: + spectext = ', '.join(str(specpart) for specpart in spec) + header(f'template<> struct internal::{internalName}<{spectext}> {"{"} using Type = ::{gnsp}::{name}; {"}"};') + header(f'template<{templatetypes}> using {name} = typename internal::{internalName}<{templateargs}>::Type;') From 311117c6242788d71db3e657e810c1f06523d81c Mon Sep 17 00:00:00 2001 From: David Schneller Date: Sat, 26 Jul 2025 21:34:44 +0200 Subject: [PATCH 2/9] Fix build --- yateto/metagen.py | 45 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/yateto/metagen.py b/yateto/metagen.py index 46dfb38..d757ce4 100644 --- a/yateto/metagen.py +++ b/yateto/metagen.py @@ -63,20 +63,43 @@ def generate(self, outputDir='', namespace='yateto'): with header.Namespace(namespace): for kernel in kernels: self.template(header, kernel, kernels[kernel], 'kernel') + + with Cpp(os.path.join(outputDir, 'tensor.cpp')) as header: + for i, gendata in enumerate(self.generators): + header.include(f'metagen{i}/tensor.cpp') + + with Cpp(os.path.join(outputDir, 'init.cpp')) as header: + for i, gendata in enumerate(self.generators): + header.include(f'metagen{i}/init.cpp') + + with Cpp(os.path.join(outputDir, 'kernel.cpp')) as header: + for i, gendata in enumerate(self.generators): + header.include(f'metagen{i}/kernel.cpp') + + def namespacing(self, header, spaces, inner): + if len(spaces) == 0: + inner() + else: + with header.Namespace(spaces[0]): + self.namespacing(header, spaces[1:], inner) def template(self, header, prename, foundin, subnsp): splitname = prename.split('::') - name = '::'.join(splitname[:-1] + [subnsp, splitname[-1]]) - - escname = name.replace(':', '_') - internalName = f'Internal_{escname}' + def inner(): + name = splitname[-1] + fullname = '::'.join(splitname[:-1] + [subnsp, splitname[-1]]) + escname = name.replace(':', '_') + internalName = f'Internal_{escname}' - templatetypes = ', '.join(f'{typ} Arg{i}' for i, typ in enumerate(self.templateType)) - templateargs = ', '.join(f'Arg{i}' for i, _ in enumerate(self.templateType)) + templatetypes = ', '.join(f'{typ} Arg{i}' for i, typ in enumerate(self.templateType)) + templateargs = ', '.join(f'Arg{i}' for i, _ in enumerate(self.templateType)) + + with header.Namespace('internal'): + header(f'template<{templatetypes}> struct {internalName};') + for gnsp, spec in foundin: + spectext = ', '.join(str(specpart) for specpart in spec) + header(f'template<> struct {internalName}<{spectext}> {"{"} using Type = ::{gnsp}::{fullname}; {"}"};') + header(f'template<{templatetypes}> using {name} = typename internal::{internalName}<{templateargs}>::Type;') - header(f'template<{templatetypes}> struct internal::{internalName};') - for gnsp, spec in foundin: - spectext = ', '.join(str(specpart) for specpart in spec) - header(f'template<> struct internal::{internalName}<{spectext}> {"{"} using Type = ::{gnsp}::{name}; {"}"};') - header(f'template<{templatetypes}> using {name} = typename internal::{internalName}<{templateargs}>::Type;') + self.namespacing(header, splitname[:-1] + [subnsp], inner) From 22b977e8c87b269a0cbdd55bcef353c6090e8b3b Mon Sep 17 00:00:00 2001 From: David Schneller Date: Sat, 26 Jul 2025 22:19:21 +0200 Subject: [PATCH 3/9] Add pre-includes to the metagen --- yateto/metagen.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/yateto/metagen.py b/yateto/metagen.py index d757ce4..1a13bb5 100644 --- a/yateto/metagen.py +++ b/yateto/metagen.py @@ -15,7 +15,7 @@ def add_generator(self, template, generator, *args, **kwargs): 'kwargs': kwargs }] - def generate(self, outputDir='', namespace='yateto'): + def generate(self, outputDir='', namespace='yateto', includes=[]): tensors = {} kernels = {} @@ -42,6 +42,8 @@ def generate(self, outputDir='', namespace='yateto'): # TODO: open meta header with Cpp(os.path.join(outputDir, 'tensor.h')) as header: with header.HeaderGuard('METAGEN_TENSOR_H_'): + for path in includes: + header.include(path) for i, gendata in enumerate(self.generators): header.include(f'metagen{i}/tensor.h') with header.Namespace(namespace): @@ -50,6 +52,8 @@ def generate(self, outputDir='', namespace='yateto'): with Cpp(os.path.join(outputDir, 'init.h')) as header: with header.HeaderGuard('METAGEN_INIT_H_'): + for path in includes: + header.include(path) for i, gendata in enumerate(self.generators): header.include(f'metagen{i}/init.h') with header.Namespace(namespace): @@ -58,6 +62,8 @@ def generate(self, outputDir='', namespace='yateto'): with Cpp(os.path.join(outputDir, 'kernel.h')) as header: with header.HeaderGuard('METAGEN_KERNEL_H_'): + for path in includes: + header.include(path) for i, gendata in enumerate(self.generators): header.include(f'metagen{i}/kernel.h') with header.Namespace(namespace): @@ -76,6 +82,10 @@ def generate(self, outputDir='', namespace='yateto'): for i, gendata in enumerate(self.generators): header.include(f'metagen{i}/kernel.cpp') + with Cpp(os.path.join(outputDir, 'test-kernels.cpp')) as header: + for i, gendata in enumerate(self.generators): + header.include(f'metagen{i}/test-kernels.cpp') + def namespacing(self, header, spaces, inner): if len(spaces) == 0: inner() From d9ad0bf7c2f1806d01a6c0de22e963a88dd64efb Mon Sep 17 00:00:00 2001 From: David Schneller Date: Sat, 2 Aug 2025 06:47:47 +0200 Subject: [PATCH 4/9] Add dummy init structs for scalars --- yateto/codegen/visitor.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/yateto/codegen/visitor.py b/yateto/codegen/visitor.py index 6162b88..2ac2283 100644 --- a/yateto/codegen/visitor.py +++ b/yateto/codegen/visitor.py @@ -795,6 +795,12 @@ def generateInitH(self, header): with header.Namespace(namespace), header.Namespace(self.INIT_NAMESPACE): for (base_name, base_name_without_namespace), tensors in tensor_dict.items(): self._init(header, base_name, base_name_without_namespace, '', tensors, False) + for namespace, scalar_dict in self.iterate_collect_scalar(): + with header.Namespace(namespace), header.Namespace(self.INIT_NAMESPACE): + for (baseName, baseNameWithoutNamespace), scalars in scalar_dict.items(): + with header.Struct('{0} : {1}::{0}'.format(baseNameWithoutNamespace, self.TENSOR_NAMESPACE)): + # empty forward declaration + pass def generateInitCpp(self, cpp): for namespace, tensor_dict in self.iterate_collect(): From 197f0f9df913e0c3f2b2da7f51c1ac9a79bdfcee Mon Sep 17 00:00:00 2001 From: David Schneller Date: Tue, 5 Aug 2025 02:39:20 +0200 Subject: [PATCH 5/9] Allow tensor+kernel forward declarations --- yateto/metagen.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/yateto/metagen.py b/yateto/metagen.py index 1a13bb5..8770ab0 100644 --- a/yateto/metagen.py +++ b/yateto/metagen.py @@ -15,10 +15,15 @@ def add_generator(self, template, generator, *args, **kwargs): 'kwargs': kwargs }] - def generate(self, outputDir='', namespace='yateto', includes=[]): + def generate(self, outputDir='', namespace='yateto', includes=[], declarationsTensors=[], declarationsKernels=[]): tensors = {} kernels = {} + for tensor in declarationsTensors: + tensors[tensor] = [] + for tensor in declarationsKernels: + kernels[tensor] = [] + namespacepfx = 'yatetometagen' for i, gendata in enumerate(self.generators): subnamespace = f'{namespace}::{namespacepfx}{i}' @@ -30,6 +35,7 @@ def generate(self, outputDir='', namespace='yateto', includes=[]): args = gendata['args'] kwargs = gendata['kwargs'] result = generator.generate(*args, **kwargs, namespace=subnamespace, outputDir=outdir) + for tensor in result['tensors']: if tensor not in tensors: tensors[tensor] = [] @@ -106,7 +112,7 @@ def inner(): templateargs = ', '.join(f'Arg{i}' for i, _ in enumerate(self.templateType)) with header.Namespace('internal'): - header(f'template<{templatetypes}> struct {internalName};') + header(f'template<{templatetypes}> struct {internalName} {"{"} using Type = void; {"}"};') for gnsp, spec in foundin: spectext = ', '.join(str(specpart) for specpart in spec) header(f'template<> struct {internalName}<{spectext}> {"{"} using Type = ::{gnsp}::{fullname}; {"}"};') From efa9dc2898a1b12b1af79c420ddd198a7b5644c8 Mon Sep 17 00:00:00 2001 From: David Schneller Date: Tue, 5 Aug 2025 03:11:24 +0200 Subject: [PATCH 6/9] Always load the correct arch first --- yateto/metagen.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/yateto/metagen.py b/yateto/metagen.py index 8770ab0..5f87614 100644 --- a/yateto/metagen.py +++ b/yateto/metagen.py @@ -1,4 +1,6 @@ +from .arch import fixArchitectureGlobal from .codegen.code import Cpp + import os class MetaGenerator: @@ -34,6 +36,8 @@ def generate(self, outputDir='', namespace='yateto', includes=[], declarationsTe template = gendata['template'] args = gendata['args'] kwargs = gendata['kwargs'] + + fixArchitectureGlobal(generator._arch) result = generator.generate(*args, **kwargs, namespace=subnamespace, outputDir=outdir) for tensor in result['tensors']: From d602959d63ae9f3c964041603aa384ff402ecbdd Mon Sep 17 00:00:00 2001 From: David Schneller Date: Fri, 10 Oct 2025 04:26:12 +0200 Subject: [PATCH 7/9] Fix metagen test kernel name; forward header namespace --- yateto/metagen.py | 78 ++++++++++++++++++----------------------------- 1 file changed, 30 insertions(+), 48 deletions(-) diff --git a/yateto/metagen.py b/yateto/metagen.py index 5f87614..4443f45 100644 --- a/yateto/metagen.py +++ b/yateto/metagen.py @@ -44,57 +44,39 @@ def generate(self, outputDir='', namespace='yateto', includes=[], declarationsTe if tensor not in tensors: tensors[tensor] = [] tensors[tensor] += [(subnamespace, template)] - for tensor in result['kernels']: - if tensor not in kernels: - kernels[tensor] = [] - kernels[tensor] += [(subnamespace, template)] - - # TODO: open meta header - with Cpp(os.path.join(outputDir, 'tensor.h')) as header: - with header.HeaderGuard('METAGEN_TENSOR_H_'): - for path in includes: - header.include(path) - for i, gendata in enumerate(self.generators): - header.include(f'metagen{i}/tensor.h') - with header.Namespace(namespace): - for tensor in tensors: - self.template(header, tensor, tensors[tensor], 'tensor') - - with Cpp(os.path.join(outputDir, 'init.h')) as header: - with header.HeaderGuard('METAGEN_INIT_H_'): - for path in includes: - header.include(path) - for i, gendata in enumerate(self.generators): - header.include(f'metagen{i}/init.h') - with header.Namespace(namespace): - for tensor in tensors: - self.template(header, tensor, tensors[tensor], 'init') + for kernel in result['kernels']: + if kernel not in kernels: + kernels[kernel] = [] + kernels[kernel] += [(subnamespace, template)] + + nspuppercase = namespace.upper() + + def headerForward(name, data): + upper = name.upper() + with Cpp(os.path.join(outputDir, f'{name}.h')) as header: + with header.HeaderGuard(f'METAGEN_{nspuppercase}_{upper}_H_'): + for path in includes: + header.include(path) + for i, gendata in enumerate(self.generators): + header.include(f'metagen{i}/{name}.h') + with header.Namespace(namespace): + for entry in data: + self.template(header, entry, data[entry], f'{name}') + - with Cpp(os.path.join(outputDir, 'kernel.h')) as header: - with header.HeaderGuard('METAGEN_KERNEL_H_'): - for path in includes: - header.include(path) + headerForward('tensor', tensors) + headerForward('init', tensors) + headerForward('kernel', kernels) + + def cppForward(name): + with Cpp(os.path.join(outputDir, f'{name}.cpp')) as header: for i, gendata in enumerate(self.generators): - header.include(f'metagen{i}/kernel.h') - with header.Namespace(namespace): - for kernel in kernels: - self.template(header, kernel, kernels[kernel], 'kernel') - - with Cpp(os.path.join(outputDir, 'tensor.cpp')) as header: - for i, gendata in enumerate(self.generators): - header.include(f'metagen{i}/tensor.cpp') - - with Cpp(os.path.join(outputDir, 'init.cpp')) as header: - for i, gendata in enumerate(self.generators): - header.include(f'metagen{i}/init.cpp') - - with Cpp(os.path.join(outputDir, 'kernel.cpp')) as header: - for i, gendata in enumerate(self.generators): - header.include(f'metagen{i}/kernel.cpp') + header.include(f'metagen{i}/{name}.cpp') - with Cpp(os.path.join(outputDir, 'test-kernels.cpp')) as header: - for i, gendata in enumerate(self.generators): - header.include(f'metagen{i}/test-kernels.cpp') + cppForward('tensor') + cppForward('init') + cppForward('kernel') + cppForward('test-kernel') def namespacing(self, header, spaces, inner): if len(spaces) == 0: From 6fcef267764ff0895b4a1882c4d483640bb89fb8 Mon Sep 17 00:00:00 2001 From: David Schneller Date: Wed, 11 Feb 2026 23:01:21 +0100 Subject: [PATCH 8/9] Update metagen for single generator --- yateto/metagen.py | 83 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 25 deletions(-) diff --git a/yateto/metagen.py b/yateto/metagen.py index 4443f45..13d7b04 100644 --- a/yateto/metagen.py +++ b/yateto/metagen.py @@ -7,17 +7,56 @@ class MetaGenerator: def __init__(self, templateType): self.templateType = templateType self.generators = [] - + def add_generator(self, template, generator, *args, **kwargs): assert len(self.templateType) == len(template) self.generators += [{ + 'name': kwargs["name"] if "name" in kwargs else str(len(self.generators)), 'template': template, 'generator': generator, 'args': args, 'kwargs': kwargs }] - - def generate(self, outputDir='', namespace='yateto', includes=[], declarationsTensors=[], declarationsKernels=[]): + + def compile_list(self): + outfiles = [] + for gendata in self.generators: + outdirname = f'metagen_{gendata["name"]}' + outdir = os.path.join(outputDir, outdirname) + + genout = [] + for file in ['tensor', 'init', 'kernel', 'test-kernel']: + genout += [file] + outfiles += [genout] + return outfiles + + def generate_single(self, index, outputDir='', namespace='yateto'): + namespacepfx = 'yatetometagen' + gendata = self.generators[index] + subnamespace = f'{namespace}::{namespacepfx}_{gendata["name"]}' + outdirname = f'metagen_{gendata["name"]}' + outdir = os.path.join(outputDir, outdirname) + os.makedirs(outdir, exist_ok=True) + + generator = gendata['generator'] + template = gendata['template'] + args = gendata['args'] + kwargs = gendata['kwargs'] + + fixArchitectureGlobal(generator._arch) + result = generator.generate(*args, **kwargs, namespace=subnamespace, outputDir=outdir) + + tensors = {} + kernels = {} + + for tensor in result['tensors']: + tensors[tensor] = (subnamespace, template) + for kernel in result['kernels']: + kernels[kernel] = (subnamespace, template) + + return tensors, kernels + + def generate(self, outputDir='', namespace='yateto', includes=[], declarationsTensors=[], declarationsKernels=[], precompiled=None): tensors = {} kernels = {} @@ -26,28 +65,20 @@ def generate(self, outputDir='', namespace='yateto', includes=[], declarationsTe for tensor in declarationsKernels: kernels[tensor] = [] - namespacepfx = 'yatetometagen' - for i, gendata in enumerate(self.generators): - subnamespace = f'{namespace}::{namespacepfx}{i}' - outdir = os.path.join(outputDir, f'metagen{i}') - os.makedirs(outdir, exist_ok=True) + for index in range(len(self.generators)): + if precompiled is None: + local_tensors, local_kernels = self.generate_single(index, outputDir, namespace) + else: + local_tensors, local_kernels = precompiled[index] - generator = gendata['generator'] - template = gendata['template'] - args = gendata['args'] - kwargs = gendata['kwargs'] - - fixArchitectureGlobal(generator._arch) - result = generator.generate(*args, **kwargs, namespace=subnamespace, outputDir=outdir) - - for tensor in result['tensors']: + for tensor in local_tensors: if tensor not in tensors: tensors[tensor] = [] - tensors[tensor] += [(subnamespace, template)] - for kernel in result['kernels']: + tensors[tensor] += [local_tensors[tensor]] + for kernel in local_kernels: if kernel not in kernels: kernels[kernel] = [] - kernels[kernel] += [(subnamespace, template)] + kernels[kernel] += [local_kernels[kernel]] nspuppercase = namespace.upper() @@ -57,8 +88,9 @@ def headerForward(name, data): with header.HeaderGuard(f'METAGEN_{nspuppercase}_{upper}_H_'): for path in includes: header.include(path) - for i, gendata in enumerate(self.generators): - header.include(f'metagen{i}/{name}.h') + for gendata in self.generators: + outdirname = f'metagen_{gendata["name"]}' + header.include(f'{outdirname}/{name}.h') with header.Namespace(namespace): for entry in data: self.template(header, entry, data[entry], f'{name}') @@ -70,14 +102,15 @@ def headerForward(name, data): def cppForward(name): with Cpp(os.path.join(outputDir, f'{name}.cpp')) as header: - for i, gendata in enumerate(self.generators): - header.include(f'metagen{i}/{name}.cpp') + for gendata in self.generators: + outdirname = f'metagen_{gendata["name"]}' + header.include(f'{outdirname}/{name}.cpp') cppForward('tensor') cppForward('init') cppForward('kernel') cppForward('test-kernel') - + def namespacing(self, header, spaces, inner): if len(spaces) == 0: inner() From 0bd73963ee43d1513d9c93a96aa26aee12fbf2f4 Mon Sep 17 00:00:00 2001 From: David Schneller Date: Thu, 12 Feb 2026 15:47:54 +0100 Subject: [PATCH 9/9] Address review --- yateto/metagen.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/yateto/metagen.py b/yateto/metagen.py index 13d7b04..cc910d2 100644 --- a/yateto/metagen.py +++ b/yateto/metagen.py @@ -18,7 +18,7 @@ def add_generator(self, template, generator, *args, **kwargs): 'kwargs': kwargs }] - def compile_list(self): + def compile_list(self, outputDir=''): outfiles = [] for gendata in self.generators: outdirname = f'metagen_{gendata["name"]}' @@ -43,7 +43,7 @@ def generate_single(self, index, outputDir='', namespace='yateto'): args = gendata['args'] kwargs = gendata['kwargs'] - fixArchitectureGlobal(generator._arch) + fixArchitectureGlobal(generator.arch()) result = generator.generate(*args, **kwargs, namespace=subnamespace, outputDir=outdir) tensors = {} @@ -121,6 +121,8 @@ def namespacing(self, header, spaces, inner): def template(self, header, prename, foundin, subnsp): splitname = prename.split('::') + assert len(splitname) > 0 + def inner(): name = splitname[-1] fullname = '::'.join(splitname[:-1] + [subnsp, splitname[-1]])