From 52495ba907fcf74ab21bc5fccc652d7b78cb499c Mon Sep 17 00:00:00 2001 From: Konata <42537566+konata-chan404@users.noreply.github.com> Date: Mon, 12 Jun 2023 18:44:48 +0300 Subject: [PATCH 1/2] Add xxMacroProcessor --- xx.py | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/xx.py b/xx.py index a0c7dd1..dd2f0e1 100644 --- a/xx.py +++ b/xx.py @@ -1,6 +1,7 @@ import sys import hashlib import argparse +import re, shlex, dis # Macro dependencies parser = argparse.ArgumentParser(description="xx") parser.add_argument('inFile', help='File to open') @@ -113,6 +114,70 @@ def testBinary(self): return self.normData = "{:02x}".format(int(bindata, 2)) +class xxMacroProcessor: + """ + Macro (pre)processor for xx language. + """ + def __init__(self): + self.macros = {} + self.define_macro("$DEF", self.def_macro) + + def define_macro(self, name, body): + """ + Defines a macro with the given name and body (can be an atom or function) + """ + self.macros[name] = body + + def def_macro(self, name, body): + """ + Internal macro definition function used for "$DEF" macro. If macro body starts with 'py' it is python code. + """ + val = body + if body.startswith("py"): + val = eval(re.sub("λ|\\\.", "lambda", body[2:])) + self.define_macro(name, val) + + def process_macros(self, line): + """ + Processes macros in the given line and returns the resulting line. It implements it's own parsing using regex for better or worse. + """ + # Remove comments from the line + for comment in asciiComments + twoCharComments: + line = line.split(comment)[0] + + # Define the pattern to match macros - example: (MACRO ARG1 ARG2 ...) + pattern = r'(? {line} -> {macro} doesn't exist!") + exit() + except SyntaxError: + print(f"> {line} -> syntax error at macro definition") + exit() + except Exception as e: + print(f"> {line} -> {e}") + exit() + if replacement is None: + replacement = "" + + line = line[:match.start()] + replacement + line[match.end():] + else: + break + + return line + ################################################################################ def getTokenAttributes(inTok): """ @@ -278,6 +343,7 @@ def tokenizeXX(xxline, lineNum): return tokens def parseXX(xxFile): + macro_processor = xxMacroProcessor() xxOut = b"" lineNum = 0 joinedLine = "" @@ -287,6 +353,7 @@ def parseXX(xxFile): multilineComment, joinedLine, line, mustContinue = filterMultLineComments(multilineComment, joinedLine, line) if mustContinue: continue + # line = macro_processor.process_macros(line) # (Pre)Process line using macros lineTokens = tokenizeXX(line, lineNum) isComment = 0 linesHexData = "" From 5c8773c31c73da584e7992e730b2884643c73d02 Mon Sep 17 00:00:00 2001 From: Konata <42537566+konata-chan404@users.noreply.github.com> Date: Mon, 12 Jun 2023 21:19:42 +0300 Subject: [PATCH 2/2] Finalize macro support <3 --- examples/macro.xx | 46 ++++++++++++++++++++++++++++++++++++++++++++++ xx.py | 16 +++++++++------- 2 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 examples/macro.xx diff --git a/examples/macro.xx b/examples/macro.xx new file mode 100644 index 0000000..fd966b4 --- /dev/null +++ b/examples/macro.xx @@ -0,0 +1,46 @@ + |\ /| /\ /~~|~~\ /~~\ /~~\ + | \/ | /__\| |__/| |`--. + | |/ \\__| \ \__/ \__/ + + + |\ _,,,---,,_ + -ZZZzz /,`.-'`' -. ;-;;,_ + |,4- ) )-,_. ,\ ( `'-' + -'---''(_/--' `-'\_) + +┌─ Macros ─────────────────────────────────────────────────────────── +│ xx now supports preprocessed macros~ +│ They're in Lisp-esque syntax and have a certain pattern: +| (MACRO ARG1 ARG2 ... ) +│ There's one single predefined macro and it is $DEF NAME BODY, +| it defines a macro and is our only interface with the macro system. +└───────────────────────────┐ +( $DEF COM "4D 45 4F 57" ) | <-- Here we define macro magic COM +( $DEF SMB "FF534D42" ) | Macros expand to xx syntax +( $DEF MBR "55 AA") | Ä< Hail Lisp >Ä +┌───────────────────────────┘ +│ Let's invoke some macros! +└────────────────────────────────┐ +(COM) ; Expands to MEOW | + | +┌─ Macro Functions ──────────────┘ +│ Defining simple macros is nice but there's much more to it. +│ You can define macro functions that operate on data, +| Code starts with py followed by minified python function <3 +└─────────────────────────────────────────────────────-────────────── +( $DEF $INC py'λ x : int(x,) + 1' ) -- Macros need to return a function, easiest way is to define a lambda function. + -- All parameters are text - cast type as you wish + -- In order to simplify the syntax both \. and λ expand to lambda :D +( $INC 19 ) --> 20 +( $INC 48 ) --> 111 + +( $DEF $ADDHEX py'λ a, b : hex(int(a, 0)+int(b, 0))' ) +( $ADDHEX 0x40 0xD ) +( $ADDHEX 0x1a 6 ) + +( $DEF $REPEAT py'λ s, n : s*int(n,0)') +C($REPEAT A 0xFF)T // >:3 + +( $DEF $MODULE_CALL py'λ m, f, *args: getattr(__import__(m), f)(*args)') +"Here are your files: ( $MODULE_CALL 'os' 'listdir' '.' )" // These Macros are inherently unsafe + // although could be integrated with interesting modules (pyasm) diff --git a/xx.py b/xx.py index dd2f0e1..91439d5 100644 --- a/xx.py +++ b/xx.py @@ -8,6 +8,7 @@ parser.add_argument('-x', dest='dumpHex', help='Dump hex instead of writing file', action="store_true") parser.add_argument('-o', dest='outFile', help='Output Filename') parser.add_argument('-r', dest='rawOut', help='Dump buffer to stdout instead of writing file', action="store_true") +parser.add_argument('-m', dest='macroEnable', help='Enables macro support for advanced use (UNSAFE)', action="store_true") xxVersion = "0.5" @@ -151,7 +152,6 @@ def process_macros(self, line): while True: match = re.search(pattern, line) if match: - print('match') macro = match.group(1) args = shlex.split(match.group(2)) # Parse arguments, handling quoted strings try: @@ -171,8 +171,8 @@ def process_macros(self, line): exit() if replacement is None: replacement = "" - - line = line[:match.start()] + replacement + line[match.end():] + + line = line[:match.start()] + str(replacement) + line[match.end():] else: break @@ -342,8 +342,8 @@ def tokenizeXX(xxline, lineNum): tokens.append(xxToken(buf, lineNum, False, isString)) # Append last token on EOL return tokens -def parseXX(xxFile): - macro_processor = xxMacroProcessor() +def parseXX(xxFile, macroEnable): + if macroEnable: macro_processor = xxMacroProcessor() xxOut = b"" lineNum = 0 joinedLine = "" @@ -353,7 +353,8 @@ def parseXX(xxFile): multilineComment, joinedLine, line, mustContinue = filterMultLineComments(multilineComment, joinedLine, line) if mustContinue: continue - # line = macro_processor.process_macros(line) # (Pre)Process line using macros + + if macroEnable: line = macro_processor.process_macros(line) # (Pre)Process line using macros lineTokens = tokenizeXX(line, lineNum) isComment = 0 linesHexData = "" @@ -372,9 +373,10 @@ def parseXX(xxFile): dumpHex = args.dumpHex outFile = args.outFile rawOut = args.rawOut + macroEnable = args.macroEnable with open(inFile,"r") as f: xxFileLines = f.readlines() - out = parseXX(xxFileLines) + out = parseXX(xxFileLines, macroEnable) if dumpHex: dHex(out) elif rawOut: