msxbatoken.py

#!/usr/bin/env python3
"""
MSX Basic Tokenizer
v1.4
Convert ASCII MSX Basic to tokenized format

Copyright (C) 2019-2022 - Fred Rique (farique)
https://github.com/farique1/MSX-Basic-Tokenizer

See also:
MSX Sublime Tools at
https://github.com/farique1/MSX-Sublime-Tools
Syntax Highlight, Theme, Build System, Comment Preference and Auto Completion.

MSX Basic Dignified at
https://github.com/farique1/msx-basic-dignified
Convert modern MSX Basic Dignified to traditional MSX Basic format.

msxbatoken.py <source> <destination> [args...]
msxbatoken.py -h for help.

New: 1.4v 10/12/2021
    WINDOWS COMPATIBILITY YEY!
        os.path() operations to improve compatibility across systems
    Changed .INI section names

Notes:
    Known discrepancies:
        MSX &b (binary notation) tokenizes anything after it as characters except when a tokenized command is reached.
            The implementation here only looks for 0 and 1, reverting back to the normal parsing when other characters are found.
        Spaces at the end of a line are removed.
            The MSX does not remove them if loading from an ASCII file, only if typed on the machine.
        The MSX seems to split overflowed numbers on branching instructions (preceded by 0e), here it throw an error.
        Syntax errors generate wildly different results from the ones generated by the MSX.
    Conversion stopping errors:
        Line number too high, Line number out of order, Lines not starting with numbers, Branching lines too high
        Numbers bigger than their explicit type (in some cases they are converted up as per on the MSX)
    Tested with over 100 random basic programs from magazines and other sources, some crated to stress the conversions
        However there should be still some (several?) fringe cases not covered here. Be careful.
"""

import re
import os.path
import binascii
import argparse
import configparser
from datetime import datetime
from os import remove as osremove

file_load = ''              # Source file
file_save = ''              # Destination file
export_list = 0             # Save a .mlt list file detailing the tokenization: [#] number of bytes per line (def 16) (max 32) (0 no)
delete_original = False     # Delete the original ASCII file
verbose_level = 3           # Verbosity level: 0 silent, 1 errors, 2 +warnings, 3 +steps(def), 4 +details, 5 +conversion dump
is_from_build = False       # Tell if it is being called from a build system (show file name on error messages and other stuff)


def show_log(line_number, text, level, **kwargs):
    bullets = ['', '*** ', '  * ', '--- ', '  - ', '    ']

    try:
        bullet = kwargs['bullet']
    except KeyError:
        bullet = level

    display_file_name = ''
    if is_from_build and (bullet == 1 or bullet == 2):
        display_file_name = os.path.basename(file_load) + ': '

    line_number = '(' + str(line_number) + '): ' if line_number != '' else ''

    if verbose_level >= level:
        print(bullets[bullet] + display_file_name + line_number + text)

    if bullet == 1:
        if is_from_build:
            print('    Tokenizing_aborted')
        else:
            print('    Execution_stoped')
            print()
        raise SystemExit(0)


local_path = os.path.split(os.path.abspath(__file__))[0]
ini_path = os.path.join(local_path, 'MSXBatoken.ini')
if os.path.isfile(ini_path):
    config = configparser.ConfigParser()
    config.sections()
    try:
        config.read(ini_path)
        file_load = config.get('CONFIGS', 'file_load') if config.get('CONFIGS', 'file_load') else file_load
        file_save = config.get('CONFIGS', 'file_save') if config.get('CONFIGS', 'file_save') else file_save
        export_list = config.getboolean('CONFIGS', 'export_list') if config.get('CONFIGS', 'export_list') else export_list
        delete_original = config.getboolean('CONFIGS', 'delete_original') if config.get('CONFIGS', 'delete_original') else delete_original
        verbose_level = config.getint('CONFIGS', 'verbose_level') if config.get('CONFIGS', 'verbose_level') else verbose_level
    except (ValueError, configparser.NoOptionError) as e:
        show_log('', 'MSXBatoken.ini: ' + str(e), 1)

parser = argparse.ArgumentParser(description='Tokenize ASCII MSX Basic')
parser.add_argument("input", nargs='?', default=file_load, help='Source file (preferible .asc)')
parser.add_argument("output", nargs='?', default=file_save, help='Destination file ([source].bas) if missing')
parser.add_argument("-el", default=export_list, const=16, type=int, nargs='?', help="Save a .mlt list file detailing the tokenization: [#] number of bytes per line (def 16) (max 32)")
parser.add_argument("-do", default=delete_original, action='store_true', help="Delete original file after conversion")
parser.add_argument("-vb", default=verbose_level, type=int, help="Verbosity level: 0 silent, 1 errors, 2 +warnings, 3 +steps(def), 4 +details, 5 +conversion dump")
parser.add_argument("-frb", default=is_from_build, action='store_true', help="Tell it is running from a build system")
args = parser.parse_args()

file_load = args.input
file_save = args.output
if args.output == '':
    save_path = os.path.dirname(file_load)
    save_path = '' if save_path == '' else save_path
    save_file = os.path.basename(file_load)
    save_file = os.path.splitext(save_file)[0] + '.bas'
    file_save = os.path.join(save_path, save_file)
bytes_width = min(abs(args.el), 32)
export_list = True if args.el > 0 else False
delete_original = args.do
verbose_level = args.vb
is_from_build = args.frb

lines_num = 0
width_byte = bytes_width * 2
width_line = bytes_width * 3 + 7
now = datetime.now()
list_path = os.path.dirname(file_load)
# list_path = '' if list_path == '' else list_path
list_file = os.path.basename(file_load)
list_file = os.path.splitext(list_file)[0] + '.mlt'
file_list = os.path.join(list_path, list_file)
tokens = [('>', 'ee'), ('PAINT', 'bf'), ('=', 'ef'), ('ERROR', 'a6'), ('ERR', 'e2'), ('<', 'f0'), ('+', 'f1'),
          ('FIELD', 'b1'), ('PLAY', 'c1'), ('-', 'f2'), ('FILES', 'b7'), ('POINT', 'ed'), ('*', 'f3'), ('POKE', '98'),
          ('/', 'f4'), ('FN', 'de'), ('^', 'f5'), ('FOR', '82'), ('PRESET', 'c3'), ('\\', 'fc'), ('PRINT', '91'), ('?', '91'),
          ('PSET', 'c2'), ('AND', 'f6'), ('GET', 'b2'), ('PUT', 'b3'), ('GOSUB', '8d'), ('READ', '87'), ('GOTO', '89'),
          ('ATTR$', 'e9'), ('RENUM', 'aa'), ('AUTO', 'a9'), ('IF', '8b'), ('RESTORE', '8c'), ('BASE', 'c9'), ('IMP', 'fa'),
          ('RESUME', 'a7'), ('BEEP', 'c0'), ('INKEY$', 'ec'), ('RETURN', '8e'), ('BLOAD', 'cf'), ('INPUT', '85'),
          ('BSAVE', 'd0'), ('INSTR', 'e5'), ('RSET', 'b9'), ('CALL', 'ca'), ('_', '5f'), ('RUN', '8a'), ('IPL', 'd5'), ('SAVE', 'ba'),
          ('KEY', 'cc'), ('SCREEN', 'c5'), ('KILL', 'd4'), ('SET', 'd2'), ('CIRCLE', 'bc'), ('CLEAR', '92'), ('CLOAD', '9b'),
          ('LET', '88'), ('SOUND', 'c4'), ('CLOSE', 'b4'), ('LFILES', 'bb'), ('CLS', '9f'), ('LINE', 'af'), ('SPC(', 'df'),
          ('CMD', 'd7'), ('LIST', '93'), ('SPRITE', 'c7'), ('COLOR', 'bd'), ('LLIST', '9e'), ('CONT', '99'), ('LOAD', 'b5'),
          ('STEP', 'dc'), ('COPY', 'd6'), ('LOCATE', 'd8'), ('STOP', '90'), ('CSAVE', '9a'), ('CSRLIN', 'e8'),
          ('STRING$', 'e3'), ('LPRINT', '9d'), ('SWAP', 'a4'), ('LSET', 'b8'), ('TAB(', 'db'), ('MAX', 'cd'), ('DATA', '84'),
          ('MERGE', 'b6'), ('THEN', 'da'), ('TIME', 'cb'), ('TO', 'd9'), ('DEFDBL', 'ae'), ('DEFINT', 'ac'), ('DEFSTR', 'ab'),
          ('TROFF', 'a3'), ('DEFSNG', 'ad'), ('TRON', 'a2'), ('DEF', '97'), ('MOD', 'fb'), ('USING', 'e4'),
          ('DELETE', 'a8'), ('MOTOR', 'ce'), ('USR', 'dd'), ('DIM', '86'), ('NAME', 'd3'), ('DRAW', 'be'), ('NEW', '94'),
          ('VARPTR', 'e7'), ('NEXT', '83'), ('VDP', 'c8'), ('DSKI$', 'ea'), ('NOT', 'e0'), ('DSKO$', 'd1'), ('VPOKE', 'c6'),
          ('OFF', 'eb'), ('WAIT', '96'), ('END', '81'), ('ON', '95'), ('WIDTH', 'a0'), ('OPEN', 'b0'), ('XOR', 'f8'),
          ('EQV', 'f9'), ('OR', 'f7'), ('ERASE', 'a5'), ('OUT', '9c'), ('ERL', 'e1'), ('REM', '8f'),

          ('PDL', 'ffa4'), ('EXP', 'ff8b'), ('PEEK', 'ff97'), ('FIX', 'ffa1'), ('POS', 'ff91'), ('FPOS', 'ffa7'),
          ('ABS', 'ff86'), ('FRE', 'ff8f'), ('ASC', 'ff95'), ('ATN', 'ff8e'), ('HEX$', 'ff9b'), ('BIN$', 'ff9d'),
          ('INP', 'ff90'), ('RIGHT$', 'ff82'), ('RND', 'ff88'), ('INT', 'ff85'), ('CDBL', 'ffa0'), ('CHR$', 'ff96'),
          ('CINT', 'ff9e'), ('LEFT$', 'ff81'), ('SGN', 'ff84'), ('LEN', 'ff92'), ('SIN', 'ff89'), ('SPACE$', 'ff99'),
          ('SQR', 'ff87'), ('LOC(', 'ffac28'), ('STICK', 'ffa2'), ('COS', 'ff8c'), ('LOF', 'ffad'), ('STR$', 'ff93'),
          ('CSNG', 'ff9f'), ('LOG', 'ff8a'), ('STRIG', 'ffa3'), ('LPOS', 'ff9c'), ('CVD', 'ffaa'), ('CVI', 'ffa8'),
          ('CVS', 'ffa9'), ('TAN', 'ff8d'), ('MID$', 'ff83'), ('MKD$', 'ffb0'), ('MKI$', 'ffae'), ('MKS$', 'ffaf'),
          ('VAL', 'ff94'), ('DSKF', 'ffa6'), ('VPEEK', 'ff98'), ('OCT$', 'ff9a'), ('EOF', 'ffab'), ('PAD', 'ffa5'),

          ("'", '3a8fe6'), ('ELSE', '3aa1'), ('AS', '4153')]
jumps = ['RESTORE', 'AUTO', 'RENUM', 'DELETE', 'RESUME', 'ERL', 'ELSE', 'RUN', 'LIST', 'LLIST', 'GOTO', 'RETURN', 'THEN', 'GOSUB']


def update_lines(source, compiled):
    global line_compiled
    global line_source
    if len(line_source) > 2:
        line_source = line_source[source:]
        line_compiled = line_compiled + compiled
        show_log('', ' '.join([line_compiled + '|' + line_source.rstrip()]), 5)


def parse_numeric_bases(nugget_comp, token, base):
    if not nugget_comp:
        nugget_comp = ''
        hexa = '0000'
    else:
        if int(nugget_comp, base) > 65535:
            show_log(line_number, ' '.join(['overflow', nugget_comp]), 1)  # Exit
        hexa = '{0:04x}'.format(int(nugget_comp, base))
    return token + hexa[2:] + hexa[:-2]


def make_list(base_prev, compiled, source):
    line_inc = 12
    next_addr = str(compiled[0:4])
    curr_line = str(compiled[4:8])
    line_byte = str(compiled[8:])
    line_splt = [line_byte[i:i + width_byte] for i in range(0, len(line_byte), width_byte)]
    for line in line_splt:
        curr_addr = str(hex(base_prev)[2:][:-2]) + str(hex(base_prev)[2:][2:])

        byte_splt = ' '.join([line[i:i + 2] for i in range(0, len(line), 2)])

        line_list = curr_addr + ': ' + next_addr + ' ' + curr_line + ' ' \
            + byte_splt + (' ' * (width_line - len(byte_splt))) + source.rstrip()

        list_code.append(line_list)
        next_addr, curr_line, source = '        ', '', ''
        base_prev += line_inc
        line_inc = len(line) // 2


def parse_sgn_dbl(header, precision, nugget_integer, nugget_fractional, nugget_group_1_orig, nugget_number):
    nugget_stripped = nugget_integer.lstrip('0')
    if nugget_stripped == '':
        if nugget_fractional == '' or int(str(nugget_fractional[1:]) + '0') == 0:
            nugget_stripped = '0'
            hexa_precision = '00'
        else:
            nugget_integer = nugget_group_1_orig
            if str(nugget_fractional[1]) == '0':
                nugget_zeros = nugget_fractional[1:].rstrip('0')
                hexa_precision = '{0:02x}'.format(64 - (len(nugget_zeros) - len(nugget_zeros.lstrip('0'))))
            else:
                hexa_precision = '40'
    else:
        hexa_precision = '{0:02x}'.format(len(nugget_stripped) + 64)
    hexa = header + hexa_precision
    cropped = str(int(nugget_number))
    round_digit = int(cropped[precision:precision + 1]) if cropped[precision:precision + 1].isdigit() else 0
    nugget_cropped = cropped[0:precision] if round_digit < 5 else str(int(cropped[:precision]) + 1)
    hexa += nugget_cropped
    return hexa, nugget_integer


show_log('', '', 3, bullet=0)

if file_save == file_load:
    show_log('', ' '.join(['destination_same_as_source', file_save]), 1)  # Exit

show_log('', 'Loading file', 3)
ascii_code = []
if file_load:
    show_log('', ' '.join(['load_file:', file_load]), 4)
    try:
        with open(file_load, 'r', encoding='latin1') as f:
            for line in f:
                if line.strip() == "" or line.strip().isdigit():
                    continue
                ascii_code.append(line.strip() + '\r\n')
    except IOError:
        show_log('', ' '.join(['source_not_found', file_load]), 1)  # Exit
else:
    show_log('', 'source_not_given', 1)  # Exit

show_log('', 'Start tokenizing', 3)
base = 0x8001
base_base = base
line_order = 0
line_number = 0
tokenized_code = ['ff']
list_code = ["' -------------------------------------",
             "' MSX Basic Tokenizer: " + '"' + os.path.basename(file_load) + '"',
             "' Date: " + now.strftime("%Y-%m-%d %H:%M:%S"),
             "' -------------------------------------", ""]
list_code.append(hex(base - 1)[2:] + ': ' + 'ff' + (' ' * (width_line + 8)) + 'start')

for line_source in ascii_code:
    if ord(line_source[0]) < 48 or ord(line_source[0]) > 57:
        if ord(line_source[0]) == 26:  # Avoid '' on last line of some listings
            continue
        else:
            show_log(line_number, ' '.join(['line_not_starting_with_number']), 1)  # Exit

    if line_source == '':
        continue

    base_source = line_source
    line_compiled = ''

    show_log('', ' '.join([line_compiled + '|' + line_source.rstrip()]), 5)

    # Get line number
    nugget = re.match(r'\s*\d+\s?', line_source).group()
    line_number = nugget.strip()
    if int(line_number) <= line_order:
        show_log(line_number, ' '.join(['line_number_out_of_order', str(line_number)]), 1)  # Exit
    if int(line_number) > 65529:
        show_log(line_number, ' '.join(['line_number_too_high', str(line_number)]), 1)  # Exit
    line_order = int(line_number)
    line_source = line_source[len(nugget):]
    hexa = '{0:04x}'.format(int(nugget))
    line_compiled += hexa[2:] + hexa[:-2]

    show_log('', ' '.join([line_compiled + '|' + line_source.rstrip()]), 5)

    # Look for instructions
    while len(line_source) > 2:
        for command, token in tokens:
            if line_source.upper().startswith(command):
                compiled = token
                source = len(command)
                update_lines(source, compiled)

                if command == 'AS':
                    nugget = re.match(r'(\s*)(\d{1,2})', line_source)
                    if nugget:
                        nugget_spaces = nugget.group(1)
                        nugget_line = nugget.group(2)
                        hex_spaces = '20' * len(nugget_spaces)
                        hexa = '{0:02x}'.format(ord(nugget_line))
                        compiled = hex_spaces + hexa
                        source = len(nugget_spaces) + len(nugget_line)
                        update_lines(source, compiled)

                # Is a jumping instructions
                if command in jumps:
                    while True:
                        nugget = re.match(r'(\s*)(\d+|,+)', line_source)
                        if nugget:
                            nugget_spaces = nugget.group(1)
                            nugget_line = nugget.group(2)
                            if nugget_line.isdigit():
                                if int(nugget_line) > 65529:
                                    show_log(line_number, ' '.join(['line_number_jump_too_high', str(nugget_line)]), 1)  # Exit
                                hex_spaces = '20' * len(nugget_spaces)
                                hexa = '{0:04x}'.format(int(nugget_line))
                                compiled = hex_spaces + '0e' + hexa[2:] + hexa[:-2]
                                source = len(nugget_spaces) + len(nugget_line)
                                update_lines(source, compiled)
                            # Has several jumps (on goto/gosub)
                            else:
                                hex_spaces = '20' * len(nugget_spaces)
                                hexa = '2c' * len(nugget_line)
                                compiled = hex_spaces + hexa
                                source = len(nugget_spaces) + len(nugget_line)
                                update_lines(source, compiled)
                        else:
                            break

                # Instruction with literal data after it
                if command == 'DATA' or command == 'REM' or command == "'" or command == 'CALL' or command == '_':
                    while True:
                        character = line_source[0]
                        if command == 'CALL' or command == '_':
                            character = character.upper()
                        hexa = '{0:02x}'.format(ord(character))
                        compiled = hexa
                        source = 1
                        update_lines(source, compiled)

                        if len(line_source) <= 2 \
                                or (command == 'DATA' and line_source[0] == ':') \
                                or (command == '_' and (line_source[0] == ':' or line_source[0] == '('))\
                                or (command == 'CALL' and (line_source[0] == ':' or line_source[0] == '(')):
                            break
                break

        # Look each character
        else:
            # Is a number
            is_int = False
            nugget = line_source[0].upper()
            if nugget.isdigit() or nugget == '.':
                nugget = re.match(r'(\d*)\s*(.)\s*(.?)', line_source)
                nugget_number = nugget.group(1)
                nugget_integer = nugget.group(1)
                nugget_fractional = ''
                nugget_signal = nugget.group(2)
                nugget_notif_confirm = nugget.group(3)

                # Is floating point
                if nugget_signal == '.':
                    nugget = re.match(r'(\d*)\s*(.)\s*(\d*)\s*(.)\s*(.?)', line_source)
                    nugget_group1 = '0' if nugget.group(1) == '' else nugget.group(1)
                    nugget_number = nugget_group1 + nugget.group(3)
                    nugget_integer = nugget_group1
                    nugget_fractional = '.' + nugget.group(3)
                    nugget_signal = nugget.group(4)
                    nugget_notif_confirm = nugget.group(5)

                # Has integer signal
                if nugget_signal == '%':
                    nugget_number = nugget_integer
                    is_int = True
                    if int(nugget_number) >= 32768:
                        show_log(line_number, ' '.join(['overflow', str(nugget_number)]), 1)  # Exit
                elif nugget_signal != '%' and nugget_signal != '!' and nugget_signal != '#' and \
                        ((nugget_signal.lower() != 'e' and nugget_signal.lower() != 'd')
                         or (nugget_notif_confirm != '-' and nugget_notif_confirm != '+')):
                    nugget_signal = ''
                    if nugget_fractional == '':
                        is_int = True

                # Is scientific notation
                if (nugget_signal.lower() == 'e' or nugget_signal.lower() == 'd') \
                        and (nugget_notif_confirm == '-' or nugget_notif_confirm == '+'):  # Avoid matching E from ELSE after a number
                    exp = re.match(r'\d*\s*.\s*\d*\s*.\s*(\+|-)\s*(\d+)', line_source)

                    nugget_exp_size = len(nugget_integer.lstrip('0')) + int(exp.group(1) + exp.group(2))
                    nugget_man_size = nugget_exp_size - len(nugget_fractional[1:]) - len(nugget_integer.lstrip('0'))

                    if nugget_exp_size > 63 or nugget_exp_size < -64:
                        show_log(line_number, ' '.join(['overflow', str(nugget_number)]), 1)  # Exit

                    fractional = abs(nugget_man_size) if nugget_man_size < 0 else 0
                    notation = '%.*f' % (fractional, int(nugget_number) * (10 ** (nugget_man_size)))
                    notation_parts = re.match(r'(\d+)(\.\d+)?', notation)
                    notation_integer = notation_parts.group(1)
                    notation_fractional = notation_parts.group(2) if notation_parts.group(2) else ''
                    notation_number = notation.replace('.', '')
                    notation_size = nugget_number.lstrip('0')

                    if nugget_signal.lower() == 'e' and len(notation_size) < 7:
                        hexa, _ = parse_sgn_dbl('1d', 6, notation_integer, notation_fractional,
                                                nugget.group(1), notation_number)
                        hexa += '0' * (10 - len(hexa))
                    else:
                        hexa, _ = parse_sgn_dbl('1f', 14, notation_integer, notation_fractional,
                                                nugget.group(1), notation_number)
                        hexa += '0' * (18 - len(hexa))
                        hexa = hexa[0:18]

                    nugget_integer = nugget.group(1) if nugget_integer.lstrip('0') == '' else nugget_integer
                    nugget_signal += exp.group(1) + exp.group(2)

                # Is single precision
                elif (int(nugget_number) >= 32768 and int(nugget_number) <= 999999 and nugget_signal != '#') \
                        or (nugget_signal == '!' and int(nugget_number) <= (10 ** 63 - 1)) \
                        or (not is_int and int(nugget_number) <= 999999 and nugget_signal != '#'):

                    hexa, nugget_integer = parse_sgn_dbl('1d', 6, nugget_integer, nugget_fractional,
                                                         nugget.group(1), nugget_number)
                    hexa += '0' * (10 - len(hexa))

                # Is double precision
                elif (int(nugget_number) >= 1000000 and int(nugget_number) <= (10 ** 63 - 1)) \
                        or (nugget_signal == '#' and int(nugget_number) <= (10 ** 63 - 1)) \
                        or (not is_int and int(nugget_number) <= (10 ** 63 - 1)):

                    hexa, nugget_integer = parse_sgn_dbl('1f', 14, nugget_integer, nugget_fractional,
                                                         nugget.group(1), nugget_number)
                    hexa += '0' * (18 - len(hexa))
                    hexa = hexa[0:18]

                # Is normal integer
                elif int(nugget_number) >= 0 and int(nugget_number) <= 9:
                    nugget_add = str(int(nugget_number) + 17)
                    hexa = '{0:02x}'.format(int(nugget_add))

                elif int(nugget_number) >= 10 and int(nugget_number) <= 255:
                    hexa = '0f' + '{0:02x}'.format(int(nugget_number))

                elif int(nugget_number) >= 256 and int(nugget_number) <= 32767:
                    hexa = '{0:04x}'.format(int(nugget_number))
                    hexa = '1c' + hexa[2:] + hexa[:-2]

                else:
                    show_log(line_number, ' '.join(['number_too_high', str(nugget_number.lstrip('0'))]), 1)  # Exit

                compiled = hexa
                source = len(nugget_integer) + len(nugget_fractional) + len(nugget_signal)
                update_lines(source, compiled)

            # Other bases
            elif nugget == '&':
                nugget = line_source[0:2].upper()
                if nugget == '&H':
                    nugget_comp = re.match(r'[0-9a-f]*', line_source[2:].lower()).group()
                    hexa = parse_numeric_bases(nugget_comp, '0c', 16)
                elif nugget == '&O':
                    nugget_comp = re.match(r'[0-7]*', line_source[2:]).group()
                    hexa = parse_numeric_bases(nugget_comp, '0b', 8)
                elif nugget == '&B':
                    nugget_comp = re.match(r'[01]*', line_source[2:]).group()
                    hexa = '2642'
                    if nugget_comp:
                        for character in nugget_comp:
                            hexa += '{0:02x}'.format(ord(character))
                    else:
                        nugget_comp = ''
                else:
                    nugget = '&'
                    hexa = '{0:02x}'.format(ord(nugget))
                    nugget_comp = ''
                compiled = hexa
                source = len(nugget) + len(nugget_comp)
                update_lines(source, compiled)

            # Quotes
            else:
                nugget = line_source[0].upper()
                if nugget == '"':
                    num_quotes = 0
                    while True:
                        if line_source[0] == '"':
                            num_quotes += 1
                        hexa = '{0:02x}'.format(ord(line_source[0]))
                        compiled = hexa
                        source = 1
                        update_lines(source, compiled)
                        if num_quotes > 1 or len(line_source) <= 2:
                            break
                # And the rest
                else:
                    if ord(nugget) >= 65 and ord(nugget) <= 90:
                        is_var = True
                        while True:
                            nugget = line_source[0].upper()
                            for command, token in tokens:
                                if line_source.upper().startswith(command):
                                    is_var = False
                            if (ord(nugget) < 48 or ord(nugget) > 57) \
                                    and (ord(nugget) < 65 or ord(nugget) > 90) \
                                    or not is_var:
                                is_var = False
                                break
                            hexa = '{0:02x}'.format(ord(line_source[0]))
                            compiled = hexa
                            source = 1
                            update_lines(source, compiled)
                    else:
                        compiled = '{0:02x}'.format(ord(nugget.upper()))
                        source = 1
                        update_lines(source, compiled)

    base_prev = base
    base += (len(line_compiled) + 6) // 2
    hexa = '{0:04x}'.format(base)
    line_compiled = hexa[2:] + hexa[:-2] + line_compiled
    line_compiled += '00'
    tokenized_code.append(line_compiled)
    if export_list:
        make_list(base_prev, line_compiled, base_source)
    lines_num += 1

show_log('', 'End tokenizing', 3)
tokenized_code.append('0000')
list_code.append(str(hexa) + ': 0000' + (' ' * (width_line + 6)) + 'end')

list_code.extend(["", "' -------------------------------------",
                      "' Statistics",
                      "' -------------------------------------", ""])
list_code.append('lines ' + str(lines_num))
list_code.append('start &h' + '{0:04x}'.format(base_base - 1) + ' > ' + str(base_base - 1))
list_code.append('end   &h' + '{0:04x}'.format(base + 1) + ' > ' + str(base + 1))
list_code.append('size  &h' + '{0:04x}'.format((base - base_base) + 3) + ' > ' + str((base - base_base) + 3))

show_log('', 'Saving file', 3)
show_log('', ' '.join(['save_file:', file_save]), 4)
with open(file_save, 'wb') as f:
    for line in tokenized_code:
        f.write(binascii.unhexlify(line))

if delete_original:
    if os.path.isfile(file_save):
        show_log('', 'Deleting source', 3)
        show_log('', ' '.join(['delete_file:', file_load]), 4)
        osremove(file_load)
    else:
        show_log('', ' '.join(['source_not_deleted', file_load]), 2)
        show_log('', ' '.join(['converted_not_found', file_save]), 2)

if export_list:
    show_log('', 'Saving list', 3)
    show_log('', ' '.join(['save_list:', file_list]), 4)
    with open(file_list, 'w') as f:
        for line in list_code:
            f.write(line + '\n')

show_log('', '', 3, bullet=0)