Skip to content

Commit

Permalink
Several important updates from Nabeel
Browse files Browse the repository at this point in the history 
Define substitutions have been optimized to only run on new code generations, rather than multiple steps repeated over the entire compiled output
Macro arguments can now spell defines to then be re-substituted at practically endless levels (until macros themselves crash at 40 levels). post_l/iterate_macro() commands are thus deprecated now
Fixed file associations to code lines imported via recursive directory imports
Macro expansion now records the top-level "calling line" into expanded lines, which allow the developer to see the line of the macro invoked to create the error
  • Loading branch information
@mkruselj
mkruselj committed Mar 31, 2024
1 parent 21d280c commit 24812cc
Show file tree
Hide file tree
Showing 2 changed files with 113 additions and 96 deletions.
76 changes: 43 additions & 33 deletions compiler/ksp_compiler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,20 +217,29 @@ def __init__(self, line, message, no_traceback = False):
if no_traceback:
utils.disable_traceback()

msg = "%s\n\n%s\n\n%s" % (message, str(line).strip(), line.get_locations_string())
if line.calling_lines:
macro_chain = '\n'.join(['=>{}'.format(l.command.strip()) for l in line.calling_lines])
line_content = 'Macro traceback:\n{}'.format(macro_chain, str(line).strip())
else:
line_content = str(line).strip()

msg = "%s\n\n%s\n\n%s" % (message, line_content, line.get_locations_string())

Exception.__init__(self, msg)
self.line = line
self.message = msg

class Line:
'''Line object used for handling lines before AST lex/yacc parsing'''

def __init__(self, s, locations = None, namespaces = None, placeholders = placeholders):
def __init__(self, s, locations = None, namespaces = None, placeholders = placeholders, calling_lines = None):
# locations should be a list of (filename, lineno) tuples
self.command = s # current line returned as string
self.locations = locations or [(None, -1)] # filename and line number
self.namespaces = namespaces or [] # a list of the namespaces (each import appends the as-name onto the stack)
self.placeholders = placeholders
self.source_locations = None
self.calling_lines = calling_lines

def get_lineno(self):
return self.locations[0][1]
Expand All @@ -242,14 +251,14 @@ def get_filename(self):
filename = property(get_filename)

def get_locations_string(self):
return '\n'.join(('%s%s:%d \r\n' % (' ' * (i * 4), filename or '<main script>', lineno)) \
return '\n'.join(('%s%s: %d' % (' ' * (i * 4), filename or '<main script>', lineno)) \
for (i, (filename, lineno)) in enumerate(reversed(self.locations)))

def copy(self, new_command = None, add_location = None):
''' Returns a copy of the line.
If the new_command parameter is specified, that will be the command of the new line
and it will get the same indentation as the old line. '''
line = Line(self.command, self.locations, self.namespaces)
line = Line(self.command, self.locations, self.namespaces, calling_lines = self.calling_lines)

if add_location:
line.locations = line.locations + [add_location]
Expand Down Expand Up @@ -498,7 +507,7 @@ def read_path(basepath, filepath):
if preprocessor_func:
preproc_s = preprocessor_func(source, namespaces)

new_lines.extend(parse_lines_and_handle_imports(basepath, preproc_s, compiler_import_cache, filename, namespaces))
new_lines.extend(parse_lines_and_handle_imports(basepath, preproc_s, compiler_import_cache, path, namespaces))
# non-import line so just add it to result line list:
else:
new_lines.append(line)
Expand Down Expand Up @@ -621,8 +630,7 @@ def extract_callback_lines(lines):

return (normal_lines, callback_lines)


def expand_macros(lines, macros, level = 0, replace_raw = True):
def expand_macros(lines, macros, level = 0, replace_raw = True, define_cache = None):
'''Inline macro invocations by the body of the macro definition (with parameters properly replaced)
returns tuple (normal_lines, callback_lines) where the latter are callbacks'''
macro_call_re = re.compile(r'(?ms)^\s*([\w_.]+)\s*(\(.*\))?%s$' % white_space_re)
Expand Down Expand Up @@ -687,15 +695,36 @@ def expand_macros(lines, macros, level = 0, replace_raw = True):
# erase any inner comments to not disturb outer
macro_call_str = re.sub(white_space, '', macro_call_str)
normal_lines, callback_lines = extract_callback_lines(macro.lines[1:-1])
new_lines.extend(normal_lines)
new_callback_lines.extend(callback_lines)

contents = normal_lines + callback_lines

from preprocessor_plugins import macro_iter_functions, post_macro_iter_functions, substituteDefines

convert_strings_to_placeholders(contents)
substituteDefines(contents, define_cache)

while macro_iter_functions(contents, placeholders):
convert_strings_to_placeholders(contents)
substituteDefines(contents, define_cache)

while post_macro_iter_functions(contents, placeholders):
convert_strings_to_placeholders(contents)
substituteDefines(contents, define_cache)

for c in contents:
if not line.calling_lines:
c.calling_lines = [line]
else:
c.calling_lines = line.calling_lines + [line]

new_lines.extend(contents)

num_substitutions += 1

if num_substitutions:
return expand_macros(new_lines + new_callback_lines, macros, level+1, replace_raw)
return expand_macros(new_lines, macros, level+1, replace_raw, define_cache)
else:
return (new_lines, new_callback_lines)
return (new_lines)

class ASTModifierBase(ksp_ast_processing.ASTModifier):
'''Class for accessing AST nodes for modification'''
Expand Down Expand Up @@ -2074,36 +2103,17 @@ def extract_macros(self):

def expand_macros(self):
from preprocessor_plugins import macro_iter_functions, post_macro_iter_functions, substituteDefines
self.lines = expand_macros(self.lines, self.macros, 0, True, self.define_cache)

# initial expansion. Macro strings are expanded
normal_lines, callback_lines = expand_macros(self.lines, self.macros, 0, True)
self.lines = normal_lines + callback_lines

# convert any strings from the macro expansion back into placeholders to prevent defines with identical names within strings being replaced
convert_strings_to_placeholders(self.lines)


# nested expansion, supports now using macros to further specify define constants used for iterate and literate macros
while macro_iter_functions(self.lines, placeholders):
normal_lines, callback_lines = expand_macros(self.lines, self.macros, 0, True)
self.lines = normal_lines + callback_lines
self.lines = expand_macros(self.lines, self.macros, 0, True, self.define_cache)

# convert any strings from the macro expansion back into placeholders to allow iter_macros to substitute strings
convert_strings_to_placeholders(self.lines)

# run define subs a second time, catch returned cache just as a formality
self.define_cache = substituteDefines(self.lines, self.define_cache)

while post_macro_iter_functions(self.lines, placeholders):
normal_lines, callback_lines = expand_macros(self.lines, self.macros, 0, True)
self.lines = normal_lines + callback_lines

# convert any strings from the macro expansion back into placeholders to allow iter_macros to substitute strings
convert_strings_to_placeholders(self.lines)

# run define subs a final time, catch returned cache just as a formality
self.define_cache = substituteDefines(self.lines, self.define_cache)

self.lines = expand_macros(self.lines, self.macros, 0, True, self.define_cache)

def examine_pragmas(self, code, namespaces):
'''Examine pragmas within code'''
Expand Down
133 changes: 70 additions & 63 deletions compiler/preprocessor_plugins.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1633,85 +1633,92 @@ def handleDefineConstants(lines, define_cache = None):

if define_cache is not None:
defineConstants = define_cache

for l in lines:
for dc in defineConstants:
l.command = dc.substituteValue(l.command, defineConstants, l)
else:
defineConstants = collections.deque()

newLines = collections.deque()
defineNames = set()
newLines = collections.deque()
defineNames = set()

# Scan through all the lines to find define declarations.
for l in lines:
command = l.command.strip()
# Scan through all the lines to find define declarations.
for l in lines:
command = l.command.strip()

if not command.startswith("define"):
newLines.append(l)
continue
if not command.startswith("define"):
newLines.append(l)
continue

define_type = 'none'
m = re.search(defineRe, command)
if m:
define_type = 'new'
define_type = 'none'
m = re.search(defineRe, command)

if define_type == 'none':
m = re.search(defineAppendRe, command)
if m:
define_type = 'append'
define_type = 'new'

if define_type == 'none':
m = re.search(definePrependRe, command)
if m:
define_type = 'prepend'
if define_type == 'none':
m = re.search(defineAppendRe, command)

if define_type == 'none':
newLines.append(l)
continue
if m:
define_type = 'append'

# count how many literals we have
num_literals = m.group("val").count(",") + 1

if num_literals > 1:
# add define for amount of entries in a literal define (.SIZE suffix)
defineSizeObj = DefineConstant(m.group("whole") + '.SIZE', str(num_literals), None, l)
defineConstants.append(defineSizeObj)

# Create define and evaluate if legitimate
defineObj = None
existing = list(filter(lambda d: d.name == m.group("name"), defineConstants))\

if define_type == 'append' and len(existing) > 0:
# If appending to existing, remove existing and concatente
defineObj = DefineConstant(m.group("whole"), existing[0].value + ', ' + m.group("val").strip(), m.group("args"), l)
defineConstants.remove(existing[0])
elif define_type == 'prepend' and len(existing) > 0:
# If appending to existing, remove existing and concatente
defineObj = DefineConstant(m.group("whole"), m.group("val").strip() + ', ' + existing[0].value, m.group("args"), l)
defineConstants.remove(existing[0])
elif define_type == 'new' and len(existing) > 0:
# If new and exists already, raise Exception
if existing[0].value != m.group("val").strip():
raise ParseException(l, "Define constant was already declared!")
else:
# All other cases, create a new define
defineObj = DefineConstant(m.group("whole"), m.group("val").strip(), m.group("args"), l)
if define_type == 'none':
m = re.search(definePrependRe, command)

if defineObj:
defineConstants.append(defineObj)
if m:
define_type = 'prepend'

if defineConstants:
# Replace all occurences where other defines are used in define values - do it a few times to catch some deeper nested defines.
if define_cache is None:
for n in range(0, 3):
for dc_i in defineConstants:
for dc_j in defineConstants:
dc_i.setValue(dc_j.substituteValue(dc_i.getValue(), defineConstants))
if define_type == 'none':
newLines.append(l)
continue

dc_i.evaluateValue()
# count how many literals we have
num_literals = m.group("val").count(",") + 1

if num_literals > 1:
# add define for amount of entries in a literal define (.SIZE suffix)
defineSizeObj = DefineConstant(m.group("whole") + '.SIZE', str(num_literals), None, l)
defineConstants.append(defineSizeObj)

# Create define and evaluate if legitimate
defineObj = None
existing = list(filter(lambda d: d.name == m.group("name"), defineConstants))\

if define_type == 'append' and len(existing) > 0:
# If appending to existing, remove existing and concatente
defineObj = DefineConstant(m.group("whole"), existing[0].value + ', ' + m.group("val").strip(), m.group("args"), l)
defineConstants.remove(existing[0])
elif define_type == 'prepend' and len(existing) > 0:
# If appending to existing, remove existing and concatente
defineObj = DefineConstant(m.group("whole"), m.group("val").strip() + ', ' + existing[0].value, m.group("args"), l)
defineConstants.remove(existing[0])
elif define_type == 'new' and len(existing) > 0:
# If new and exists already, raise Exception
if existing[0].value != m.group("val").strip():
raise ParseException(l, "Define constant was already declared!")
else:
# All other cases, create a new define
defineObj = DefineConstant(m.group("whole"), m.group("val").strip(), m.group("args"), l)

for l in newLines:
for dc in defineConstants:
l.command = dc.substituteValue(l.command, defineConstants, l)
if defineObj:
defineConstants.append(defineObj)

replaceLines(lines, newLines)
if defineConstants:
# Replace all occurences where other defines are used in define values - do it a few times to catch some deeper nested defines.
if define_cache is None:
for n in range(0, 3):
for dc_i in defineConstants:
for dc_j in defineConstants:
dc_i.setValue(dc_j.substituteValue(dc_i.getValue(), defineConstants))

dc_i.evaluateValue()

for l in newLines:
for dc in defineConstants:
l.command = dc.substituteValue(l.command, defineConstants, l)

replaceLines(lines, newLines)

return defineConstants

Expand Down

0 comments on commit 24812cc

Please sign in to comment.