Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Several important updates from Nabeel #449

Merged
merged 1 commit into from
Apr 2, 2024
Merged

Several important updates from Nabeel #449

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 42 additions & 25 deletions compiler/ksp_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,20 +217,29 @@ def __init__(self, line, message, no_traceback = False):
if no_traceback:
utils.disable_traceback()

msg = "%s\n\n%s\n\n%s" % (message, str(line).strip(), line.get_locations_string())
if line.calling_lines:
macro_chain = '\n'.join(['=>{}'.format(l.command.strip()) for l in line.calling_lines])
line_content = 'Macro traceback:\n{}'.format(macro_chain, str(line).strip())
else:
line_content = str(line).strip()

msg = "%s\n\n%s\n\n%s" % (message, line_content, line.get_locations_string())

Exception.__init__(self, msg)
self.line = line
self.message = msg

class Line:
'''Line object used for handling lines before AST lex/yacc parsing'''

def __init__(self, s, locations = None, namespaces = None, placeholders = placeholders):
def __init__(self, s, locations = None, namespaces = None, placeholders = placeholders, calling_lines = None):
# locations should be a list of (filename, lineno) tuples
self.command = s # current line returned as string
self.locations = locations or [(None, -1)] # filename and line number
self.namespaces = namespaces or [] # a list of the namespaces (each import appends the as-name onto the stack)
self.placeholders = placeholders
self.source_locations = None
self.calling_lines = calling_lines

def get_lineno(self):
return self.locations[0][1]
Expand All @@ -242,14 +251,14 @@ def get_filename(self):
filename = property(get_filename)

def get_locations_string(self):
return '\n'.join(('%s%s:%d \r\n' % (' ' * (i * 4), filename or '<main script>', lineno)) \
return '\n'.join(('%s%s: %d' % (' ' * (i * 4), filename or '<main script>', lineno)) \
for (i, (filename, lineno)) in enumerate(reversed(self.locations)))

def copy(self, new_command = None, add_location = None):
''' Returns a copy of the line.
If the new_command parameter is specified, that will be the command of the new line
and it will get the same indentation as the old line. '''
line = Line(self.command, self.locations, self.namespaces)
line = Line(self.command, self.locations, self.namespaces, calling_lines = self.calling_lines)

if add_location:
line.locations = line.locations + [add_location]
Expand Down Expand Up @@ -498,7 +507,7 @@ def read_path(basepath, filepath):
if preprocessor_func:
preproc_s = preprocessor_func(source, namespaces)

new_lines.extend(parse_lines_and_handle_imports(basepath, preproc_s, compiler_import_cache, filename, namespaces))
new_lines.extend(parse_lines_and_handle_imports(basepath, preproc_s, compiler_import_cache, path, namespaces))
# non-import line so just add it to result line list:
else:
new_lines.append(line)
Expand Down Expand Up @@ -621,8 +630,27 @@ def extract_callback_lines(lines):

return (normal_lines, callback_lines)

def sub_defines(lines, cur_line, define_cache):
from preprocessor_plugins import macro_iter_functions, post_macro_iter_functions, substituteDefines

convert_strings_to_placeholders(lines)
substituteDefines(lines, define_cache)

while macro_iter_functions(lines, placeholders):
convert_strings_to_placeholders(lines)
substituteDefines(lines, define_cache)

while post_macro_iter_functions(lines, placeholders):
convert_strings_to_placeholders(lines)
substituteDefines(lines, define_cache)

for c in lines:
if not cur_line.calling_lines:
c.calling_lines = [cur_line]
else:
c.calling_lines = cur_line.calling_lines + [cur_line]

def expand_macros(lines, macros, level = 0, replace_raw = True):
def expand_macros(lines, macros, level = 0, replace_raw = True, define_cache = None):
'''Inline macro invocations by the body of the macro definition (with parameters properly replaced)
returns tuple (normal_lines, callback_lines) where the latter are callbacks'''
macro_call_re = re.compile(r'(?ms)^\s*([\w_.]+)\s*(\(.*\))?%s$' % white_space_re)
Expand Down Expand Up @@ -687,13 +715,17 @@ def expand_macros(lines, macros, level = 0, replace_raw = True):
# erase any inner comments to not disturb outer
macro_call_str = re.sub(white_space, '', macro_call_str)
normal_lines, callback_lines = extract_callback_lines(macro.lines[1:-1])

sub_defines(normal_lines, line, define_cache)
sub_defines(callback_lines, line, define_cache)

new_lines.extend(normal_lines)
new_callback_lines.extend(callback_lines)

num_substitutions += 1

if num_substitutions:
return expand_macros(new_lines + new_callback_lines, macros, level+1, replace_raw)
return expand_macros(new_lines + new_callback_lines, macros, level + 1, replace_raw, define_cache)
else:
return (new_lines, new_callback_lines)

Expand Down Expand Up @@ -2075,36 +2107,21 @@ def extract_macros(self):
def expand_macros(self):
from preprocessor_plugins import macro_iter_functions, post_macro_iter_functions, substituteDefines

# initial expansion. Macro strings are expanded
normal_lines, callback_lines = expand_macros(self.lines, self.macros, 0, True)
normal_lines, callback_lines = expand_macros(self.lines, self.macros, 0, True, self.define_cache)
self.lines = normal_lines + callback_lines

# convert any strings from the macro expansion back into placeholders to prevent defines with identical names within strings being replaced
convert_strings_to_placeholders(self.lines)


# nested expansion, supports now using macros to further specify define constants used for iterate and literate macros
while macro_iter_functions(self.lines, placeholders):
normal_lines, callback_lines = expand_macros(self.lines, self.macros, 0, True)
normal_lines, callback_lines = expand_macros(self.lines, self.macros, 0, True, self.define_cache)
self.lines = normal_lines + callback_lines

# convert any strings from the macro expansion back into placeholders to allow iter_macros to substitute strings
convert_strings_to_placeholders(self.lines)

# run define subs a second time, catch returned cache just as a formality
self.define_cache = substituteDefines(self.lines, self.define_cache)

while post_macro_iter_functions(self.lines, placeholders):
normal_lines, callback_lines = expand_macros(self.lines, self.macros, 0, True)
normal_lines, callback_lines = expand_macros(self.lines, self.macros, 0, True, self.define_cache)
self.lines = normal_lines + callback_lines

# convert any strings from the macro expansion back into placeholders to allow iter_macros to substitute strings
convert_strings_to_placeholders(self.lines)

# run define subs a final time, catch returned cache just as a formality
self.define_cache = substituteDefines(self.lines, self.define_cache)


def examine_pragmas(self, code, namespaces):
'''Examine pragmas within code'''

Expand Down
133 changes: 70 additions & 63 deletions compiler/preprocessor_plugins.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1633,85 +1633,92 @@ def handleDefineConstants(lines, define_cache = None):

if define_cache is not None:
defineConstants = define_cache

for l in lines:
for dc in defineConstants:
l.command = dc.substituteValue(l.command, defineConstants, l)
else:
defineConstants = collections.deque()

newLines = collections.deque()
defineNames = set()
newLines = collections.deque()
defineNames = set()

# Scan through all the lines to find define declarations.
for l in lines:
command = l.command.strip()
# Scan through all the lines to find define declarations.
for l in lines:
command = l.command.strip()

if not command.startswith("define"):
newLines.append(l)
continue
if not command.startswith("define"):
newLines.append(l)
continue

define_type = 'none'
m = re.search(defineRe, command)
if m:
define_type = 'new'
define_type = 'none'
m = re.search(defineRe, command)

if define_type == 'none':
m = re.search(defineAppendRe, command)
if m:
define_type = 'append'
define_type = 'new'

if define_type == 'none':
m = re.search(definePrependRe, command)
if m:
define_type = 'prepend'
if define_type == 'none':
m = re.search(defineAppendRe, command)

if define_type == 'none':
newLines.append(l)
continue
if m:
define_type = 'append'

# count how many literals we have
num_literals = m.group("val").count(",") + 1

if num_literals > 1:
# add define for amount of entries in a literal define (.SIZE suffix)
defineSizeObj = DefineConstant(m.group("whole") + '.SIZE', str(num_literals), None, l)
defineConstants.append(defineSizeObj)

# Create define and evaluate if legitimate
defineObj = None
existing = list(filter(lambda d: d.name == m.group("name"), defineConstants))\

if define_type == 'append' and len(existing) > 0:
# If appending to existing, remove existing and concatente
defineObj = DefineConstant(m.group("whole"), existing[0].value + ', ' + m.group("val").strip(), m.group("args"), l)
defineConstants.remove(existing[0])
elif define_type == 'prepend' and len(existing) > 0:
# If appending to existing, remove existing and concatente
defineObj = DefineConstant(m.group("whole"), m.group("val").strip() + ', ' + existing[0].value, m.group("args"), l)
defineConstants.remove(existing[0])
elif define_type == 'new' and len(existing) > 0:
# If new and exists already, raise Exception
if existing[0].value != m.group("val").strip():
raise ParseException(l, "Define constant was already declared!")
else:
# All other cases, create a new define
defineObj = DefineConstant(m.group("whole"), m.group("val").strip(), m.group("args"), l)
if define_type == 'none':
m = re.search(definePrependRe, command)

if defineObj:
defineConstants.append(defineObj)
if m:
define_type = 'prepend'

if defineConstants:
# Replace all occurences where other defines are used in define values - do it a few times to catch some deeper nested defines.
if define_cache is None:
for n in range(0, 3):
for dc_i in defineConstants:
for dc_j in defineConstants:
dc_i.setValue(dc_j.substituteValue(dc_i.getValue(), defineConstants))
if define_type == 'none':
newLines.append(l)
continue

dc_i.evaluateValue()
# count how many literals we have
num_literals = m.group("val").count(",") + 1

if num_literals > 1:
# add define for amount of entries in a literal define (.SIZE suffix)
defineSizeObj = DefineConstant(m.group("whole") + '.SIZE', str(num_literals), None, l)
defineConstants.append(defineSizeObj)

# Create define and evaluate if legitimate
defineObj = None
existing = list(filter(lambda d: d.name == m.group("name"), defineConstants))\

if define_type == 'append' and len(existing) > 0:
# If appending to existing, remove existing and concatente
defineObj = DefineConstant(m.group("whole"), existing[0].value + ', ' + m.group("val").strip(), m.group("args"), l)
defineConstants.remove(existing[0])
elif define_type == 'prepend' and len(existing) > 0:
# If appending to existing, remove existing and concatente
defineObj = DefineConstant(m.group("whole"), m.group("val").strip() + ', ' + existing[0].value, m.group("args"), l)
defineConstants.remove(existing[0])
elif define_type == 'new' and len(existing) > 0:
# If new and exists already, raise Exception
if existing[0].value != m.group("val").strip():
raise ParseException(l, "Define constant was already declared!")
else:
# All other cases, create a new define
defineObj = DefineConstant(m.group("whole"), m.group("val").strip(), m.group("args"), l)

for l in newLines:
for dc in defineConstants:
l.command = dc.substituteValue(l.command, defineConstants, l)
if defineObj:
defineConstants.append(defineObj)

replaceLines(lines, newLines)
if defineConstants:
# Replace all occurences where other defines are used in define values - do it a few times to catch some deeper nested defines.
if define_cache is None:
for n in range(0, 3):
for dc_i in defineConstants:
for dc_j in defineConstants:
dc_i.setValue(dc_j.substituteValue(dc_i.getValue(), defineConstants))

dc_i.evaluateValue()

for l in newLines:
for dc in defineConstants:
l.command = dc.substituteValue(l.command, defineConstants, l)

replaceLines(lines, newLines)

return defineConstants

Expand Down
Loading