chg: Changing the basics of the preprocessor !wip

Breaking the property preprocessor into several independent fragments, building the first interfaces and implementations around this idea.

examples/accessors_generator/main.py

@@ -129,7 +129,7 @@ def main():
         header_file = SourceFile()
         header_file.type = SourceFileType.HEADER
 
-        # Convert guard from 'EXAMPLE.H' to 'EXAMPLE_H'
+        # Convert guard against 'EXAMPLE.H' to 'EXAMPLE_H'
         header_file.header_guard = file_name.upper().replace(".", "_")
         header_file.includes.extend([string_include, iostream_include])
 

src/devana/preprocessing/components/extractors/__init__.py

@@ -0,0 +1,3 @@
+"""A set of ready-made generators that accept a set of data (usually just C++ files to be parsed) and extract from them
+the content that is of interest to the user, which constitutes the basis for processing, for example, attributes or
+comments."""

src/devana/preprocessing/components/property/__init__.py

@@ -0,0 +1,6 @@
+"""A property is a set of data that can be assigned to any element of code and serves as a description of how the
+preprocessor should transform the code or generate side effects.
+
+Typically, property comes from comments or attributes. This module includes components that allow you to easily
+complement your own property support - configure how they are to be parsed, how to operate, and add further
+processing functions."""

src/devana/preprocessing/components/property/parsers/__init__.py

@@ -0,0 +1,2 @@
+"""Auxiliary classes that allow you to transform (parse) input data containing the text definition of a property
+into a set of Python data that can be used in the executor module."""

src/devana/preprocessing/components/property/parsers/attributeparser.py

@@ -0,0 +1,43 @@
+from typing import Type, Iterable, Optional
+from devana.preprocessing.components.property.parsers.types import *
+from devana.preprocessing.components.property.parsers.descriptions import IDescribedProperty, IDescribedType
+from devana.preprocessing.preprocessor import IGenerator
+from devana.syntax_abstraction.attribute import Attribute
+from devana.preprocessing.components.property.parsers.result import Result
+from devana.preprocessing.components.property.parsers.configuration import Configuration
+
+
+class AttributeParser(IGenerator):
+    """Parser implementation using C++ standard attributes."""
+
+    def __init__(self, properties: Optional[List[IDescribedProperty]] = None,
+                 configuration: Optional[Configuration] = None):
+        self._properties = properties if properties is not None else []
+        self._configuration = configuration if configuration else Configuration(ignore_unknown=True)
+        if self._configuration.ignore_unknown is False:
+            raise ValueError("The attribute parser does not allow a restrictive approach to unknown attributes due to "
+                             "the existence of attributes that are compiler extensions.")
+        for p in self._properties:
+            self._validate_property(p)
+
+    def add_property(self, prop: IDescribedProperty):
+        self._validate_property(prop)
+        self._properties.append(prop)
+
+    @classmethod
+    def get_required_type(cls) -> Type:
+        return Attribute
+
+    @classmethod
+    def get_produced_type(cls) -> Type:
+        return Result
+
+    def generate(self, data: Iterable[Attribute]) -> Iterable[Result]:
+        return []
+
+    @staticmethod
+    def _validate_property(prop: IDescribedProperty):
+        """Check if property is able to usage."""
+        if not prop.namespace:
+            raise ValueError("Due to the existence of compiler extension attributes and standard standard "
+                             "attributes, it is required that preprocessor attributes use namespace.")

src/devana/preprocessing/components/property/parsers/configuration.py

@@ -0,0 +1,20 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class Configuration:
+    """Parser core configuration."""
+    orphaned_enum_values_allowed: bool = False
+    """An orphaned enum is an enum value not preceded by an enum name. For example, EnumTypeName.Value1 is the standard
+    value notation, but the user can only use Value1 in a given context - a parser based on the expected argument type
+    can match the argument type.
+    This option is mainly useful for simple preprocessors that do not use unions."""
+    property_overload_allowed: bool = False
+    """Allows multiple properties with the same name (within namespace) with different argument types.
+    Does not affect the ability to use default arguments."""
+    ignore_unknown: bool = False
+    """An option that determines whether to ignore a syntactically valid property with an unknown name without error.
+    This does not apply to cases where unknown values assigned to a property with a known name.
+    This option can be partially ignored for special parsers. In particular, C++11 attribute-based parsers may partially
+    ignore this setting in order to more easily implement compiler extension attributes - this applies to the global
+    namespace, so it is recommended for such parsers to always use their own namespaces."""

src/devana/preprocessing/components/property/parsers/descriptions.py

@@ -0,0 +1,70 @@
+from abc import ABC, abstractmethod
+from typing import Optional, List, Any
+
+
+class IDescribedType(ABC):
+    """An interface containing a basic description of the type assumed by the property for unambiguous identification.
+    Types can be (and often should be) created dynamically along with the creation of properties to provide support
+    for example enums."""
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """The full name of the type, including the names of the generic specialization types"""
+
+
+class IDescribedValue(ABC):
+    """Interface describing the possible value taken by property. Its use is to describe property definitions in the
+    context of default values, making the parser's work easier and ensuring diagnostic messages of appropriate quality.
+    This interface does not apply to actual property values."""
+
+    @property
+    @abstractmethod
+    def type(self) -> IDescribedType:
+        """The type of the current value."""
+
+    @property
+    @abstractmethod
+    def content_as_string(self) -> str:
+        """Test representation of a stored value - without a type, object data, etc. For example, holding the number
+        7.5, the text representation will be the string 7.5.
+        If no content value is provided, the string should be paired to obtain the correct value."""
+
+    @property
+    @abstractmethod
+    def content(self) -> Optional[Any]:
+        """Provides a value if you need to provide a more reliable or non-standard way to provide values instead of
+        relying on the parser when providing default values."""
+
+
+class IDescribedArgument(ABC):
+    """Interface describing the arguments accepted by property."""
+
+    @property
+    @abstractmethod
+    def type(self) -> IDescribedType:
+        """Type of argument."""
+
+    @property
+    @abstractmethod
+    def default_value(self) -> Optional[IDescribedValue]:
+        """Default value of argument, if any."""
+
+
+class IDescribedProperty(ABC):
+    """Description property for parsing and diagnostic messages."""
+
+    @property
+    @abstractmethod
+    def namespace(self) -> Optional[str]:
+        """Namespace in which the property is located (if it is located in any)."""
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Name of property."""
+
+    @property
+    @abstractmethod
+    def arguments(self) -> List[IDescribedArgument]:
+        """List of arguments for this property."""

src/devana/preprocessing/components/property/parsers/parser.py

@@ -0,0 +1,28 @@
+from typing import Optional, Any
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from devana.preprocessing.components.property.parsers.types import *
+
+
+@dataclass
+class ParsableElementError:
+    """Possible parsing errors"""
+    what: Optional[str] = None
+    """Error description. In most cases it will be empty allowing fallback to other types."""
+
+    @property
+    def is_meaningless(self) -> bool:
+        """Indicates when an error is used for fallback - it is not an actual user syntax error."""
+        return self.what is None
+
+class IParsableElement(ABC):
+    """A class that performs basic data parsing - the smallest component, for example, a type."""
+
+    @abstractmethod
+    def parse(self, text: str) -> Union[ParsableElementError, Any]:
+        """Extraxt python data from text."""
+
+    @property
+    @abstractmethod
+    def result_type(self) -> Type:
+        """Python type of result."""

src/devana/preprocessing/components/property/parsers/result.py

@@ -0,0 +1,38 @@
+from dataclasses import dataclass, field
+from typing import List, Type, Dict, Any
+from devana.syntax_abstraction.syntax import ISyntaxElement
+
+
+@dataclass
+class Value:
+    """The current value of the property argument with which it will be called."""
+    content: Any
+    """Python value."""
+
+    @property
+    def type(self) -> Type:
+        """Python type of content."""
+        return type(self.content)
+
+
+@dataclass
+class PropertySignature:
+    """Invoking target property identification data."""
+    name: str
+    namespaces: List[str] = field(default_factory=lambda: [])
+    arguments: List[Type] = field(default_factory=lambda: [])
+
+
+@dataclass
+class Arguments:
+    """Calling arguments."""
+    positional: List[Value]
+    named: Dict[str, Value]
+
+
+@dataclass
+class Result:
+    """Call frame - target function, arguments and current context"""
+    property: PropertySignature
+    arguments: Arguments
+    target: ISyntaxElement

src/devana/preprocessing/components/property/parsers/types.py

@@ -0,0 +1,145 @@
+from typing import List, Union, Type
+import re
+from devana.preprocessing.components.property.parsers.descriptions import IDescribedType
+from devana.preprocessing.components.property.parsers.parser import IParsableElement, ParsableElementError
+
+
+class IntegerType(IDescribedType, IParsableElement):
+    """Representation of integer number."""
+
+    @property
+    def name(self) -> str:
+        return "Integer"
+
+    __pattern = re.compile(r"^\s*([-+]?\d+)\s*$")
+
+    def parse(self, text: str) -> Union[ParsableElementError, int]:
+        match = self.__pattern.match(text)
+        if match:
+            return int(match.group(1))
+        return ParsableElementError()
+
+    @property
+    def result_type(self) -> Type:
+        return int
+
+
+class FloatType(IDescribedType, IParsableElement):
+    """Representation of floating point number."""
+
+    @property
+    def name(self) -> str:
+        return "Float"
+
+    __pattern = re.compile(r"^\s*([+-]?\d+\.?\d*)\s*$")
+
+    def parse(self, text: str) -> Union[ParsableElementError, float]:
+        match = self.__pattern.match(text)
+        if match:
+            return float(match.group(1))
+        return ParsableElementError()
+
+    @property
+    def result_type(self) -> Type:
+        return float
+
+
+class StringType(IDescribedType, IParsableElement):
+    """Representation of text."""
+
+    @property
+    def name(self) -> str:
+        return "String"
+
+    __pattern = re.compile(r'^\s*\"(.*)\"\s*$')
+
+    def parse(self, text: str) -> Union[ParsableElementError, str]:
+        match = self.__pattern.match(text)
+        if match:
+            return match.group(1)
+        return ParsableElementError()
+
+    @property
+    def result_type(self) -> Type:
+        return str
+
+
+class BooleanType(IDescribedType, IParsableElement):
+    """Representation of true/false."""
+
+    @property
+    def name(self) -> str:
+        return "Boolean"
+
+    __pattern = re.compile(r"^\s*(true|false)\s*$")
+
+    def parse(self, text: str) -> Union[ParsableElementError, bool]:
+        match = self.__pattern.match(text)
+        if match:
+            if match.group(1) == "true":
+                return True
+            elif match.group(1) == "false":
+                return False
+            else:
+                return ParsableElementError()
+        return ParsableElementError()
+
+    @property
+    def result_type(self) -> Type:
+        return bool
+
+
+class EnumType(IDescribedType):
+    """Representation of enum."""
+    def __init__(self, name: str, values: List[str]):
+        self._name = name
+        self._values = values
+
+    @property
+    def values(self) -> List[str]:
+        return self._values
+
+    @property
+    def name(self) -> str:
+        return f"Enum {self.name}"
+
+
+class _GenericType(IDescribedType):
+    """internal mixin for generic implementation."""
+    def __init__(self, name: str, specialization: IDescribedType):
+        self._name = name
+        self._specialization = specialization
+
+    @property
+    def specialization(self) -> IDescribedType:
+        return self._specialization
+
+    @property
+    def name(self) -> str:
+        return f"{self.name}<{self.specialization.name}>"
+
+
+class ListType(_GenericType):
+    """internal of list like []"""
+    def __init__(self, specialization: IDescribedType):
+        super().__init__("List", specialization)
+
+
+class OptionalType(_GenericType):
+    """internal of optional can be value of specialization type or none."""
+    def __init__(self, specialization: IDescribedType):
+        super().__init__("Optional", specialization)
+
+
+class UnionType(IDescribedType):
+    """internal of union type can be value of many types."""
+    def __init__(self, types: List[IDescribedType]):
+        self._types = types
+
+    @property
+    def types(self) -> List[IDescribedType]:
+        return self._types
+
+    @property
+    def name(self) -> str:
+        return f"Union<{'|'.join([s.name for s in self._types])}>"