diff --git a/.gitignore b/.gitignore index 25f5cc4..7ca62d0 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ venv*/ /build/ /tests/res/build/ !/tests/res/build/empty.txt +/_sandbox/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index de04855..d21e9a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,13 +6,25 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [5.1.0] - 2024-04-30 + +### Added + +- A class attribute called `__servicer_cls__` to any service interfaces (for + a gRPC build) with reference to that services `GrpcServicer` class +- Added a bunch of useful sub-tests to unittests + +### Fixed + +- A bunch of unittesting issues + ## [5.0.0] - 2024-04-15 ### Changed - Moved this entire project over to Github - Bumped the version in order to not confuse older stuff that doesn't expect - protoplasm to exist in Pypi.org (if we end up migriting this there and + protoplasm to exist in Pypi.org (if we end up migrating this there and just open-sourcing the whole thing) - Also in case something changes in the API while migrating, cause I tend to fiddle with the code and tidy up and refactor when moving stuff diff --git a/_sandbox.py b/_sandbox.py new file mode 100644 index 0000000..7907868 --- /dev/null +++ b/_sandbox.py @@ -0,0 +1,17 @@ +from neobuilder.neobuilder import NeoBuilder + + +if __name__ == '__main__': + #n = NeoBuilder( + # package='efrit', + # protopath='./_sandbox/proto/', + # build_root='./_sandbox/build/', + # verbose=True, + #) + n = NeoBuilder( + package='sandbox', + protopath=r'D:\Code\github\ccpgames\neobuilder\tests\res\proto', + build_root='./_sandbox/build/', + verbose=True, + ) + n.build() diff --git a/neobuilder/__init__.py b/neobuilder/__init__.py index a0f6658..a5e4513 100644 --- a/neobuilder/__init__.py +++ b/neobuilder/__init__.py @@ -1 +1 @@ -__version__ = '5.0.0' +__version__ = '5.1.0' diff --git a/neobuilder/data/templates/grpc_receiver.jinja2 b/neobuilder/data/templates/grpc_receiver.jinja2 index 490abbf..4a6a228 100644 --- a/neobuilder/data/templates/grpc_receiver.jinja2 +++ b/neobuilder/data/templates/grpc_receiver.jinja2 @@ -11,6 +11,7 @@ from protoplasm import plasm {{ imports }} if TYPE_CHECKING: from grpc import ServicerContext + {{ api_import }} import logging log = logging.getLogger(__name__) diff --git a/neobuilder/data/templates/parts/_interface_service.jinja2 b/neobuilder/data/templates/parts/_interface_service.jinja2 index b799efe..7168eba 100644 --- a/neobuilder/data/templates/parts/_interface_service.jinja2 +++ b/neobuilder/data/templates/parts/_interface_service.jinja2 @@ -1,5 +1,9 @@ {% from 'macros/indentclude.jinja2' import indentclude with context %} class {{ service_name }}Interface: +{% if is_grpc %} + __servicer_cls__ = {{ service_name }}GrpcServicer + +{% endif %} {% for method in method_list %} {{ indentclude('parts/_interface_method.jinja2', (indent_spaces + 4), method) }} diff --git a/neobuilder/generators/servicebuilders/base.py b/neobuilder/generators/servicebuilders/base.py index 33ab558..1026ac2 100644 --- a/neobuilder/generators/servicebuilders/base.py +++ b/neobuilder/generators/servicebuilders/base.py @@ -20,7 +20,7 @@ def get_template_context(self) -> Dict: } def render_top(self) -> str: - # TODO(thordurm@ccpgames.com) 2022-06-24: Remove! + # TODO(thordurm@ccpgames.com) 2022-06-24: Remove... why? return ('# Auto-Generated file - DO NOT EDIT!\n' f'# Source module: {self.module.get_module_full_name()}\n' f'# Generated at: {datetime.datetime.now().isoformat()}\n') @@ -49,6 +49,7 @@ def __init__(self, service: symbols_service.ProtoService, indent_level: int = 0) def get_template_context(self) -> Dict: return { + 'is_grpc': self.service.module.is_grpc_file(), 'service': self.service, 'service_name': self.service.service_descriptor.name, '_indent_level': self.indent_level, diff --git a/neobuilder/generators/servicebuilders/grpc_receiver.py b/neobuilder/generators/servicebuilders/grpc_receiver.py index e557a45..442fe57 100644 --- a/neobuilder/generators/servicebuilders/grpc_receiver.py +++ b/neobuilder/generators/servicebuilders/grpc_receiver.py @@ -15,7 +15,6 @@ def render_imports(self) -> str: f'from {self.module.get_package()} import {self.module.get_module_name()} as pb2', f'from {self.module.get_package()} import {self.module.get_module_name()}_grpc as pb2_grpc', f'from {self.module.get_package()} import {self.module.get_module_name()[:-3]}dc as dc', - f'from {self.module.get_package()} import {self.module.get_module_name()[:-3]}api as api', } for s in self.module.service_map.values(): @@ -42,6 +41,7 @@ def render_imports(self) -> str: def get_template_context(self) -> Dict: d = super().get_template_context() d.update({ + 'api_import': f'from {self.module.get_package()} import {self.module.get_module_name()[:-3]}api as api', 'services': self.render_services(), 'imports': self.render_imports(), 'all_list': [f'{svc.service_descriptor.name}GrpcServicer' for svc in self.module.service_map.values()], @@ -60,7 +60,7 @@ def render(self) -> str: i = self.base_indent return ( f'{i}class {self.service.service_descriptor.name}GrpcServicer(plasm.BaseGrpcServicer, pb2_grpc.{self.service.service_descriptor.name}Servicer):\n' - f'{i}{__}def __init__(self, implementation: api.{self.service.service_descriptor.name}Interface):\n' + f"{i}{__}def __init__(self, implementation: 'api.{self.service.service_descriptor.name}Interface'):\n" f'{i}{__}{__}super().__init__(implementation)\n' '\n' f'{i}{__}def add_to_server(self, server):\n' diff --git a/neobuilder/generators/servicebuilders/interface.py b/neobuilder/generators/servicebuilders/interface.py index d31edc5..9c6d9d0 100644 --- a/neobuilder/generators/servicebuilders/interface.py +++ b/neobuilder/generators/servicebuilders/interface.py @@ -16,6 +16,10 @@ def get_import_lines(self) -> List[str]: f'from {self.module.get_package()} import {self.module.get_module_name()[:-3]}dc as dc' } for s in self.module.service_map.values(): + + if self.module.is_grpc_file(): + import_set.add(f'from {self.module.get_package()}.{self.module.get_module_name()[:-3]}grpc_receiver import {s.service_descriptor.name}GrpcServicer') + for m in s.method_map.values(): # From 3.2 if m.input.self_import.get_package() != 'google.protobuf': diff --git a/neobuilder/generators/symbols/modules.py b/neobuilder/generators/symbols/modules.py index 2c44816..db77a97 100644 --- a/neobuilder/generators/symbols/modules.py +++ b/neobuilder/generators/symbols/modules.py @@ -163,6 +163,7 @@ def render_dataclass_file(self): }) def write_dataclass_file(self): + log.debug(f' - - write_dataclass_file() {self.get_render_file_name()}') self.write_file(self.get_render_file_name(), self.render_dataclass_file()) # @@ -182,6 +183,7 @@ def render_grpc_server_file(self): def write_api_file(self): b = servicebuilders.get_module_builder('interface')(self) + log.debug(f' - - write_api_file() {b.get_render_filename()}') self.write_file(b.get_render_filename(), b.render()) def render_api_file(self): @@ -193,6 +195,7 @@ def render_api_file(self): def write_grpc_file(self): b = servicebuilders.get_module_builder('grpc_receiver')(self) + log.debug(f' - - write_grpc_file() {b.get_render_filename()}') self.write_file(b.get_render_filename(), b.render()) def render_grpc_file(self): @@ -204,6 +207,7 @@ def render_grpc_file(self): def write_grpc_impl_file(self): b = servicebuilders.get_module_builder('grpc_sender')(self) + log.debug(f' - - write_grpc_impl_file() {b.get_render_filename()}') self.write_file(b.get_render_filename(), b.render()) def render_grpc_impl_file(self): diff --git a/neobuilder/neobuilder/__init__.py b/neobuilder/neobuilder/__init__.py index ee5423e..c17380e 100644 --- a/neobuilder/neobuilder/__init__.py +++ b/neobuilder/neobuilder/__init__.py @@ -61,6 +61,11 @@ def __init__(self, format='%(levelname)8s - %(message)s') log.info(f'Initializing Neobuilder {self.neobuilder_version()}') + log.debug(f'{self.package=}') + log.debug(f'{self.protopath=}') + log.debug(f'{self.build_root=}') + log.debug(f'{self.proto_include=}') + @staticmethod def _get_basic_proto_path() -> str: if _PY_3_9_PLUS: @@ -337,6 +342,7 @@ def plasm_build(self, protofile: ProtoFile): else: try: p = modules.ProtoModule(m) + log.debug(f' - Writing ProtoModule: {p.get_module_full_name()}') p.write_rendered_file() except Exception as ex: log.exception(f'ERROR! Bad stuff happened! %r' % ex) diff --git a/tests/res/expected/sandbox/test/illnamedservice_api.py b/tests/res/expected/sandbox/test/illnamedservice_api.py index cd33e5d..1c3ae03 100644 --- a/tests/res/expected/sandbox/test/illnamedservice_api.py +++ b/tests/res/expected/sandbox/test/illnamedservice_api.py @@ -10,12 +10,15 @@ from protoplasm import plasm from sandbox.test import illnamedservice_dc as dc +from sandbox.test.illnamedservice_grpc_receiver import ServiceWithBadRequestNamesGrpcServicer import logging log = logging.getLogger(__name__) class ServiceWithBadRequestNamesInterface: + __servicer_cls__ = ServiceWithBadRequestNamesGrpcServicer + def do_something(self, foo: str = None) -> str: raise plasm.Unimplemented() diff --git a/tests/res/expected/sandbox/test/illnamedservice_grpc_receiver.py b/tests/res/expected/sandbox/test/illnamedservice_grpc_receiver.py index 7b01efd..90e8a3e 100644 --- a/tests/res/expected/sandbox/test/illnamedservice_grpc_receiver.py +++ b/tests/res/expected/sandbox/test/illnamedservice_grpc_receiver.py @@ -7,19 +7,19 @@ ] from typing import * from protoplasm import plasm -from sandbox.test import illnamedservice_api as api from sandbox.test import illnamedservice_dc as dc from sandbox.test import illnamedservice_pb2 as pb2 from sandbox.test import illnamedservice_pb2_grpc as pb2_grpc if TYPE_CHECKING: from grpc import ServicerContext + from sandbox.test import illnamedservice_api as api import logging log = logging.getLogger(__name__) class ServiceWithBadRequestNamesGrpcServicer(plasm.BaseGrpcServicer, pb2_grpc.ServiceWithBadRequestNamesServicer): - def __init__(self, implementation: api.ServiceWithBadRequestNamesInterface): + def __init__(self, implementation: 'api.ServiceWithBadRequestNamesInterface'): super().__init__(implementation) def add_to_server(self, server): diff --git a/tests/res/expected/sandbox/test/river_api.py b/tests/res/expected/sandbox/test/river_api.py index 9d79cd3..8afc4c4 100644 --- a/tests/res/expected/sandbox/test/river_api.py +++ b/tests/res/expected/sandbox/test/river_api.py @@ -10,12 +10,15 @@ from protoplasm import plasm from sandbox.test import river_dc as dc +from sandbox.test.river_grpc_receiver import StreamingServiceGrpcServicer import logging log = logging.getLogger(__name__) class StreamingServiceInterface: + __servicer_cls__ = StreamingServiceGrpcServicer + def reverse_my_shit(self, shit: str = None) -> str: raise plasm.Unimplemented() diff --git a/tests/res/expected/sandbox/test/river_grpc_receiver.py b/tests/res/expected/sandbox/test/river_grpc_receiver.py index 12b0205..4ebbc2d 100644 --- a/tests/res/expected/sandbox/test/river_grpc_receiver.py +++ b/tests/res/expected/sandbox/test/river_grpc_receiver.py @@ -7,19 +7,19 @@ ] from typing import * from protoplasm import plasm -from sandbox.test import river_api as api from sandbox.test import river_dc as dc from sandbox.test import river_pb2 as pb2 from sandbox.test import river_pb2_grpc as pb2_grpc if TYPE_CHECKING: from grpc import ServicerContext + from sandbox.test import river_api as api import logging log = logging.getLogger(__name__) class StreamingServiceGrpcServicer(plasm.BaseGrpcServicer, pb2_grpc.StreamingServiceServicer): - def __init__(self, implementation: api.StreamingServiceInterface): + def __init__(self, implementation: 'api.StreamingServiceInterface'): super().__init__(implementation) def add_to_server(self, server): diff --git a/tests/res/expected/sandbox/test/service_api.py b/tests/res/expected/sandbox/test/service_api.py index 5abe935..d4b2ee7 100644 --- a/tests/res/expected/sandbox/test/service_api.py +++ b/tests/res/expected/sandbox/test/service_api.py @@ -11,12 +11,16 @@ from protoplasm import plasm from sandbox.test import service_dc as dc +from sandbox.test.service_grpc_receiver import MathGrpcServicer +from sandbox.test.service_grpc_receiver import SimpleServiceGrpcServicer import logging log = logging.getLogger(__name__) class SimpleServiceInterface: + __servicer_cls__ = SimpleServiceGrpcServicer + def hello(self, greeting: str = None) -> str: raise plasm.Unimplemented() @@ -28,6 +32,8 @@ def empty_hello(self) -> NoReturn: class MathInterface: + __servicer_cls__ = MathGrpcServicer + def add(self, x: int = None, y: int = None) -> int: raise plasm.Unimplemented() diff --git a/tests/res/expected/sandbox/test/service_grpc_receiver.py b/tests/res/expected/sandbox/test/service_grpc_receiver.py index 637d9fe..db0b15f 100644 --- a/tests/res/expected/sandbox/test/service_grpc_receiver.py +++ b/tests/res/expected/sandbox/test/service_grpc_receiver.py @@ -8,19 +8,20 @@ ] from typing import * from protoplasm import plasm -from sandbox.test import service_api as api + from sandbox.test import service_dc as dc from sandbox.test import service_pb2 as pb2 from sandbox.test import service_pb2_grpc as pb2_grpc if TYPE_CHECKING: from grpc import ServicerContext + from sandbox.test import service_api as api import logging log = logging.getLogger(__name__) class SimpleServiceGrpcServicer(plasm.BaseGrpcServicer, pb2_grpc.SimpleServiceServicer): - def __init__(self, implementation: api.SimpleServiceInterface): + def __init__(self, implementation: 'api.SimpleServiceInterface'): super().__init__(implementation) def add_to_server(self, server): @@ -37,7 +38,7 @@ def EmptyHello(self, request: dc.pb2.EmptyHelloRequest, context: 'ServicerContex class MathGrpcServicer(plasm.BaseGrpcServicer, pb2_grpc.MathServicer): - def __init__(self, implementation: api.MathInterface): + def __init__(self, implementation: 'api.MathInterface'): super().__init__(implementation) def add_to_server(self, server): diff --git a/tests/res/expected/sandbox/test/service_with_imported_io_api.py b/tests/res/expected/sandbox/test/service_with_imported_io_api.py index ec3c8f0..95c59dd 100644 --- a/tests/res/expected/sandbox/test/service_with_imported_io_api.py +++ b/tests/res/expected/sandbox/test/service_with_imported_io_api.py @@ -15,12 +15,15 @@ from sandbox.test import nested_dc as sandbox__test__nested_dc from sandbox.test import rainbow_dc as sandbox__test__rainbow_dc from sandbox.test import service_with_imported_io_dc as dc +from sandbox.test.service_with_imported_io_grpc_receiver import ServiceWithImportedInputAndOutputGrpcServicer import logging log = logging.getLogger(__name__) class ServiceWithImportedInputAndOutputInterface: + __servicer_cls__ = ServiceWithImportedInputAndOutputGrpcServicer + def simple(self, my_string: str = None, my_number: int = None, my_level_three_message: sandbox__test__beta_dc.BetaMessage = None) -> Tuple[str, str]: raise plasm.Unimplemented() diff --git a/tests/res/expected/sandbox/test/service_with_imported_io_grpc_receiver.py b/tests/res/expected/sandbox/test/service_with_imported_io_grpc_receiver.py index dc44956..737af2d 100644 --- a/tests/res/expected/sandbox/test/service_with_imported_io_grpc_receiver.py +++ b/tests/res/expected/sandbox/test/service_with_imported_io_grpc_receiver.py @@ -14,19 +14,19 @@ from sandbox.test import delta_dc as sandbox__test__delta_dc from sandbox.test import nested_dc as sandbox__test__nested_dc from sandbox.test import rainbow_dc as sandbox__test__rainbow_dc -from sandbox.test import service_with_imported_io_api as api from sandbox.test import service_with_imported_io_dc as dc from sandbox.test import service_with_imported_io_pb2 as pb2 from sandbox.test import service_with_imported_io_pb2_grpc as pb2_grpc if TYPE_CHECKING: from grpc import ServicerContext + from sandbox.test import service_with_imported_io_api as api import logging log = logging.getLogger(__name__) class ServiceWithImportedInputAndOutputGrpcServicer(plasm.BaseGrpcServicer, pb2_grpc.ServiceWithImportedInputAndOutputServicer): - def __init__(self, implementation: api.ServiceWithImportedInputAndOutputInterface): + def __init__(self, implementation: 'api.ServiceWithImportedInputAndOutputInterface'): super().__init__(implementation) def add_to_server(self, server): diff --git a/tests/res/expected/sandbox/test/service_with_oneof_api.py b/tests/res/expected/sandbox/test/service_with_oneof_api.py index b4c7254..319d8a4 100644 --- a/tests/res/expected/sandbox/test/service_with_oneof_api.py +++ b/tests/res/expected/sandbox/test/service_with_oneof_api.py @@ -10,12 +10,15 @@ from protoplasm import plasm from sandbox.test import service_with_oneof_dc as dc +from sandbox.test.service_with_oneof_grpc_receiver import SimpleOneOfServiceGrpcServicer import logging log = logging.getLogger(__name__) class SimpleOneOfServiceInterface: + __servicer_cls__ = SimpleOneOfServiceGrpcServicer + def hello_again(self, greeting: str = None) -> Tuple[str, int]: raise plasm.Unimplemented() diff --git a/tests/res/expected/sandbox/test/service_with_oneof_grpc_receiver.py b/tests/res/expected/sandbox/test/service_with_oneof_grpc_receiver.py index 3f7a02f..644d3f5 100644 --- a/tests/res/expected/sandbox/test/service_with_oneof_grpc_receiver.py +++ b/tests/res/expected/sandbox/test/service_with_oneof_grpc_receiver.py @@ -7,19 +7,19 @@ ] from typing import * from protoplasm import plasm -from sandbox.test import service_with_oneof_api as api from sandbox.test import service_with_oneof_dc as dc from sandbox.test import service_with_oneof_pb2 as pb2 from sandbox.test import service_with_oneof_pb2_grpc as pb2_grpc if TYPE_CHECKING: from grpc import ServicerContext + from sandbox.test import service_with_oneof_api as api import logging log = logging.getLogger(__name__) class SimpleOneOfServiceGrpcServicer(plasm.BaseGrpcServicer, pb2_grpc.SimpleOneOfServiceServicer): - def __init__(self, implementation: api.SimpleOneOfServiceInterface): + def __init__(self, implementation: 'api.SimpleOneOfServiceInterface'): super().__init__(implementation) def add_to_server(self, server): diff --git a/tests/test_neobuilder.py b/tests/test_neobuilder.py index 48aa957..242446e 100644 --- a/tests/test_neobuilder.py +++ b/tests/test_neobuilder.py @@ -22,6 +22,9 @@ class NeobuilderTest(unittest.TestCase): + def setUp(self): + self.maxDiff = None + @classmethod def setUpClass(cls) -> None: # Remove old stuff... @@ -45,45 +48,64 @@ def assert_exists(file_to_check): assert_exists(os.path.join(dirpath, f)) def test_code_in_files(self): - def assert_files_are_same(file_to_check): - file_to_check = file_to_check.replace('\\', '/') - lines_to_check = [] - lines_skipped = 0 - with open(os.path.join(BUILD_ROOT, file_to_check), 'r') as fin: - for line in fin: - if not line.strip().startswith('# '): - stripped_line = line.rstrip() - if stripped_line: - lines_to_check.append(stripped_line) - else: - lines_skipped += 1 - - lines_should_be = [] - with open(os.path.join(EXPECTED_ROOT, file_to_check), 'r') as fin: - for line in fin: - if not line.strip().startswith('# '): - if file_to_check == 'sandbox/__init__.py': - if line.startswith('__protoplasm_version__ = ('): - line = f'__protoplasm_version__ = {protoplasm_version}' - if line.startswith('__neobuilder_version__ = ('): - line = f'__neobuilder_version__ = {neobuilder_version}' - stripped_line = line.rstrip() - if stripped_line: - lines_should_be.append(stripped_line) - - self.assertEqual(len(lines_should_be), len(lines_to_check), msg=f'Line number mismatch in {file_to_check}') - - if len(lines_to_check) == len(lines_should_be): - for i in range(len(lines_should_be)): - self.assertEqual(lines_should_be[i], lines_to_check[i], - msg=f'Line no {i+lines_skipped+1} in {file_to_check} does not match!') - file_count = 0 for (dirpath, dirnames, filenames) in os.walk(EXPECTED_ROOT): for f in filenames: if not (f.endswith('_pb2.py') or f.endswith('_pb2_grpc.py')): # We don't test the pb2 files! - rel_f = os.path.relpath(os.path.join(dirpath, f), EXPECTED_ROOT) + rel_f = os.path.relpath(os.path.join(dirpath, f), EXPECTED_ROOT) # noqa file_count += 1 - assert_files_are_same(rel_f) + + file_to_check = rel_f.replace('\\', '/') + lines_to_check = [] + lines_skipped = 0 + with open(os.path.join(BUILD_ROOT, file_to_check), 'r') as fin: # noqa + for line in fin: + if not line.strip().startswith('# '): + stripped_line = line.rstrip() + if stripped_line: + lines_to_check.append(stripped_line) + else: + lines_skipped += 1 + + lines_should_be = [] + with open(os.path.join(EXPECTED_ROOT, file_to_check), 'r') as fin: # noqa + for line in fin: + if not line.strip().startswith('# '): + if file_to_check == 'sandbox/__init__.py': + if line.startswith("__protoplasm_version__ = '"): + line = f"__protoplasm_version__ = '{protoplasm_version}'" + if line.startswith("__neobuilder_version__ = '"): + line = f"__neobuilder_version__ = '{neobuilder_version}'" + stripped_line = line.rstrip() + if stripped_line: + lines_should_be.append(stripped_line) + + line_count_should_be = len(lines_should_be) + line_count_is = len(lines_to_check) + with self.subTest(line_count_should_be=line_count_should_be, line_count_is=line_count_is, file_to_check=file_to_check): + self.assertEqual(line_count_should_be, line_count_is, + msg=f'Line number mismatch in {file_to_check}') + + with self.subTest(join_comparison_of_file=file_to_check): + self.assertEqual('\n'.join(lines_should_be), + '\n'.join(lines_to_check), msg=f'Content difference in {file_to_check}') + + if line_count_is == line_count_should_be: + for i in range(len(lines_should_be)): + with self.subTest(line_no=i + lines_skipped + 1, file_to_check=file_to_check): + self.assertEqual(lines_should_be[i], lines_to_check[i], + msg=f'Line no {i + lines_skipped + 1} in {file_to_check} does not match!') + + else: + lines_should_be_set = set(lines_should_be) + lines_are_set = set(lines_to_check) + missing_lines = lines_should_be_set - lines_are_set + extra_lines = lines_are_set - lines_should_be_set + log.warning(f'There are {len(missing_lines)} missing lines in {file_to_check}:') + for l in missing_lines: + log.warning(f' - "{l}"') + log.warning(f'There are {len(extra_lines)} extra lines in {file_to_check}:') + for l in extra_lines: + log.warning(f' + "{l}"') self.assertEqual(EXPECTED_NUMBER_OF_FILES_CHECKED, file_count, msg='Did not check the expected number of files!')