From 2b13a42a1d486a67291d1153bdd52b6c2f7825b8 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Thu, 26 Sep 2024 18:55:42 +0200 Subject: [PATCH 01/51] [TASK] use meson build system Removing setuptools specific parts --- MANIFEST.in | 4 +- bact_archiver/glob.sh | 5 + bact_archiver/meson.build | 10 + cysetuptools.py | 408 -------------------------------------- meson.build | 16 ++ proto/epics_event.pyx | 8 +- proto/meson.build | 43 ++++ proto_gen/00README.txt | 1 - protocol_buffer.py | 79 -------- pyproject.toml | 9 +- setup.cfg | 8 - setup.py | 30 --- 12 files changed, 86 insertions(+), 535 deletions(-) create mode 100644 bact_archiver/glob.sh create mode 100644 bact_archiver/meson.build delete mode 100644 cysetuptools.py create mode 100644 meson.build create mode 100644 proto/meson.build delete mode 100644 proto_gen/00README.txt delete mode 100644 protocol_buffer.py delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/MANIFEST.in b/MANIFEST.in index 9672084..d14cfc3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,8 +1,8 @@ include protocol_buffer.py # the source files -include proto/* +include proto/ * # the files generated -include proto_gen/* +include proto_gen/ * # configuration files include bact_archiver/*.cfg # documentation files diff --git a/bact_archiver/glob.sh b/bact_archiver/glob.sh new file mode 100644 index 0000000..71474ca --- /dev/null +++ b/bact_archiver/glob.sh @@ -0,0 +1,5 @@ +#!/bin/sh +for i in *.py; +do + echo "$i" +done diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build new file mode 100644 index 0000000..8927e9f --- /dev/null +++ b/bact_archiver/meson.build @@ -0,0 +1,10 @@ +py_sources = run_command('/bin/sh', +['glob.sh'], + check : true +).stdout().strip().split('\n') + +py.install_sources( + py_sources, + pure: false, + subdir: 'timepix_sort' +) \ No newline at end of file diff --git a/cysetuptools.py b/cysetuptools.py deleted file mode 100644 index 5d96fd8..0000000 --- a/cysetuptools.py +++ /dev/null @@ -1,408 +0,0 @@ -import subprocess -import os -import os.path as op -import sys -import shlex -import argparse - -import setuptools - -PY3 = sys.version_info[0] == 3 -if PY3: - import configparser -else: - import ConfigParser as configparser - - -DEFAULTS_SECTION = 'cython-defaults' -MODULE_SECTION_PREFIX = 'cython-module:' -CYTHON_EXT = '.pyx' -C_EXT = '.c' -CPP_EXT = '.cpp' - - -def setup(cythonize=True, **kwargs): - """ - Drop-in replacement for :func:`setuptools.setup`, adding Cython niceties. - - Cython modules are described in setup.cfg, for example:: - - [cython-module: foo.bar] - sources = foo.pyx - bar.cpp - include_dirs = eval(__import__('numpy').get_include()) - /usr/include/foo - language = c++ - pkg_config_packages = opencv - - You still need to provide a ``setup.py``:: - - from cysetuptools import setup - - setup() - - The modules sections support the following entries: - - sources - The list of Cython and C/C++ source files that are compiled to build - the module. - - libraries - A list of libraries to link with the module. - - include_dirs - A list of directories to find include files. This entry supports - python expressions with ``eval()``; in the example above this is used - to retrieve the numpy include directory. - - library_dirs - A list of directories to find libraries. This entry supports - python expressions with ``eval()`` like ``include_dirs``. - - extra_compile_args - Extra arguments passed to the compiler. - - extra_link_args - Extra arguments passed to the linker. - - language - Typically "c" or "c++". - - pkg_config_packages - A list of ``pkg-config`` package names to link with the module. - - pkg_config_dirs - A list of directories to add to the pkg-config search paths (extends - the ``PKG_CONFIG_PATH`` environment variable). - - Defaults can also be specified in the ``[cython-defaults]`` section, for - example:: - - [cython-defaults] - include_dirs = /usr/include/bar - - [cython-module: foo.one] - sources = foo/one.pyx - - [cython-module: foo.two] - sources = foo/two.pyx - include_dirs = /usr/include/foo - - Here, ``foo.one`` and ``foo.two`` both will have ``/usr/include/bar`` in - their ``include_dirs``. List parameters in defaults are extended, so in the - example above, module ``foo.two`` ``include_dirs`` will be - ``['/usr/include/bar', '/usr/include/foo']``. - - There are two approaches when distributing Cython modules: with or without - the C files. Both approaches have their advantages and inconvenients: - - * not distributing the C files means they are generated on the fly when - compiling the modules. Cython needs to be installed on the system, - and it makes your package a bit more future proof, as Cython evolves - to support newer Python versions. It also introduces some variance in - the builds, as they now depend on the Cython version installed on the - system; - - * when you distribute the C files with your package, the modules can be - compiled directly with the host compiler, no Cython required. It also - makes your tarball heavier, as Cython generates quite verbose code. - It might also be good for performance-critical code, when you want to - make sure the generated code is optimal, regardless of version of - Cython installed on the host system. - - In the first case, you can make Cython available to pip for compilation by - adding it to your ``setup.cfg``:: - - [options] - install_requires = cython - - This way people who just want to install your package won't need to have - Cython installed in their system/venv. - - It is up to you to choose one option or the other. The *cythonize* argument - controls the default mode of operation: set it to ``True`` if you don't - distribute C files with your package (the default), and ``False`` if you - do. - - Packages that distribute C files may use the ``CYTHONIZE`` environment - variable to create or update the C files:: - - CYTHONIZE=1 python setup.py build_ext --inplace - - You can also enable profiling for the Cython modules with the - ``PROFILE_CYTHON`` environment variable:: - - PROFILE_CYTHON=1 python setup.py build_ext --inplace - - Debugging symbols can be added with:: - - DEBUG=1 python setup.py build_ext --inplace - - """ - this_dir = op.dirname(__file__) - setup_cfg_file = op.join(this_dir, 'setup.cfg') - cythonize = _str_to_bool(os.environ.get('CYTHONIZE', cythonize)) - profile_cython = _str_to_bool(os.environ.get('PROFILE_CYTHON', False)) - debug = _str_to_bool(os.environ.get('DEBUG', False)) - if op.exists(setup_cfg_file): - # Create Cython Extension objects - with open(setup_cfg_file) as fp: - parsed_setup_cfg = parse_setup_cfg(fp, cythonize=cythonize) - cython_ext_modules = create_cython_ext_modules( - parsed_setup_cfg, - profile_cython=profile_cython, - debug=debug - ) - - if cythonize: - try: - from Cython.Build import cythonize - except ImportError: - pass - else: - cython_ext_modules = cythonize(cython_ext_modules) - - ext_modules = kwargs.setdefault('ext_modules', []) - ext_modules.extend(cython_ext_modules) - - setuptools.setup(**kwargs) - - -def create_cython_ext_modules(cython_modules, profile_cython=False, - debug=False): - """ - Create :class:`~distutils.extension.Extension` objects from - *cython_modules*. - - *cython_modules* must be a dict, as returned by :func:`parse_setup_cfg`. - - If *profile_cython* is true, Cython modules are compiled to support Python - proiflers. - - Debug symbols are included if *debug* is true. - """ - if profile_cython: - from Cython.Distutils import Extension - else: - from distutils.extension import Extension - - ret = [] - for name, mod_data in cython_modules.items(): - kwargs = {'name': name} - kwargs.update(mod_data) - if profile_cython: - cython_directives = kwargs.setdefault('cython_directives', {}) - cython_directives['profile'] = True - if debug: - for args_name in ('extra_compile_args', 'extra_link_args'): - args = kwargs.setdefault(args_name, []) - if '-g' not in args: - args.append('-g') - ext = Extension(**kwargs) - ret.append(ext) - return ret - - -def parse_setup_cfg(fp, cythonize=False, pkg_config=None, base_dir=''): - """ - Parse the cython specific bits in a setup.cfg file. - - *fp* must be a file-like object opened for reading. - - *pkg_config* may be a callable taking a list of library names and returning - a ``pkg-config`` like string (e.g. ``-I/foo -L/bar -lbaz``). The default is - to use an internal function that actually runs ``pkg-config`` (normally - used for testing). - - *base_dir* can be used to make relative paths absolute. - """ - if pkg_config is None: - pkg_config = _run_pkg_config - config = configparser.SafeConfigParser() - config.readfp(fp) - return _expand_cython_modules(config, cythonize, pkg_config, base_dir) - - -class _StoreOrderedArgs(argparse.Action): - - def __call__(self, parser, namespace, values, option_string=None): - if 'ordered_args' not in namespace: - setattr(namespace, 'ordered_args', []) - namespace.ordered_args.append((self.dest, values)) - - -def extract_args(args_str, args): - """ - Extract *args* from arguments string *args_str*. - - Return a *(extracted_args, remaining_args_str)* tuple, where - *extracted_args* is a dict containing the extracted arguments, and - *remaining_args_str* a string containing the remaining arguments. - """ - parser = argparse.ArgumentParser() - for arg in args: - parser.add_argument(arg, action=_StoreOrderedArgs) - args_list = shlex.split(args_str) - try: - args_ns, other_args = parser.parse_known_args(args_list) - except SystemExit: - raise Exception('args parsing failed') - extracted_args = {} - for arg_name, value in getattr(args_ns, 'ordered_args', []): - arg_values = extracted_args.setdefault(arg_name, []) - arg_values.append(value) - return extracted_args, ' '.join(other_args) - - -def _expand_cython_modules(config, cythonize, pkg_config, base_dir): - ret = {} - for section in config.sections(): - if section.startswith(MODULE_SECTION_PREFIX): - module_name = section[len(MODULE_SECTION_PREFIX):].strip() - module_dict = _expand_one_cython_module(config, section, cythonize, - pkg_config, base_dir) - ret[module_name] = module_dict - return ret - - -def _expand_one_cython_module(config, section, cythonize, pkg_config, - base_dir): - (pc_include_dirs, - pc_extra_compile_args, - pc_library_dirs, - pc_libraries, - pc_extra_link_args) = _expand_pkg_config_pkgs(config, section, pkg_config) - - module = {} - module['language'] = _get_config_opt(config, section, 'language', None) - module['extra_compile_args'] = \ - _get_config_list(config, section, 'extra_compile_args') + \ - pc_extra_compile_args - module['extra_link_args'] = \ - _get_config_list(config, section, 'extra_link_args') + \ - pc_extra_link_args - module['sources'] = _expand_sources(config, section, module['language'], - cythonize) - include_dirs = _get_config_list(config, section, 'include_dirs') - include_dirs += pc_include_dirs - include_dirs = _eval_strings(include_dirs) - include_dirs = _make_paths_absolute(include_dirs, base_dir) - library_dirs = _get_config_list(config, section, 'library_dirs') - library_dirs += pc_library_dirs - library_dirs = _eval_strings(library_dirs) - library_dirs = _make_paths_absolute(library_dirs, base_dir) - libraries = _get_config_list(config, section, 'libraries') - module['include_dirs'] = include_dirs - module['library_dirs'] = library_dirs - module['libraries'] = libraries + pc_libraries - all_conf_items = config.items(section) - try: - all_conf_items += config.items(DEFAULTS_SECTION) - except configparser.NoSectionError: - pass - for key, value in all_conf_items: - if key != 'pkg_config_packages' and key not in module: - module[key] = value - return module - - -def _make_paths_absolute(paths, base_dir): - return [op.join(base_dir, p) if not p.startswith('/') else p - for p in paths] - - -def _eval_strings(values): - ret = [] - for value in values: - if value.startswith('eval(') and value.endswith(')'): - ret.append(eval(value[5:-1])) - else: - ret.append(value) - return ret - - -def _expand_pkg_config_pkgs(config, section, pkg_config): - pkg_names = _get_config_list(config, section, 'pkg_config_packages') - if not pkg_names: - return [], [], [], [], [] - - original_pkg_config_path = os.environ.get('PKG_CONFIG_PATH', '') - pkg_config_path = original_pkg_config_path.split(":") - pkg_config_path.extend(_get_config_list(config, section, - 'pkg_config_dirs')) - env = os.environ.copy() - env['PKG_CONFIG_PATH'] = ":".join(pkg_config_path) - - extra_compile_args = pkg_config(pkg_names, '--cflags', env) - extra_link_args = pkg_config(pkg_names, '--libs', env) - - extracted_args, extra_compile_args = extract_args(extra_compile_args, - ['-I']) - include_dirs = extracted_args.get('I', []) - extracted_args, extra_link_args = extract_args(extra_link_args, - ['-L', '-l']) - library_dirs = extracted_args.get('L', []) - libraries = extracted_args.get('l', []) - - extra_compile_args = shlex.split(extra_compile_args) - extra_link_args = shlex.split(extra_link_args) - - return (include_dirs, extra_compile_args, library_dirs, libraries, - extra_link_args) - - -def _run_pkg_config(pkg_names, command, env): - return subprocess.check_output(['pkg-config', command] + pkg_names, - env=env).decode('utf8') - - -def _expand_sources(config, section, language, cythonize): - if cythonize: - ext = CYTHON_EXT - elif language == 'c++': - ext = CPP_EXT - else: - ext = C_EXT - sources = _get_config_list(config, section, 'sources') - return [_replace_cython_ext(s, ext) for s in sources] - - -def _replace_cython_ext(filename, target_ext): - root, ext = op.splitext(filename) - if ext == CYTHON_EXT: - return root + target_ext - return filename - - -def _get_default(config, option, default): - try: - return config.get(DEFAULTS_SECTION, option) - except (configparser.NoOptionError, configparser.NoSectionError): - return default - - -def _get_config_opt(config, section, option, default): - try: - return config.get(section, option) - except configparser.NoOptionError: - return _get_default(config, option, default) - - -def _get_config_list(config, section, option): - defaults_value = _get_default(config, option, '') - try: - value = config.get(section, option) - except configparser.NoOptionError: - value = '' - return ('%s %s' % (defaults_value, value)).split() - - -def _str_to_bool(value): - if isinstance(value, bool): - return value - value = value.lower() - if value in ('1', 'on', 'true', 'yes'): - return True - elif value in ('0', 'off', 'false', 'no'): - return False - raise ValueError('invalid boolean string %r' % value) diff --git a/meson.build b/meson.build new file mode 100644 index 0000000..6bbb574 --- /dev/null +++ b/meson.build @@ -0,0 +1,16 @@ +project( + 'purelib-and-platlib', + 'cpp', + 'cython', + default_options: [ + ], +) + +py = import('python').find_installation(pure: false) + + + +# need to build the proto generated files before +# I can build the cython extension +subdir('proto') +subdir('bact_archiver') \ No newline at end of file diff --git a/proto/epics_event.pyx b/proto/epics_event.pyx index 56ee129..7bdaefb 100644 --- a/proto/epics_event.pyx +++ b/proto/epics_event.pyx @@ -17,10 +17,10 @@ The following functions are expected to be called by external modules? """ # read EPICSEvent.pxd definition of Protocol-Buffer code -from proto.epics_event cimport PayloadInfo, string -from proto.epics_event cimport ScalarDouble, ScalarString, ScalarEnum, ScalarInt -from proto.epics_event cimport VectorDouble, VectorFloat -from proto.epics_event cimport VectorInt, VectorShort, VectorChar +from epics_event cimport PayloadInfo, string +from epics_event cimport ScalarDouble, ScalarString, ScalarEnum, ScalarInt +from epics_event cimport VectorDouble, VectorFloat +from epics_event cimport VectorInt, VectorShort, VectorChar import numpy as np cimport numpy as np diff --git a/proto/meson.build b/proto/meson.build new file mode 100644 index 0000000..da40905 --- /dev/null +++ b/proto/meson.build @@ -0,0 +1,43 @@ +# use protoc to create buffer +protoc = find_program('protoc', required : true) +deps = dependency('protobuf', required : true) +cpp_gen = generator(protoc, + output : ['@BASENAME@.pb.cc', '@BASENAME@.pb.h'], + arguments : ['--proto_path=@CURRENT_SOURCE_DIR@', '--cpp_out=@BUILD_DIR@', '@INPUT@'] + ) +py_gen = generator(protoc, + output : ['@BASENAME@_pb2.py'], + arguments : ['--proto_path=@CURRENT_SOURCE_DIR@', '--python_out=@BUILD_DIR@', '@INPUT@'] + ) +cpp_generated = cpp_gen.process('epics_event.proto') +py_generated = py_gen.process('epics_event.proto') + +# Why do I need to copy the files here? +# by default ninja will put them into its own dir? +cpp_proc = custom_target('cpp_proto', + command: [ 'cp', '@INPUT@', '@OUTPUT@' ], + input : cpp_generated, + output : '.', + build_by_default : true) + +py_proc = custom_target('py_proto', + command: [ 'cp', '@INPUT@', '@OUTPUT@' ], + input : py_generated, + output : 'epics_event_pb2.py', + build_by_default : true) + +# cpython interface +incdir_numpy = run_command(py, + ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], + check : true +).stdout().strip() +inc_dir = include_directories(incdir_numpy, '.', ) + +py.extension_module( + 'epics_event', + 'epics_event.pyx', + install: true, + include_directories : inc_dir, + subdir: 'bact_archiver', + override_options : ['cython_language=cpp'] + ) diff --git a/proto_gen/00README.txt b/proto_gen/00README.txt deleted file mode 100644 index 987a856..0000000 --- a/proto_gen/00README.txt +++ /dev/null @@ -1 +0,0 @@ -Directory for the file that proto buf generates. diff --git a/protocol_buffer.py b/protocol_buffer.py deleted file mode 100644 index 89ca8bc..0000000 --- a/protocol_buffer.py +++ /dev/null @@ -1,79 +0,0 @@ -# -*- coding: utf-8 -*- -# Author: Andreas Schälicke -# Pierre Schnizer -# Date: 2017, 2020 -"""Generate the interface code for Google Protobuf - -For Google protocol buffer see -https://developers.google.com/protocol-buffers -""" - -import os.path -import setuptools -import logging - -logger = logging.getLogger('setup') -_log = setuptools.distutils.log - -class GenerateProtocolBuffer(setuptools.Command): - """Custom build step for protobuf - - Generate the python and c++ wrapper code from the protocol buffer specifications - """ - - description = "Run protoc to generate python and cpp wrapper files" - - user_options = [ - # The format is (long option, short option, description). - ('inplace', None, 'create files inplace of proto-file (default)'), - ('python', None, 'create python wrapper'), - ('cpp', None, 'create C++ wrapper'), - ('source', None, 'files to be processed by protoc'), - ('src-dir', None, 'directory, where the input files are located'), - ('pydir', None, 'directroy, where the python output will be placed'), - ('build-dir', None, 'directroy, where the output will be placed'), - ('protoc=', None, 'protoc executable to use'), - ] - - def initialize_options(self): - """Set default values for options.""" - # Each user option must be listed here with their default value. - self.inplace = True - self.python = False - self.cpp = False - self.protoc = None - cwd = os.getcwd() - self.source = '' - self.pydir = cwd - self.src_dir = cwd - self.build_dir = cwd - - - def finalize_options(self): - """Post-process options.""" - if not self.inplace: - self.announce('inplace False is not support yet', - level=_log.WARN) - - if not self.python and not self.cpp: - self.python = True - self.cpp = True - self.announce('select python and C++ wrapper', - level=_log.INFO) - - def run(self): - self.announce("creating Wrapper for Archiver Protocol Buffers", - level=_log.INFO) - - if self.protoc is None: - protoc = 'protoc' - else: - protoc = self.protoc - - #self.announce('using protoc "%s"' %(protoc,), level=_log.INFO) - args = [protoc, self.source, '--proto_path={}'.format(self.src_dir)] - - if self.cpp: - self.spawn(args + ['--cpp_out={}'.format(self.build_dir)]) - if self.python: - self.spawn(args + ['--python_out={}'.format(self.pydir)]) diff --git a/pyproject.toml b/pyproject.toml index 4779476..9770ea0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,11 @@ # -*- coding: utf-8 -*- +# [build-system] +# requires = ["setuptools >= 61.0", "setuptools-scm", "cython", "numpy"] +# build-backend = "setuptools.build_meta" + [build-system] -requires = ["setuptools >= 61.0", "setuptools-scm", "cython", "numpy"] -build-backend = "setuptools.build_meta" +build-backend = 'mesonpy' +requires = ['meson-python', 'cython', 'numpy'] [project] name = "bact-archiver" @@ -40,4 +44,3 @@ homepage = "https://github.com/hz-b/bact-archiver" [tool.setuptools.packages.find] exclude = ["*.cc", "*.h", "proto/*"] namespaces = false - diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index abbf747..0000000 --- a/setup.cfg +++ /dev/null @@ -1,8 +0,0 @@ -# Howto include this section to pyproject.toml -[build_proto_c] -python = True -cpp = True -src_dir = proto -build_dir = proto_gen -pydir = bact_archiver -source = proto/epics_event.proto diff --git a/setup.py b/setup.py deleted file mode 100644 index 50296ca..0000000 --- a/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- -# Authors : Andreas Schälicke -# Pierre Schnizer -# Date : 2017, 2020, 2023, 2024 -import os -import sys - -from setuptools import Extension, setup -from numpy import get_include -# required that protocol_buffer is found -sys.path.append(os.path.dirname(__file__)) -from protocol_buffer import GenerateProtocolBuffer - -setup( - cmdclass=dict(build_proto_c=GenerateProtocolBuffer), - ext_modules=[ - Extension( - name="bact_archiver.epics_event", - sources=[ - "proto/epics_event.pyx", - "proto_gen/epics_event.pb.cc", - ], - include_dirs=[".", "proto/", "proto_gen/", get_include()], - libraries=[ - "protobuf", - ], - language="c++", - ) - ], -) From 6afe4cf57a8dfc754a31d21ad348fea17cb205d1 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Fri, 27 Sep 2024 10:32:44 +0200 Subject: [PATCH 02/51] [FIX] use build, no extra step for protoc --- .github/workflows/python-package.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index e2c86ae..2ced473 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: False matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] numpy-version: [ "numpy<2.0" , "numpy>=2.0" ] steps: @@ -37,11 +37,10 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install flake8 pytest wheel setuptools build + python -m pip install flake8 pytest wheel build if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: build and binary module run: | - python setup.py build_proto_c python -m pip wheel ./ python -m pip install ./ From 2b4cdacd463061fde6007f96bbd37811bec4a82f Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Fri, 27 Sep 2024 10:51:23 +0200 Subject: [PATCH 03/51] [TASK] documented change to meson, new version bumped up minor number: build system changed --- Changelog | 4 ++++ bact_archiver/meson.build | 2 +- pyproject.toml | 15 ++------------- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/Changelog b/Changelog index 0966e74..8402244 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,7 @@ +2024-09-27 Pierre Schnizer + * meson build system: replaced setuptools with meson + build. facilites compiling + 2021-04-16 Pierre Schnizer * archiver.getData: t0, t1 now required to be timezone aware diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build index 8927e9f..dc7fc25 100644 --- a/bact_archiver/meson.build +++ b/bact_archiver/meson.build @@ -6,5 +6,5 @@ py_sources = run_command('/bin/sh', py.install_sources( py_sources, pure: false, - subdir: 'timepix_sort' + subdir: 'bact_archiver' ) \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 9770ea0..06ef541 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,17 +1,12 @@ # -*- coding: utf-8 -*- -# [build-system] -# requires = ["setuptools >= 61.0", "setuptools-scm", "cython", "numpy"] -# build-backend = "setuptools.build_meta" - [build-system] build-backend = 'mesonpy' requires = ['meson-python', 'cython', 'numpy'] + [project] name = "bact-archiver" -# url = -# fullname = "BACT epics archiver appliance access" -version = "0.2.3" +version = "0.3.0" description = "EPICS archiver appliance access using google protobuf" readme="README.rst" authors = [ @@ -38,9 +33,3 @@ dependencies = [ [project.urls] homepage = "https://github.com/hz-b/bact-archiver" - - - -[tool.setuptools.packages.find] -exclude = ["*.cc", "*.h", "proto/*"] -namespaces = false From a504b3fe876810b4a368adcc97da7f9910764b96 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Fri, 27 Sep 2024 11:59:38 +0200 Subject: [PATCH 04/51] [FIX] build system: dependencies --- bact_archiver/meson.build | 1 + proto/meson.build | 21 ++++++++------------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build index dc7fc25..ae915b9 100644 --- a/bact_archiver/meson.build +++ b/bact_archiver/meson.build @@ -5,6 +5,7 @@ py_sources = run_command('/bin/sh', py.install_sources( py_sources, + 'archiver.cfg', pure: false, subdir: 'bact_archiver' ) \ No newline at end of file diff --git a/proto/meson.build b/proto/meson.build index da40905..16f4176 100644 --- a/proto/meson.build +++ b/proto/meson.build @@ -1,6 +1,7 @@ -# use protoc to create buffer +# protoc: created required files protoc = find_program('protoc', required : true) -deps = dependency('protobuf', required : true) +protobuf = dependency('protobuf', required : true) + cpp_gen = generator(protoc, output : ['@BASENAME@.pb.cc', '@BASENAME@.pb.h'], arguments : ['--proto_path=@CURRENT_SOURCE_DIR@', '--cpp_out=@BUILD_DIR@', '@INPUT@'] @@ -12,32 +13,26 @@ py_gen = generator(protoc, cpp_generated = cpp_gen.process('epics_event.proto') py_generated = py_gen.process('epics_event.proto') -# Why do I need to copy the files here? -# by default ninja will put them into its own dir? -cpp_proc = custom_target('cpp_proto', - command: [ 'cp', '@INPUT@', '@OUTPUT@' ], - input : cpp_generated, - output : '.', - build_by_default : true) - py_proc = custom_target('py_proto', command: [ 'cp', '@INPUT@', '@OUTPUT@' ], input : py_generated, output : 'epics_event_pb2.py', build_by_default : true) -# cpython interface +# cython extension incdir_numpy = run_command(py, ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], check : true ).stdout().strip() -inc_dir = include_directories(incdir_numpy, '.', ) +inc_dir = include_directories(incdir_numpy) py.extension_module( 'epics_event', 'epics_event.pyx', + cpp_generated, install: true, include_directories : inc_dir, subdir: 'bact_archiver', - override_options : ['cython_language=cpp'] + override_options : ['cython_language=cpp'], + dependencies : protobuf ) From cc30f5da54c6d543ae42f9f6f3703a367afcd81d Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Fri, 27 Sep 2024 12:09:28 +0200 Subject: [PATCH 05/51] [FIX] included created python file --- bact_archiver/meson.build | 1 + 1 file changed, 1 insertion(+) diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build index ae915b9..e67fc89 100644 --- a/bact_archiver/meson.build +++ b/bact_archiver/meson.build @@ -5,6 +5,7 @@ py_sources = run_command('/bin/sh', py.install_sources( py_sources, + 'epics_event_pb2.py', 'archiver.cfg', pure: false, subdir: 'bact_archiver' From 5cfd80cd991e456f72dcb1d066d7c96084597530 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Thu, 26 Sep 2024 18:55:42 +0200 Subject: [PATCH 06/51] [TASK] use meson build system Removing setuptools specific parts --- MANIFEST.in | 4 +- bact_archiver/glob.sh | 5 + bact_archiver/meson.build | 10 + cysetuptools.py | 408 -------------------------------------- meson.build | 16 ++ proto/epics_event.pyx | 8 +- proto/meson.build | 43 ++++ proto_gen/00README.txt | 1 - protocol_buffer.py | 79 -------- pyproject.toml | 9 +- setup.cfg | 8 - setup.py | 30 --- 12 files changed, 86 insertions(+), 535 deletions(-) create mode 100644 bact_archiver/glob.sh create mode 100644 bact_archiver/meson.build delete mode 100644 cysetuptools.py create mode 100644 meson.build create mode 100644 proto/meson.build delete mode 100644 proto_gen/00README.txt delete mode 100644 protocol_buffer.py delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/MANIFEST.in b/MANIFEST.in index 9672084..d14cfc3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,8 +1,8 @@ include protocol_buffer.py # the source files -include proto/* +include proto/ * # the files generated -include proto_gen/* +include proto_gen/ * # configuration files include bact_archiver/*.cfg # documentation files diff --git a/bact_archiver/glob.sh b/bact_archiver/glob.sh new file mode 100644 index 0000000..71474ca --- /dev/null +++ b/bact_archiver/glob.sh @@ -0,0 +1,5 @@ +#!/bin/sh +for i in *.py; +do + echo "$i" +done diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build new file mode 100644 index 0000000..8927e9f --- /dev/null +++ b/bact_archiver/meson.build @@ -0,0 +1,10 @@ +py_sources = run_command('/bin/sh', +['glob.sh'], + check : true +).stdout().strip().split('\n') + +py.install_sources( + py_sources, + pure: false, + subdir: 'timepix_sort' +) \ No newline at end of file diff --git a/cysetuptools.py b/cysetuptools.py deleted file mode 100644 index 5d96fd8..0000000 --- a/cysetuptools.py +++ /dev/null @@ -1,408 +0,0 @@ -import subprocess -import os -import os.path as op -import sys -import shlex -import argparse - -import setuptools - -PY3 = sys.version_info[0] == 3 -if PY3: - import configparser -else: - import ConfigParser as configparser - - -DEFAULTS_SECTION = 'cython-defaults' -MODULE_SECTION_PREFIX = 'cython-module:' -CYTHON_EXT = '.pyx' -C_EXT = '.c' -CPP_EXT = '.cpp' - - -def setup(cythonize=True, **kwargs): - """ - Drop-in replacement for :func:`setuptools.setup`, adding Cython niceties. - - Cython modules are described in setup.cfg, for example:: - - [cython-module: foo.bar] - sources = foo.pyx - bar.cpp - include_dirs = eval(__import__('numpy').get_include()) - /usr/include/foo - language = c++ - pkg_config_packages = opencv - - You still need to provide a ``setup.py``:: - - from cysetuptools import setup - - setup() - - The modules sections support the following entries: - - sources - The list of Cython and C/C++ source files that are compiled to build - the module. - - libraries - A list of libraries to link with the module. - - include_dirs - A list of directories to find include files. This entry supports - python expressions with ``eval()``; in the example above this is used - to retrieve the numpy include directory. - - library_dirs - A list of directories to find libraries. This entry supports - python expressions with ``eval()`` like ``include_dirs``. - - extra_compile_args - Extra arguments passed to the compiler. - - extra_link_args - Extra arguments passed to the linker. - - language - Typically "c" or "c++". - - pkg_config_packages - A list of ``pkg-config`` package names to link with the module. - - pkg_config_dirs - A list of directories to add to the pkg-config search paths (extends - the ``PKG_CONFIG_PATH`` environment variable). - - Defaults can also be specified in the ``[cython-defaults]`` section, for - example:: - - [cython-defaults] - include_dirs = /usr/include/bar - - [cython-module: foo.one] - sources = foo/one.pyx - - [cython-module: foo.two] - sources = foo/two.pyx - include_dirs = /usr/include/foo - - Here, ``foo.one`` and ``foo.two`` both will have ``/usr/include/bar`` in - their ``include_dirs``. List parameters in defaults are extended, so in the - example above, module ``foo.two`` ``include_dirs`` will be - ``['/usr/include/bar', '/usr/include/foo']``. - - There are two approaches when distributing Cython modules: with or without - the C files. Both approaches have their advantages and inconvenients: - - * not distributing the C files means they are generated on the fly when - compiling the modules. Cython needs to be installed on the system, - and it makes your package a bit more future proof, as Cython evolves - to support newer Python versions. It also introduces some variance in - the builds, as they now depend on the Cython version installed on the - system; - - * when you distribute the C files with your package, the modules can be - compiled directly with the host compiler, no Cython required. It also - makes your tarball heavier, as Cython generates quite verbose code. - It might also be good for performance-critical code, when you want to - make sure the generated code is optimal, regardless of version of - Cython installed on the host system. - - In the first case, you can make Cython available to pip for compilation by - adding it to your ``setup.cfg``:: - - [options] - install_requires = cython - - This way people who just want to install your package won't need to have - Cython installed in their system/venv. - - It is up to you to choose one option or the other. The *cythonize* argument - controls the default mode of operation: set it to ``True`` if you don't - distribute C files with your package (the default), and ``False`` if you - do. - - Packages that distribute C files may use the ``CYTHONIZE`` environment - variable to create or update the C files:: - - CYTHONIZE=1 python setup.py build_ext --inplace - - You can also enable profiling for the Cython modules with the - ``PROFILE_CYTHON`` environment variable:: - - PROFILE_CYTHON=1 python setup.py build_ext --inplace - - Debugging symbols can be added with:: - - DEBUG=1 python setup.py build_ext --inplace - - """ - this_dir = op.dirname(__file__) - setup_cfg_file = op.join(this_dir, 'setup.cfg') - cythonize = _str_to_bool(os.environ.get('CYTHONIZE', cythonize)) - profile_cython = _str_to_bool(os.environ.get('PROFILE_CYTHON', False)) - debug = _str_to_bool(os.environ.get('DEBUG', False)) - if op.exists(setup_cfg_file): - # Create Cython Extension objects - with open(setup_cfg_file) as fp: - parsed_setup_cfg = parse_setup_cfg(fp, cythonize=cythonize) - cython_ext_modules = create_cython_ext_modules( - parsed_setup_cfg, - profile_cython=profile_cython, - debug=debug - ) - - if cythonize: - try: - from Cython.Build import cythonize - except ImportError: - pass - else: - cython_ext_modules = cythonize(cython_ext_modules) - - ext_modules = kwargs.setdefault('ext_modules', []) - ext_modules.extend(cython_ext_modules) - - setuptools.setup(**kwargs) - - -def create_cython_ext_modules(cython_modules, profile_cython=False, - debug=False): - """ - Create :class:`~distutils.extension.Extension` objects from - *cython_modules*. - - *cython_modules* must be a dict, as returned by :func:`parse_setup_cfg`. - - If *profile_cython* is true, Cython modules are compiled to support Python - proiflers. - - Debug symbols are included if *debug* is true. - """ - if profile_cython: - from Cython.Distutils import Extension - else: - from distutils.extension import Extension - - ret = [] - for name, mod_data in cython_modules.items(): - kwargs = {'name': name} - kwargs.update(mod_data) - if profile_cython: - cython_directives = kwargs.setdefault('cython_directives', {}) - cython_directives['profile'] = True - if debug: - for args_name in ('extra_compile_args', 'extra_link_args'): - args = kwargs.setdefault(args_name, []) - if '-g' not in args: - args.append('-g') - ext = Extension(**kwargs) - ret.append(ext) - return ret - - -def parse_setup_cfg(fp, cythonize=False, pkg_config=None, base_dir=''): - """ - Parse the cython specific bits in a setup.cfg file. - - *fp* must be a file-like object opened for reading. - - *pkg_config* may be a callable taking a list of library names and returning - a ``pkg-config`` like string (e.g. ``-I/foo -L/bar -lbaz``). The default is - to use an internal function that actually runs ``pkg-config`` (normally - used for testing). - - *base_dir* can be used to make relative paths absolute. - """ - if pkg_config is None: - pkg_config = _run_pkg_config - config = configparser.SafeConfigParser() - config.readfp(fp) - return _expand_cython_modules(config, cythonize, pkg_config, base_dir) - - -class _StoreOrderedArgs(argparse.Action): - - def __call__(self, parser, namespace, values, option_string=None): - if 'ordered_args' not in namespace: - setattr(namespace, 'ordered_args', []) - namespace.ordered_args.append((self.dest, values)) - - -def extract_args(args_str, args): - """ - Extract *args* from arguments string *args_str*. - - Return a *(extracted_args, remaining_args_str)* tuple, where - *extracted_args* is a dict containing the extracted arguments, and - *remaining_args_str* a string containing the remaining arguments. - """ - parser = argparse.ArgumentParser() - for arg in args: - parser.add_argument(arg, action=_StoreOrderedArgs) - args_list = shlex.split(args_str) - try: - args_ns, other_args = parser.parse_known_args(args_list) - except SystemExit: - raise Exception('args parsing failed') - extracted_args = {} - for arg_name, value in getattr(args_ns, 'ordered_args', []): - arg_values = extracted_args.setdefault(arg_name, []) - arg_values.append(value) - return extracted_args, ' '.join(other_args) - - -def _expand_cython_modules(config, cythonize, pkg_config, base_dir): - ret = {} - for section in config.sections(): - if section.startswith(MODULE_SECTION_PREFIX): - module_name = section[len(MODULE_SECTION_PREFIX):].strip() - module_dict = _expand_one_cython_module(config, section, cythonize, - pkg_config, base_dir) - ret[module_name] = module_dict - return ret - - -def _expand_one_cython_module(config, section, cythonize, pkg_config, - base_dir): - (pc_include_dirs, - pc_extra_compile_args, - pc_library_dirs, - pc_libraries, - pc_extra_link_args) = _expand_pkg_config_pkgs(config, section, pkg_config) - - module = {} - module['language'] = _get_config_opt(config, section, 'language', None) - module['extra_compile_args'] = \ - _get_config_list(config, section, 'extra_compile_args') + \ - pc_extra_compile_args - module['extra_link_args'] = \ - _get_config_list(config, section, 'extra_link_args') + \ - pc_extra_link_args - module['sources'] = _expand_sources(config, section, module['language'], - cythonize) - include_dirs = _get_config_list(config, section, 'include_dirs') - include_dirs += pc_include_dirs - include_dirs = _eval_strings(include_dirs) - include_dirs = _make_paths_absolute(include_dirs, base_dir) - library_dirs = _get_config_list(config, section, 'library_dirs') - library_dirs += pc_library_dirs - library_dirs = _eval_strings(library_dirs) - library_dirs = _make_paths_absolute(library_dirs, base_dir) - libraries = _get_config_list(config, section, 'libraries') - module['include_dirs'] = include_dirs - module['library_dirs'] = library_dirs - module['libraries'] = libraries + pc_libraries - all_conf_items = config.items(section) - try: - all_conf_items += config.items(DEFAULTS_SECTION) - except configparser.NoSectionError: - pass - for key, value in all_conf_items: - if key != 'pkg_config_packages' and key not in module: - module[key] = value - return module - - -def _make_paths_absolute(paths, base_dir): - return [op.join(base_dir, p) if not p.startswith('/') else p - for p in paths] - - -def _eval_strings(values): - ret = [] - for value in values: - if value.startswith('eval(') and value.endswith(')'): - ret.append(eval(value[5:-1])) - else: - ret.append(value) - return ret - - -def _expand_pkg_config_pkgs(config, section, pkg_config): - pkg_names = _get_config_list(config, section, 'pkg_config_packages') - if not pkg_names: - return [], [], [], [], [] - - original_pkg_config_path = os.environ.get('PKG_CONFIG_PATH', '') - pkg_config_path = original_pkg_config_path.split(":") - pkg_config_path.extend(_get_config_list(config, section, - 'pkg_config_dirs')) - env = os.environ.copy() - env['PKG_CONFIG_PATH'] = ":".join(pkg_config_path) - - extra_compile_args = pkg_config(pkg_names, '--cflags', env) - extra_link_args = pkg_config(pkg_names, '--libs', env) - - extracted_args, extra_compile_args = extract_args(extra_compile_args, - ['-I']) - include_dirs = extracted_args.get('I', []) - extracted_args, extra_link_args = extract_args(extra_link_args, - ['-L', '-l']) - library_dirs = extracted_args.get('L', []) - libraries = extracted_args.get('l', []) - - extra_compile_args = shlex.split(extra_compile_args) - extra_link_args = shlex.split(extra_link_args) - - return (include_dirs, extra_compile_args, library_dirs, libraries, - extra_link_args) - - -def _run_pkg_config(pkg_names, command, env): - return subprocess.check_output(['pkg-config', command] + pkg_names, - env=env).decode('utf8') - - -def _expand_sources(config, section, language, cythonize): - if cythonize: - ext = CYTHON_EXT - elif language == 'c++': - ext = CPP_EXT - else: - ext = C_EXT - sources = _get_config_list(config, section, 'sources') - return [_replace_cython_ext(s, ext) for s in sources] - - -def _replace_cython_ext(filename, target_ext): - root, ext = op.splitext(filename) - if ext == CYTHON_EXT: - return root + target_ext - return filename - - -def _get_default(config, option, default): - try: - return config.get(DEFAULTS_SECTION, option) - except (configparser.NoOptionError, configparser.NoSectionError): - return default - - -def _get_config_opt(config, section, option, default): - try: - return config.get(section, option) - except configparser.NoOptionError: - return _get_default(config, option, default) - - -def _get_config_list(config, section, option): - defaults_value = _get_default(config, option, '') - try: - value = config.get(section, option) - except configparser.NoOptionError: - value = '' - return ('%s %s' % (defaults_value, value)).split() - - -def _str_to_bool(value): - if isinstance(value, bool): - return value - value = value.lower() - if value in ('1', 'on', 'true', 'yes'): - return True - elif value in ('0', 'off', 'false', 'no'): - return False - raise ValueError('invalid boolean string %r' % value) diff --git a/meson.build b/meson.build new file mode 100644 index 0000000..6bbb574 --- /dev/null +++ b/meson.build @@ -0,0 +1,16 @@ +project( + 'purelib-and-platlib', + 'cpp', + 'cython', + default_options: [ + ], +) + +py = import('python').find_installation(pure: false) + + + +# need to build the proto generated files before +# I can build the cython extension +subdir('proto') +subdir('bact_archiver') \ No newline at end of file diff --git a/proto/epics_event.pyx b/proto/epics_event.pyx index 56ee129..7bdaefb 100644 --- a/proto/epics_event.pyx +++ b/proto/epics_event.pyx @@ -17,10 +17,10 @@ The following functions are expected to be called by external modules? """ # read EPICSEvent.pxd definition of Protocol-Buffer code -from proto.epics_event cimport PayloadInfo, string -from proto.epics_event cimport ScalarDouble, ScalarString, ScalarEnum, ScalarInt -from proto.epics_event cimport VectorDouble, VectorFloat -from proto.epics_event cimport VectorInt, VectorShort, VectorChar +from epics_event cimport PayloadInfo, string +from epics_event cimport ScalarDouble, ScalarString, ScalarEnum, ScalarInt +from epics_event cimport VectorDouble, VectorFloat +from epics_event cimport VectorInt, VectorShort, VectorChar import numpy as np cimport numpy as np diff --git a/proto/meson.build b/proto/meson.build new file mode 100644 index 0000000..da40905 --- /dev/null +++ b/proto/meson.build @@ -0,0 +1,43 @@ +# use protoc to create buffer +protoc = find_program('protoc', required : true) +deps = dependency('protobuf', required : true) +cpp_gen = generator(protoc, + output : ['@BASENAME@.pb.cc', '@BASENAME@.pb.h'], + arguments : ['--proto_path=@CURRENT_SOURCE_DIR@', '--cpp_out=@BUILD_DIR@', '@INPUT@'] + ) +py_gen = generator(protoc, + output : ['@BASENAME@_pb2.py'], + arguments : ['--proto_path=@CURRENT_SOURCE_DIR@', '--python_out=@BUILD_DIR@', '@INPUT@'] + ) +cpp_generated = cpp_gen.process('epics_event.proto') +py_generated = py_gen.process('epics_event.proto') + +# Why do I need to copy the files here? +# by default ninja will put them into its own dir? +cpp_proc = custom_target('cpp_proto', + command: [ 'cp', '@INPUT@', '@OUTPUT@' ], + input : cpp_generated, + output : '.', + build_by_default : true) + +py_proc = custom_target('py_proto', + command: [ 'cp', '@INPUT@', '@OUTPUT@' ], + input : py_generated, + output : 'epics_event_pb2.py', + build_by_default : true) + +# cpython interface +incdir_numpy = run_command(py, + ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], + check : true +).stdout().strip() +inc_dir = include_directories(incdir_numpy, '.', ) + +py.extension_module( + 'epics_event', + 'epics_event.pyx', + install: true, + include_directories : inc_dir, + subdir: 'bact_archiver', + override_options : ['cython_language=cpp'] + ) diff --git a/proto_gen/00README.txt b/proto_gen/00README.txt deleted file mode 100644 index 987a856..0000000 --- a/proto_gen/00README.txt +++ /dev/null @@ -1 +0,0 @@ -Directory for the file that proto buf generates. diff --git a/protocol_buffer.py b/protocol_buffer.py deleted file mode 100644 index 89ca8bc..0000000 --- a/protocol_buffer.py +++ /dev/null @@ -1,79 +0,0 @@ -# -*- coding: utf-8 -*- -# Author: Andreas Schälicke -# Pierre Schnizer -# Date: 2017, 2020 -"""Generate the interface code for Google Protobuf - -For Google protocol buffer see -https://developers.google.com/protocol-buffers -""" - -import os.path -import setuptools -import logging - -logger = logging.getLogger('setup') -_log = setuptools.distutils.log - -class GenerateProtocolBuffer(setuptools.Command): - """Custom build step for protobuf - - Generate the python and c++ wrapper code from the protocol buffer specifications - """ - - description = "Run protoc to generate python and cpp wrapper files" - - user_options = [ - # The format is (long option, short option, description). - ('inplace', None, 'create files inplace of proto-file (default)'), - ('python', None, 'create python wrapper'), - ('cpp', None, 'create C++ wrapper'), - ('source', None, 'files to be processed by protoc'), - ('src-dir', None, 'directory, where the input files are located'), - ('pydir', None, 'directroy, where the python output will be placed'), - ('build-dir', None, 'directroy, where the output will be placed'), - ('protoc=', None, 'protoc executable to use'), - ] - - def initialize_options(self): - """Set default values for options.""" - # Each user option must be listed here with their default value. - self.inplace = True - self.python = False - self.cpp = False - self.protoc = None - cwd = os.getcwd() - self.source = '' - self.pydir = cwd - self.src_dir = cwd - self.build_dir = cwd - - - def finalize_options(self): - """Post-process options.""" - if not self.inplace: - self.announce('inplace False is not support yet', - level=_log.WARN) - - if not self.python and not self.cpp: - self.python = True - self.cpp = True - self.announce('select python and C++ wrapper', - level=_log.INFO) - - def run(self): - self.announce("creating Wrapper for Archiver Protocol Buffers", - level=_log.INFO) - - if self.protoc is None: - protoc = 'protoc' - else: - protoc = self.protoc - - #self.announce('using protoc "%s"' %(protoc,), level=_log.INFO) - args = [protoc, self.source, '--proto_path={}'.format(self.src_dir)] - - if self.cpp: - self.spawn(args + ['--cpp_out={}'.format(self.build_dir)]) - if self.python: - self.spawn(args + ['--python_out={}'.format(self.pydir)]) diff --git a/pyproject.toml b/pyproject.toml index 4779476..9770ea0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,11 @@ # -*- coding: utf-8 -*- +# [build-system] +# requires = ["setuptools >= 61.0", "setuptools-scm", "cython", "numpy"] +# build-backend = "setuptools.build_meta" + [build-system] -requires = ["setuptools >= 61.0", "setuptools-scm", "cython", "numpy"] -build-backend = "setuptools.build_meta" +build-backend = 'mesonpy' +requires = ['meson-python', 'cython', 'numpy'] [project] name = "bact-archiver" @@ -40,4 +44,3 @@ homepage = "https://github.com/hz-b/bact-archiver" [tool.setuptools.packages.find] exclude = ["*.cc", "*.h", "proto/*"] namespaces = false - diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index abbf747..0000000 --- a/setup.cfg +++ /dev/null @@ -1,8 +0,0 @@ -# Howto include this section to pyproject.toml -[build_proto_c] -python = True -cpp = True -src_dir = proto -build_dir = proto_gen -pydir = bact_archiver -source = proto/epics_event.proto diff --git a/setup.py b/setup.py deleted file mode 100644 index 50296ca..0000000 --- a/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- -# Authors : Andreas Schälicke -# Pierre Schnizer -# Date : 2017, 2020, 2023, 2024 -import os -import sys - -from setuptools import Extension, setup -from numpy import get_include -# required that protocol_buffer is found -sys.path.append(os.path.dirname(__file__)) -from protocol_buffer import GenerateProtocolBuffer - -setup( - cmdclass=dict(build_proto_c=GenerateProtocolBuffer), - ext_modules=[ - Extension( - name="bact_archiver.epics_event", - sources=[ - "proto/epics_event.pyx", - "proto_gen/epics_event.pb.cc", - ], - include_dirs=[".", "proto/", "proto_gen/", get_include()], - libraries=[ - "protobuf", - ], - language="c++", - ) - ], -) From e03dc4d5e19f2ca7c67509f559fbff449602354a Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Fri, 27 Sep 2024 10:32:44 +0200 Subject: [PATCH 07/51] [FIX] use build, no extra step for protoc --- .github/workflows/python-package.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index e2c86ae..2ced473 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: False matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] numpy-version: [ "numpy<2.0" , "numpy>=2.0" ] steps: @@ -37,11 +37,10 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install flake8 pytest wheel setuptools build + python -m pip install flake8 pytest wheel build if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: build and binary module run: | - python setup.py build_proto_c python -m pip wheel ./ python -m pip install ./ From 9bdef6a38f6772c8c3698fa159677ff950f1df16 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Fri, 27 Sep 2024 10:51:23 +0200 Subject: [PATCH 08/51] [TASK] documented change to meson, new version bumped up minor number: build system changed --- Changelog | 4 ++++ bact_archiver/meson.build | 2 +- pyproject.toml | 15 ++------------- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/Changelog b/Changelog index 0966e74..8402244 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,7 @@ +2024-09-27 Pierre Schnizer + * meson build system: replaced setuptools with meson + build. facilites compiling + 2021-04-16 Pierre Schnizer * archiver.getData: t0, t1 now required to be timezone aware diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build index 8927e9f..dc7fc25 100644 --- a/bact_archiver/meson.build +++ b/bact_archiver/meson.build @@ -6,5 +6,5 @@ py_sources = run_command('/bin/sh', py.install_sources( py_sources, pure: false, - subdir: 'timepix_sort' + subdir: 'bact_archiver' ) \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 9770ea0..06ef541 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,17 +1,12 @@ # -*- coding: utf-8 -*- -# [build-system] -# requires = ["setuptools >= 61.0", "setuptools-scm", "cython", "numpy"] -# build-backend = "setuptools.build_meta" - [build-system] build-backend = 'mesonpy' requires = ['meson-python', 'cython', 'numpy'] + [project] name = "bact-archiver" -# url = -# fullname = "BACT epics archiver appliance access" -version = "0.2.3" +version = "0.3.0" description = "EPICS archiver appliance access using google protobuf" readme="README.rst" authors = [ @@ -38,9 +33,3 @@ dependencies = [ [project.urls] homepage = "https://github.com/hz-b/bact-archiver" - - - -[tool.setuptools.packages.find] -exclude = ["*.cc", "*.h", "proto/*"] -namespaces = false From 2d924d0375a94144eaad938d90efbe7be5469eca Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Fri, 27 Sep 2024 11:59:38 +0200 Subject: [PATCH 09/51] [FIX] build system: dependencies --- bact_archiver/meson.build | 1 + proto/meson.build | 21 ++++++++------------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build index dc7fc25..ae915b9 100644 --- a/bact_archiver/meson.build +++ b/bact_archiver/meson.build @@ -5,6 +5,7 @@ py_sources = run_command('/bin/sh', py.install_sources( py_sources, + 'archiver.cfg', pure: false, subdir: 'bact_archiver' ) \ No newline at end of file diff --git a/proto/meson.build b/proto/meson.build index da40905..16f4176 100644 --- a/proto/meson.build +++ b/proto/meson.build @@ -1,6 +1,7 @@ -# use protoc to create buffer +# protoc: created required files protoc = find_program('protoc', required : true) -deps = dependency('protobuf', required : true) +protobuf = dependency('protobuf', required : true) + cpp_gen = generator(protoc, output : ['@BASENAME@.pb.cc', '@BASENAME@.pb.h'], arguments : ['--proto_path=@CURRENT_SOURCE_DIR@', '--cpp_out=@BUILD_DIR@', '@INPUT@'] @@ -12,32 +13,26 @@ py_gen = generator(protoc, cpp_generated = cpp_gen.process('epics_event.proto') py_generated = py_gen.process('epics_event.proto') -# Why do I need to copy the files here? -# by default ninja will put them into its own dir? -cpp_proc = custom_target('cpp_proto', - command: [ 'cp', '@INPUT@', '@OUTPUT@' ], - input : cpp_generated, - output : '.', - build_by_default : true) - py_proc = custom_target('py_proto', command: [ 'cp', '@INPUT@', '@OUTPUT@' ], input : py_generated, output : 'epics_event_pb2.py', build_by_default : true) -# cpython interface +# cython extension incdir_numpy = run_command(py, ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], check : true ).stdout().strip() -inc_dir = include_directories(incdir_numpy, '.', ) +inc_dir = include_directories(incdir_numpy) py.extension_module( 'epics_event', 'epics_event.pyx', + cpp_generated, install: true, include_directories : inc_dir, subdir: 'bact_archiver', - override_options : ['cython_language=cpp'] + override_options : ['cython_language=cpp'], + dependencies : protobuf ) From d7b0b52f004c8e269ee91174a6e191d2135798ee Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Fri, 27 Sep 2024 12:09:28 +0200 Subject: [PATCH 10/51] [FIX] included created python file --- bact_archiver/meson.build | 1 + 1 file changed, 1 insertion(+) diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build index ae915b9..e67fc89 100644 --- a/bact_archiver/meson.build +++ b/bact_archiver/meson.build @@ -5,6 +5,7 @@ py_sources = run_command('/bin/sh', py.install_sources( py_sources, + 'epics_event_pb2.py', 'archiver.cfg', pure: false, subdir: 'bact_archiver' From a88935bffd434448c9575790e9824af992d23ac9 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Mon, 30 Sep 2024 08:58:56 +0200 Subject: [PATCH 11/51] [TASK] as far as I could get: configure_file hack for handling created file --- bact_archiver/meson.build | 5 +++-- meson.build | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build index e67fc89..0e6cb08 100644 --- a/bact_archiver/meson.build +++ b/bact_archiver/meson.build @@ -5,8 +5,9 @@ py_sources = run_command('/bin/sh', py.install_sources( py_sources, - 'epics_event_pb2.py', + # 'epics_event_pb2.py', 'archiver.cfg', pure: false, subdir: 'bact_archiver' -) \ No newline at end of file +) +subdir('proto') diff --git a/meson.build b/meson.build index 6bbb574..d68725e 100644 --- a/meson.build +++ b/meson.build @@ -12,5 +12,4 @@ py = import('python').find_installation(pure: false) # need to build the proto generated files before # I can build the cython extension -subdir('proto') subdir('bact_archiver') \ No newline at end of file From 9404a076e1ec3c1470d89549b525da7a535f4a56 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Mon, 16 Dec 2024 16:23:21 +0100 Subject: [PATCH 12/51] [Snapshot] current development --- .../proto}/epics_event.proto | 0 .../proto}/epics_event.pxd | 0 .../proto}/epics_event.pyx | 0 {proto => bact_archiver/proto}/meson.build | 21 +++++++++++++------ 4 files changed, 15 insertions(+), 6 deletions(-) rename {proto => bact_archiver/proto}/epics_event.proto (100%) rename {proto => bact_archiver/proto}/epics_event.pxd (100%) rename {proto => bact_archiver/proto}/epics_event.pyx (100%) rename {proto => bact_archiver/proto}/meson.build (62%) diff --git a/proto/epics_event.proto b/bact_archiver/proto/epics_event.proto similarity index 100% rename from proto/epics_event.proto rename to bact_archiver/proto/epics_event.proto diff --git a/proto/epics_event.pxd b/bact_archiver/proto/epics_event.pxd similarity index 100% rename from proto/epics_event.pxd rename to bact_archiver/proto/epics_event.pxd diff --git a/proto/epics_event.pyx b/bact_archiver/proto/epics_event.pyx similarity index 100% rename from proto/epics_event.pyx rename to bact_archiver/proto/epics_event.pyx diff --git a/proto/meson.build b/bact_archiver/proto/meson.build similarity index 62% rename from proto/meson.build rename to bact_archiver/proto/meson.build index 16f4176..8591939 100644 --- a/proto/meson.build +++ b/bact_archiver/proto/meson.build @@ -13,12 +13,21 @@ py_gen = generator(protoc, cpp_generated = cpp_gen.process('epics_event.proto') py_generated = py_gen.process('epics_event.proto') -py_proc = custom_target('py_proto', - command: [ 'cp', '@INPUT@', '@OUTPUT@' ], - input : py_generated, - output : 'epics_event_pb2.py', - build_by_default : true) - +# py_proc = custom_target('py_proto', +# command: [ 'cp', '@INPUT@', '@OUTPUT@' ], +# input : py_generated, +# output : 'epics_event_pb2.py', +# build_by_default : true) +message('install dir') +message(py.get_install_dir()) +py_proc2 = configure_file(#'python_proto_as_config', + # command: [ 'proto_wrap.sh', '@INPUT@', '@OUTPUT@' ], + copy : true, + input : 'epics_event.proto', + output : 'epics_event_pb2.py', + install : true, + install_dir: py.get_install_dir(), +) # cython extension incdir_numpy = run_command(py, ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], From 5fac4911e842b2b9adf1580d8dd9e3a844b94bcc Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Mon, 16 Dec 2024 17:44:18 +0100 Subject: [PATCH 13/51] [Snapshot] on including generated python file --- bact_archiver/carchiver.py | 2 +- bact_archiver/proto/meson.build | 8 ++++---- meson.build | 3 +-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/bact_archiver/carchiver.py b/bact_archiver/carchiver.py index ae52def..2cab07b 100644 --- a/bact_archiver/carchiver.py +++ b/bact_archiver/carchiver.py @@ -5,7 +5,7 @@ """ from .epics_event import read_chunk, decode -from . import epics_event_pb2 as proto +import bact_archiver_epics_event_pb2 as proto from .protocol_buffer import (Chunk, dtypes as _dtypes, decoder as _decoder, dbrtypes as _dbrtypes, dsize as _dsize) from .archiver import ArchiverBasis, convert_datetime_to_timestamp diff --git a/bact_archiver/proto/meson.build b/bact_archiver/proto/meson.build index 8591939..dcecdd8 100644 --- a/bact_archiver/proto/meson.build +++ b/bact_archiver/proto/meson.build @@ -20,11 +20,11 @@ py_generated = py_gen.process('epics_event.proto') # build_by_default : true) message('install dir') message(py.get_install_dir()) -py_proc2 = configure_file(#'python_proto_as_config', - # command: [ 'proto_wrap.sh', '@INPUT@', '@OUTPUT@' ], - copy : true, +py_proc2 = configure_file(# 'python_proto_as_config', + command: [ 'proto_wrap.sh', '@INPUT@', '@OUTPUT@' ], + # copy : true, input : 'epics_event.proto', - output : 'epics_event_pb2.py', + output : 'bact_archiver_epics_event_pb2.py', install : true, install_dir: py.get_install_dir(), ) diff --git a/meson.build b/meson.build index 6bbb574..24b6034 100644 --- a/meson.build +++ b/meson.build @@ -12,5 +12,4 @@ py = import('python').find_installation(pure: false) # need to build the proto generated files before # I can build the cython extension -subdir('proto') -subdir('bact_archiver') \ No newline at end of file +subdir('bact_archiver') From 94ccb479aab5541515b28fdad4b71b8c741bd6c6 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Thu, 26 Sep 2024 18:55:42 +0200 Subject: [PATCH 14/51] [TASK] use meson build system Removing setuptools specific parts --- bact_archiver/proto/meson.build | 38 ++++++++++++++++++--------------- pyproject.toml | 5 ++++- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/bact_archiver/proto/meson.build b/bact_archiver/proto/meson.build index 8591939..6cbb0a1 100644 --- a/bact_archiver/proto/meson.build +++ b/bact_archiver/proto/meson.build @@ -1,7 +1,13 @@ +<<<<<<< HEAD:bact_archiver/proto/meson.build # protoc: created required files protoc = find_program('protoc', required : true) protobuf = dependency('protobuf', required : true) +======= +# use protoc to create buffer +protoc = find_program('protoc', required : true) +deps = dependency('protobuf', required : true) +>>>>>>> 2b13a42 ([TASK] use meson build system):proto/meson.build cpp_gen = generator(protoc, output : ['@BASENAME@.pb.cc', '@BASENAME@.pb.h'], arguments : ['--proto_path=@CURRENT_SOURCE_DIR@', '--cpp_out=@BUILD_DIR@', '@INPUT@'] @@ -13,27 +19,25 @@ py_gen = generator(protoc, cpp_generated = cpp_gen.process('epics_event.proto') py_generated = py_gen.process('epics_event.proto') -# py_proc = custom_target('py_proto', -# command: [ 'cp', '@INPUT@', '@OUTPUT@' ], -# input : py_generated, -# output : 'epics_event_pb2.py', -# build_by_default : true) -message('install dir') -message(py.get_install_dir()) -py_proc2 = configure_file(#'python_proto_as_config', - # command: [ 'proto_wrap.sh', '@INPUT@', '@OUTPUT@' ], - copy : true, - input : 'epics_event.proto', - output : 'epics_event_pb2.py', - install : true, - install_dir: py.get_install_dir(), -) -# cython extension +# Why do I need to copy the files here? +# by default ninja will put them into its own dir? +cpp_proc = custom_target('cpp_proto', + command: [ 'cp', '@INPUT@', '@OUTPUT@' ], + input : cpp_generated, + output : '.', + build_by_default : true) + +py_proc = custom_target('py_proto', + command: [ 'cp', '@INPUT@', '@OUTPUT@' ], + input : py_generated, + output : 'epics_event_pb2.py', + build_by_default : true) + incdir_numpy = run_command(py, ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], check : true ).stdout().strip() -inc_dir = include_directories(incdir_numpy) +inc_dir = include_directories(incdir_numpy, '.', ) py.extension_module( 'epics_event', diff --git a/pyproject.toml b/pyproject.toml index 06ef541..ee8a911 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,12 @@ # -*- coding: utf-8 -*- +# [build-system] +# requires = ["setuptools >= 61.0", "setuptools-scm", "cython", "numpy"] +# build-backend = "setuptools.build_meta" + [build-system] build-backend = 'mesonpy' requires = ['meson-python', 'cython', 'numpy'] - [project] name = "bact-archiver" version = "0.3.0" From c37405a1d665068df48a8a5ea9e5bdda0bcaa6f7 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Fri, 27 Sep 2024 10:51:23 +0200 Subject: [PATCH 15/51] [TASK] documented change to meson, new version bumped up minor number: build system changed --- pyproject.toml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ee8a911..06ef541 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,9 @@ # -*- coding: utf-8 -*- -# [build-system] -# requires = ["setuptools >= 61.0", "setuptools-scm", "cython", "numpy"] -# build-backend = "setuptools.build_meta" - [build-system] build-backend = 'mesonpy' requires = ['meson-python', 'cython', 'numpy'] + [project] name = "bact-archiver" version = "0.3.0" From 4b8e12f11083184b4944041596b9d8ec89d7b1dc Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Fri, 27 Sep 2024 11:59:38 +0200 Subject: [PATCH 16/51] [FIX] build system: dependencies --- bact_archiver/proto/meson.build | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/bact_archiver/proto/meson.build b/bact_archiver/proto/meson.build index 6cbb0a1..bfac3d7 100644 --- a/bact_archiver/proto/meson.build +++ b/bact_archiver/proto/meson.build @@ -4,10 +4,11 @@ protoc = find_program('protoc', required : true) protobuf = dependency('protobuf', required : true) ======= -# use protoc to create buffer +# protoc: created required files protoc = find_program('protoc', required : true) -deps = dependency('protobuf', required : true) ->>>>>>> 2b13a42 ([TASK] use meson build system):proto/meson.build +protobuf = dependency('protobuf', required : true) + +>>>>>>> a504b3f ([FIX] build system: dependencies):proto/meson.build cpp_gen = generator(protoc, output : ['@BASENAME@.pb.cc', '@BASENAME@.pb.h'], arguments : ['--proto_path=@CURRENT_SOURCE_DIR@', '--cpp_out=@BUILD_DIR@', '@INPUT@'] @@ -19,25 +20,18 @@ py_gen = generator(protoc, cpp_generated = cpp_gen.process('epics_event.proto') py_generated = py_gen.process('epics_event.proto') -# Why do I need to copy the files here? -# by default ninja will put them into its own dir? -cpp_proc = custom_target('cpp_proto', - command: [ 'cp', '@INPUT@', '@OUTPUT@' ], - input : cpp_generated, - output : '.', - build_by_default : true) - py_proc = custom_target('py_proto', command: [ 'cp', '@INPUT@', '@OUTPUT@' ], input : py_generated, output : 'epics_event_pb2.py', build_by_default : true) +# cython extension incdir_numpy = run_command(py, ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], check : true ).stdout().strip() -inc_dir = include_directories(incdir_numpy, '.', ) +inc_dir = include_directories(incdir_numpy) py.extension_module( 'epics_event', From cccc165c2b4f97d599fb76dd20ff76c5adaabc66 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Mon, 16 Dec 2024 19:04:46 +0100 Subject: [PATCH 17/51] [FIX] included created python file --- bact_archiver/meson.build | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build index 0e6cb08..d2b7558 100644 --- a/bact_archiver/meson.build +++ b/bact_archiver/meson.build @@ -5,7 +5,11 @@ py_sources = run_command('/bin/sh', py.install_sources( py_sources, +<<<<<<< HEAD # 'epics_event_pb2.py', +======= + 'epics_event_pb2.py', +>>>>>>> cc30f5d ([FIX] included created python file) 'archiver.cfg', pure: false, subdir: 'bact_archiver' From abe23f0184ff8acf985118950edce07ba409d438 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Mon, 16 Dec 2024 17:44:18 +0100 Subject: [PATCH 18/51] [Snapshot] on including generated python file --- bact_archiver/carchiver.py | 2 +- bact_archiver/proto/meson.build | 28 +++++++++++++++------------- meson.build | 2 +- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/bact_archiver/carchiver.py b/bact_archiver/carchiver.py index ae52def..2cab07b 100644 --- a/bact_archiver/carchiver.py +++ b/bact_archiver/carchiver.py @@ -5,7 +5,7 @@ """ from .epics_event import read_chunk, decode -from . import epics_event_pb2 as proto +import bact_archiver_epics_event_pb2 as proto from .protocol_buffer import (Chunk, dtypes as _dtypes, decoder as _decoder, dbrtypes as _dbrtypes, dsize as _dsize) from .archiver import ArchiverBasis, convert_datetime_to_timestamp diff --git a/bact_archiver/proto/meson.build b/bact_archiver/proto/meson.build index bfac3d7..dcecdd8 100644 --- a/bact_archiver/proto/meson.build +++ b/bact_archiver/proto/meson.build @@ -1,14 +1,7 @@ -<<<<<<< HEAD:bact_archiver/proto/meson.build # protoc: created required files protoc = find_program('protoc', required : true) protobuf = dependency('protobuf', required : true) -======= -# protoc: created required files -protoc = find_program('protoc', required : true) -protobuf = dependency('protobuf', required : true) - ->>>>>>> a504b3f ([FIX] build system: dependencies):proto/meson.build cpp_gen = generator(protoc, output : ['@BASENAME@.pb.cc', '@BASENAME@.pb.h'], arguments : ['--proto_path=@CURRENT_SOURCE_DIR@', '--cpp_out=@BUILD_DIR@', '@INPUT@'] @@ -20,12 +13,21 @@ py_gen = generator(protoc, cpp_generated = cpp_gen.process('epics_event.proto') py_generated = py_gen.process('epics_event.proto') -py_proc = custom_target('py_proto', - command: [ 'cp', '@INPUT@', '@OUTPUT@' ], - input : py_generated, - output : 'epics_event_pb2.py', - build_by_default : true) - +# py_proc = custom_target('py_proto', +# command: [ 'cp', '@INPUT@', '@OUTPUT@' ], +# input : py_generated, +# output : 'epics_event_pb2.py', +# build_by_default : true) +message('install dir') +message(py.get_install_dir()) +py_proc2 = configure_file(# 'python_proto_as_config', + command: [ 'proto_wrap.sh', '@INPUT@', '@OUTPUT@' ], + # copy : true, + input : 'epics_event.proto', + output : 'bact_archiver_epics_event_pb2.py', + install : true, + install_dir: py.get_install_dir(), +) # cython extension incdir_numpy = run_command(py, ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], diff --git a/meson.build b/meson.build index d68725e..24b6034 100644 --- a/meson.build +++ b/meson.build @@ -12,4 +12,4 @@ py = import('python').find_installation(pure: false) # need to build the proto generated files before # I can build the cython extension -subdir('bact_archiver') \ No newline at end of file +subdir('bact_archiver') From c167ce277139b38bb901539756c4cfdf3eaace56 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Mon, 16 Dec 2024 18:19:18 +0100 Subject: [PATCH 19/51] [Snapshot] compiling --- bact_archiver/proto/meson.build | 5 +++-- bact_archiver/proto/proto_wrap.sh | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100755 bact_archiver/proto/proto_wrap.sh diff --git a/bact_archiver/proto/meson.build b/bact_archiver/proto/meson.build index dcecdd8..3832a04 100644 --- a/bact_archiver/proto/meson.build +++ b/bact_archiver/proto/meson.build @@ -21,10 +21,10 @@ py_generated = py_gen.process('epics_event.proto') message('install dir') message(py.get_install_dir()) py_proc2 = configure_file(# 'python_proto_as_config', - command: [ 'proto_wrap.sh', '@INPUT@', '@OUTPUT@' ], + command: [ 'proto_wrap.sh', '@INPUT@', '@OUTPUT@' ], # copy : true, input : 'epics_event.proto', - output : 'bact_archiver_epics_event_pb2.py', + output : 'epics_event_pb2.py', install : true, install_dir: py.get_install_dir(), ) @@ -39,6 +39,7 @@ py.extension_module( 'epics_event', 'epics_event.pyx', cpp_generated, + py_generated, install: true, include_directories : inc_dir, subdir: 'bact_archiver', diff --git a/bact_archiver/proto/proto_wrap.sh b/bact_archiver/proto/proto_wrap.sh new file mode 100755 index 0000000..e5e9687 --- /dev/null +++ b/bact_archiver/proto/proto_wrap.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -xv + +input=$1 +output=$2 +PROTOC=protoc +DIRNAME=dirname + +SRC_DIR=`"$DIRNAME" "$input"` +BUILD_DIR=`"$DIRNAME" "$output"` + +echo "$PROTOC" --proto_path="$SRC_DIR" --python_out="$BUILD_DIR" "$input" +"$PROTOC" --proto_path="$SRC_DIR" --python_out="$BUILD_DIR" "$input" + + From 53f72f5a3f304ef0a7835fcc12dad54da6c8b2ae Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Mon, 16 Dec 2024 18:32:20 +0100 Subject: [PATCH 20/51] [Snapshot] compiling and installing Caveat: search path to proto_gen ... needs to be revisited --- bact_archiver/carchiver.py | 2 +- bact_archiver/proto/meson.build | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/bact_archiver/carchiver.py b/bact_archiver/carchiver.py index 2cab07b..ae52def 100644 --- a/bact_archiver/carchiver.py +++ b/bact_archiver/carchiver.py @@ -5,7 +5,7 @@ """ from .epics_event import read_chunk, decode -import bact_archiver_epics_event_pb2 as proto +from . import epics_event_pb2 as proto from .protocol_buffer import (Chunk, dtypes as _dtypes, decoder as _decoder, dbrtypes as _dbrtypes, dsize as _dsize) from .archiver import ArchiverBasis, convert_datetime_to_timestamp diff --git a/bact_archiver/proto/meson.build b/bact_archiver/proto/meson.build index 3832a04..c01f41e 100644 --- a/bact_archiver/proto/meson.build +++ b/bact_archiver/proto/meson.build @@ -26,8 +26,10 @@ py_proc2 = configure_file(# 'python_proto_as_config', input : 'epics_event.proto', output : 'epics_event_pb2.py', install : true, - install_dir: py.get_install_dir(), + install_dir: py.get_install_dir() / 'bact_archiver', + install_tag: 'python-runtime', ) + # cython extension incdir_numpy = run_command(py, ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], @@ -39,10 +41,9 @@ py.extension_module( 'epics_event', 'epics_event.pyx', cpp_generated, - py_generated, install: true, include_directories : inc_dir, subdir: 'bact_archiver', override_options : ['cython_language=cpp'], dependencies : protobuf - ) + ) \ No newline at end of file From a8cdd4cf63cbfe4f1100abc93dbc91ecdaa5922b Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Mon, 16 Dec 2024 18:55:59 +0100 Subject: [PATCH 21/51] [TASK] build wheel without using some shell --- bact_archiver/proto/meson.build | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/bact_archiver/proto/meson.build b/bact_archiver/proto/meson.build index c01f41e..b98b407 100644 --- a/bact_archiver/proto/meson.build +++ b/bact_archiver/proto/meson.build @@ -13,23 +13,6 @@ py_gen = generator(protoc, cpp_generated = cpp_gen.process('epics_event.proto') py_generated = py_gen.process('epics_event.proto') -# py_proc = custom_target('py_proto', -# command: [ 'cp', '@INPUT@', '@OUTPUT@' ], -# input : py_generated, -# output : 'epics_event_pb2.py', -# build_by_default : true) -message('install dir') -message(py.get_install_dir()) -py_proc2 = configure_file(# 'python_proto_as_config', - command: [ 'proto_wrap.sh', '@INPUT@', '@OUTPUT@' ], - # copy : true, - input : 'epics_event.proto', - output : 'epics_event_pb2.py', - install : true, - install_dir: py.get_install_dir() / 'bact_archiver', - install_tag: 'python-runtime', -) - # cython extension incdir_numpy = run_command(py, ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], @@ -37,6 +20,17 @@ incdir_numpy = run_command(py, ).stdout().strip() inc_dir = include_directories(incdir_numpy) +py_gen2 = custom_target('epics_event_build_py', + output: ['epics_event_pb2.py'], + input: 'epics_event.proto', + command: [ + protoc, '--proto_path=@SRCDIR@', '--python_out=@OUTDIR@', '@INPUT@' + ], + install: true, + install_dir: py.get_install_dir() / 'bact_archiver', # need to install _umfpack.py + install_tag: 'python-runtime', +) + py.extension_module( 'epics_event', 'epics_event.pyx', @@ -46,4 +40,5 @@ py.extension_module( subdir: 'bact_archiver', override_options : ['cython_language=cpp'], dependencies : protobuf - ) \ No newline at end of file + ) + From 8e8c2c263fe07991b0dcd1e17f3cbe78abe1037d Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Tue, 17 Dec 2024 09:13:20 +0100 Subject: [PATCH 22/51] [TASK] now epics_event_pb2 ends up where it should --- bact_archiver/carchiver.py | 2 +- bact_archiver/proto/meson.build | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bact_archiver/carchiver.py b/bact_archiver/carchiver.py index 2cab07b..ae52def 100644 --- a/bact_archiver/carchiver.py +++ b/bact_archiver/carchiver.py @@ -5,7 +5,7 @@ """ from .epics_event import read_chunk, decode -import bact_archiver_epics_event_pb2 as proto +from . import epics_event_pb2 as proto from .protocol_buffer import (Chunk, dtypes as _dtypes, decoder as _decoder, dbrtypes as _dbrtypes, dsize as _dsize) from .archiver import ArchiverBasis, convert_datetime_to_timestamp diff --git a/bact_archiver/proto/meson.build b/bact_archiver/proto/meson.build index 8c84415..78d3fcb 100644 --- a/bact_archiver/proto/meson.build +++ b/bact_archiver/proto/meson.build @@ -19,7 +19,7 @@ py_gen2 = custom_target('epics_event_build_py', output: ['epics_event_pb2.py'], input: 'epics_event.proto', command: [ - protoc, '--proto_path=@SRCDIR@', '--python_out=@OUTDIR@', '@INPUT@' + protoc, '--proto_path=@CURRENT_SOURCE_DIR@', '--python_out=@OUTDIR@', '@INPUT@' ], install: true, install_dir: py.get_install_dir() / 'bact_archiver', # need to instalall epics_event_pb2.py From e0e011b0a6e1b7e9924c1d35a6ece0f41eec1ca6 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Tue, 17 Dec 2024 09:15:39 +0100 Subject: [PATCH 23/51] [TASK] cleanup, proto back where it should be using custom command, no external shell hack needed --- bact_archiver/meson.build | 1 - bact_archiver/proto/proto_wrap.sh | 15 --------------- meson.build | 1 + {bact_archiver/proto => proto}/epics_event.proto | 0 {bact_archiver/proto => proto}/epics_event.pxd | 0 {bact_archiver/proto => proto}/epics_event.pyx | 0 {bact_archiver/proto => proto}/meson.build | 0 7 files changed, 1 insertion(+), 16 deletions(-) delete mode 100755 bact_archiver/proto/proto_wrap.sh rename {bact_archiver/proto => proto}/epics_event.proto (100%) rename {bact_archiver/proto => proto}/epics_event.pxd (100%) rename {bact_archiver/proto => proto}/epics_event.pyx (100%) rename {bact_archiver/proto => proto}/meson.build (100%) diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build index 0d3e2e8..37c4b44 100644 --- a/bact_archiver/meson.build +++ b/bact_archiver/meson.build @@ -9,4 +9,3 @@ py.install_sources( pure: false, subdir: 'bact_archiver' ) -subdir('proto') diff --git a/bact_archiver/proto/proto_wrap.sh b/bact_archiver/proto/proto_wrap.sh deleted file mode 100755 index e5e9687..0000000 --- a/bact_archiver/proto/proto_wrap.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -set -xv - -input=$1 -output=$2 -PROTOC=protoc -DIRNAME=dirname - -SRC_DIR=`"$DIRNAME" "$input"` -BUILD_DIR=`"$DIRNAME" "$output"` - -echo "$PROTOC" --proto_path="$SRC_DIR" --python_out="$BUILD_DIR" "$input" -"$PROTOC" --proto_path="$SRC_DIR" --python_out="$BUILD_DIR" "$input" - - diff --git a/meson.build b/meson.build index 24b6034..55a2323 100644 --- a/meson.build +++ b/meson.build @@ -12,4 +12,5 @@ py = import('python').find_installation(pure: false) # need to build the proto generated files before # I can build the cython extension +subdir('proto') subdir('bact_archiver') diff --git a/bact_archiver/proto/epics_event.proto b/proto/epics_event.proto similarity index 100% rename from bact_archiver/proto/epics_event.proto rename to proto/epics_event.proto diff --git a/bact_archiver/proto/epics_event.pxd b/proto/epics_event.pxd similarity index 100% rename from bact_archiver/proto/epics_event.pxd rename to proto/epics_event.pxd diff --git a/bact_archiver/proto/epics_event.pyx b/proto/epics_event.pyx similarity index 100% rename from bact_archiver/proto/epics_event.pyx rename to proto/epics_event.pyx diff --git a/bact_archiver/proto/meson.build b/proto/meson.build similarity index 100% rename from bact_archiver/proto/meson.build rename to proto/meson.build From 6ac84f2648c38dd094c8d06869547641940793ce Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Thu, 26 Sep 2024 18:55:42 +0200 Subject: [PATCH 24/51] [TASK] use meson build system Removing setuptools specific parts --- MANIFEST.in | 4 +- bact_archiver/glob.sh | 5 + bact_archiver/meson.build | 10 + cysetuptools.py | 408 -------------------------------------- meson.build | 16 ++ proto/epics_event.pyx | 8 +- proto/meson.build | 43 ++++ proto_gen/00README.txt | 1 - protocol_buffer.py | 79 -------- pyproject.toml | 9 +- setup.cfg | 8 - setup.py | 30 --- 12 files changed, 86 insertions(+), 535 deletions(-) create mode 100644 bact_archiver/glob.sh create mode 100644 bact_archiver/meson.build delete mode 100644 cysetuptools.py create mode 100644 meson.build create mode 100644 proto/meson.build delete mode 100644 proto_gen/00README.txt delete mode 100644 protocol_buffer.py delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/MANIFEST.in b/MANIFEST.in index 9672084..d14cfc3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,8 +1,8 @@ include protocol_buffer.py # the source files -include proto/* +include proto/ * # the files generated -include proto_gen/* +include proto_gen/ * # configuration files include bact_archiver/*.cfg # documentation files diff --git a/bact_archiver/glob.sh b/bact_archiver/glob.sh new file mode 100644 index 0000000..71474ca --- /dev/null +++ b/bact_archiver/glob.sh @@ -0,0 +1,5 @@ +#!/bin/sh +for i in *.py; +do + echo "$i" +done diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build new file mode 100644 index 0000000..8927e9f --- /dev/null +++ b/bact_archiver/meson.build @@ -0,0 +1,10 @@ +py_sources = run_command('/bin/sh', +['glob.sh'], + check : true +).stdout().strip().split('\n') + +py.install_sources( + py_sources, + pure: false, + subdir: 'timepix_sort' +) \ No newline at end of file diff --git a/cysetuptools.py b/cysetuptools.py deleted file mode 100644 index 5d96fd8..0000000 --- a/cysetuptools.py +++ /dev/null @@ -1,408 +0,0 @@ -import subprocess -import os -import os.path as op -import sys -import shlex -import argparse - -import setuptools - -PY3 = sys.version_info[0] == 3 -if PY3: - import configparser -else: - import ConfigParser as configparser - - -DEFAULTS_SECTION = 'cython-defaults' -MODULE_SECTION_PREFIX = 'cython-module:' -CYTHON_EXT = '.pyx' -C_EXT = '.c' -CPP_EXT = '.cpp' - - -def setup(cythonize=True, **kwargs): - """ - Drop-in replacement for :func:`setuptools.setup`, adding Cython niceties. - - Cython modules are described in setup.cfg, for example:: - - [cython-module: foo.bar] - sources = foo.pyx - bar.cpp - include_dirs = eval(__import__('numpy').get_include()) - /usr/include/foo - language = c++ - pkg_config_packages = opencv - - You still need to provide a ``setup.py``:: - - from cysetuptools import setup - - setup() - - The modules sections support the following entries: - - sources - The list of Cython and C/C++ source files that are compiled to build - the module. - - libraries - A list of libraries to link with the module. - - include_dirs - A list of directories to find include files. This entry supports - python expressions with ``eval()``; in the example above this is used - to retrieve the numpy include directory. - - library_dirs - A list of directories to find libraries. This entry supports - python expressions with ``eval()`` like ``include_dirs``. - - extra_compile_args - Extra arguments passed to the compiler. - - extra_link_args - Extra arguments passed to the linker. - - language - Typically "c" or "c++". - - pkg_config_packages - A list of ``pkg-config`` package names to link with the module. - - pkg_config_dirs - A list of directories to add to the pkg-config search paths (extends - the ``PKG_CONFIG_PATH`` environment variable). - - Defaults can also be specified in the ``[cython-defaults]`` section, for - example:: - - [cython-defaults] - include_dirs = /usr/include/bar - - [cython-module: foo.one] - sources = foo/one.pyx - - [cython-module: foo.two] - sources = foo/two.pyx - include_dirs = /usr/include/foo - - Here, ``foo.one`` and ``foo.two`` both will have ``/usr/include/bar`` in - their ``include_dirs``. List parameters in defaults are extended, so in the - example above, module ``foo.two`` ``include_dirs`` will be - ``['/usr/include/bar', '/usr/include/foo']``. - - There are two approaches when distributing Cython modules: with or without - the C files. Both approaches have their advantages and inconvenients: - - * not distributing the C files means they are generated on the fly when - compiling the modules. Cython needs to be installed on the system, - and it makes your package a bit more future proof, as Cython evolves - to support newer Python versions. It also introduces some variance in - the builds, as they now depend on the Cython version installed on the - system; - - * when you distribute the C files with your package, the modules can be - compiled directly with the host compiler, no Cython required. It also - makes your tarball heavier, as Cython generates quite verbose code. - It might also be good for performance-critical code, when you want to - make sure the generated code is optimal, regardless of version of - Cython installed on the host system. - - In the first case, you can make Cython available to pip for compilation by - adding it to your ``setup.cfg``:: - - [options] - install_requires = cython - - This way people who just want to install your package won't need to have - Cython installed in their system/venv. - - It is up to you to choose one option or the other. The *cythonize* argument - controls the default mode of operation: set it to ``True`` if you don't - distribute C files with your package (the default), and ``False`` if you - do. - - Packages that distribute C files may use the ``CYTHONIZE`` environment - variable to create or update the C files:: - - CYTHONIZE=1 python setup.py build_ext --inplace - - You can also enable profiling for the Cython modules with the - ``PROFILE_CYTHON`` environment variable:: - - PROFILE_CYTHON=1 python setup.py build_ext --inplace - - Debugging symbols can be added with:: - - DEBUG=1 python setup.py build_ext --inplace - - """ - this_dir = op.dirname(__file__) - setup_cfg_file = op.join(this_dir, 'setup.cfg') - cythonize = _str_to_bool(os.environ.get('CYTHONIZE', cythonize)) - profile_cython = _str_to_bool(os.environ.get('PROFILE_CYTHON', False)) - debug = _str_to_bool(os.environ.get('DEBUG', False)) - if op.exists(setup_cfg_file): - # Create Cython Extension objects - with open(setup_cfg_file) as fp: - parsed_setup_cfg = parse_setup_cfg(fp, cythonize=cythonize) - cython_ext_modules = create_cython_ext_modules( - parsed_setup_cfg, - profile_cython=profile_cython, - debug=debug - ) - - if cythonize: - try: - from Cython.Build import cythonize - except ImportError: - pass - else: - cython_ext_modules = cythonize(cython_ext_modules) - - ext_modules = kwargs.setdefault('ext_modules', []) - ext_modules.extend(cython_ext_modules) - - setuptools.setup(**kwargs) - - -def create_cython_ext_modules(cython_modules, profile_cython=False, - debug=False): - """ - Create :class:`~distutils.extension.Extension` objects from - *cython_modules*. - - *cython_modules* must be a dict, as returned by :func:`parse_setup_cfg`. - - If *profile_cython* is true, Cython modules are compiled to support Python - proiflers. - - Debug symbols are included if *debug* is true. - """ - if profile_cython: - from Cython.Distutils import Extension - else: - from distutils.extension import Extension - - ret = [] - for name, mod_data in cython_modules.items(): - kwargs = {'name': name} - kwargs.update(mod_data) - if profile_cython: - cython_directives = kwargs.setdefault('cython_directives', {}) - cython_directives['profile'] = True - if debug: - for args_name in ('extra_compile_args', 'extra_link_args'): - args = kwargs.setdefault(args_name, []) - if '-g' not in args: - args.append('-g') - ext = Extension(**kwargs) - ret.append(ext) - return ret - - -def parse_setup_cfg(fp, cythonize=False, pkg_config=None, base_dir=''): - """ - Parse the cython specific bits in a setup.cfg file. - - *fp* must be a file-like object opened for reading. - - *pkg_config* may be a callable taking a list of library names and returning - a ``pkg-config`` like string (e.g. ``-I/foo -L/bar -lbaz``). The default is - to use an internal function that actually runs ``pkg-config`` (normally - used for testing). - - *base_dir* can be used to make relative paths absolute. - """ - if pkg_config is None: - pkg_config = _run_pkg_config - config = configparser.SafeConfigParser() - config.readfp(fp) - return _expand_cython_modules(config, cythonize, pkg_config, base_dir) - - -class _StoreOrderedArgs(argparse.Action): - - def __call__(self, parser, namespace, values, option_string=None): - if 'ordered_args' not in namespace: - setattr(namespace, 'ordered_args', []) - namespace.ordered_args.append((self.dest, values)) - - -def extract_args(args_str, args): - """ - Extract *args* from arguments string *args_str*. - - Return a *(extracted_args, remaining_args_str)* tuple, where - *extracted_args* is a dict containing the extracted arguments, and - *remaining_args_str* a string containing the remaining arguments. - """ - parser = argparse.ArgumentParser() - for arg in args: - parser.add_argument(arg, action=_StoreOrderedArgs) - args_list = shlex.split(args_str) - try: - args_ns, other_args = parser.parse_known_args(args_list) - except SystemExit: - raise Exception('args parsing failed') - extracted_args = {} - for arg_name, value in getattr(args_ns, 'ordered_args', []): - arg_values = extracted_args.setdefault(arg_name, []) - arg_values.append(value) - return extracted_args, ' '.join(other_args) - - -def _expand_cython_modules(config, cythonize, pkg_config, base_dir): - ret = {} - for section in config.sections(): - if section.startswith(MODULE_SECTION_PREFIX): - module_name = section[len(MODULE_SECTION_PREFIX):].strip() - module_dict = _expand_one_cython_module(config, section, cythonize, - pkg_config, base_dir) - ret[module_name] = module_dict - return ret - - -def _expand_one_cython_module(config, section, cythonize, pkg_config, - base_dir): - (pc_include_dirs, - pc_extra_compile_args, - pc_library_dirs, - pc_libraries, - pc_extra_link_args) = _expand_pkg_config_pkgs(config, section, pkg_config) - - module = {} - module['language'] = _get_config_opt(config, section, 'language', None) - module['extra_compile_args'] = \ - _get_config_list(config, section, 'extra_compile_args') + \ - pc_extra_compile_args - module['extra_link_args'] = \ - _get_config_list(config, section, 'extra_link_args') + \ - pc_extra_link_args - module['sources'] = _expand_sources(config, section, module['language'], - cythonize) - include_dirs = _get_config_list(config, section, 'include_dirs') - include_dirs += pc_include_dirs - include_dirs = _eval_strings(include_dirs) - include_dirs = _make_paths_absolute(include_dirs, base_dir) - library_dirs = _get_config_list(config, section, 'library_dirs') - library_dirs += pc_library_dirs - library_dirs = _eval_strings(library_dirs) - library_dirs = _make_paths_absolute(library_dirs, base_dir) - libraries = _get_config_list(config, section, 'libraries') - module['include_dirs'] = include_dirs - module['library_dirs'] = library_dirs - module['libraries'] = libraries + pc_libraries - all_conf_items = config.items(section) - try: - all_conf_items += config.items(DEFAULTS_SECTION) - except configparser.NoSectionError: - pass - for key, value in all_conf_items: - if key != 'pkg_config_packages' and key not in module: - module[key] = value - return module - - -def _make_paths_absolute(paths, base_dir): - return [op.join(base_dir, p) if not p.startswith('/') else p - for p in paths] - - -def _eval_strings(values): - ret = [] - for value in values: - if value.startswith('eval(') and value.endswith(')'): - ret.append(eval(value[5:-1])) - else: - ret.append(value) - return ret - - -def _expand_pkg_config_pkgs(config, section, pkg_config): - pkg_names = _get_config_list(config, section, 'pkg_config_packages') - if not pkg_names: - return [], [], [], [], [] - - original_pkg_config_path = os.environ.get('PKG_CONFIG_PATH', '') - pkg_config_path = original_pkg_config_path.split(":") - pkg_config_path.extend(_get_config_list(config, section, - 'pkg_config_dirs')) - env = os.environ.copy() - env['PKG_CONFIG_PATH'] = ":".join(pkg_config_path) - - extra_compile_args = pkg_config(pkg_names, '--cflags', env) - extra_link_args = pkg_config(pkg_names, '--libs', env) - - extracted_args, extra_compile_args = extract_args(extra_compile_args, - ['-I']) - include_dirs = extracted_args.get('I', []) - extracted_args, extra_link_args = extract_args(extra_link_args, - ['-L', '-l']) - library_dirs = extracted_args.get('L', []) - libraries = extracted_args.get('l', []) - - extra_compile_args = shlex.split(extra_compile_args) - extra_link_args = shlex.split(extra_link_args) - - return (include_dirs, extra_compile_args, library_dirs, libraries, - extra_link_args) - - -def _run_pkg_config(pkg_names, command, env): - return subprocess.check_output(['pkg-config', command] + pkg_names, - env=env).decode('utf8') - - -def _expand_sources(config, section, language, cythonize): - if cythonize: - ext = CYTHON_EXT - elif language == 'c++': - ext = CPP_EXT - else: - ext = C_EXT - sources = _get_config_list(config, section, 'sources') - return [_replace_cython_ext(s, ext) for s in sources] - - -def _replace_cython_ext(filename, target_ext): - root, ext = op.splitext(filename) - if ext == CYTHON_EXT: - return root + target_ext - return filename - - -def _get_default(config, option, default): - try: - return config.get(DEFAULTS_SECTION, option) - except (configparser.NoOptionError, configparser.NoSectionError): - return default - - -def _get_config_opt(config, section, option, default): - try: - return config.get(section, option) - except configparser.NoOptionError: - return _get_default(config, option, default) - - -def _get_config_list(config, section, option): - defaults_value = _get_default(config, option, '') - try: - value = config.get(section, option) - except configparser.NoOptionError: - value = '' - return ('%s %s' % (defaults_value, value)).split() - - -def _str_to_bool(value): - if isinstance(value, bool): - return value - value = value.lower() - if value in ('1', 'on', 'true', 'yes'): - return True - elif value in ('0', 'off', 'false', 'no'): - return False - raise ValueError('invalid boolean string %r' % value) diff --git a/meson.build b/meson.build new file mode 100644 index 0000000..6bbb574 --- /dev/null +++ b/meson.build @@ -0,0 +1,16 @@ +project( + 'purelib-and-platlib', + 'cpp', + 'cython', + default_options: [ + ], +) + +py = import('python').find_installation(pure: false) + + + +# need to build the proto generated files before +# I can build the cython extension +subdir('proto') +subdir('bact_archiver') \ No newline at end of file diff --git a/proto/epics_event.pyx b/proto/epics_event.pyx index 56ee129..7bdaefb 100644 --- a/proto/epics_event.pyx +++ b/proto/epics_event.pyx @@ -17,10 +17,10 @@ The following functions are expected to be called by external modules? """ # read EPICSEvent.pxd definition of Protocol-Buffer code -from proto.epics_event cimport PayloadInfo, string -from proto.epics_event cimport ScalarDouble, ScalarString, ScalarEnum, ScalarInt -from proto.epics_event cimport VectorDouble, VectorFloat -from proto.epics_event cimport VectorInt, VectorShort, VectorChar +from epics_event cimport PayloadInfo, string +from epics_event cimport ScalarDouble, ScalarString, ScalarEnum, ScalarInt +from epics_event cimport VectorDouble, VectorFloat +from epics_event cimport VectorInt, VectorShort, VectorChar import numpy as np cimport numpy as np diff --git a/proto/meson.build b/proto/meson.build new file mode 100644 index 0000000..da40905 --- /dev/null +++ b/proto/meson.build @@ -0,0 +1,43 @@ +# use protoc to create buffer +protoc = find_program('protoc', required : true) +deps = dependency('protobuf', required : true) +cpp_gen = generator(protoc, + output : ['@BASENAME@.pb.cc', '@BASENAME@.pb.h'], + arguments : ['--proto_path=@CURRENT_SOURCE_DIR@', '--cpp_out=@BUILD_DIR@', '@INPUT@'] + ) +py_gen = generator(protoc, + output : ['@BASENAME@_pb2.py'], + arguments : ['--proto_path=@CURRENT_SOURCE_DIR@', '--python_out=@BUILD_DIR@', '@INPUT@'] + ) +cpp_generated = cpp_gen.process('epics_event.proto') +py_generated = py_gen.process('epics_event.proto') + +# Why do I need to copy the files here? +# by default ninja will put them into its own dir? +cpp_proc = custom_target('cpp_proto', + command: [ 'cp', '@INPUT@', '@OUTPUT@' ], + input : cpp_generated, + output : '.', + build_by_default : true) + +py_proc = custom_target('py_proto', + command: [ 'cp', '@INPUT@', '@OUTPUT@' ], + input : py_generated, + output : 'epics_event_pb2.py', + build_by_default : true) + +# cpython interface +incdir_numpy = run_command(py, + ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], + check : true +).stdout().strip() +inc_dir = include_directories(incdir_numpy, '.', ) + +py.extension_module( + 'epics_event', + 'epics_event.pyx', + install: true, + include_directories : inc_dir, + subdir: 'bact_archiver', + override_options : ['cython_language=cpp'] + ) diff --git a/proto_gen/00README.txt b/proto_gen/00README.txt deleted file mode 100644 index 987a856..0000000 --- a/proto_gen/00README.txt +++ /dev/null @@ -1 +0,0 @@ -Directory for the file that proto buf generates. diff --git a/protocol_buffer.py b/protocol_buffer.py deleted file mode 100644 index 89ca8bc..0000000 --- a/protocol_buffer.py +++ /dev/null @@ -1,79 +0,0 @@ -# -*- coding: utf-8 -*- -# Author: Andreas Schälicke -# Pierre Schnizer -# Date: 2017, 2020 -"""Generate the interface code for Google Protobuf - -For Google protocol buffer see -https://developers.google.com/protocol-buffers -""" - -import os.path -import setuptools -import logging - -logger = logging.getLogger('setup') -_log = setuptools.distutils.log - -class GenerateProtocolBuffer(setuptools.Command): - """Custom build step for protobuf - - Generate the python and c++ wrapper code from the protocol buffer specifications - """ - - description = "Run protoc to generate python and cpp wrapper files" - - user_options = [ - # The format is (long option, short option, description). - ('inplace', None, 'create files inplace of proto-file (default)'), - ('python', None, 'create python wrapper'), - ('cpp', None, 'create C++ wrapper'), - ('source', None, 'files to be processed by protoc'), - ('src-dir', None, 'directory, where the input files are located'), - ('pydir', None, 'directroy, where the python output will be placed'), - ('build-dir', None, 'directroy, where the output will be placed'), - ('protoc=', None, 'protoc executable to use'), - ] - - def initialize_options(self): - """Set default values for options.""" - # Each user option must be listed here with their default value. - self.inplace = True - self.python = False - self.cpp = False - self.protoc = None - cwd = os.getcwd() - self.source = '' - self.pydir = cwd - self.src_dir = cwd - self.build_dir = cwd - - - def finalize_options(self): - """Post-process options.""" - if not self.inplace: - self.announce('inplace False is not support yet', - level=_log.WARN) - - if not self.python and not self.cpp: - self.python = True - self.cpp = True - self.announce('select python and C++ wrapper', - level=_log.INFO) - - def run(self): - self.announce("creating Wrapper for Archiver Protocol Buffers", - level=_log.INFO) - - if self.protoc is None: - protoc = 'protoc' - else: - protoc = self.protoc - - #self.announce('using protoc "%s"' %(protoc,), level=_log.INFO) - args = [protoc, self.source, '--proto_path={}'.format(self.src_dir)] - - if self.cpp: - self.spawn(args + ['--cpp_out={}'.format(self.build_dir)]) - if self.python: - self.spawn(args + ['--python_out={}'.format(self.pydir)]) diff --git a/pyproject.toml b/pyproject.toml index 4779476..9770ea0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,11 @@ # -*- coding: utf-8 -*- +# [build-system] +# requires = ["setuptools >= 61.0", "setuptools-scm", "cython", "numpy"] +# build-backend = "setuptools.build_meta" + [build-system] -requires = ["setuptools >= 61.0", "setuptools-scm", "cython", "numpy"] -build-backend = "setuptools.build_meta" +build-backend = 'mesonpy' +requires = ['meson-python', 'cython', 'numpy'] [project] name = "bact-archiver" @@ -40,4 +44,3 @@ homepage = "https://github.com/hz-b/bact-archiver" [tool.setuptools.packages.find] exclude = ["*.cc", "*.h", "proto/*"] namespaces = false - diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index abbf747..0000000 --- a/setup.cfg +++ /dev/null @@ -1,8 +0,0 @@ -# Howto include this section to pyproject.toml -[build_proto_c] -python = True -cpp = True -src_dir = proto -build_dir = proto_gen -pydir = bact_archiver -source = proto/epics_event.proto diff --git a/setup.py b/setup.py deleted file mode 100644 index 50296ca..0000000 --- a/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- -# Authors : Andreas Schälicke -# Pierre Schnizer -# Date : 2017, 2020, 2023, 2024 -import os -import sys - -from setuptools import Extension, setup -from numpy import get_include -# required that protocol_buffer is found -sys.path.append(os.path.dirname(__file__)) -from protocol_buffer import GenerateProtocolBuffer - -setup( - cmdclass=dict(build_proto_c=GenerateProtocolBuffer), - ext_modules=[ - Extension( - name="bact_archiver.epics_event", - sources=[ - "proto/epics_event.pyx", - "proto_gen/epics_event.pb.cc", - ], - include_dirs=[".", "proto/", "proto_gen/", get_include()], - libraries=[ - "protobuf", - ], - language="c++", - ) - ], -) From 9ea55da55894a6b5cfa9170d005065d7bd1150fc Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Fri, 27 Sep 2024 10:32:44 +0200 Subject: [PATCH 25/51] [FIX] use build, no extra step for protoc --- .github/workflows/python-package.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 18001c6..c99c2a8 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -37,11 +37,10 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install flake8 pytest wheel setuptools build + python -m pip install flake8 pytest wheel build if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: build and binary module run: | - python setup.py build_proto_c python -m pip wheel ./ python -m pip install ./ From 932f70b2fa7487a1c683954938d1589b2d1e6cf4 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Fri, 27 Sep 2024 10:51:23 +0200 Subject: [PATCH 26/51] [TASK] documented change to meson, new version bumped up minor number: build system changed --- Changelog | 4 ++++ bact_archiver/meson.build | 2 +- pyproject.toml | 15 ++------------- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/Changelog b/Changelog index 0966e74..8402244 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,7 @@ +2024-09-27 Pierre Schnizer + * meson build system: replaced setuptools with meson + build. facilites compiling + 2021-04-16 Pierre Schnizer * archiver.getData: t0, t1 now required to be timezone aware diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build index 8927e9f..dc7fc25 100644 --- a/bact_archiver/meson.build +++ b/bact_archiver/meson.build @@ -6,5 +6,5 @@ py_sources = run_command('/bin/sh', py.install_sources( py_sources, pure: false, - subdir: 'timepix_sort' + subdir: 'bact_archiver' ) \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 9770ea0..06ef541 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,17 +1,12 @@ # -*- coding: utf-8 -*- -# [build-system] -# requires = ["setuptools >= 61.0", "setuptools-scm", "cython", "numpy"] -# build-backend = "setuptools.build_meta" - [build-system] build-backend = 'mesonpy' requires = ['meson-python', 'cython', 'numpy'] + [project] name = "bact-archiver" -# url = -# fullname = "BACT epics archiver appliance access" -version = "0.2.3" +version = "0.3.0" description = "EPICS archiver appliance access using google protobuf" readme="README.rst" authors = [ @@ -38,9 +33,3 @@ dependencies = [ [project.urls] homepage = "https://github.com/hz-b/bact-archiver" - - - -[tool.setuptools.packages.find] -exclude = ["*.cc", "*.h", "proto/*"] -namespaces = false From 8d964012d4dc59ecd94e08ec272aa2510dd4999e Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Fri, 27 Sep 2024 11:59:38 +0200 Subject: [PATCH 27/51] [FIX] build system: dependencies --- bact_archiver/meson.build | 1 + proto/meson.build | 21 ++++++++------------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build index dc7fc25..ae915b9 100644 --- a/bact_archiver/meson.build +++ b/bact_archiver/meson.build @@ -5,6 +5,7 @@ py_sources = run_command('/bin/sh', py.install_sources( py_sources, + 'archiver.cfg', pure: false, subdir: 'bact_archiver' ) \ No newline at end of file diff --git a/proto/meson.build b/proto/meson.build index da40905..16f4176 100644 --- a/proto/meson.build +++ b/proto/meson.build @@ -1,6 +1,7 @@ -# use protoc to create buffer +# protoc: created required files protoc = find_program('protoc', required : true) -deps = dependency('protobuf', required : true) +protobuf = dependency('protobuf', required : true) + cpp_gen = generator(protoc, output : ['@BASENAME@.pb.cc', '@BASENAME@.pb.h'], arguments : ['--proto_path=@CURRENT_SOURCE_DIR@', '--cpp_out=@BUILD_DIR@', '@INPUT@'] @@ -12,32 +13,26 @@ py_gen = generator(protoc, cpp_generated = cpp_gen.process('epics_event.proto') py_generated = py_gen.process('epics_event.proto') -# Why do I need to copy the files here? -# by default ninja will put them into its own dir? -cpp_proc = custom_target('cpp_proto', - command: [ 'cp', '@INPUT@', '@OUTPUT@' ], - input : cpp_generated, - output : '.', - build_by_default : true) - py_proc = custom_target('py_proto', command: [ 'cp', '@INPUT@', '@OUTPUT@' ], input : py_generated, output : 'epics_event_pb2.py', build_by_default : true) -# cpython interface +# cython extension incdir_numpy = run_command(py, ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], check : true ).stdout().strip() -inc_dir = include_directories(incdir_numpy, '.', ) +inc_dir = include_directories(incdir_numpy) py.extension_module( 'epics_event', 'epics_event.pyx', + cpp_generated, install: true, include_directories : inc_dir, subdir: 'bact_archiver', - override_options : ['cython_language=cpp'] + override_options : ['cython_language=cpp'], + dependencies : protobuf ) From 55ae2e6445d37f9146c64491f38cbdf94a70be77 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Fri, 27 Sep 2024 12:09:28 +0200 Subject: [PATCH 28/51] [FIX] included created python file --- bact_archiver/meson.build | 1 + 1 file changed, 1 insertion(+) diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build index ae915b9..e67fc89 100644 --- a/bact_archiver/meson.build +++ b/bact_archiver/meson.build @@ -5,6 +5,7 @@ py_sources = run_command('/bin/sh', py.install_sources( py_sources, + 'epics_event_pb2.py', 'archiver.cfg', pure: false, subdir: 'bact_archiver' From 15c04347659b64bd01e31220979a7f77a98a5a97 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Mon, 30 Sep 2024 08:58:56 +0200 Subject: [PATCH 29/51] [TASK] as far as I could get: configure_file hack for handling created file --- bact_archiver/meson.build | 5 +++-- meson.build | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build index e67fc89..0e6cb08 100644 --- a/bact_archiver/meson.build +++ b/bact_archiver/meson.build @@ -5,8 +5,9 @@ py_sources = run_command('/bin/sh', py.install_sources( py_sources, - 'epics_event_pb2.py', + # 'epics_event_pb2.py', 'archiver.cfg', pure: false, subdir: 'bact_archiver' -) \ No newline at end of file +) +subdir('proto') diff --git a/meson.build b/meson.build index 6bbb574..d68725e 100644 --- a/meson.build +++ b/meson.build @@ -12,5 +12,4 @@ py = import('python').find_installation(pure: false) # need to build the proto generated files before # I can build the cython extension -subdir('proto') subdir('bact_archiver') \ No newline at end of file From 2a502d4f2dfcb78f429fae96945db9b81cffcf98 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Mon, 16 Dec 2024 16:23:21 +0100 Subject: [PATCH 30/51] [Snapshot] current development --- .../proto}/epics_event.proto | 0 .../proto}/epics_event.pxd | 0 .../proto}/epics_event.pyx | 0 {proto => bact_archiver/proto}/meson.build | 21 +++++++++++++------ 4 files changed, 15 insertions(+), 6 deletions(-) rename {proto => bact_archiver/proto}/epics_event.proto (100%) rename {proto => bact_archiver/proto}/epics_event.pxd (100%) rename {proto => bact_archiver/proto}/epics_event.pyx (100%) rename {proto => bact_archiver/proto}/meson.build (62%) diff --git a/proto/epics_event.proto b/bact_archiver/proto/epics_event.proto similarity index 100% rename from proto/epics_event.proto rename to bact_archiver/proto/epics_event.proto diff --git a/proto/epics_event.pxd b/bact_archiver/proto/epics_event.pxd similarity index 100% rename from proto/epics_event.pxd rename to bact_archiver/proto/epics_event.pxd diff --git a/proto/epics_event.pyx b/bact_archiver/proto/epics_event.pyx similarity index 100% rename from proto/epics_event.pyx rename to bact_archiver/proto/epics_event.pyx diff --git a/proto/meson.build b/bact_archiver/proto/meson.build similarity index 62% rename from proto/meson.build rename to bact_archiver/proto/meson.build index 16f4176..8591939 100644 --- a/proto/meson.build +++ b/bact_archiver/proto/meson.build @@ -13,12 +13,21 @@ py_gen = generator(protoc, cpp_generated = cpp_gen.process('epics_event.proto') py_generated = py_gen.process('epics_event.proto') -py_proc = custom_target('py_proto', - command: [ 'cp', '@INPUT@', '@OUTPUT@' ], - input : py_generated, - output : 'epics_event_pb2.py', - build_by_default : true) - +# py_proc = custom_target('py_proto', +# command: [ 'cp', '@INPUT@', '@OUTPUT@' ], +# input : py_generated, +# output : 'epics_event_pb2.py', +# build_by_default : true) +message('install dir') +message(py.get_install_dir()) +py_proc2 = configure_file(#'python_proto_as_config', + # command: [ 'proto_wrap.sh', '@INPUT@', '@OUTPUT@' ], + copy : true, + input : 'epics_event.proto', + output : 'epics_event_pb2.py', + install : true, + install_dir: py.get_install_dir(), +) # cython extension incdir_numpy = run_command(py, ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], From 8351d3b89188bf81716b1a8c95b0ff1f40c05c40 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Thu, 26 Sep 2024 18:55:42 +0200 Subject: [PATCH 31/51] [TASK] use meson build system Removing setuptools specific parts --- bact_archiver/proto/meson.build | 38 ++++++++++++++++++--------------- pyproject.toml | 5 ++++- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/bact_archiver/proto/meson.build b/bact_archiver/proto/meson.build index 8591939..6cbb0a1 100644 --- a/bact_archiver/proto/meson.build +++ b/bact_archiver/proto/meson.build @@ -1,7 +1,13 @@ +<<<<<<< HEAD:bact_archiver/proto/meson.build # protoc: created required files protoc = find_program('protoc', required : true) protobuf = dependency('protobuf', required : true) +======= +# use protoc to create buffer +protoc = find_program('protoc', required : true) +deps = dependency('protobuf', required : true) +>>>>>>> 2b13a42 ([TASK] use meson build system):proto/meson.build cpp_gen = generator(protoc, output : ['@BASENAME@.pb.cc', '@BASENAME@.pb.h'], arguments : ['--proto_path=@CURRENT_SOURCE_DIR@', '--cpp_out=@BUILD_DIR@', '@INPUT@'] @@ -13,27 +19,25 @@ py_gen = generator(protoc, cpp_generated = cpp_gen.process('epics_event.proto') py_generated = py_gen.process('epics_event.proto') -# py_proc = custom_target('py_proto', -# command: [ 'cp', '@INPUT@', '@OUTPUT@' ], -# input : py_generated, -# output : 'epics_event_pb2.py', -# build_by_default : true) -message('install dir') -message(py.get_install_dir()) -py_proc2 = configure_file(#'python_proto_as_config', - # command: [ 'proto_wrap.sh', '@INPUT@', '@OUTPUT@' ], - copy : true, - input : 'epics_event.proto', - output : 'epics_event_pb2.py', - install : true, - install_dir: py.get_install_dir(), -) -# cython extension +# Why do I need to copy the files here? +# by default ninja will put them into its own dir? +cpp_proc = custom_target('cpp_proto', + command: [ 'cp', '@INPUT@', '@OUTPUT@' ], + input : cpp_generated, + output : '.', + build_by_default : true) + +py_proc = custom_target('py_proto', + command: [ 'cp', '@INPUT@', '@OUTPUT@' ], + input : py_generated, + output : 'epics_event_pb2.py', + build_by_default : true) + incdir_numpy = run_command(py, ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], check : true ).stdout().strip() -inc_dir = include_directories(incdir_numpy) +inc_dir = include_directories(incdir_numpy, '.', ) py.extension_module( 'epics_event', diff --git a/pyproject.toml b/pyproject.toml index 06ef541..ee8a911 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,12 @@ # -*- coding: utf-8 -*- +# [build-system] +# requires = ["setuptools >= 61.0", "setuptools-scm", "cython", "numpy"] +# build-backend = "setuptools.build_meta" + [build-system] build-backend = 'mesonpy' requires = ['meson-python', 'cython', 'numpy'] - [project] name = "bact-archiver" version = "0.3.0" From 1abb098c64ccb596ef927ef9009fea7e6a78a552 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Fri, 27 Sep 2024 10:51:23 +0200 Subject: [PATCH 32/51] [TASK] documented change to meson, new version bumped up minor number: build system changed --- pyproject.toml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ee8a911..06ef541 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,9 @@ # -*- coding: utf-8 -*- -# [build-system] -# requires = ["setuptools >= 61.0", "setuptools-scm", "cython", "numpy"] -# build-backend = "setuptools.build_meta" - [build-system] build-backend = 'mesonpy' requires = ['meson-python', 'cython', 'numpy'] + [project] name = "bact-archiver" version = "0.3.0" From 63409f7d68ea9b8697d0172ffb8644217ac33807 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Fri, 27 Sep 2024 11:59:38 +0200 Subject: [PATCH 33/51] [FIX] build system: dependencies --- bact_archiver/proto/meson.build | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/bact_archiver/proto/meson.build b/bact_archiver/proto/meson.build index 6cbb0a1..bfac3d7 100644 --- a/bact_archiver/proto/meson.build +++ b/bact_archiver/proto/meson.build @@ -4,10 +4,11 @@ protoc = find_program('protoc', required : true) protobuf = dependency('protobuf', required : true) ======= -# use protoc to create buffer +# protoc: created required files protoc = find_program('protoc', required : true) -deps = dependency('protobuf', required : true) ->>>>>>> 2b13a42 ([TASK] use meson build system):proto/meson.build +protobuf = dependency('protobuf', required : true) + +>>>>>>> a504b3f ([FIX] build system: dependencies):proto/meson.build cpp_gen = generator(protoc, output : ['@BASENAME@.pb.cc', '@BASENAME@.pb.h'], arguments : ['--proto_path=@CURRENT_SOURCE_DIR@', '--cpp_out=@BUILD_DIR@', '@INPUT@'] @@ -19,25 +20,18 @@ py_gen = generator(protoc, cpp_generated = cpp_gen.process('epics_event.proto') py_generated = py_gen.process('epics_event.proto') -# Why do I need to copy the files here? -# by default ninja will put them into its own dir? -cpp_proc = custom_target('cpp_proto', - command: [ 'cp', '@INPUT@', '@OUTPUT@' ], - input : cpp_generated, - output : '.', - build_by_default : true) - py_proc = custom_target('py_proto', command: [ 'cp', '@INPUT@', '@OUTPUT@' ], input : py_generated, output : 'epics_event_pb2.py', build_by_default : true) +# cython extension incdir_numpy = run_command(py, ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], check : true ).stdout().strip() -inc_dir = include_directories(incdir_numpy, '.', ) +inc_dir = include_directories(incdir_numpy) py.extension_module( 'epics_event', From dedb5efd230bf100a6707a7521b34d193ea1c5a7 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Mon, 16 Dec 2024 19:04:46 +0100 Subject: [PATCH 34/51] [FIX] included created python file --- bact_archiver/meson.build | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build index 0e6cb08..d2b7558 100644 --- a/bact_archiver/meson.build +++ b/bact_archiver/meson.build @@ -5,7 +5,11 @@ py_sources = run_command('/bin/sh', py.install_sources( py_sources, +<<<<<<< HEAD # 'epics_event_pb2.py', +======= + 'epics_event_pb2.py', +>>>>>>> cc30f5d ([FIX] included created python file) 'archiver.cfg', pure: false, subdir: 'bact_archiver' From dfc508cc7621b7d0bd990fc8f7fdb87a339b2f73 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Mon, 16 Dec 2024 17:44:18 +0100 Subject: [PATCH 35/51] [Snapshot] on including generated python file --- bact_archiver/carchiver.py | 2 +- bact_archiver/proto/meson.build | 28 +++++++++++++++------------- meson.build | 2 +- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/bact_archiver/carchiver.py b/bact_archiver/carchiver.py index ae52def..2cab07b 100644 --- a/bact_archiver/carchiver.py +++ b/bact_archiver/carchiver.py @@ -5,7 +5,7 @@ """ from .epics_event import read_chunk, decode -from . import epics_event_pb2 as proto +import bact_archiver_epics_event_pb2 as proto from .protocol_buffer import (Chunk, dtypes as _dtypes, decoder as _decoder, dbrtypes as _dbrtypes, dsize as _dsize) from .archiver import ArchiverBasis, convert_datetime_to_timestamp diff --git a/bact_archiver/proto/meson.build b/bact_archiver/proto/meson.build index bfac3d7..dcecdd8 100644 --- a/bact_archiver/proto/meson.build +++ b/bact_archiver/proto/meson.build @@ -1,14 +1,7 @@ -<<<<<<< HEAD:bact_archiver/proto/meson.build # protoc: created required files protoc = find_program('protoc', required : true) protobuf = dependency('protobuf', required : true) -======= -# protoc: created required files -protoc = find_program('protoc', required : true) -protobuf = dependency('protobuf', required : true) - ->>>>>>> a504b3f ([FIX] build system: dependencies):proto/meson.build cpp_gen = generator(protoc, output : ['@BASENAME@.pb.cc', '@BASENAME@.pb.h'], arguments : ['--proto_path=@CURRENT_SOURCE_DIR@', '--cpp_out=@BUILD_DIR@', '@INPUT@'] @@ -20,12 +13,21 @@ py_gen = generator(protoc, cpp_generated = cpp_gen.process('epics_event.proto') py_generated = py_gen.process('epics_event.proto') -py_proc = custom_target('py_proto', - command: [ 'cp', '@INPUT@', '@OUTPUT@' ], - input : py_generated, - output : 'epics_event_pb2.py', - build_by_default : true) - +# py_proc = custom_target('py_proto', +# command: [ 'cp', '@INPUT@', '@OUTPUT@' ], +# input : py_generated, +# output : 'epics_event_pb2.py', +# build_by_default : true) +message('install dir') +message(py.get_install_dir()) +py_proc2 = configure_file(# 'python_proto_as_config', + command: [ 'proto_wrap.sh', '@INPUT@', '@OUTPUT@' ], + # copy : true, + input : 'epics_event.proto', + output : 'bact_archiver_epics_event_pb2.py', + install : true, + install_dir: py.get_install_dir(), +) # cython extension incdir_numpy = run_command(py, ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], diff --git a/meson.build b/meson.build index d68725e..24b6034 100644 --- a/meson.build +++ b/meson.build @@ -12,4 +12,4 @@ py = import('python').find_installation(pure: false) # need to build the proto generated files before # I can build the cython extension -subdir('bact_archiver') \ No newline at end of file +subdir('bact_archiver') From e9f662a24e3e8600da2dd6ae0ba0e9815c8456ed Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Mon, 16 Dec 2024 18:19:18 +0100 Subject: [PATCH 36/51] [Snapshot] compiling --- bact_archiver/proto/meson.build | 5 +++-- bact_archiver/proto/proto_wrap.sh | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100755 bact_archiver/proto/proto_wrap.sh diff --git a/bact_archiver/proto/meson.build b/bact_archiver/proto/meson.build index dcecdd8..3832a04 100644 --- a/bact_archiver/proto/meson.build +++ b/bact_archiver/proto/meson.build @@ -21,10 +21,10 @@ py_generated = py_gen.process('epics_event.proto') message('install dir') message(py.get_install_dir()) py_proc2 = configure_file(# 'python_proto_as_config', - command: [ 'proto_wrap.sh', '@INPUT@', '@OUTPUT@' ], + command: [ 'proto_wrap.sh', '@INPUT@', '@OUTPUT@' ], # copy : true, input : 'epics_event.proto', - output : 'bact_archiver_epics_event_pb2.py', + output : 'epics_event_pb2.py', install : true, install_dir: py.get_install_dir(), ) @@ -39,6 +39,7 @@ py.extension_module( 'epics_event', 'epics_event.pyx', cpp_generated, + py_generated, install: true, include_directories : inc_dir, subdir: 'bact_archiver', diff --git a/bact_archiver/proto/proto_wrap.sh b/bact_archiver/proto/proto_wrap.sh new file mode 100755 index 0000000..e5e9687 --- /dev/null +++ b/bact_archiver/proto/proto_wrap.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -xv + +input=$1 +output=$2 +PROTOC=protoc +DIRNAME=dirname + +SRC_DIR=`"$DIRNAME" "$input"` +BUILD_DIR=`"$DIRNAME" "$output"` + +echo "$PROTOC" --proto_path="$SRC_DIR" --python_out="$BUILD_DIR" "$input" +"$PROTOC" --proto_path="$SRC_DIR" --python_out="$BUILD_DIR" "$input" + + From 0b9ad2e448e192b4bacdd33201dad89f8170f3bc Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Mon, 16 Dec 2024 18:32:20 +0100 Subject: [PATCH 37/51] [Snapshot] compiling and installing Caveat: search path to proto_gen ... needs to be revisited --- bact_archiver/carchiver.py | 2 +- bact_archiver/proto/meson.build | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/bact_archiver/carchiver.py b/bact_archiver/carchiver.py index 2cab07b..ae52def 100644 --- a/bact_archiver/carchiver.py +++ b/bact_archiver/carchiver.py @@ -5,7 +5,7 @@ """ from .epics_event import read_chunk, decode -import bact_archiver_epics_event_pb2 as proto +from . import epics_event_pb2 as proto from .protocol_buffer import (Chunk, dtypes as _dtypes, decoder as _decoder, dbrtypes as _dbrtypes, dsize as _dsize) from .archiver import ArchiverBasis, convert_datetime_to_timestamp diff --git a/bact_archiver/proto/meson.build b/bact_archiver/proto/meson.build index 3832a04..c01f41e 100644 --- a/bact_archiver/proto/meson.build +++ b/bact_archiver/proto/meson.build @@ -26,8 +26,10 @@ py_proc2 = configure_file(# 'python_proto_as_config', input : 'epics_event.proto', output : 'epics_event_pb2.py', install : true, - install_dir: py.get_install_dir(), + install_dir: py.get_install_dir() / 'bact_archiver', + install_tag: 'python-runtime', ) + # cython extension incdir_numpy = run_command(py, ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], @@ -39,10 +41,9 @@ py.extension_module( 'epics_event', 'epics_event.pyx', cpp_generated, - py_generated, install: true, include_directories : inc_dir, subdir: 'bact_archiver', override_options : ['cython_language=cpp'], dependencies : protobuf - ) + ) \ No newline at end of file From b56bc43d86c8dcf9e184df50974cf211eba094c4 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Mon, 16 Dec 2024 18:55:59 +0100 Subject: [PATCH 38/51] [TASK] build wheel without using some shell --- bact_archiver/proto/meson.build | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/bact_archiver/proto/meson.build b/bact_archiver/proto/meson.build index c01f41e..b98b407 100644 --- a/bact_archiver/proto/meson.build +++ b/bact_archiver/proto/meson.build @@ -13,23 +13,6 @@ py_gen = generator(protoc, cpp_generated = cpp_gen.process('epics_event.proto') py_generated = py_gen.process('epics_event.proto') -# py_proc = custom_target('py_proto', -# command: [ 'cp', '@INPUT@', '@OUTPUT@' ], -# input : py_generated, -# output : 'epics_event_pb2.py', -# build_by_default : true) -message('install dir') -message(py.get_install_dir()) -py_proc2 = configure_file(# 'python_proto_as_config', - command: [ 'proto_wrap.sh', '@INPUT@', '@OUTPUT@' ], - # copy : true, - input : 'epics_event.proto', - output : 'epics_event_pb2.py', - install : true, - install_dir: py.get_install_dir() / 'bact_archiver', - install_tag: 'python-runtime', -) - # cython extension incdir_numpy = run_command(py, ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], @@ -37,6 +20,17 @@ incdir_numpy = run_command(py, ).stdout().strip() inc_dir = include_directories(incdir_numpy) +py_gen2 = custom_target('epics_event_build_py', + output: ['epics_event_pb2.py'], + input: 'epics_event.proto', + command: [ + protoc, '--proto_path=@SRCDIR@', '--python_out=@OUTDIR@', '@INPUT@' + ], + install: true, + install_dir: py.get_install_dir() / 'bact_archiver', # need to install _umfpack.py + install_tag: 'python-runtime', +) + py.extension_module( 'epics_event', 'epics_event.pyx', @@ -46,4 +40,5 @@ py.extension_module( subdir: 'bact_archiver', override_options : ['cython_language=cpp'], dependencies : protobuf - ) \ No newline at end of file + ) + From fa3583a1c2bf971e6f736d8c1ae8ad23c4664844 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Thu, 26 Sep 2024 18:55:42 +0200 Subject: [PATCH 39/51] [TASK] use meson build system Removing setuptools specific parts --- bact_archiver/meson.build | 5 ----- bact_archiver/proto/meson.build | 16 +++++++++++++++- pyproject.toml | 4 ++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build index d2b7558..0d3e2e8 100644 --- a/bact_archiver/meson.build +++ b/bact_archiver/meson.build @@ -5,11 +5,6 @@ py_sources = run_command('/bin/sh', py.install_sources( py_sources, -<<<<<<< HEAD - # 'epics_event_pb2.py', -======= - 'epics_event_pb2.py', ->>>>>>> cc30f5d ([FIX] included created python file) 'archiver.cfg', pure: false, subdir: 'bact_archiver' diff --git a/bact_archiver/proto/meson.build b/bact_archiver/proto/meson.build index b98b407..7f172fb 100644 --- a/bact_archiver/proto/meson.build +++ b/bact_archiver/proto/meson.build @@ -14,6 +14,21 @@ cpp_generated = cpp_gen.process('epics_event.proto') py_generated = py_gen.process('epics_event.proto') # cython extension +# Why do I need to copy the files here? +# by default ninja will put them into its own dir? +cpp_proc = custom_target('cpp_proto', + command: [ 'cp', '@INPUT@', '@OUTPUT@' ], + input : cpp_generated, + output : '.', + build_by_default : true) + +py_proc = custom_target('py_proto', + command: [ 'cp', '@INPUT@', '@OUTPUT@' ], + input : py_generated, + output : 'epics_event_pb2.py', + build_by_default : true) + +# cpython interface incdir_numpy = run_command(py, ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], check : true @@ -41,4 +56,3 @@ py.extension_module( override_options : ['cython_language=cpp'], dependencies : protobuf ) - diff --git a/pyproject.toml b/pyproject.toml index 06ef541..149d6aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,8 @@ # -*- coding: utf-8 -*- +# [build-system] +# requires = ["setuptools >= 61.0", "setuptools-scm", "cython", "numpy"] +# build-backend = "setuptools.build_meta" + [build-system] build-backend = 'mesonpy' requires = ['meson-python', 'cython', 'numpy'] From f40965e3d7a0ee063ec2972fabafac81faa16c75 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Fri, 27 Sep 2024 10:51:23 +0200 Subject: [PATCH 40/51] [TASK] documented change to meson, new version bumped up minor number: build system changed --- pyproject.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 149d6aa..06ef541 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,4 @@ # -*- coding: utf-8 -*- -# [build-system] -# requires = ["setuptools >= 61.0", "setuptools-scm", "cython", "numpy"] -# build-backend = "setuptools.build_meta" - [build-system] build-backend = 'mesonpy' requires = ['meson-python', 'cython', 'numpy'] From f13083e18739c49a14318e60b1921eff91fdd5e8 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Fri, 27 Sep 2024 11:59:38 +0200 Subject: [PATCH 41/51] [FIX] build system: dependencies --- bact_archiver/proto/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bact_archiver/proto/meson.build b/bact_archiver/proto/meson.build index 7f172fb..88cce94 100644 --- a/bact_archiver/proto/meson.build +++ b/bact_archiver/proto/meson.build @@ -28,7 +28,7 @@ py_proc = custom_target('py_proto', output : 'epics_event_pb2.py', build_by_default : true) -# cpython interface +# cython extension incdir_numpy = run_command(py, ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], check : true From d6cae76489e6674020e6ebbd7b9ab878736e4b29 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Fri, 27 Sep 2024 12:09:28 +0200 Subject: [PATCH 42/51] [FIX] included created python file --- bact_archiver/meson.build | 1 + 1 file changed, 1 insertion(+) diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build index 0d3e2e8..08f7806 100644 --- a/bact_archiver/meson.build +++ b/bact_archiver/meson.build @@ -5,6 +5,7 @@ py_sources = run_command('/bin/sh', py.install_sources( py_sources, + 'epics_event_pb2.py', 'archiver.cfg', pure: false, subdir: 'bact_archiver' From 486bb7495b69def0212f83f7688935b8d0af7d6d Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Mon, 16 Dec 2024 17:44:18 +0100 Subject: [PATCH 43/51] [Snapshot] on including generated python file --- bact_archiver/carchiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bact_archiver/carchiver.py b/bact_archiver/carchiver.py index ae52def..2cab07b 100644 --- a/bact_archiver/carchiver.py +++ b/bact_archiver/carchiver.py @@ -5,7 +5,7 @@ """ from .epics_event import read_chunk, decode -from . import epics_event_pb2 as proto +import bact_archiver_epics_event_pb2 as proto from .protocol_buffer import (Chunk, dtypes as _dtypes, decoder as _decoder, dbrtypes as _dbrtypes, dsize as _dsize) from .archiver import ArchiverBasis, convert_datetime_to_timestamp From a07f72f3dfd86788fba3b3576436317624781164 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Tue, 17 Dec 2024 09:13:20 +0100 Subject: [PATCH 44/51] [TASK] now epics_event_pb2 ends up where it should --- bact_archiver/carchiver.py | 2 +- bact_archiver/proto/meson.build | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/bact_archiver/carchiver.py b/bact_archiver/carchiver.py index 2cab07b..ae52def 100644 --- a/bact_archiver/carchiver.py +++ b/bact_archiver/carchiver.py @@ -5,7 +5,7 @@ """ from .epics_event import read_chunk, decode -import bact_archiver_epics_event_pb2 as proto +from . import epics_event_pb2 as proto from .protocol_buffer import (Chunk, dtypes as _dtypes, decoder as _decoder, dbrtypes as _dbrtypes, dsize as _dsize) from .archiver import ArchiverBasis, convert_datetime_to_timestamp diff --git a/bact_archiver/proto/meson.build b/bact_archiver/proto/meson.build index 88cce94..025218b 100644 --- a/bact_archiver/proto/meson.build +++ b/bact_archiver/proto/meson.build @@ -13,14 +13,16 @@ py_gen = generator(protoc, cpp_generated = cpp_gen.process('epics_event.proto') py_generated = py_gen.process('epics_event.proto') -# cython extension -# Why do I need to copy the files here? -# by default ninja will put them into its own dir? -cpp_proc = custom_target('cpp_proto', - command: [ 'cp', '@INPUT@', '@OUTPUT@' ], - input : cpp_generated, - output : '.', - build_by_default : true) +py_gen2 = custom_target('epics_event_build_py', + output: ['epics_event_pb2.py'], + input: 'epics_event.proto', + command: [ + protoc, '--proto_path=@CURRENT_SOURCE_DIR@', '--python_out=@OUTDIR@', '@INPUT@' + ], + install: true, + install_dir: py.get_install_dir() / 'bact_archiver', # need to instalall epics_event_pb2.py + install_tag: 'python-runtime', +) py_proc = custom_target('py_proto', command: [ 'cp', '@INPUT@', '@OUTPUT@' ], From 3fab14dcc1398147b82d2b3f2c08572eb76ab867 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Tue, 17 Dec 2024 09:15:39 +0100 Subject: [PATCH 45/51] [TASK] cleanup, proto back where it should be using custom command, no external shell hack needed --- bact_archiver/meson.build | 1 - bact_archiver/proto/proto_wrap.sh | 15 --------------- meson.build | 1 + {bact_archiver/proto => proto}/epics_event.proto | 0 {bact_archiver/proto => proto}/epics_event.pxd | 0 {bact_archiver/proto => proto}/epics_event.pyx | 0 {bact_archiver/proto => proto}/meson.build | 0 7 files changed, 1 insertion(+), 16 deletions(-) delete mode 100755 bact_archiver/proto/proto_wrap.sh rename {bact_archiver/proto => proto}/epics_event.proto (100%) rename {bact_archiver/proto => proto}/epics_event.pxd (100%) rename {bact_archiver/proto => proto}/epics_event.pyx (100%) rename {bact_archiver/proto => proto}/meson.build (100%) diff --git a/bact_archiver/meson.build b/bact_archiver/meson.build index 08f7806..4b2a37a 100644 --- a/bact_archiver/meson.build +++ b/bact_archiver/meson.build @@ -10,4 +10,3 @@ py.install_sources( pure: false, subdir: 'bact_archiver' ) -subdir('proto') diff --git a/bact_archiver/proto/proto_wrap.sh b/bact_archiver/proto/proto_wrap.sh deleted file mode 100755 index e5e9687..0000000 --- a/bact_archiver/proto/proto_wrap.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -set -xv - -input=$1 -output=$2 -PROTOC=protoc -DIRNAME=dirname - -SRC_DIR=`"$DIRNAME" "$input"` -BUILD_DIR=`"$DIRNAME" "$output"` - -echo "$PROTOC" --proto_path="$SRC_DIR" --python_out="$BUILD_DIR" "$input" -"$PROTOC" --proto_path="$SRC_DIR" --python_out="$BUILD_DIR" "$input" - - diff --git a/meson.build b/meson.build index 24b6034..55a2323 100644 --- a/meson.build +++ b/meson.build @@ -12,4 +12,5 @@ py = import('python').find_installation(pure: false) # need to build the proto generated files before # I can build the cython extension +subdir('proto') subdir('bact_archiver') diff --git a/bact_archiver/proto/epics_event.proto b/proto/epics_event.proto similarity index 100% rename from bact_archiver/proto/epics_event.proto rename to proto/epics_event.proto diff --git a/bact_archiver/proto/epics_event.pxd b/proto/epics_event.pxd similarity index 100% rename from bact_archiver/proto/epics_event.pxd rename to proto/epics_event.pxd diff --git a/bact_archiver/proto/epics_event.pyx b/proto/epics_event.pyx similarity index 100% rename from bact_archiver/proto/epics_event.pyx rename to proto/epics_event.pyx diff --git a/bact_archiver/proto/meson.build b/proto/meson.build similarity index 100% rename from bact_archiver/proto/meson.build rename to proto/meson.build From b1c435e98f590222959625b9ed892f31680deef3 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Tue, 17 Dec 2024 09:30:07 +0100 Subject: [PATCH 46/51] [FIX] cleanup --- proto/meson.build | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/proto/meson.build b/proto/meson.build index 682a16d..edf8dec 100644 --- a/proto/meson.build +++ b/proto/meson.build @@ -7,14 +7,8 @@ cpp_gen = generator(protoc, arguments : ['--proto_path=@CURRENT_SOURCE_DIR@', '--cpp_out=@BUILD_DIR@', '@INPUT@'] ) cpp_generated = cpp_gen.process('epics_event.proto') -# does not fully work .. -# how do I install it -py_gen = generator(protoc, - output : ['@BASENAME@_pb2.py'], - arguments : ['--proto_path=@CURRENT_SOURCE_DIR@', '--python_out=@BUILD_DIR@', '@INPUT@'] - ) -py_generated = py_gen.process('epics_event.proto') +# Got generator not to do the job py_gen2 = custom_target('epics_event_build_py', output: ['epics_event_pb2.py'], input: 'epics_event.proto', @@ -26,12 +20,6 @@ py_gen2 = custom_target('epics_event_build_py', install_tag: 'python-runtime', ) -py_proc = custom_target('py_proto', - command: [ 'cp', '@INPUT@', '@OUTPUT@' ], - input : py_generated, - output : 'epics_event_pb2.py', - build_by_default : true) - # cython extension incdir_numpy = run_command(py, ['-c', 'import os; os.chdir(".."); import numpy; print(numpy.get_include())'], From 1d0ce7743ce81e71cf90f4198277a6e8112f0fa3 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Tue, 17 Dec 2024 10:00:11 +0100 Subject: [PATCH 47/51] [TASK] now only using pypi-publish --- .github/workflows/python-publish.yml | 35 ---------------------------- 1 file changed, 35 deletions(-) delete mode 100644 .github/workflows/python-publish.yml diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml deleted file mode 100644 index 42cbb6d..0000000 --- a/.github/workflows/python-publish.yml +++ /dev/null @@ -1,35 +0,0 @@ -# This workflows will upload a Python Package using Twine when a release is created -# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries - -name: Upload Python Package - -on: - release: - types: [created] - -jobs: - deploy: - - runs-on: ubuntu-latest - - steps: - - uses: ConorMacBride/install-package@v1 - with: - apt: protobuf-compiler - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip build wheel setuptools twine numpy - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py build_proto_c - python -m build -w - python -m build -s - python -m twine upload --repository testpypi dist/* From d479bf33f5d0255b8b3a8ce30f8a7cdebc506a99 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Tue, 17 Dec 2024 10:04:34 +0100 Subject: [PATCH 48/51] [TASK] add protoc, cleanup --- .github/workflows/pypi-publish.yml | 3 +++ .github/workflows/python-package.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 124cbaa..64b72aa 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -8,6 +8,9 @@ jobs: runs-on: ubuntu-latest steps: + - uses: ConorMacBride/install-package@v1 + with: + apt: protobuf-compiler - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index cab36b6..b5f2fbc 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: False matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: [ "3.12" ] # ["3.9", "3.10", "3.11", "3.12"] numpy-version: [ "numpy<2.0" , "numpy>=2.0" ] steps: From d58a2e91374dc8b40e998a5e4b49a8a72e3c05a8 Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Tue, 17 Dec 2024 10:11:36 +0100 Subject: [PATCH 49/51] [FIX] spelling --- MANIFEST.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index d14cfc3..9672084 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,8 +1,8 @@ include protocol_buffer.py # the source files -include proto/ * +include proto/* # the files generated -include proto_gen/ * +include proto_gen/* # configuration files include bact_archiver/*.cfg # documentation files From b2c3362ebb66e50cd3c69a570deb517168b0722a Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Tue, 17 Dec 2024 10:11:49 +0100 Subject: [PATCH 50/51] [TASK] MANIFEST.in setup tools specific? --- MANIFEST.in | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 9672084..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,13 +0,0 @@ -include protocol_buffer.py -# the source files -include proto/* -# the files generated -include proto_gen/* -# configuration files -include bact_archiver/*.cfg -# documentation files -include doc/conf.py -include doc/*.rst - -include THANKS -include Changelog From aee2c69c9898b1ce96eaa44457b19b0b6ab54c3f Mon Sep 17 00:00:00 2001 From: Pierre Schnizer Date: Tue, 17 Dec 2024 10:13:30 +0100 Subject: [PATCH 51/51] [TASK] actiavated matrix again --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index b5f2fbc..cab36b6 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: False matrix: - python-version: [ "3.12" ] # ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] numpy-version: [ "numpy<2.0" , "numpy>=2.0" ] steps: