From 2053635c5cf7b746c1b4805abbc0fc6d792e07ba Mon Sep 17 00:00:00 2001 From: Omar Zapata Date: Mon, 29 Jul 2024 00:24:56 -0500 Subject: [PATCH 1/2] implemented issue https://github.com/colav/impactu/issues/208 --- bin/kahi_generate | 19 ++++ kahi/templates/__init__.py | 0 kahi/templates/plugin/Kahi_template/LICENSE | 30 +++++++ .../plugin/Kahi_template/MANIFEST.in | 2 + kahi/templates/plugin/Kahi_template/README.md | 43 +++++++++ .../kahi_template/Kahi_template.py | 14 +++ .../Kahi_template/kahi_template/__init__.py | 0 .../Kahi_template/kahi_template/_version.py | 6 ++ kahi/templates/plugin/Kahi_template/setup.py | 89 +++++++++++++++++++ setup.py | 2 +- 10 files changed, 204 insertions(+), 1 deletion(-) create mode 100755 bin/kahi_generate create mode 100644 kahi/templates/__init__.py create mode 100644 kahi/templates/plugin/Kahi_template/LICENSE create mode 100644 kahi/templates/plugin/Kahi_template/MANIFEST.in create mode 100644 kahi/templates/plugin/Kahi_template/README.md create mode 100644 kahi/templates/plugin/Kahi_template/kahi_template/Kahi_template.py create mode 100644 kahi/templates/plugin/Kahi_template/kahi_template/__init__.py create mode 100644 kahi/templates/plugin/Kahi_template/kahi_template/_version.py create mode 100644 kahi/templates/plugin/Kahi_template/setup.py diff --git a/bin/kahi_generate b/bin/kahi_generate new file mode 100755 index 0000000..921bfd3 --- /dev/null +++ b/bin/kahi_generate @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 + +import argparse +from kahi.PluginGenerator import PluginGenerator + +parser = argparse.ArgumentParser( + description='ETL for bibliographic data.') + +parser.add_argument('--plugin', type=str, + help='Generate a plugin package directory, please provide the plugin name ex: --plugin test , the output is Kahi_test') + + +args = parser.parse_args() + +if __name__ == '__main__': + # create instance of kahi with the workflow file as a parameter + if args.plugin: + pg = PluginGenerator(args.plugin) + pg.generate() diff --git a/kahi/templates/__init__.py b/kahi/templates/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kahi/templates/plugin/Kahi_template/LICENSE b/kahi/templates/plugin/Kahi_template/LICENSE new file mode 100644 index 0000000..cef2ea0 --- /dev/null +++ b/kahi/templates/plugin/Kahi_template/LICENSE @@ -0,0 +1,30 @@ +Copyright (c) 2005-2020, Colav Developers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the NumPy Developers nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/kahi/templates/plugin/Kahi_template/MANIFEST.in b/kahi/templates/plugin/Kahi_template/MANIFEST.in new file mode 100644 index 0000000..ce7f6ed --- /dev/null +++ b/kahi/templates/plugin/Kahi_template/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include kahi_template/ *.py +recursive-include kahi_template/ *.* \ No newline at end of file diff --git a/kahi/templates/plugin/Kahi_template/README.md b/kahi/templates/plugin/Kahi_template/README.md new file mode 100644 index 0000000..66c2c5c --- /dev/null +++ b/kahi/templates/plugin/Kahi_template/README.md @@ -0,0 +1,43 @@ +
+ +# Kahi template plugin +This is a template for xyz project +replace template for the name of the plugin everywhere. + +# Description +Write something meaningful here ;) + +# Installation + +## Dependencies +What do I need fot this plugin?, it could be external services etc.. + +## Package +Write here how to install this plugin +usauly is + +`pip install kahi_template` + + +# Usage +what should I know? +put it here. + +Additional parameters for kahi_run in the workflow should be here as well. +example : + +``` +template: + - my_param_example: value +``` +Those parameters are not really needed in the workflow file, it is just for illustration. + + +# License +BSD-3-Clause License + +# Links +http://colav.udea.edu.co/ + + + diff --git a/kahi/templates/plugin/Kahi_template/kahi_template/Kahi_template.py b/kahi/templates/plugin/Kahi_template/kahi_template/Kahi_template.py new file mode 100644 index 0000000..cae2f35 --- /dev/null +++ b/kahi/templates/plugin/Kahi_template/kahi_template/Kahi_template.py @@ -0,0 +1,14 @@ +from kahi.KahiBase import KahiBase + + +class Kahi_template(KahiBase): + + config = {} + + def __init__(self, config): + self.config = config + + def run(self): + # entry point for the execution of the plugin + # magic happens here... + pass diff --git a/kahi/templates/plugin/Kahi_template/kahi_template/__init__.py b/kahi/templates/plugin/Kahi_template/kahi_template/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kahi/templates/plugin/Kahi_template/kahi_template/_version.py b/kahi/templates/plugin/Kahi_template/kahi_template/_version.py new file mode 100644 index 0000000..7b6ecff --- /dev/null +++ b/kahi/templates/plugin/Kahi_template/kahi_template/_version.py @@ -0,0 +1,6 @@ +# flake8: noqa +__version__ = '0.0.1-alpha' + + +def get_version(): + return __version__ diff --git a/kahi/templates/plugin/Kahi_template/setup.py b/kahi/templates/plugin/Kahi_template/setup.py new file mode 100644 index 0000000..3bd3a30 --- /dev/null +++ b/kahi/templates/plugin/Kahi_template/setup.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# coding: utf-8 + +# Copyright (c) Colav. +# Distributed under the terms of the Modified BSD License. + +# ----------------------------------------------------------------------------- +# Minimal Python version sanity check (from IPython) +# ----------------------------------------------------------------------------- + +# See https://stackoverflow.com/a/26737258/2268280 +# sudo pip3 install twine +# python3 setup.py sdist bdist_wheel +# twine upload dist/* +# For test purposes +# twine upload --repository-url https://test.pypi.org/legacy/ dist/* + +from __future__ import print_function +from setuptools import setup, find_packages + +import os +import sys +import codecs + + +v = sys.version_info + + +def read(rel_path): + here = os.path.abspath(os.path.dirname(__file__)) + with codecs.open(os.path.join(here, rel_path), 'r') as fp: + return fp.read() + + +def get_version(rel_path): + for line in read(rel_path).splitlines(): + if line.startswith('__version__'): + delim = '"' if '"' in line else "'" + return line.split(delim)[1] + else: + raise RuntimeError("Unable to find version string.") + + +shell = False +if os.name in ('nt', 'dos'): + shell = True + warning = "WARNING: Windows is not officially supported" + print(warning, file=sys.stderr) + + +def main(): + setup( + # Application name: + name="Kahi_template", + + # Version number (initial): + version=get_version('kahi_template/_version.py'), + + # Application author details: + author="Colav", + author_email="colav@udea.edu.co", + + # Packages + packages=find_packages(exclude=['tests']), + + # Include additional files into the package + include_package_data=True, + + # Details + url="https://github.com/colav/Kahi_xyz", + # + license="BSD", + + description="Kahi plugin template", + + long_description=open("README.md").read(), + + long_description_content_type="text/markdown", + + # Dependent packages (distributions) + # put you packages here + install_requires=[ + 'kahi' + ], + ) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 3702665..ec15708 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ def main(): # Details url="https://github.com/colav/Kahi", - scripts=['bin/kahi_run'], + scripts=['bin/kahi_run','bin/kahi_generate'], license="BSD", From 21e8cd4c3a7523de94cab45ac3d99a66ca03fa8d Mon Sep 17 00:00:00 2001 From: Omar Zapata Date: Mon, 29 Jul 2024 00:32:58 -0500 Subject: [PATCH 2/2] updated readme --- README.md | 36 +++++++++++------------------------- setup.py | 2 +- 2 files changed, 12 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index cdfc767..1eb0319 100644 --- a/README.md +++ b/README.md @@ -2,35 +2,12 @@
# Kahi -KAHI is a powerful ETL (Extract, Transform, Load) application designed to construct an academic database by merging databases and files from various sources. It simplifies the database construction process by offering a framework to define a workflow of sequential tasks using a plugin system that KAHI understands. +KAHI is a powerful tool by offering a framework to define a workflow of sequential tasks using a plugin system that KAHI understands. # Plugins Take a look on plugins examples in the repository https://github.com/colav/Kahi_plugins -List of available plugins: - -* kahi_doaj_sources -* kahi_minciencias_opendata_affiliations -* kahi_minciencias_opendata_person -* kahi_openalex_affiliations -* kahi_openalex_person -* kahi_openalex_sources -* kahi_openalex_subjects -* kahi_openalex_works -* kahi_ranking_udea_works -* kahi_ror_affiliations -* kahi_scholar_works -* kahi_scienti_affiliations -* kahi_scienti_person -* kahi_scienti_sources -* kahi_scholar_works -* kahi_scimago_sources -* kahi_scopus_works -* kahi_staff_udea_affiliations -* kahi_staff_udea_person -* kahi_wos_works - ## Installation To install KAHI, follow these simple steps: @@ -82,13 +59,22 @@ kahi_run --workflow worflow.yaml ``` Replace workflow.yaml with the path to your YAML file. -Suggested workflows can be found on [our worflow repository](https://github.com/colav/kahi_workflows). +Suggested workflows can be found on [our worflow repository](https://github.com/colav/impactu/tree/main/workflows). # Logging KAHI keeps a detailed log of each plugin's execution in a mongodb collection, including the name, execution time, elapsed time, execution status, and error messages. This information is valuable for both users and developers, and it enables the ability to resume the workflow from the last successful task. Plugins can take advantage of a researved parameter **task**. When the reserved paramer task is used, the log entry becomes unique with the name of the plugin and the task as a suffix. +# Creating a template package + +You can create a template package structure by running: +``` +kahi_generate --plugin myplugin +``` +and a basic installable template called Kahi_myplugin will be created. + + # Contributing If you are interested in contributing to KAHI or creating your own plugins, please refer to the kahi-plugins repository. It contains the necessary resources and documentation to implement new plugins easily. Feel free to submit pull requests or report any issues you encounter. diff --git a/setup.py b/setup.py index ec15708..b8eb12d 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ def main(): # Details url="https://github.com/colav/Kahi", - scripts=['bin/kahi_run','bin/kahi_generate'], + scripts=['bin/kahi_run', 'bin/kahi_generate'], license="BSD",