From d34c3b714bc808ae73dda67fefbe104eeea5c377 Mon Sep 17 00:00:00 2001 From: Leslie VanDeMark Date: Fri, 12 Jan 2024 16:31:19 +0000 Subject: [PATCH 1/9] bump backoff for pyton 3.11 compatibility --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 3d95c5d..5f6512d 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import subprocess setup(name="singer-python", - version='5.13.0', + version='5.14.0', description="Singer.io utility library", author="Stitch", classifiers=['Programming Language :: Python :: 3 :: Only'], @@ -14,7 +14,7 @@ 'jsonschema==2.6.0', 'simplejson==3.11.1', 'python-dateutil>=2.6.0', - 'backoff==1.8.0', + 'backoff==2.2.1', 'ciso8601', ], extras_require={ From aac9a0f1a4aebc56ed18c40e4baf8c7087d1d45f Mon Sep 17 00:00:00 2001 From: Leslie VanDeMark Date: Fri, 12 Jan 2024 18:31:06 +0000 Subject: [PATCH 2/9] update pip version --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ae6734c..347d0bf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,7 @@ jobs: mkdir -p ~/.virtualenvs python3 -m venv ~/.virtualenvs/singer-python source ~/.virtualenvs/singer-python/bin/activate - pip install -U 'pip<19.2' 'setuptools<51.0.0' + pip install -U 'pip<20.3.4' 'setuptools<51.0.0' make install - run: name: 'Run tests' From ad4b75570f120badbb3561b0227512bae435c624 Mon Sep 17 00:00:00 2001 From: Leslie VanDeMark Date: Fri, 12 Jan 2024 18:37:30 +0000 Subject: [PATCH 3/9] try new circleci yml --- .circleci/config.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 347d0bf..018488a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,19 +1,22 @@ -version: 2 +version: 2.1 + +workflows: + build: + jobs: + - build: + context: + - circleci-user + jobs: build: docker: - - image: ubuntu:16.04 + - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/sources-python:1.1.0 steps: - checkout - - run: - name: 'Install python 3.5.2' - command: | - apt update - apt install --yes python3 python3-pip python3-venv - run: name: 'Setup virtualenv' command: | - mkdir -p ~/.virtualenvs + pyenv global 3.11.7 python3 -m venv ~/.virtualenvs/singer-python source ~/.virtualenvs/singer-python/bin/activate pip install -U 'pip<20.3.4' 'setuptools<51.0.0' @@ -21,6 +24,5 @@ jobs: - run: name: 'Run tests' command: | - # Need to re-activate the virtualenv source ~/.virtualenvs/singer-python/bin/activate make test From 96f2ed603ec6fd0497cfb81bc3761b13e1f94509 Mon Sep 17 00:00:00 2001 From: Leslie VanDeMark Date: Fri, 12 Jan 2024 18:43:20 +0000 Subject: [PATCH 4/9] remove 'make' --- .circleci/config.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 018488a..b5005f4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,10 +19,17 @@ jobs: pyenv global 3.11.7 python3 -m venv ~/.virtualenvs/singer-python source ~/.virtualenvs/singer-python/bin/activate - pip install -U 'pip<20.3.4' 'setuptools<51.0.0' - make install + pip install -U 'pip==20.3.4' 'setuptools<51.0.0' + pip install .[dev] - run: - name: 'Run tests' + name: 'Pylint' command: | source ~/.virtualenvs/singer-python/bin/activate - make test + pip install pylint + pylint singer --extension-pkg-whitelist=ciso8601 -d missing-docstring,broad-except,bare-except,too-many-return-statements,too-many-branches,too-many-arguments,no-else-return,too-few-public-methods,fixme,protected-access + - run: + name: 'Run Tests' + command: | + source ~/.virtualenvs/singer-python/bin/activate + pip install nose2 + nose2 -v -s tests From 049f37e8f90a9c78480fc57c15f17efe6c81cce4 Mon Sep 17 00:00:00 2001 From: Leslie VanDeMark Date: Fri, 12 Jan 2024 19:10:57 +0000 Subject: [PATCH 5/9] make pylint happy --- .circleci/config.yml | 2 +- singer/catalog.py | 2 +- singer/exceptions.py | 2 +- singer/messages.py | 8 ++++---- singer/transform.py | 12 ++++++------ singer/utils.py | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b5005f4..3cb95fa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,7 +26,7 @@ jobs: command: | source ~/.virtualenvs/singer-python/bin/activate pip install pylint - pylint singer --extension-pkg-whitelist=ciso8601 -d missing-docstring,broad-except,bare-except,too-many-return-statements,too-many-branches,too-many-arguments,no-else-return,too-few-public-methods,fixme,protected-access + pylint singer --extension-pkg-whitelist=ciso8601 -d missing-docstring,broad-exception-raised,bare-except,too-many-return-statements,too-many-branches,too-many-arguments,no-else-return,too-few-public-methods,fixme,protected-access - run: name: 'Run Tests' command: | diff --git a/singer/catalog.py b/singer/catalog.py index 1767ff1..373a606 100644 --- a/singer/catalog.py +++ b/singer/catalog.py @@ -92,7 +92,7 @@ def __eq__(self, other): @classmethod def load(cls, filename): - with open(filename) as fp: # pylint: disable=invalid-name + with open(filename, encoding="utf-8") as fp: return Catalog.from_dict(json.load(fp)) @classmethod diff --git a/singer/exceptions.py b/singer/exceptions.py index 9231328..f11b12c 100644 --- a/singer/exceptions.py +++ b/singer/exceptions.py @@ -11,7 +11,7 @@ def __init__(self, message): The first line is the error's class name. The subsequent lines are the message that class was created with. """ - super().__init__('{}\n{}'.format(self.__class__.__name__, message)) + super().__init__(f'{(self.__class__.__name__}\n{message}') class SingerConfigurationError(SingerError): diff --git a/singer/messages.py b/singer/messages.py index 3848801..9607072 100644 --- a/singer/messages.py +++ b/singer/messages.py @@ -11,16 +11,16 @@ class Message(): '''Base class for messages.''' - def asdict(self): # pylint: disable=no-self-use + def asdict(self): raise Exception('Not implemented') def __eq__(self, other): return isinstance(other, Message) and self.asdict() == other.asdict() def __repr__(self): - pairs = ["{}={}".format(k, v) for k, v in self.asdict().items()] + pairs = [f"{k}={v}" for k, v in self.asdict().items()] attrstr = ", ".join(pairs) - return "{}({})".format(self.__class__.__name__, attrstr) + return f"{self.__class__.__name__}({attrstr})" def __str__(self): return str(self.asdict()) @@ -169,7 +169,7 @@ def asdict(self): def _required_key(msg, k): if k not in msg: - raise Exception("Message is missing required key '{}': {}".format(k, msg)) + raise Exception(f"Message is missing required key '{k}': {msg}" return msg[k] diff --git a/singer/transform.py b/singer/transform.py index 3fdefdf..39132eb 100644 --- a/singer/transform.py +++ b/singer/transform.py @@ -54,8 +54,8 @@ def __init__(self, errors): else: estrs = [e.tostr() for e in errors] - msg = "Errors during transform\n\t{}".format("\n\t".join(estrs)) - msg += "\n\n\nErrors during transform: [{}]".format(", ".join(estrs)) + msg = f"Errors during transform\n\t{"\n\t".join(estrs)}" + msg += f"\n\n\nErrors during transform: [{", ".join(estrs)}]" super().__init__(msg) @@ -77,16 +77,16 @@ def tostr(self): path = ".".join(map(str, self.path)) if self.schema: if self.logging_level >= logging.INFO: - msg = "data does not match {}".format(self.schema) + msg = f"data does not match {self.schema}" else: - msg = "does not match {}".format(self.schema) + msg = f"does not match {self.schema}" else: msg = "not in schema" if self.logging_level >= logging.INFO: - output = "{}: {}".format(path, msg) + output = f"{path}: {msg}" else: - output = "{}: {} {}".format(path, self.data, msg) + output = f"{path}: {self.data} {msg}" return output diff --git a/singer/utils.py b/singer/utils.py index 48675eb..14df37c 100644 --- a/singer/utils.py +++ b/singer/utils.py @@ -105,7 +105,7 @@ def chunk(array, num): def load_json(path): - with open(path) as fil: + with open(path, encoding="utf-8") as fil: return json.load(fil) @@ -193,7 +193,7 @@ def parse_args(required_config_keys): def check_config(config, required_keys): missing_keys = [key for key in required_keys if key not in config] if missing_keys: - raise Exception("Config is missing required keys: {}".format(missing_keys)) + raise Exception(f"Config is missing required keys: {missing_keys}" def backoff(exceptions, giveup): From 2ed2a3e673163210b82e3e2cbbc0a1055b291ce1 Mon Sep 17 00:00:00 2001 From: Leslie VanDeMark Date: Fri, 12 Jan 2024 19:38:58 +0000 Subject: [PATCH 6/9] make pylint happy --- .circleci/config.yml | 2 +- singer/exceptions.py | 2 +- singer/transform.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3cb95fa..41e2986 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,7 +26,7 @@ jobs: command: | source ~/.virtualenvs/singer-python/bin/activate pip install pylint - pylint singer --extension-pkg-whitelist=ciso8601 -d missing-docstring,broad-exception-raised,bare-except,too-many-return-statements,too-many-branches,too-many-arguments,no-else-return,too-few-public-methods,fixme,protected-access + pylint singer --extension-pkg-whitelist=ciso8601 -d missing-docstring,broad-exception-raised,bare-except,too-many-return-statements,too-many-branches,too-many-arguments,no-else-return,too-few-public-methods,fixme,protected-access,consider-using-f-string - run: name: 'Run Tests' command: | diff --git a/singer/exceptions.py b/singer/exceptions.py index f11b12c..b13016d 100644 --- a/singer/exceptions.py +++ b/singer/exceptions.py @@ -11,7 +11,7 @@ def __init__(self, message): The first line is the error's class name. The subsequent lines are the message that class was created with. """ - super().__init__(f'{(self.__class__.__name__}\n{message}') + super().__init__(f"{self.__class__.__name__}\n{message}") class SingerConfigurationError(SingerError): diff --git a/singer/transform.py b/singer/transform.py index 39132eb..69f812a 100644 --- a/singer/transform.py +++ b/singer/transform.py @@ -54,8 +54,8 @@ def __init__(self, errors): else: estrs = [e.tostr() for e in errors] - msg = f"Errors during transform\n\t{"\n\t".join(estrs)}" - msg += f"\n\n\nErrors during transform: [{", ".join(estrs)}]" + msg = "Errors during transform\n\t{}".format("\n\t".join(estrs)) + msg += "\n\n\nErrors during transform: [{}]".format(", ".join(estrs)) super().__init__(msg) From 066c17a5ec4789cd7f1dd7e1f17491b11fe0d847 Mon Sep 17 00:00:00 2001 From: Leslie VanDeMark Date: Fri, 12 Jan 2024 19:43:52 +0000 Subject: [PATCH 7/9] make pylint happy again --- .circleci/config.yml | 2 +- singer/messages.py | 2 +- singer/utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 41e2986..a64745e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,7 +26,7 @@ jobs: command: | source ~/.virtualenvs/singer-python/bin/activate pip install pylint - pylint singer --extension-pkg-whitelist=ciso8601 -d missing-docstring,broad-exception-raised,bare-except,too-many-return-statements,too-many-branches,too-many-arguments,no-else-return,too-few-public-methods,fixme,protected-access,consider-using-f-string + pylint singer --extension-pkg-whitelist=ciso8601 -d missing-docstring,broad-exception-raised,broad-exception-caught,bare-except,too-many-return-statements,too-many-branches,too-many-arguments,no-else-return,too-few-public-methods,fixme,protected-access,consider-using-f-string - run: name: 'Run Tests' command: | diff --git a/singer/messages.py b/singer/messages.py index 9607072..4a87235 100644 --- a/singer/messages.py +++ b/singer/messages.py @@ -169,7 +169,7 @@ def asdict(self): def _required_key(msg, k): if k not in msg: - raise Exception(f"Message is missing required key '{k}': {msg}" + raise Exception(f"Message is missing required key '{k}': {msg}") return msg[k] diff --git a/singer/utils.py b/singer/utils.py index 14df37c..6620005 100644 --- a/singer/utils.py +++ b/singer/utils.py @@ -193,7 +193,7 @@ def parse_args(required_config_keys): def check_config(config, required_keys): missing_keys = [key for key in required_keys if key not in config] if missing_keys: - raise Exception(f"Config is missing required keys: {missing_keys}" + raise Exception(f"Config is missing required keys: {missing_keys}") def backoff(exceptions, giveup): From a7d8ee958658b29577e5bd33a7c4d253c550406c Mon Sep 17 00:00:00 2001 From: Leslie VanDeMark Date: Fri, 12 Jan 2024 20:13:01 +0000 Subject: [PATCH 8/9] backoff version is a breaking change for old python versions --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5f6512d..48305ab 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import subprocess setup(name="singer-python", - version='5.14.0', + version='6.0.0', description="Singer.io utility library", author="Stitch", classifiers=['Programming Language :: Python :: 3 :: Only'], From 8ce2c0790e6bb52befbe350bc2fe1c0d95a72fa8 Mon Sep 17 00:00:00 2001 From: Leslie VanDeMark Date: Fri, 12 Jan 2024 20:15:33 +0000 Subject: [PATCH 9/9] Changelog update --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d06a7a7..21dbe09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 6.0.0 + * Bump backoff version to 2.2.1. This version drops support for python 3.5, but adds it for 3.1o [#165](https://github.com/singer-io/singer-python/pull/165) + ## 5.13.0 * Add support for dev mode argument parsing [#158](https://github.com/singer-io/singer-python/pull/158)