From 26047a1b39bb71e943404ef7a21b1516d2246947 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Sun, 18 Jun 2023 12:40:32 -0600 Subject: [PATCH 1/5] parse_timestamp test coverage for negative and post-2038 times --- tests/unit/test_utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 7cceaa8ec5..bb4e350dc5 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -424,6 +424,18 @@ def test_parse_epoch_zero_time(self): datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc()), ) + def test_parse_epoch_negative_time(self): + self.assertEqual( + parse_timestamp(-2208988800), + datetime.datetime(1900, 1, 1, 0, 0, 0, tzinfo=tzutc()), + ) + + def test_parse_epoch_beyond_2038(self): + self.assertEqual( + parse_timestamp(2524608000), + datetime.datetime(2050, 1, 1, 0, 0, 0, tzinfo=tzutc()), + ) + def test_parse_epoch_as_string(self): self.assertEqual( parse_timestamp('1222172800'), From 82b6751c6dab1313d40ddcb99132baf3391553d3 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Sun, 18 Jun 2023 12:41:37 -0600 Subject: [PATCH 2/5] parse_timestamp to fall back to timedelta method for negative and post-2038 times --- botocore/utils.py | 43 ++++++++++++++++++++++++++++++++++++++-- tests/unit/test_utils.py | 37 ++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/botocore/utils.py b/botocore/utils.py index 484dd0f8f6..f40afe0c9c 100644 --- a/botocore/utils.py +++ b/botocore/utils.py @@ -904,6 +904,22 @@ def percent_encode(input_str, safe=SAFE_CHARS): return quote(input_str, safe=safe) +def _epoch_seconds_to_datetime(value, tzinfo): + """Parse numerical epoch timestamps (seconds since 1970) into a + ``datetime.datetime`` in UTC using ``datetime.timedelta``. This is intended + as fallback when ``fromtimestamp`` raises ``OverflowError`` or ``OSError``. + + :type value: float or int + :param value: The Unix timestamps as number. + + :type tzinfo: callable + :param tzinfo: A ``datetime.tzinfo`` class or compatible callable. + """ + epoch_zero = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc()) + epoch_zero_localized = epoch_zero.astimezone(tzinfo()) + return epoch_zero_localized + datetime.timedelta(seconds=value) + + def _parse_timestamp_with_tzinfo(value, tzinfo): """Parse timestamp with pluggable tzinfo options.""" if isinstance(value, (int, float)): @@ -935,15 +951,38 @@ def parse_timestamp(value): This will return a ``datetime.datetime`` object. """ - for tzinfo in get_tzinfo_options(): + tzinfo_options = get_tzinfo_options() + for tzinfo in tzinfo_options: try: + print(f"_parse_timestamp_with_tzinfo({value}, {tzinfo})") return _parse_timestamp_with_tzinfo(value, tzinfo) - except OSError as e: + except (OSError, OverflowError) as e: logger.debug( 'Unable to parse timestamp with "%s" timezone info.', tzinfo.__name__, exc_info=e, ) + # For numeric values attempt fallback to using fromtimestamp-free method. + # From Python's ``datetime.datetime.fromtimestamp`` documentation: "This + # may raise ``OverflowError``, if the timestamp is out of the range of + # values supported by the platform C localtime() function, and ``OSError`` + # on localtime() failure. It's common for this to be restricted to years + # from 1970 through 2038." + try: + numeric_value = float(value) + except (TypeError, ValueError): + pass + else: + try: + for tzinfo in tzinfo_options: + return _epoch_seconds_to_datetime(numeric_value, tzinfo=tzinfo) + except (OSError, OverflowError) as e: + logger.debug( + 'Unable to parse timestamp using fallback method with "%s" ' + 'timezone info.', + tzinfo.__name__, + exc_info=e, + ) raise RuntimeError( 'Unable to calculate correct timezone offset for "%s"' % value ) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index bb4e350dc5..6b1a33ba78 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -14,6 +14,7 @@ import datetime import io import operator +from contextlib import contextmanager from sys import getrefcount import pytest @@ -477,6 +478,42 @@ def test_parse_timestamp_fails_with_bad_tzinfo(self): with self.assertRaises(RuntimeError): parse_timestamp(0) + @contextmanager + def mocked_fromtimestamp_that_raises(self, exception_type): + class MockDatetime(datetime.datetime): + @classmethod + def fromtimestamp(cls, *args, **kwargs): + raise exception_type() + + mock_fromtimestamp = mock.Mock() + mock_fromtimestamp.side_effect = OverflowError() + + with mock.patch('datetime.datetime', MockDatetime): + yield + + def test_parse_timestamp_succeeds_with_fromtimestamp_overflowerror(self): + # ``datetime.fromtimestamp()`` fails with OverflowError on some systems + # for timestamps beyond 2038. See + # https://docs.python.org/3/library/datetime.html#datetime.datetime.fromtimestamp + # This test mocks fromtimestamp() to always raise an OverflowError and + # checks that the fallback method returns the same time and timezone + # as fromtimestamp. + wout_fallback = parse_timestamp(0) + with self.mocked_fromtimestamp_that_raises(OverflowError): + with_fallback = parse_timestamp(0) + self.assertEqual(with_fallback, wout_fallback) + self.assertEqual(with_fallback.tzinfo, wout_fallback.tzinfo) + + def test_parse_timestamp_succeeds_with_fromtimestamp_oserror(self): + # Same as test_parse_timestamp_succeeds_with_fromtimestamp_overflowerror + # but for systems where datetime.fromtimestamp() fails with OSerror for + # negative timestamps that represent times before 1970. + wout_fallback = parse_timestamp(0) + with self.mocked_fromtimestamp_that_raises(OSError): + with_fallback = parse_timestamp(0) + self.assertEqual(with_fallback, wout_fallback) + self.assertEqual(with_fallback.tzinfo, wout_fallback.tzinfo) + class TestDatetime2Timestamp(unittest.TestCase): def test_datetime2timestamp_naive(self): From e8604cba59e79b557c606a416796b015af99fa08 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Sun, 18 Jun 2023 15:16:07 -0600 Subject: [PATCH 3/5] changelog --- .changes/next-release/bugfix-Parsers-42740.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changes/next-release/bugfix-Parsers-42740.json diff --git a/.changes/next-release/bugfix-Parsers-42740.json b/.changes/next-release/bugfix-Parsers-42740.json new file mode 100644 index 0000000000..1820b2bb04 --- /dev/null +++ b/.changes/next-release/bugfix-Parsers-42740.json @@ -0,0 +1,5 @@ +{ + "type": "bugfix", + "category": "Parsers", + "description": "fixes `#3642 `__, `#2564 `__, `#2355 `__, `#1783 `__, boto/boto3`#2069 `__" +} From 8f5b698016f32581d8049acb4543b1fc5ce8651c Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Mon, 26 Jun 2023 16:56:54 -0600 Subject: [PATCH 4/5] changelog --- .changes/next-release/bugfix-Parsers-42740.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/next-release/bugfix-Parsers-42740.json b/.changes/next-release/bugfix-Parsers-42740.json index 1820b2bb04..24a44416bb 100644 --- a/.changes/next-release/bugfix-Parsers-42740.json +++ b/.changes/next-release/bugfix-Parsers-42740.json @@ -1,5 +1,5 @@ { "type": "bugfix", "category": "Parsers", - "description": "fixes `#3642 `__, `#2564 `__, `#2355 `__, `#1783 `__, boto/boto3`#2069 `__" + "description": "Fixes parsing of negative and out-of-range timestamp values (`#2564 `__)." } From 768b88a3a76397f73a9b9d81e61514e3740591f7 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Mon, 26 Jun 2023 16:58:29 -0600 Subject: [PATCH 5/5] changelog --- .changes/next-release/bugfix-Parsers-42740.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/next-release/bugfix-Parsers-42740.json b/.changes/next-release/bugfix-Parsers-42740.json index 24a44416bb..7244ed9404 100644 --- a/.changes/next-release/bugfix-Parsers-42740.json +++ b/.changes/next-release/bugfix-Parsers-42740.json @@ -1,5 +1,5 @@ { "type": "bugfix", "category": "Parsers", - "description": "Fixes parsing of negative and out-of-range timestamp values (`#2564 `__)." + "description": "Fixes datetime parse error handling for out-of-range and negative timestamps (`#2564 `__)." }