Merge branch 'master' into cond-propedit

changes/3ef24cd8510986bf45e39e390b383077.yaml

@@ -0,0 +1,6 @@
+---
+desc: Added ``indent`` kwarg to ``$lib.json.save()`` to indent serialized json with a number of spaces or a specified string.
+prs:
+- 4052
+type: feat
+...

changes/bd45773cff22d0cf547370e90918ba58.yaml

@@ -0,0 +1,5 @@
+---
+desc: Added a patch for Python ``http.cookies`` module to address CVE-2024-7592 exposure.
+prs: []
+type: bug
+...

synapse/common.py

@@ -29,6 +29,8 @@
 import contextlib
 import collections
 
+import http.cookies
+
 import yaml
 import regex
 
@@ -38,6 +40,8 @@
 import synapse.lib.structlog as s_structlog
 
 import synapse.vendor.cpython.lib.ipaddress as ipaddress
+import synapse.vendor.cpython.lib.http.cookies as v_cookies
+
 
 try:
     from yaml import CSafeLoader as Loader
@@ -1218,6 +1222,17 @@ def trimText(text: str, n: int = 256, placeholder: str = '...') -> str:
     assert n > plen
     return f'{text[:mlen]}{placeholder}'
 
+def _patch_http_cookies():
+    '''
+    Patch stdlib http.cookies._unquote from the 3.11.10 implementation if
+    the interpreter we are using is not patched for CVE-2024-7592.
+    '''
+    if not hasattr(http.cookies, '_QuotePatt'):
+        return
+    http.cookies._unquote = v_cookies._unquote
+
+_patch_http_cookies()
+
 # TODO:  Switch back to using asyncio.wait_for when we are using py 3.12+
 # This is a workaround for a race where asyncio.wait_for can end up
 # ignoring cancellation https://github.com/python/cpython/issues/86296

synapse/exc.py

@@ -56,9 +56,14 @@ def setdefault(self, name, valu):
         self.errinfo[name] = valu
         self._setExcMesg()
 
+    def update(self, items: dict):
+        '''Update multiple items in the errinfo dict at once.'''
+        self.errinfo.update(**items)
+        self._setExcMesg()
+
 class StormRaise(SynErr):
     '''
-    This represents a user provided exception inside of a Storm runtime. It requires a errname key.
+    This represents a user provided exception raised in the Storm runtime. It requires a errname key.
     '''
     def __init__(self, *args, **info):
         SynErr.__init__(self, *args, **info)

synapse/lib/ast.py

@@ -70,7 +70,7 @@ def getPosInfo(self):
 
     def addExcInfo(self, exc):
         if 'highlight' not in exc.errinfo:
-            exc.errinfo['highlight'] = self.getPosInfo()
+            exc.set('highlight', self.getPosInfo())
         return exc
 
     def repr(self):

synapse/lib/parser.py

@@ -508,7 +508,7 @@ def _larkToSynExc(self, e):
             origexc = e.orig_exc
             if not isinstance(origexc, s_exc.SynErr):
                 raise e.orig_exc # pragma: no cover
-            origexc.errinfo['text'] = self.text
+            origexc.set('text', self.text)
             return s_exc.BadSyntax(**origexc.errinfo)
 
         elif isinstance(e, lark.exceptions.UnexpectedCharacters):  # pragma: no cover

synapse/lib/snap.py

@@ -363,9 +363,9 @@ async def _set(self, prop, valu, norminfo=None, ignore_ro=False):
                 valu, norminfo = prop.type.norm(valu)
             except s_exc.BadTypeValu as e:
                 oldm = e.errinfo.get('mesg')
-                e.errinfo['prop'] = prop.name
-                e.errinfo['form'] = prop.form.name
-                e.errinfo['mesg'] = f'Bad prop value {prop.full}={valu!r} : {oldm}'
+                e.update({'prop': prop.name,
+                          'form': prop.form.name,
+                          'mesg': f'Bad prop value {prop.full}={valu!r} : {oldm}'})
                 if self.ctx.snap.strict:
                     raise e
                 await self.ctx.snap.warn(e)
@@ -493,7 +493,7 @@ async def _addNode(self, form, valu, props=None, norminfo=None):
             try:
                 valu, norminfo = form.type.norm(valu)
             except s_exc.BadTypeValu as e:
-                e.errinfo['form'] = form.name
+                e.set('form', form.name)
                 if self.snap.strict: raise e
                 await self.snap.warn(f'addNode() BadTypeValu {form.name}={valu} {e}')
                 return None

synapse/lib/stormlib/json.py

@@ -93,6 +93,7 @@ class JsonLib(s_stormtypes.Lib):
          'type': {'type': 'function', '_funcname': '_jsonSave',
                   'args': (
                       {'name': 'item', 'type': 'any', 'desc': 'The item to be serialized as a JSON string.', },
+                      {'name': 'indent', 'type': 'int', 'desc': 'Specify a number of spaces to indent with.', 'default': None},
                   ),
                   'returns': {'type': 'str', 'desc': 'The JSON serialized object.', }}},
         {'name': 'schema', 'desc': 'Get a JS schema validation object.',
@@ -115,10 +116,12 @@ def getObjLocals(self):
         }
 
     @s_stormtypes.stormfunc(readonly=True)
-    async def _jsonSave(self, item):
+    async def _jsonSave(self, item, indent=None):
+        indent = await s_stormtypes.toint(indent, noneok=True)
+
         try:
             item = await s_stormtypes.toprim(item)
-            return json.dumps(item)
+            return json.dumps(item, indent=indent)
         except Exception as e:
             mesg = f'Argument is not JSON compatible: {item}'
             raise s_exc.MustBeJsonSafe(mesg=mesg)

synapse/tests/test_exc.py

@@ -27,6 +27,9 @@ def test_basic(self):
         e.setdefault('defv', 2)
         self.eq("SynErr: defv=1 foo='words' hehe=1234 mesg='words'", str(e))
 
+        e.update({'foo': 'newwords', 'bar': 'baz'})
+        self.eq("SynErr: bar='baz' defv=1 foo='newwords' hehe=1234 mesg='words'", str(e))
+
         self.eq(e.errname, 'SynErr')
 
         e2 = s_exc.BadTypeValu(mesg='haha')

synapse/tests/test_lib_stormlib_json.py

@@ -12,6 +12,26 @@ async def test_stormlib_json(self):
 
             self.eq(((1, 2, 3)), await core.callStorm('return($lib.json.load("[1, 2, 3]"))'))
             self.eq(('["foo", "bar", "baz"]'), await core.callStorm('return($lib.json.save((foo, bar, baz)))'))
+            self.eq(('{"foo": 1, "bar": {"baz": "hello"}}'), await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}})))'))
+            self.eq(('{"foo": 1, "bar": {"baz": "hello"}}'), await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), (null)))'))
+            self.eq((
+'''{
+    "foo": 1,
+    "bar": {
+        "baz": "hello"
+    }
+}'''), await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), indent=(4)))'))
+
+            self.eq((
+'''{
+  "foo": 1,
+  "bar": {
+    "baz": "hello"
+  }
+}'''), await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), indent=2))'))
+
+            with self.raises(s_exc.BadCast):
+                await core.callStorm('return($lib.json.save(({"foo": 1, "bar": {"baz": "hello"}}), indent=x))')
 
             with self.raises(s_exc.BadJsonText):
                 await core.callStorm('return($lib.json.load(foo))')

synapse/vendor/cpython/lib/http/cookies.py

@@ -0,0 +1,59 @@
+##############################################################################
+# Taken from the cpython 3.11 source branch after the 3.11.10 release.
+##############################################################################
+####
+# Copyright 2000 by Timothy O'Malley <timo@alum.mit.edu>
+#
+#                All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software
+# and its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of
+# Timothy O'Malley  not be used in advertising or publicity
+# pertaining to distribution of the software without specific, written
+# prior permission.
+#
+# Timothy O'Malley DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
+# SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL Timothy O'Malley BE LIABLE FOR
+# ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+# PERFORMANCE OF THIS SOFTWARE.
+#
+
+#
+# Import our required modules
+#
+import re
+
+_unquote_sub = re.compile(r'\\(?:([0-3][0-7][0-7])|(.))').sub
+
+def _unquote_replace(m):
+    if m[1]:
+        return chr(int(m[1], 8))
+    else:
+        return m[2]
+
+def _unquote(str):
+    # If there aren't any doublequotes,
+    # then there can't be any special characters.  See RFC 2109.
+    if str is None or len(str) < 2:
+        return str
+    if str[0] != '"' or str[-1] != '"':
+        return str
+
+    # We have to assume that we must decode this string.
+    # Down to work.
+
+    # Remove the "s
+    str = str[1:-1]
+
+    # Check for special sequences.  Examples:
+    #    \012 --> \n
+    #    \"   --> "
+    #
+    return _unquote_sub(_unquote_replace, str)

synapse/vendor/cpython/lib/test/test_http_cookies.py

@@ -0,0 +1,49 @@
+##############################################################################
+# Taken from the cpython 3.11 source branch after the 3.11.10 release.
+# It has been modified for vendored imports and vendored test harness.
+##############################################################################
+
+# Simple test suite for http/cookies.py
+
+from http import cookies
+
+# s_v_utils runs the monkeypatch
+import synapse.vendor.utils as s_v_utils
+
+class CookieTests(s_v_utils.VendorTest):
+
+    def test_unquote(self):
+        cases = [
+            (r'a="b=\""', 'b="'),
+            (r'a="b=\\"', 'b=\\'),
+            (r'a="b=\="', 'b=='),
+            (r'a="b=\n"', 'b=n'),
+            (r'a="b=\042"', 'b="'),
+            (r'a="b=\134"', 'b=\\'),
+            (r'a="b=\377"', 'b=\xff'),
+            (r'a="b=\400"', 'b=400'),
+            (r'a="b=\42"', 'b=42'),
+            (r'a="b=\\042"', 'b=\\042'),
+            (r'a="b=\\134"', 'b=\\134'),
+            (r'a="b=\\\""', 'b=\\"'),
+            (r'a="b=\\\042"', 'b=\\"'),
+            (r'a="b=\134\""', 'b=\\"'),
+            (r'a="b=\134\042"', 'b=\\"'),
+        ]
+        for encoded, decoded in cases:
+            with self.subTest(encoded):
+                C = cookies.SimpleCookie()
+                C.load(encoded)
+                self.assertEqual(C['a'].value, decoded)
+
+    def test_unquote_large(self):
+        n = 10**6
+        for encoded in r'\\', r'\134':
+            with self.subTest(encoded):
+                data = 'a="b=' + encoded * n + ';"'
+                C = cookies.SimpleCookie()
+                C.load(data)
+                value = C['a'].value
+                self.assertEqual(value[:3], 'b=\\')
+                self.assertEqual(value[-2:], '\\;')
+                self.assertEqual(len(value), n + 3)