Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add conditional edit operator syntax (SYN-7486) #4046

Open
wants to merge 15 commits into
base: master
Choose a base branch
from
5 changes: 5 additions & 0 deletions changes/e1b7c7693e7454c32ca657a2bed734d5.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
desc: Added syntax for conditional node property edit operators in Storm.
prs: []
type: feat
...
30 changes: 30 additions & 0 deletions docs/synapse/userguides/storm_ref_data_mod.rstorm
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ what changes should be made and to what data:
- `Edit Brackets`_
- `Edit Parentheses`_
- `"Try" Operator`_
- `Conditional Edit Operators`_
- `Autoadds and Depadds`_

.. _edit-brackets:
Expand Down Expand Up @@ -180,6 +181,31 @@ where the variable may contain unexpected values. For example:

See the :ref:`type-syn-tag` section of the :ref:`storm-ref-type-specific` for additional detail on tags / ``syn:tag`` forms.

Conditional Edit Operators
++++++++++++++++++++++++++

The conditional edit operators ( ``*unset=`` and ``*$<varvalue>=`` ) can be used to only set properties when certain
Cisphyx marked this conversation as resolved.
Show resolved Hide resolved
conditions are met.

The ``*unset=`` operator will only set a property when it does not already have a value to prevent overwriting
existing data. For example:

``inet:ipv4 = 1.2.3.4 [ :asn *unset= 12345 ]``

will only set the ``:asn`` property on the ``inet:ipv4`` node if it is not already set. The conditional edit operators
can also be combined with the "try" operator ( ``*unset?=`` ) to prevent failures due to bad data:

``inet:ipv4 = 1.2.3.4 [ :asn *unset?= invalid ]``

Variable values may also be used to control the conditional edit behavior, and allow two more values in addition to
``unset``; ``always`` and ``never``. For example:

``$asn = 'always' $loc = 'never' inet:ipv4 = 1.2.4.5 [ :loc *$loc= us :asn *$asn?= 12345 ]``

will never set the ``:loc`` property and will always attempt to set the ``:asn`` property. This behavior is useful
when creating Storm ingest functions where fine tuned control over specific property edit behavior is needed. Rather
than creating variations of the same ingest function with different combinations of property set behavior, one function
can use a dictionary of configuration options to control the edit behavior used during each execution.

.. _autoadds-depadds:

Expand Down Expand Up @@ -312,6 +338,10 @@ The same syntax is used to apply a new property or modify an existing property.

*<query>* **[ :** *<prop>* **=** | **?=** *<pval>* ... **]**

*<query>* **[ :** *<prop>* ***unset=** | ***unset?** *<pval>* ... **]**

*<query>* **[ :** *<prop>* ***$<varvalue>=** | ***$<varvalue>?=** *<pval>* ... **]**
Cisphyx marked this conversation as resolved.
Show resolved Hide resolved

.. TIP::

You can optionally use the :ref:`edit-try` ( ``?=`` ) when setting or modifying properties.
Expand Down
9 changes: 6 additions & 3 deletions synapse/datamodel.py
Original file line number Diff line number Diff line change
Expand Up @@ -445,14 +445,17 @@ def prop(self, name: str):
'''
return self.props.get(name)

def reqProp(self, name):
def reqProp(self, name, extra=None):
prop = self.props.get(name)
if prop is not None:
return prop

full = f'{self.name}:{name}'
mesg = f'No property named {full}.'
raise s_exc.NoSuchProp(mesg=mesg, name=full)
exc = s_exc.NoSuchProp.init(full)
if extra is not None:
exc = extra(exc)

raise exc

def pack(self):
props = {p.name: p.pack() for p in self.props.values()}
Expand Down
156 changes: 111 additions & 45 deletions synapse/lib/ast.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,16 @@

from synapse.lib.stormtypes import tobool, toint, toprim, tostr, tonumber, tocmprvalu, undef

SET_ALWAYS = 0
SET_UNSET = 1
SET_NEVER = 2

COND_EDIT_SET = {
'always': SET_ALWAYS,
'unset': SET_UNSET,
'never': SET_NEVER,
}

logger = logging.getLogger(__name__)

def parseNumber(x):
Expand Down Expand Up @@ -136,7 +146,7 @@ def hasAstClass(self, clss):
retn = True
break

if isinstance(kid, (EditPropSet, Function, CmdOper)):
if isinstance(kid, (EditPropSet, EditCondPropSet, Function, CmdOper)):
continue

if kid.hasAstClass(clss):
Expand Down Expand Up @@ -179,6 +189,12 @@ def reqRuntSafe(self, runt, mesg):

todo.extend(nkid.kids)

def reqNotReadOnly(self, runt, mesg=None):
if runt.readonly:
if mesg is None:
mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))

def hasVarName(self, name):
return any(k.hasVarName(name) for k in self.kids)

Expand Down Expand Up @@ -238,9 +254,8 @@ def __init__(self, astinfo, kids, autoadd=False):

async def run(self, runt, genr):

if runt.readonly and self.autoadd:
mesg = 'Autoadd may not be executed in readonly Storm runtime.'
raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
if self.autoadd:
self.reqNotReadOnly(runt)

async def getnode(form, valu):
try:
Expand Down Expand Up @@ -1269,15 +1284,17 @@ async def run(self, runt, genr):
item = s_stormtypes.fromprim(await self.kids[0].compute(runt, path), basetypes=False)

if runt.readonly and not getattr(item.setitem, '_storm_readonly', False):
mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
raise self.kids[0].addExcInfo(s_exc.IsReadOnly(mesg=mesg))
self.kids[0].reqNotReadOnly(runt)

name = await self.kids[1].compute(runt, path)
valu = await self.kids[2].compute(runt, path)

# TODO: ditch this when storm goes full heavy object
with s_scope.enter({'runt': runt}):
await item.setitem(name, valu)
try:
await item.setitem(name, valu)
except s_exc.SynErr as e:
raise self.kids[0].addExcInfo(e)

yield node, path

Expand All @@ -1289,12 +1306,14 @@ async def run(self, runt, genr):
valu = await self.kids[2].compute(runt, None)

if runt.readonly and not getattr(item.setitem, '_storm_readonly', False):
mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
raise self.kids[0].addExcInfo(s_exc.IsReadOnly(mesg=mesg))
self.kids[0].reqNotReadOnly(runt)

# TODO: ditch this when storm goes full heavy object
with s_scope.enter({'runt': runt}):
await item.setitem(name, valu)
try:
await item.setitem(name, valu)
except s_exc.SynErr as e:
raise self.kids[0].addExcInfo(e)

class VarListSetOper(Oper):

Expand Down Expand Up @@ -3572,7 +3591,8 @@ async def compute(self, runt, path):
raise self.addExcInfo(s_exc.StormRuntimeError(mesg=mesg))

if runt.readonly and not getattr(func, '_storm_readonly', False):
mesg = f'Function ({func.__name__}) is not marked readonly safe.'
funcname = getattr(func, '_storm_funcpath', func.__name__)
mesg = f'{funcname}() is not marked readonly safe.'
raise self.kids[0].addExcInfo(s_exc.IsReadOnly(mesg=mesg))

argv = await self.kids[1].compute(runt, path)
Expand Down Expand Up @@ -3998,9 +4018,7 @@ class EditParens(Edit):

async def run(self, runt, genr):

if runt.readonly:
mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
self.reqNotReadOnly(runt)

nodeadd = self.kids[0]
assert isinstance(nodeadd, EditNodeAdd)
Expand Down Expand Up @@ -4093,9 +4111,7 @@ async def run(self, runt, genr):
# case 2: <query> [ foo:bar=($node, 20) ]
# case 2: <query> $blah=:baz [ foo:bar=($blah, 20) ]

if runt.readonly:
mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
self.reqNotReadOnly(runt)

runtsafe = self.isRuntSafe(runt)

Expand Down Expand Up @@ -4161,13 +4177,79 @@ async def feedfunc():
async for item in agen:
yield item

class CondSetOper(Oper):
def __init__(self, astinfo, kids, errok=False):
Value.__init__(self, astinfo, kids=kids)
self.errok = errok

def prepare(self):
self.isconst = False
if isinstance(self.kids[0], Const):
self.isconst = True
self.valu = COND_EDIT_SET.get(self.kids[0].value())

async def compute(self, runt, path):
if self.isconst:
return self.valu

valu = await self.kids[0].compute(runt, path)
if (retn := COND_EDIT_SET.get(valu)) is not None:
return retn

mesg = f'Invalid conditional set operator ({valu}).'
exc = s_exc.StormRuntimeError(mesg=mesg)
raise self.addExcInfo(exc)

class EditCondPropSet(Edit):

async def run(self, runt, genr):

self.reqNotReadOnly(runt)

excignore = (s_exc.BadTypeValu,) if self.kids[1].errok else ()
rval = self.kids[2]

async for node, path in genr:

propname = await self.kids[0].compute(runt, path)
name = await tostr(propname)

prop = node.form.reqProp(name, extra=self.kids[0].addExcInfo)

oper = await self.kids[1].compute(runt, path)
if oper == SET_NEVER or (oper == SET_UNSET and (oldv := node.get(name)) is not None):
yield node, path
await asyncio.sleep(0)
continue

if not node.form.isrunt:
# runt node property permissions are enforced by the callback
runt.confirmPropSet(prop)

isndef = isinstance(prop.type, s_types.Ndef)

try:
valu = await rval.compute(runt, path)
valu = await s_stormtypes.tostor(valu, isndef=isndef)

if isinstance(prop.type, s_types.Ival) and oldv is not None:
valu, _ = prop.type.norm(valu)
valu = prop.type.merge(oldv, valu)

await node.set(name, valu)

except excignore:
pass

yield node, path

await asyncio.sleep(0)

class EditPropSet(Edit):

async def run(self, runt, genr):

if runt.readonly:
mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
self.reqNotReadOnly(runt)

oper = await self.kids[1].compute(runt, None)
excignore = (s_exc.BadTypeValu,) if oper in ('?=', '?+=', '?-=') else ()
Expand Down Expand Up @@ -4212,7 +4294,7 @@ async def run(self, runt, genr):

if not isarray:
mesg = f'Property set using ({oper}) is only valid on arrays.'
exc = s_exc.StormRuntimeError(mesg)
exc = s_exc.StormRuntimeError(mesg=mesg)
raise self.kids[0].addExcInfo(exc)

arry = node.get(name)
Expand Down Expand Up @@ -4260,9 +4342,7 @@ class EditPropDel(Edit):

async def run(self, runt, genr):

if runt.readonly:
mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
self.reqNotReadOnly(runt)

async for node, path in genr:
propname = await self.kids[0].compute(runt, path)
Expand All @@ -4288,9 +4368,7 @@ class EditUnivDel(Edit):

async def run(self, runt, genr):

if runt.readonly:
mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
self.reqNotReadOnly(runt)

univprop = self.kids[0]
assert isinstance(univprop, UnivProp)
Expand Down Expand Up @@ -4466,9 +4544,7 @@ def __init__(self, astinfo, kids=(), n2=False):

async def run(self, runt, genr):

if runt.readonly:
mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
self.reqNotReadOnly(runt)

# SubQuery -> Query
query = self.kids[1].kids[0]
Expand Down Expand Up @@ -4531,9 +4607,7 @@ def __init__(self, astinfo, kids=(), n2=False):

async def run(self, runt, genr):

if runt.readonly:
mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
self.reqNotReadOnly(runt)

query = self.kids[1].kids[0]

Expand Down Expand Up @@ -4589,9 +4663,7 @@ class EditTagAdd(Edit):

async def run(self, runt, genr):

if runt.readonly:
mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
self.reqNotReadOnly(runt)

if len(self.kids) > 1 and isinstance(self.kids[0], Const) and (await self.kids[0].compute(runt, None)) == '?':
oper_offset = 1
Expand Down Expand Up @@ -4635,9 +4707,7 @@ class EditTagDel(Edit):

async def run(self, runt, genr):

if runt.readonly:
mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
self.reqNotReadOnly(runt)

async for node, path in genr:

Expand All @@ -4661,9 +4731,7 @@ class EditTagPropSet(Edit):
'''
async def run(self, runt, genr):

if runt.readonly:
mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
self.reqNotReadOnly(runt)

oper = await self.kids[1].compute(runt, None)
excignore = s_exc.BadTypeValu if oper == '?=' else ()
Expand Down Expand Up @@ -4697,9 +4765,7 @@ class EditTagPropDel(Edit):
'''
async def run(self, runt, genr):

if runt.readonly:
mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
self.reqNotReadOnly(runt)

async for node, path in genr:

Expand Down
Loading
Loading