Merge pull request #100 from opencybersecurityalliance/develop

2.3.24

CONTRIBUTING.rst

@@ -15,7 +15,7 @@ Types of Contributions
 Report Bugs
 ~~~~~~~~~~~
 
-Report bugs at https://github.com/IBM/firepit/issues.
+Report bugs at https://github.com/opencybersecurityalliance/firepit/issues.
 
 If you are reporting a bug, please include:
 
@@ -45,7 +45,7 @@ articles, and such.
 Submit Feedback
 ~~~~~~~~~~~~~~~
 
-The best way to send feedback is to file an issue at https://github.com/IBM/firepit/issues.
+The best way to send feedback is to file an issue at https://github.com/opencybersecurityalliance/firepit/issues.
 
 If you are proposing a feature:
 
@@ -67,7 +67,7 @@ Ready to contribute? Here's how to set up `firepit` for local development.
 3. Install your local copy into a virtualenv. Assuming you have pyenv installed, this is how you set up your fork for local development::
 
     $ cd firepit/
-    $ pyenv virtualenv python-3.6.8 firepit
+    $ pyenv virtualenv python-3.9.2 firepit
     $ pyenv local firepit
     $ make setup
 
@@ -90,6 +90,8 @@ Ready to contribute? Here's how to set up `firepit` for local development.
     $ git push origin name-of-your-bugfix-or-feature
 
 7. Submit a pull request through the GitHub website.
+8. Request a review from one of the maintainers.
+
 
 Pull Request Guidelines
 -----------------------
@@ -100,7 +102,8 @@ Before you submit a pull request, check that it meets these guidelines:
 2. If the pull request adds functionality, the docs should be updated. Put
    your new functionality into a function with a docstring, and add the
    feature to the list in README.rst.
-3. The pull request should work for Python 3.7, 3.8 and 3.9.
+3. The pull request should work for Python 3.8, 3.9, 3.10, and 3.11.
+   There is a GitHub workflow that will check for this automatically.
 
 All contributions must be covered by a `Contributor's License Agreement`_ (CLA) and `Entity CLA`_ (if you are contributing on behalf of your employer). You will get a prompt to sign CLA when you submit your first PR.
 
@@ -109,13 +112,17 @@ Tips
 
 To run a subset of tests::
 
-$ pytest tests.test_firepit
+$ pytest tests/test_storage.py
+
+Or::
+
+$ pytest -k test_something...
 
 
-Deploying
+Releasing
 ---------
 
-A reminder for the maintainers on how to deploy.
+A reminder for the maintainers on how to release.
 Make sure all your changes are committed (including an entry in HISTORY.rst).
 Then run::
 

firepit/__init__.py

@@ -2,7 +2,7 @@
 
 __author__ = """IBM Security"""
 __email__ = 'pcoccoli@us.ibm.com'
-__version__ = '2.3.23'
+__version__ = '2.3.24'
 
 
 import re

firepit/sqlstorage.py

@@ -910,7 +910,8 @@ def extract_observeddata_attribute(
             run=True):
         """
         Get the observations of `value` in `viewname`.`path`
-        Returns list of dicts like {'name_of_SDO_attribute': '...', '{column}': '...'}
+        Returns list of dicts like {'name_of_attribute': '...', '{column}': '...'}
+        `name_of_attribute` can be a str or list of str (to get multiple attrs)
         """
         qry = Query([
             Table(viewname),
@@ -937,11 +938,22 @@ def extract_observeddata_attribute(
             proj.append(Column(column, table, p))
         if column and value is not None:
             qry.append(Filter([Predicate(column, '=', value)]))
-        new_col = Column(name_of_attribute, 'observed-data')
-        qry.append(Order([new_col]))
+        if isinstance(name_of_attribute, str):
+            attrs = [name_of_attribute]
+        elif isinstance(name_of_attribute, list):
+            attrs = name_of_attribute
+        else:
+            raise TypeError('name_of_attribute must be str or list[str]')
+        new_cols = []
+        for attr in attrs:
+            if attr == 'id':
+                new_cols.append(Column(attr, 'observed-data', 'observation_id'))
+            else:
+                new_cols.append(Column(attr, 'observed-data'))
+        qry.append(Order(new_cols))
         if not proj:
             proj = [Column('*', viewname)]
-        qry.append(Projection([new_col] + proj))
+        qry.append(Projection(new_cols + proj))
         if limit:
             qry.append(Limit(limit))
 

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.3.23
+current_version = 2.3.24
 commit = True
 tag = True
 

setup.py

@@ -14,7 +14,7 @@
     'anytree',
     'python-dateutil',
     'ijson',
-    'lark-parser',
+    'lark',
     'tabulate',
     'typer',
     'ujson'
@@ -65,6 +65,6 @@
     test_suite='tests',
     tests_require=test_requirements,
     url='https://github.com/opencybersecurityalliance/firepit',
-    version='2.3.23',
+    version='2.3.24',
     zip_safe=False,
 )

tests/test_obs_attr.py

@@ -0,0 +1,40 @@
+from .helpers import tmp_storage
+
+
+def test_obs_attr_url(tmpdir, fake_bundle_file):
+    store = tmp_storage(tmpdir)    
+    store.cache('q1', fake_bundle_file)
+    data = store.extract_observeddata_attribute(
+        'url',
+        'last_observed')
+    assert len(data) == 31
+    assert 'last_observed' in data[0]
+
+
+def test_obs_attr_url_only(tmpdir, fake_bundle_file):
+    store = tmp_storage(tmpdir)
+    store.cache('q1', fake_bundle_file)
+    data = store.extract_observeddata_attribute('url', 'id', path=['value'])
+    assert len(data) == 31
+    assert set(data[0].keys()) == {'observation_id', 'value'}
+
+
+def test_obs_attr_multiple(tmpdir, fake_bundle_file):
+    store = tmp_storage(tmpdir)
+    store.cache('q1', fake_bundle_file)
+    attrs = [
+        'number_observed',
+        'first_observed',
+        'last_observed',
+        'id',
+    ]
+    data = store.extract_observeddata_attribute('url', attrs)
+    assert len(data) == 31
+    assert set(data[0].keys()) == {
+        'number_observed',
+        'first_observed',
+        'last_observed',
+        'observation_id',
+        'value',
+        'id',  # The original url:id
+    }