Issues 134 133

.github/workflows/main.yml

@@ -7,7 +7,9 @@ on:
   push:
     branches: [ master ]
   pull_request:
-    branches: [ master ]
+    branches:
+      - master
+      - 'release-*'
   workflow_dispatch:
 
 jobs:

snowexsql/api.py

@@ -11,7 +11,7 @@
 from snowexsql.conversions import query_to_geopandas, raster_to_rasterio
 from snowexsql.db import get_db
 from snowexsql.tables import ImageData, LayerData, PointData, Instrument, \
-    Observer, Site, Campaign
+    Observer, Site, Campaign, MeasurementType, DOI
 
 
 LOG = logging.getLogger(__name__)
@@ -49,9 +49,9 @@ class BaseDataset:
     DB_NAME = DB_NAME
 
     ALLOWED_QRY_KWARGS = [
-        "campaign", "site_id", "date", "instrument", "type",
+        "campaign", "date", "instrument", "type",
         "utm_zone", "date_greater_equal", "date_less_equal",
-        "value_greater_equal", 'value_less_equal',
+        "value_greater_equal", 'value_less_equal', "doi"
     ]
     SPECIAL_KWARGS = ["limit"]
     # Default max record count
@@ -87,6 +87,13 @@ def _check_size(cls, qry, kwargs):
                 f" to the desired number of records."
             )
 
+    @classmethod
+    def _filter_campaign(cls, qry, v):
+        qry = qry.filter(
+            Site.campaign.has(Campaign.name == v)
+        )
+        return qry
+
     @classmethod
     def extend_qry(cls, qry, check_size=True, **kwargs):
         if cls.MODEL is None:
@@ -96,9 +103,15 @@ def extend_qry(cls, qry, check_size=True, **kwargs):
         for k, v in kwargs.items():
             # Handle special operations
             if k in cls.ALLOWED_QRY_KWARGS:
+                # Logic for filtering on date with LayerData
+                if "date" in k and cls.MODEL == LayerData:
+                    qry = qry.join(LayerData.site)
+                    qry_model = Site
+                else:
+                    qry_model = cls.MODEL
                 # standard filtering using qry.filter
                 if isinstance(v, list):
-                    filter_col = getattr(cls.MODEL, k)
+                    filter_col = getattr(qry_model, k)
                     if k == "date":
                         raise ValueError(
                             "We cannot search for a list of dates"
@@ -116,34 +129,38 @@ def extend_qry(cls, qry, check_size=True, **kwargs):
                     # Filter boundary
                     if "_greater_equal" in k:
                         key = k.split("_greater_equal")[0]
-                        filter_col = getattr(cls.MODEL, key)
+                        filter_col = getattr(qry_model, key)
                         qry = qry.filter(filter_col >= v)
                     elif "_less_equal" in k:
                         key = k.split("_less_equal")[0]
-                        filter_col = getattr(cls.MODEL, key)
+                        filter_col = getattr(qry_model, key)
                         qry = qry.filter(filter_col <= v)
                     # Filter linked columns
                     elif k == "instrument":
                         qry = qry.filter(
-                            cls.MODEL.instrument.has(name=v)
+                            qry_model.instrument.has(name=v)
                         )
                     elif k == "campaign":
-                        qry = qry.join(
-                            cls.MODEL.site
-                        ).filter(
-                            Site.campaign.has(Campaign.name == v)
-                        )
+                        qry = cls._filter_campaign(qry, v)
                     elif k == "site_id":
                         qry = qry.filter(
-                            cls.MODEL.site.has(name=v)
+                            qry_model.site.has(name=v)
                         )
                     elif k == "observer":
                         qry = qry.join(
-                            LayerData.observers
+                            qry_model.observers
                         ).filter(Observer.name == v)
+                    elif k == "doi":
+                        qry = qry.join(
+                            qry_model.doi
+                        ).filter(DOI.doi == v)
+                    elif k == "type":
+                        qry = qry.join(
+                            qry_model.measurement
+                        ).filter(MeasurementType.name == v)
                     # Filter to exact value
                     else:
-                        filter_col = getattr(cls.MODEL, k)
+                        filter_col = getattr(qry_model, k)
                         qry = qry.filter(filter_col == v)
                     LOG.debug(
                         f"Filtering {k} to list {v}"
@@ -194,23 +211,13 @@ def all_site_names(self):
             result = qry.all()
         return self.retrieve_single_value_result(result)
 
-    @property
-    def all_site_ids(self):
-        """
-        Return all specific site names
-        """
-        with db_session(self.DB_NAME) as (session, engine):
-            qry = session.query(Site.name).distinct()
-            result = qry.all()
-        return self.retrieve_single_value_result(result)
-
     @property
     def all_types(self):
         """
         Return all types of the data
         """
         with db_session(self.DB_NAME) as (session, engine):
-            qry = session.query(self.MODEL.type).distinct()
+            qry = session.query(MeasurementType.name).distinct()
             result = qry.all()
         return self.retrieve_single_value_result(result)
 
@@ -234,6 +241,16 @@ def all_observers(self):
             result = qry.all()
         return self.retrieve_single_value_result(result)
 
+    @property
+    def all_dois(self):
+        """
+        Return all distinct DOIs in the data
+        """
+        with db_session(self.DB_NAME) as (session, engine):
+            qry = session.query(DOI.doi).distinct()
+            result = qry.all()
+        return self.retrieve_single_value_result(result)
+
     @property
     def all_units(self):
         """
@@ -346,9 +363,37 @@ class LayerMeasurements(PointMeasurements):
     MODEL = LayerData
     ALLOWED_QRY_KWARGS = [
         "campaign", "site_id", "date", "instrument", "observer", "type",
-        "utm_zone", "pit_id", "date_greater_equal", "date_less_equal"
+        "utm_zone", "pit_id", "date_greater_equal", "date_less_equal",
+        "doi", "value_greater_equal", 'value_less_equal'
     ]
 
+    @classmethod
+    def _filter_campaign(cls, qry, v):
+
+        qry = qry.join(cls.MODEL.site).join(
+            Site.campaign).filter(Campaign.name == v)
+        return qry
+
+    @property
+    def all_site_ids(self):
+        """
+        Return all specific site names
+        """
+        with db_session(self.DB_NAME) as (session, engine):
+            qry = session.query(Site.name).distinct()
+            result = qry.all()
+        return self.retrieve_single_value_result(result)
+
+    @property
+    def all_dates(self):
+        """
+        Return all distinct dates in the data
+        """
+        with db_session(self.DB_NAME) as (session, engine):
+            qry = session.query(Site.date).distinct()
+            result = qry.all()
+        return self.retrieve_single_value_result(result)
+
 
 class RasterMeasurements(BaseDataset):
     MODEL = ImageData

snowexsql/tables/__init__.py

@@ -6,12 +6,16 @@
 from .instrument import Instrument
 from .campaign import Campaign
 from .site import Site
+from .doi import DOI
+from .measurement_type import MeasurementType
 
 __all__ = [
     "Campaign",
+    "DOI",
     "ImageData",
     "Instrument",
     "LayerData",
+    "MeasurementType",
     "Observer",
     "PointData",
     "Site",

snowexsql/tables/base.py

@@ -6,9 +6,8 @@
 """
 
 from geoalchemy2 import Geometry
-from sqlalchemy import Column, Date, DateTime, Float, Integer, String, Time
+from sqlalchemy import Column, Float, Integer, Time, Date
 from sqlalchemy.orm import DeclarativeBase
-from sqlalchemy.sql import func
 
 
 class Base(DeclarativeBase):
@@ -17,17 +16,9 @@ class Base(DeclarativeBase):
     """
     # SQL Alchemy
     __table_args__ = {"schema": "public"}
-
     # Primary Key
     id = Column(Integer, primary_key=True)
-
-    # Standard table columns
-    time_created = Column(DateTime(timezone=True), server_default=func.now())
-    time_updated = Column(DateTime(timezone=True), onupdate=func.now())
-
     date_accessed = Column(Date)
-    date = Column(Date)
-    doi = Column(String(50))
 
 
 class SingleLocationData:
@@ -39,9 +30,4 @@ class SingleLocationData:
     time = Column(Time(timezone=True))
 
 
-class Measurement(object):
-    """
-    Base Class providing attributes required for a measurement of any kind
-    """
-    type = Column(String(50))
-    units = Column(String(50))
+

snowexsql/tables/campaign.py

@@ -5,7 +5,8 @@
 @author: jtmaz
 """
 
-from sqlalchemy import Column, String, Integer
+from sqlalchemy import Column, String, Integer, ForeignKey
+from sqlalchemy.orm import declared_attr, relationship
 
 from .base import Base 
 
@@ -22,3 +23,18 @@ class Campaign(Base):
     id = Column(Integer, primary_key=True)
     name = Column(String())
     description = Column(String())
+
+
+class InCampaign:
+    """
+    Class to extend when including a Campaign
+    """
+
+    @declared_attr
+    def campaign_id(cls):
+        return Column(Integer, ForeignKey('public.campaigns.id'),
+                      index=True)
+
+    @declared_attr
+    def campaign(cls):
+        return relationship('Campaign')

snowexsql/tables/doi.py

@@ -0,0 +1,26 @@
+from sqlalchemy import Column, String, Integer, ForeignKey
+from sqlalchemy.orm import relationship, declared_attr
+
+from .base import Base
+
+
+class DOI(Base):
+    """
+    Table stores DOI values for each unique publication
+    """
+    __tablename__ = 'dois'
+
+    doi = Column(String())
+
+
+class HasDOI:
+    """
+    Class to extend when including a DOI
+    """
+    @declared_attr
+    def doi_id(cls):
+        return Column(Integer, ForeignKey('public.dois.id'), index=True)
+
+    @declared_attr
+    def doi(cls):
+        return relationship('DOI')

snowexsql/tables/image_data.py

@@ -1,13 +1,20 @@
 from geoalchemy2 import Raster
-from sqlalchemy import Column, String
+from sqlalchemy import Column, String, Date
 
-from .base import Base, Measurement
+from .base import Base
+from .campaign import InCampaign
+from .instrument import HasInstrument
+from .measurement_type import HasMeasurement
+from .doi import HasDOI
 
 
-class ImageData(Base, Measurement):
+class ImageData(Base, HasMeasurement, HasInstrument, HasDOI, InCampaign):
     """
     Class representing the images table. This table holds all images/rasters
     """
     __tablename__ = 'images'
+    # Date of the measurement
+    date = Column(Date)
     raster = Column(Raster)
-    description = Column(String(1000))
+    description = Column(String())
+    units = Column(String(50))

snowexsql/tables/instrument.py

@@ -1,12 +1,26 @@
-from sqlalchemy import Column, Integer, String
+from sqlalchemy import Column, Integer, String, ForeignKey
+from sqlalchemy.orm import relationship, declared_attr
 from .base import Base
 
 
 class Instrument(Base):
     __tablename__ = 'instruments'
-    # auto created id
-    id = Column(Integer, primary_key=True)
     # Name of the instrument
     name = Column(String(), index=True)
     model = Column(String())
     specifications = Column(String())
+
+
+class HasInstrument:
+    """
+    Class to extend when including an Instrument
+    """
+
+    @declared_attr
+    def instrument_id(cls):
+        return Column(Integer, ForeignKey('public.instruments.id'),
+                      index=True)
+
+    @declared_attr
+    def instrument(cls):
+        return relationship('Instrument')