Merge pull request #13 from LCOGT/test-deploy

Setup automated deploys

.github/workflows/main-deploy.yml

@@ -0,0 +1,75 @@
+name: Deploy to Production
+on:
+  push:
+    branches:
+      - main
+jobs:
+  unit-test:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v1
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.SLS_AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.SLS_AWS_SECRET_ACCESS_KEY }}
+        aws-region: us-east-1
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7 
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Run unit tests 
+      run: |
+        python -m pytest
+
+  deploy-prod:
+    # Only run on creation of a new tag
+    if: github.event_name == 'create' && github.event.ref_type == 'tag' && github.event.repository.fork == false
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Check out repository
+      uses: actions/checkout@v1
+
+    - name: Create public_key file
+      run: |
+        cat > /home/runner/work/photonranch-api/photonranch-api/public_key << EOF
+        -----BEGIN CERTIFICATE-----
+        ${{ secrets.AUTH0_PUBLIC_KEY }}
+        -----END CERTIFICATE-----
+        EOF
+        
+    - name: Create Auth0 secrets file
+      run: |
+        cat > /home/runner/work/photonranch-api/photonranch-api/secrets.json << EOF
+        {
+          "AUTH0_CLIENT_ID": "${{ secrets.AUTH0_CLIENT_ID }}"
+        }
+        EOF
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.SLS_AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.SLS_AWS_SECRET_ACCESS_KEY }}
+        aws-region: us-east-1
+
+    - name: Set up Node
+      uses: actions/setup-node@v1
+      with:
+        node-version: 12
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7 
+
+    - name: Serverless Deploy
+      uses: dhollerbach/github-action-serverless-with-python-requirements@master
+      env:
+        SERVERLESS_ACCESS_KEY: ${{ secrets.SLS_SECRET_KEY }}

.github/workflows/test-deploy.yml

@@ -0,0 +1,75 @@
+name: Deploy Test Stage
+on:
+  push:
+    branches:
+      - test-deploy
+jobs:
+  unit-test:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v1
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.SLS_AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.SLS_AWS_SECRET_ACCESS_KEY }}
+        aws-region: us-east-1
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7 
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Run unit tests 
+      run: |
+        python -m pytest
+
+  deploy-test:
+    needs: unit-test
+    runs-on: ubuntu-latest
+    steps:
+
+    - uses: actions/checkout@v1
+
+    - name: Create public_key file
+      run: |
+        cat > /home/runner/work/photonranch-api/photonranch-api/public_key << EOF
+        -----BEGIN CERTIFICATE-----
+        ${{ secrets.AUTH0_PUBLIC_KEY }}
+        -----END CERTIFICATE-----
+        EOF
+        
+    - name: Create Auth0 secrets file
+      run: |
+        cat > /home/runner/work/photonranch-api/photonranch-api/secrets.json << EOF
+        {
+          "AUTH0_CLIENT_ID": "${{ secrets.AUTH0_CLIENT_ID }}"
+        }
+        EOF
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.SLS_AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.SLS_AWS_SECRET_ACCESS_KEY }}
+        aws-region: us-east-1
+
+    - name: Set up Node
+      uses: actions/setup-node@v1
+      with:
+        node-version: 12
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7  
+
+    - name: Serverless Deploy
+      uses: dhollerbach/github-action-serverless-with-python-requirements@master
+      with:
+        args: '--stage test'
+      env:
+        SERVERLESS_ACCESS_KEY: ${{ secrets.SLS_SECRET_KEY }}

api/db.py

@@ -50,27 +50,28 @@ def get_session(db_address):
 class Image(Base):
     __tablename__ = 'images'
 
-    image_id         = Column(Integer, primary_key=True)
-    base_filename    = Column(String)
-    site             = Column(String)
-    capture_date     = Column(DateTime, default=datetime.utcnow)
-    sort_date        = Column(DateTime, default=datetime.utcnow)
-    right_ascension  = Column(Float)
-    declination      = Column(Float)
-    ex00_fits_exists = Column(Boolean)
-    ex01_fits_exists = Column(Boolean)
-    ex10_fits_exists = Column(Boolean)
-    ex13_fits_exists = Column(Boolean)
-    ex10_jpg_exists  = Column(Boolean)
-    ex13_jpg_exists  = Column(Boolean)
-    altitude         = Column(Float)
-    azimuth          = Column(Float)
-    filter_used      = Column(String)
-    airmass          = Column(Float)
-    exposure_time    = Column(Float)
-    username         = Column(String)
-    user_id          = Column(String)
-    header           = Column(String)
+    image_id          = Column(Integer, primary_key=True)
+    base_filename     = Column(String)
+    data_type         = Column(String)
+    site              = Column(String)
+    capture_date      = Column(DateTime, default=datetime.utcnow)
+    sort_date         = Column(DateTime, default=datetime.utcnow)
+    right_ascension   = Column(Float)
+    declination       = Column(Float)
+
+    fits_01_exists    = Column(Boolean)
+    fits_10_exists    = Column(Boolean)
+    jpg_medium_exists = Column(Boolean)
+    jpg_small_exists  = Column(Boolean)
+
+    altitude          = Column(Float)
+    azimuth           = Column(Float)
+    filter_used       = Column(String)
+    airmass           = Column(Float)
+    exposure_time     = Column(Float)
+    username          = Column(String)
+    user_id           = Column(String)
+    header            = Column(String)
 
     def __init__(self, base_filename):
         self.base_filename = base_filename
@@ -89,6 +90,7 @@ def get_image_pkg(self):
         package = {
             "image_id": self.image_id,
             "base_filename": self.base_filename,
+            "data_type": self.data_type,
             "site": self.site, 
 
             "exposure_time": self.exposure_time,
@@ -99,11 +101,10 @@ def get_image_pkg(self):
             "altitude": self.altitude,
             "airmass": self.airmass,
 
-            "ex01_fits_exists": self.ex01_fits_exists,
-            "ex10_fits_exists": self.ex10_fits_exists,
-            "ex10_jpg_exists": self.ex10_jpg_exists,
-            "ex13_jpg_exists": self.ex13_jpg_exists,
-            "ex00_fits_exists": self.ex00_fits_exists,
+            "fits_01_exists": self.fits_01_exists,
+            "fits_10_exists": self.fits_10_exists,
+            "jpg_medium_exists": self.jpg_medium_exists,
+            "jpg_small_exists": self.jpg_small_exists,
 
             "username": self.username,
             "user_id": self.user_id,
@@ -115,11 +116,17 @@ def get_image_pkg(self):
 
         # Include a url to the jpg
         package["jpg_url"] = ""
-        if self.ex10_jpg_exists:
-            path = get_s3_image_path(self.base_filename, "EX10", "jpg")
+        if self.jpg_medium_exists:
+            path = get_s3_image_path(self.base_filename, self.data_type, "10", "jpg")
             url = get_s3_file_url(path)
             package["jpg_url"] = url
 
+        package["jpg_thumbnail_url"] = ""
+        if self.jpg_small_exists:
+            path = get_s3_image_path(self.base_filename, self.data_type, "11", "jpg")
+            url = get_s3_file_url(path)
+            package["jpg_thumbnail_url"] = url
+
         return package
 
 
@@ -142,7 +149,7 @@ def get_latest_site_images(db_address, site, number_of_images, user_id=None):
     # Filter by site, and by user_id if provided.
     query_filters = [
         Image.site==site,
-        Image.ex10_jpg_exists.is_(True) # the jpg preview must exist
+        Image.jpg_medium_exists.is_(True) # the jpg preview must exist
     ]
     if user_id:
         query_filters.append(Image.user_id==user_id)
@@ -194,7 +201,7 @@ def filtered_images_query(db_address: str, query_filters: list):
 
     """
     # Add the condition that the jpg must exist
-    query_filters.append(Image.ex10_jpg_exists.is_(True))
+    query_filters.append(Image.jpg_medium_exists.is_(True))
     with get_session(db_address=db_address) as session:
         images = session.query(Image)\
             .order_by(Image.sort_date.desc())\
@@ -222,7 +229,7 @@ def get_image_by_filename(db_address, base_filename):
     """
     query_filters = [
         Image.base_filename == base_filename, # match filenames
-        Image.ex10_jpg_exists.is_(True)       # the jpg must exist
+        Image.jpg_medium_exists.is_(True)       # the jpg must exist
     ]
     with get_session(db_address=db_address) as session:
         image = session.query(Image)\
@@ -231,6 +238,23 @@ def get_image_by_filename(db_address, base_filename):
         return image.get_image_pkg()
 
 
+def remove_image_by_filename(base_filename):
+    """ Remove an entire row represented by the data's base filename.
+
+    Args:
+        base_filename (str): identifies what to delete. 
+            Example: wmd-ea03-20190621-00000007
+    """
+
+    with get_session(db_address=DB_ADDRESS) as session:
+        Image.query\
+            .filter(Image.base_filename == base_filename)\
+            .delete()
+        session.commit()
+
+
+
+
 #####################################################
 ##############    Endpoint Handlers    ##############
 #####################################################
@@ -319,3 +343,17 @@ def filtered_images_query_handler(event, context):
         return http_response(HTTPStatus.NOT_FOUND, error_msg)
 
     return http_response(HTTPStatus.OK, images)
+
+
+
+def remove_image_by_filename_handler(event, context):
+    base_filename = event['pathParameters']['base_filename']
+
+    try: 
+        remove_image_by_filename(base_filename)
+    except Exception as e:
+        error_msg = f"Could not delete {base_filename}. Error: {e}"
+        logger.exception(error_msg)
+        return http_response(HTTPStatus.NOT_FOUND, error_msg)
+
+    return http_response(HTTPStatus.OK, image)

api/helpers.py

@@ -73,8 +73,8 @@ def get_db_connection():
     return connection
 
 
-def get_s3_image_path(base_filename, ex_value, file_extension):
-    full_filename = f"{base_filename}-{ex_value}.{file_extension}"
+def get_s3_image_path(base_filename, datatype, reduction_level, file_extension):
+    full_filename = f"{base_filename}-{datatype}{reduction_level}.{file_extension}"
     path = f"data/{full_filename}"
     return path
 

api/tests/integration/test_integration.py

@@ -14,7 +14,7 @@
 
 VALID_FILTER_REQUEST_EVENT = {
     "queryStringParameters": {
-        "site": "wmd",
+        "site": "mrc",
         "filter": "up",
         "start_date": "2020-08-30+12",
         "filename": "-"
@@ -70,14 +70,14 @@ def test_get_image_invalid_filename(base_url):
 
 
 def test_latest_images_no_user(base_url):
-    site = "wmd"
+    site = "mrc"
     url = f"{base_url}/{site}/latest_images/1"
     response = requests.get(url)
     assert response.status_code == HTTPStatus.OK
 
 
 def test_latest_images_fake_user(base_url):
-    site = "wmd"
+    site = "mrc"
     query_params = {
         "userid": "fake user id"
     }
@@ -89,7 +89,7 @@ def test_latest_images_fake_user(base_url):
 def test_filtered_image_query(base_url):
     url = f"{base_url}/filtered_images"
     params = {
-        "site": "wmd",
+        "site": "mrc",
         "filter": "up",
         "start_date": "2020-08-30+12",
     }
@@ -105,7 +105,7 @@ def test_get_fits_header_from_db(base_url):
 
 
 def test_get_config(base_url):
-    site = "wmd"
+    site = "mrc"
     url = f"{base_url}/{site}/config"
     response = requests.get(url)
     assert response.status_code == HTTPStatus.OK
@@ -156,7 +156,7 @@ def test_add_weather_status(base_url):
 
 
 def test_site_events(base_url):
-    site = "wmd"
+    site = "mrc"
     url = f"{base_url}/events?site={site}"
     response = requests.get(url)
     assert response.status_code == HTTPStatus.OK