Add ability to turn off health check, added deployment guide (#80)

Author: czgu

datahub/server/app/server.py

@@ -1,4 +1,5 @@
-from flask import send_from_directory
+import os
+from flask import send_from_directory, abort
 
 from app import auth
 from app.datasource import register, abort_request
@@ -24,6 +25,8 @@ def datasource_four_oh_four(*args, **kwargs):
 @limiter.exempt
 def get_health_check():
     """This is a health check endpoint"""
+    if os.path.exists("/tmp/datahub/deploying"):
+        abort(503)
     return "pong"
 
 

datahub/tests/test_app/test_server.py

@@ -1,3 +1,4 @@
+import os
 from app.server import get_health_check
 
 get_health_check
@@ -7,6 +8,16 @@ def test_ping_api(flask_client):
     assert b"pong" == flask_client.get("/ping/").get_data()
 
 
+def test_ping_deploy_api(flask_client, monkeypatch):
+    with monkeypatch.context() as m:
+        m.setattr(
+            os.path,
+            "exists",
+            lambda p: True if p == "/tmp/datahub/deploying" else False,
+        )
+        assert 503 == flask_client.get("/ping/").status_code
+
+
 def test_four_oh_four(flask_client, fake_user):
     assert 404 == flask_client.get("/ds/some/random/path/").status_code
 

docs/admin_guide/deployment_guide.md

@@ -0,0 +1,15 @@
+---
+id: deployment_guide
+title: Deployment Guide
+sidebar_label: Deployment Guide
+---
+
+While there are many ways to deploy datahub to production, there are some general principles
+that are recommended when setting up your own production deployment.
+
+1. Please make sure the web server, the beat scheduler, and the celery worker are using the production Docker image. Please refer to `containers/docker-compose.prod.yml` to see how to launch these images for production.
+2. To get logs, make sure /var/log/datahub/ is mounted as a Docker volume so that the logs can be moved to the host machine.
+3. Use the /ping/ endpoint for health checks.
+    1. During deployments, you can create a file that has the path `/tmp/datahub/deploying` to make the health check return 503 and remove it after completion.
+4. Please make sure celery worker is ran with concurrent mode as it is the only mode that can have a memory limit.
+5. During worker deployments, you can run the following first to make the celery worker stop receving new tasks and exit once all current tasks are finished: `celery multi stopwait datahub_worker@%h -A tasks.all_tasks --pidfile=/opt/celery_%n.pid`. This will make deployment time take much longer but users' running queries won't be killed.