Merge pull request #19 from erik-leven/orchestrator

Orchestrator

README.md

@@ -156,4 +156,35 @@ in the case of needing to rollback? Enough with excuses, this could be implement
     * Key vault must support login with git token
         * Git token used for kv2 must have permissions: read:org & write:org
 * Comparison now happens by loading the JSON inside of the files instead of straight directory comparison.
-* Added 'off' option for simplicity's sake.
+* Added 'off' option for simplicity's sake.
+
+## Example Sesam System config using version 2.1.0
+```
+{
+  "_id": "extra-node-watcher",
+  "type": "system:microservice",
+    "docker": {
+    "environment": {
+      "AUTODEPLOYER_PATH": "systems/extra-node-watcher.conf.json",
+      "BRANCH": "master",                       <--- CAN ALSO BE A TAG
+      "DEPLOY_TOKEN": "$SECRET(GIT_TOKEN)",     <--- DEPLOY_TOKEN if GIT_USERNAME is NOT set. ACCESS_TOKEN if it is.
+      "GIT_REPO": "$ENV(EXTRA_NODE_GIT_REPO)",
+      "GIT_USERNAME": "<YOUR_GITHUB_USERNAME>", <--- IF THIS IS SET 'DEPLOY_TOKEN' MUST BE A GIT ACCESS_TOKEN!
+      "JWT": "$SECRET(EXTRA_NODE_JWT)",
+      "LOG_LEVEL": "DEBUG",
+      "SYNC_ROOT": "/",
+      "VARIABLES_FILE_PATH": "variables/variables-<ENV>.json",  OPTIONAL
+      "VAULT_GIT_TOKEN": "$SECRET(GIT_TOKEN)",                  OPTIONAL
+      "VAULT_MOUNTING_POINT": "sesam/kv2",                      OPTIONAL
+      "VAULT_URL": "https://vault.<ORGANIZATION>.io",           OPTIONAL
+      "off": "false"                                            OPTIONAL, default false.
+      "ORCHESTRATOR": true                                      OPTIONAL, default false
+    },
+    "image": "sesamcommunity/github-autodeployer:2.1.0",
+    "port": 5000
+  }
+}
+```
+### Notes on version 2.1.0:
+* It is backwards compatable with previous versions as the new functionality needs the new environment variables to run.
+* If ORCHESTRATOR is set to true the microservice will overwrite all registered changes ue to the orchestrator and copy the old config.

service/github-autodeployer.py

@@ -30,6 +30,7 @@
 vault_git_token = os.environ.get('VAULT_GIT_TOKEN')
 vault_mounting_point = os.environ.get('VAULT_MOUNTING_POINT')
 vault_url = os.environ.get('VAULT_URL')
+orchestrator = os.environ.get('ORCHESTRATOR', False)
 
 git_username = os.environ.get('GIT_USERNAME', None)  # Needed if using clone_git_repov3
 
@@ -285,6 +286,36 @@ def check_for_unknown():
         logging.warning("else, prepare for unexpected behaviour. Hic Sunt Leones. You have been warned.")
         logging.warning("\n")
 
+def check_and_replace_orchestrator_pipes():
+    for old_filename in os.listdir(sesam_checkout_dir + "/unpacked/pipes/"):
+        with open(os.path.join(sesam_checkout_dir + "/unpacked/pipes/", old_filename), 'r') as f: # open in readonly mode
+            old_file = load_json(f.read())
+            try:
+                old_file["metadata"]["orchestrator"]["original_configuration"]
+                for new_filename in os.listdir(git_cloned_dir + "/sesam-node/pipes/"):
+                    with open(os.path.join(git_cloned_dir + "/sesam-node/pipes/", new_filename), 'r') as g: # open in readonly mode
+                        new_file = load_json(g.read())
+                        if old_file["metadata"]["orchestrator"]["original_configuration"] == new_file:
+                            logging.info("The pipe %s is restored to orchestrator mode" % new_file["_id"])
+                            with open(os.path.join(payload_dir + "/pipes/", new_filename), 'w') as h:
+                                h.write(dump_json(old_file))
+            except KeyError:
+                None
+def check_and_replace_orchestrator_systems():
+    for old_filename in os.listdir(sesam_checkout_dir + "/unpacked/systems/"):
+        with open(os.path.join(sesam_checkout_dir + "/unpacked/systems/", old_filename), 'r') as f: # open in readonly mode
+            old_file = load_json(f.read())
+            try:
+                old_file["metadata"]["orchestrator"]["original_configuration"]
+                for new_filename in os.listdir(git_cloned_dir + "/sesam-node/systems/"):
+                    with open(os.path.join(git_cloned_dir + "/sesam-node/systems/", new_filename), 'r') as g: # open in readonly mode
+                        new_file = load_json(g.read())
+                        if old_file["metadata"]["orchestrator"]["original_configuration"] == new_file:
+                            logging.info("The system %s is restored to orchestrator mode" % new_file["_id"])
+                            with open(os.path.join(payload_dir + "/systems/", new_filename), 'w') as h:
+                                h.write(dump_json(old_file))
+            except KeyError:
+                None
 
 if __name__ == '__main__':
     os.chdir("/service")
@@ -303,7 +334,7 @@ def check_for_unknown():
     check_for_unknown()
     copy_autodeployer()
 
-    new_node = load_sesam_files_as_json(git_cloned_dir + "/" + sync_root + '/node')
+    new_node = load_sesam_files_as_json(git_cloned_dir + "/" + sync_root)
     old_node = load_sesam_files_as_json(sesam_checkout_dir + "/" + "unpacked")
     if not compare_json_dict_list(old_node, new_node):
         # Verify variables & secrets if specified
@@ -322,6 +353,9 @@ def check_for_unknown():
                     logging.error('Failed to upload variables to node!')
             elif upload_variables and variables is None:
                 logging.error('Upload variables is true but could not get variables to upload!')
+        if orchestrator:
+            check_and_replace_orchestrator_pipes()
+            check_and_replace_orchestrator_systems()
         logging.info(f"Uploading new configuration from github to node {sesam_api}")
         zip_payload()
         upload_payload()

service/requirements.txt

@@ -1,4 +1,4 @@
 requests==2.20.0
 PyGithub==1.35
-GitPython==2.1.8
-hvac==0.9.6
+GitPython==3.0.6
+hvac==0.9.6