From 64c2ccd4df9fd3f9bdd8198b7374fb873a4a2be5 Mon Sep 17 00:00:00 2001 From: Erik Date: Tue, 16 Jun 2020 11:43:53 +0200 Subject: [PATCH 1/2] added support for orchestrator and updated readme --- README.md | 33 ++++++++++++++++++++++++++++++++- service/github-autodeployer.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a76b595..bb3b7be 100644 --- a/README.md +++ b/README.md @@ -156,4 +156,35 @@ in the case of needing to rollback? Enough with excuses, this could be implement * Key vault must support login with git token * Git token used for kv2 must have permissions: read:org & write:org * Comparison now happens by loading the JSON inside of the files instead of straight directory comparison. -* Added 'off' option for simplicity's sake. \ No newline at end of file +* Added 'off' option for simplicity's sake. + +## Example Sesam System config using version 2.1.0 +``` +{ + "_id": "extra-node-watcher", + "type": "system:microservice", + "docker": { + "environment": { + "AUTODEPLOYER_PATH": "systems/extra-node-watcher.conf.json", + "BRANCH": "master", <--- CAN ALSO BE A TAG + "DEPLOY_TOKEN": "$SECRET(GIT_TOKEN)", <--- DEPLOY_TOKEN if GIT_USERNAME is NOT set. ACCESS_TOKEN if it is. + "GIT_REPO": "$ENV(EXTRA_NODE_GIT_REPO)", + "GIT_USERNAME": "", <--- IF THIS IS SET 'DEPLOY_TOKEN' MUST BE A GIT ACCESS_TOKEN! + "JWT": "$SECRET(EXTRA_NODE_JWT)", + "LOG_LEVEL": "DEBUG", + "SYNC_ROOT": "/", + "VARIABLES_FILE_PATH": "variables/variables-.json", OPTIONAL + "VAULT_GIT_TOKEN": "$SECRET(GIT_TOKEN)", OPTIONAL + "VAULT_MOUNTING_POINT": "sesam/kv2", OPTIONAL + "VAULT_URL": "https://vault..io", OPTIONAL + "off": "false" OPTIONAL, default false. + "ORCHESTRATOR": true OPTIONAL, default false + }, + "image": "sesamcommunity/github-autodeployer:2.1.0", + "port": 5000 + } +} +``` +### Notes on version 2.1.0: +* It is backwards compatable with previous versions as the new functionality needs the new environment variables to run. +* If ORCHESTRATOR is set to true the microservice will overwrite all registered changes ue to the orchestrator and copy the old config. \ No newline at end of file diff --git a/service/github-autodeployer.py b/service/github-autodeployer.py index bd28393..08b306c 100755 --- a/service/github-autodeployer.py +++ b/service/github-autodeployer.py @@ -30,6 +30,7 @@ vault_git_token = os.environ.get('VAULT_GIT_TOKEN') vault_mounting_point = os.environ.get('VAULT_MOUNTING_POINT') vault_url = os.environ.get('VAULT_URL') +orchestrator = os.environ.get('ORCHESTRATOR', False) git_username = os.environ.get('GIT_USERNAME', None) # Needed if using clone_git_repov3 @@ -285,6 +286,36 @@ def check_for_unknown(): logging.warning("else, prepare for unexpected behaviour. Hic Sunt Leones. You have been warned.") logging.warning("\n") +def check_and_replace_orchestrator_pipes(): + for old_filename in os.listdir(sesam_checkout_dir + "/unpacked/pipes/"): + with open(os.path.join(sesam_checkout_dir + "/unpacked/pipes/", old_filename), 'r') as f: # open in readonly mode + old_file = json.loads(f.read()) + try: + old_file["metadata"]["orchestrator"]["original_configuration"] + for new_filename in os.listdir(git_cloned_dir + "/sesam-node/pipes/"): + with open(os.path.join(git_cloned_dir + "/sesam-node/pipes/", new_filename), 'r') as g: # open in readonly mode + new_file = json.loads(g.read()) + if old_file["metadata"]["orchestrator"]["original_configuration"] == new_file: + logging.info("The pipe %s is restored to orchestrator mode" % new_file["_id"]) + with open(os.path.join(payload_dir + "/pipes/", new_filename), 'w') as h: + h.write(json.dumps(old_file)) + except KeyError: + None +def check_and_replace_orchestrator_systems(): + for old_filename in os.listdir(sesam_checkout_dir + "/unpacked/systems/"): + with open(os.path.join(sesam_checkout_dir + "/unpacked/systems/", old_filename), 'r') as f: # open in readonly mode + old_file = json.loads(f.read()) + try: + old_file["metadata"]["orchestrator"]["original_configuration"] + for new_filename in os.listdir(git_cloned_dir + "/sesam-node/systems/"): + with open(os.path.join(git_cloned_dir + "/sesam-node/systems/", new_filename), 'r') as g: # open in readonly mode + new_file = json.loads(g.read()) + if old_file["metadata"]["orchestrator"]["original_configuration"] == new_file: + logging.info("The system %s is restored to orchestrator mode" % new_file["_id"]) + with open(os.path.join(payload_dir + "/systems/", new_filename), 'w') as h: + h.write(json.dumps(old_file)) + except KeyError: + None if __name__ == '__main__': os.chdir("/service") @@ -322,6 +353,9 @@ def check_for_unknown(): logging.error('Failed to upload variables to node!') elif upload_variables and variables is None: logging.error('Upload variables is true but could not get variables to upload!') + if orchestrator: + check_and_replace_orchestrator_pipes() + check_and_replace_orchestrator_systems() logging.info(f"Uploading new configuration from github to node {sesam_api}") zip_payload() upload_payload() From 391932b16d7c0be5f774173071055b48a5962aa4 Mon Sep 17 00:00:00 2001 From: Erik Date: Tue, 16 Jun 2020 12:23:48 +0200 Subject: [PATCH 2/2] changed one version in req. Removed a hardcoded 'node' statement, and changed json commands to imported statements --- service/github-autodeployer.py | 14 +++++++------- service/requirements.txt | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/service/github-autodeployer.py b/service/github-autodeployer.py index 08b306c..bbcd656 100755 --- a/service/github-autodeployer.py +++ b/service/github-autodeployer.py @@ -289,31 +289,31 @@ def check_for_unknown(): def check_and_replace_orchestrator_pipes(): for old_filename in os.listdir(sesam_checkout_dir + "/unpacked/pipes/"): with open(os.path.join(sesam_checkout_dir + "/unpacked/pipes/", old_filename), 'r') as f: # open in readonly mode - old_file = json.loads(f.read()) + old_file = load_json(f.read()) try: old_file["metadata"]["orchestrator"]["original_configuration"] for new_filename in os.listdir(git_cloned_dir + "/sesam-node/pipes/"): with open(os.path.join(git_cloned_dir + "/sesam-node/pipes/", new_filename), 'r') as g: # open in readonly mode - new_file = json.loads(g.read()) + new_file = load_json(g.read()) if old_file["metadata"]["orchestrator"]["original_configuration"] == new_file: logging.info("The pipe %s is restored to orchestrator mode" % new_file["_id"]) with open(os.path.join(payload_dir + "/pipes/", new_filename), 'w') as h: - h.write(json.dumps(old_file)) + h.write(dump_json(old_file)) except KeyError: None def check_and_replace_orchestrator_systems(): for old_filename in os.listdir(sesam_checkout_dir + "/unpacked/systems/"): with open(os.path.join(sesam_checkout_dir + "/unpacked/systems/", old_filename), 'r') as f: # open in readonly mode - old_file = json.loads(f.read()) + old_file = load_json(f.read()) try: old_file["metadata"]["orchestrator"]["original_configuration"] for new_filename in os.listdir(git_cloned_dir + "/sesam-node/systems/"): with open(os.path.join(git_cloned_dir + "/sesam-node/systems/", new_filename), 'r') as g: # open in readonly mode - new_file = json.loads(g.read()) + new_file = load_json(g.read()) if old_file["metadata"]["orchestrator"]["original_configuration"] == new_file: logging.info("The system %s is restored to orchestrator mode" % new_file["_id"]) with open(os.path.join(payload_dir + "/systems/", new_filename), 'w') as h: - h.write(json.dumps(old_file)) + h.write(dump_json(old_file)) except KeyError: None @@ -334,7 +334,7 @@ def check_and_replace_orchestrator_systems(): check_for_unknown() copy_autodeployer() - new_node = load_sesam_files_as_json(git_cloned_dir + "/" + sync_root + '/node') + new_node = load_sesam_files_as_json(git_cloned_dir + "/" + sync_root) old_node = load_sesam_files_as_json(sesam_checkout_dir + "/" + "unpacked") if not compare_json_dict_list(old_node, new_node): # Verify variables & secrets if specified diff --git a/service/requirements.txt b/service/requirements.txt index 0145f01..5579620 100644 --- a/service/requirements.txt +++ b/service/requirements.txt @@ -1,4 +1,4 @@ requests==2.20.0 PyGithub==1.35 -GitPython==2.1.8 -hvac==0.9.6 +GitPython==3.0.6 +hvac==0.9.6 \ No newline at end of file