From 652ab206d53bfdbbd4a9f603ba613f5e6f20de70 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Mon, 4 Sep 2023 23:17:17 +0200 Subject: [PATCH 01/20] Update checkout workflow action to v4 --- .github/workflows/build_job.yml | 2 +- .github/workflows/ci_job.yml | 4 ++-- .github/workflows/docker_job.yml | 2 +- .github/workflows/pypi_job.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_job.yml b/.github/workflows/build_job.yml index db94445d..28906874 100644 --- a/.github/workflows/build_job.yml +++ b/.github/workflows/build_job.yml @@ -50,7 +50,7 @@ jobs: OUT_FILE_NAME: deploy-freva.exe ASSET_MIME: application/vnd.microsoft.portable-executable steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.11 uses: actions/setup-python@v4 with: diff --git a/.github/workflows/ci_job.yml b/.github/workflows/ci_job.yml index f3af4f6a..7cdf070a 100644 --- a/.github/workflows/ci_job.yml +++ b/.github/workflows/ci_job.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.11 uses: actions/setup-python@v4 @@ -34,7 +34,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 diff --git a/.github/workflows/docker_job.yml b/.github/workflows/docker_job.yml index 75a7ab83..cd16ed95 100644 --- a/.github/workflows/docker_job.yml +++ b/.github/workflows/docker_job.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/pypi_job.yml b/.github/workflows/pypi_job.yml index aba209ae..6e5ee5c2 100644 --- a/.github/workflows/pypi_job.yml +++ b/.github/workflows/pypi_job.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v4 From de339849f2f49bbb3373849f382c969e0212d09b Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Tue, 5 Sep 2023 06:46:38 +0200 Subject: [PATCH 02/20] Add instructions for adding the databrowser-api --- assets/playbooks/solr-server-playbook.yml | 78 +++++++++++++++++--- src/freva_deployment/deploy.py | 90 +++++++++++++++++------ src/freva_deployment/utils.py | 51 +++++++++---- 3 files changed, 170 insertions(+), 49 deletions(-) diff --git a/assets/playbooks/solr-server-playbook.yml b/assets/playbooks/solr-server-playbook.yml index fb9b0b30..d4c87cb3 100644 --- a/assets/playbooks/solr-server-playbook.yml +++ b/assets/playbooks/solr-server-playbook.yml @@ -3,20 +3,48 @@ vars: ansible_python_interpreter: "{{ solr_ansible_python_interpreter }}" - continer_name: "{{ solr_name }}" - docker_cmd: > + container_name: "{{ project_name }}-databrowser" + solr_name: "{{project_name}}-solr" + mongo_name: "{{project_name}}-mongo" + docker_solr_cmd: > -e CORE=files -e NUM_BACKUPS=7 -e SOLR_HEAP={{solr_mem}} --rm + --net {{project_name}} -v /opt/freva/{{project_name}}/solr_service:/var/solr/data:z -v /opt/freva/freva-service-config/solr/managed-schema.xml:/opt/solr/managed-schema.xml:z -v /opt/freva/freva-service-config/solr/create_cores.sh:/docker-entrypoint-initdb.d/create_cores.sh:z -v /opt/freva/freva-service-config/solr/synonyms.txt:/opt/solr/synonyms.txt:z -v /opt/freva/freva-service-config/solr/daily_backup.sh:/usr/local/bin/daily_backup:z - -p {{ solr_port }}:8983 -t - --name {{solr_name}} + -p {{solr_port}}:8983 -t + --name {{project_name}}-solr -t solr:latest + docker_databrowser_cmd: > + --net {{project_name}} + -e SOLR_CORE=files + -e SOLR_HOST={{solr_hostname}}:{{solr_port}} + -e MONGO_HOST={{solr_hostname}}:27017 + -e MONGO_PASSWORD={{ root_passwd }} + -e MONGO_DB=search_stats + -e MONGO_USER=mongo + --rm + -p 7777:8080 -t + --name {{project_name}}-databrowser + ghcr.io/freva-clint/databrowserapi:latest + docker_mongo_cmd: > + -e MONGO_INITDB_ROOT_USERNAME=mongo + -e MONGO_INITDB_ROOT_PASSWORD={{ root_passwd }} + -e MONGO_INITDB_DATABASE=search_stats + -p 27017:27017 + --net {{project_name}} + -v /opt/freva/{{project_name}}/databrowser/stats:/data/db:z + --rm + --name {{project_name}}-mongodb -t + mongo:latest + + + ansible_become_user: "{{ solr_ansible_become_user | default('root') }}" use_become: "{{ solr_ansible_become_user is defined and solr_ansible_become_user != '' }}" tasks: @@ -36,10 +64,19 @@ - name: Stopping services and deleting existing containers become: "{{use_become}}" shell: > - systemctl stop {{solr_name}}; - /tmp/docker-or-podman rm {{solr_name}}; - /tmp/docker-or-podman rmi -f {{solr_name}}; + systemctl stop {{project_name}}-solr; + systemctl stop {{project_name}}-mongo; + systemctl stop {{project_name}}-databrowser; + /tmp/docker-or-podman rm {{project_name}}-solr; + /tmp/docker-or-podman rm {{project_name}}-mongo; + /tmp/docker-or-podman rm {{project_name}}-databrowser; + /tmp/docker-or-podman rmi -f {{project_name}}-solr; + /tmp/docker-or-podman rmi -f {{project_name}}-mongo; + /tmp/docker-or-podman rmi -f {{project_name}}-databrowser; + /tmp/docker-or-podman rmi -f solr; /tmp/docker-or-podman rmi -f solr; + /tmp/docker-or-podman rmi -f mongo; + /tmp/docker-or-podman rmi -f ghcr.io/freva-clint/databrowserapi exit 0 ignore_errors: true - pause: seconds=5 @@ -57,11 +94,14 @@ become: "{{use_become}}" - name: Cleaning existing directory structure file: - path: /opt/freva/{{ project_name }}/solr_service + path: "{items}" state: absent + with_items: + - "/opt/freva/{{ project_name }}/solr_service" + - "/opt/freva/{{ project_name }}/databrowser" when: solr_wipe == true become: "{{use_become}}" - - name: Creating directory structure + - name: Creating solr directory structure file: path: /opt/freva/{{ project_name }}/solr_service state: directory @@ -69,6 +109,12 @@ group: 8983 recurse: true become: "{{use_become}}" + - name: Creating mongo directory structure + file: + path: /opt/freva/{{ project_name }}/databrowser/stats + state: directory + recurse: true + become: "{{use_become}}" - name: Getting additional configurations git: repo: https://github.com/FREVA-CLINT/freva-service-config.git @@ -82,13 +128,21 @@ mode: "0755" become: "{{use_become}}" - name: Pulling container - shell: /tmp/docker-or-podman pull docker.io/solr:latest + shell: + cmd: /tmp/docker-or-podman pull {{item}} become: "{{use_become}}" + with_items: + - "docker.io/solr:latest" + - "docker.io/mongo:latest" + - "ghcr.io/freva-clint/databrowserapi:latest" - name: Copy cron create script to target machine copy: src="{{ asset_dir }}/scripts/create_cron.sh" dest=/tmp/solr_service/ become: "{{use_become}}" - - name: Starting solr service - shell: /tmp/create_systemd.py {{solr_name}} run --enable {{docker_cmd}} + - name: Creating systemd services + shell: | + /tmp/create_systemd.py {{solr_name}} run {{docker_solr_cmd | trim}} + /tmp/create_systemd.py {{mongo_name}} run {{docker_mongo_cmd | trim}} + /tmp/create_systemd.py {{container_name}} run --enable {{docker_databrowser_cmd | trim}} --requires {{solr_name}} {{mongo_name}} when: systemctl.stat.exists == true become: "{{use_become}}" - pause: seconds=3 diff --git a/src/freva_deployment/deploy.py b/src/freva_deployment/deploy.py index 2f794af2..46ae130b 100644 --- a/src/freva_deployment/deploy.py +++ b/src/freva_deployment/deploy.py @@ -61,7 +61,9 @@ def __init__( self.email_password: str = "" self._td: TemporaryDirectory = TemporaryDirectory(prefix="inventory") self.inventory_file: Path = Path(self._td.name) / "inventory.yaml" - self.eval_conf_file: Path = Path(self._td.name) / "evaluation_system.conf" + self.eval_conf_file: Path = ( + Path(self._td.name) / "evaluation_system.conf" + ) self.web_conf_file: Path = Path(self._td.name) / "freva_web.toml" self.apache_config: Path = Path(self._td.name) / "freva_web.conf" self._db_pass: str = "" @@ -106,7 +108,9 @@ def _prep_vault(self) -> None: self._config_keys.append("vault") self.cfg["vault"] = self.cfg["db"].copy() self.cfg["vault"]["config"].setdefault("ansible_become_user", "root") - self.playbooks["vault"] = self.cfg["db"]["config"].get("vault_playbook") + self.playbooks["vault"] = self.cfg["db"]["config"].get( + "vault_playbook" + ) if not self.master_pass: self.master_pass = get_passwd() self.cfg["vault"]["config"]["root_passwd"] = self.master_pass @@ -127,7 +131,9 @@ def _prep_db(self) -> None: self.cfg["db"]["config"]["passwd"] = self.db_pass self.cfg["db"]["config"]["keyfile"] = self.public_key_file for key in ("name", "user", "db"): - self.cfg["db"]["config"][key] = self.cfg["db"]["config"].get(key) or "freva" + self.cfg["db"]["config"][key] = ( + self.cfg["db"]["config"].get(key) or "freva" + ) db_host = self.cfg["db"]["config"].get("host", "") if not db_host: self.cfg["db"]["config"]["host"] = host @@ -140,10 +146,15 @@ def _prep_db(self) -> None: def _prep_solr(self) -> None: """prepare the apache solr service.""" + if not self.master_pass: + self.master_pass = get_passwd() self._config_keys.append("solr") self.cfg["solr"]["config"].setdefault("ansible_become_user", "root") + self.cfg["solr"]["config"]["root_passwd"] = self.master_pass self.cfg["solr"]["config"].pop("core", None) - self.playbooks["solr"] = self.cfg["solr"]["config"].get("solr_playbook") + self.playbooks["solr"] = self.cfg["solr"]["config"].get( + "solr_playbook" + ) for key, default in dict(mem="4g", port=8983).items(): self.cfg["solr"]["config"][key] = ( self.cfg["solr"]["config"].get(key) or default @@ -156,7 +167,9 @@ def _prep_core(self) -> None: """prepare the core deployment.""" self._config_keys.append("core") self.cfg["core"]["config"].setdefault("ansible_become_user", "") - self.playbooks["core"] = self.cfg["core"]["config"].get("core_playbook") + self.playbooks["core"] = self.cfg["core"]["config"].get( + "core_playbook" + ) self.cfg["core"]["config"]["admins"] = ( self.cfg["core"]["config"].get("admins") or getuser() ) @@ -167,11 +180,15 @@ def _prep_core(self) -> None: if not root_dir: self.cfg["core"]["config"]["root_dir"] = install_dir preview_path = self.cfg["core"]["config"].get("preview_path", "") - base_dir_location = self.cfg["core"]["config"].get("base_dir_location", "") + base_dir_location = self.cfg["core"]["config"].get( + "base_dir_location", "" + ) scheduler_output_dir = self.cfg["core"]["config"].get( "scheduler_output_dir", "" ) - scheduler_system = self.cfg["core"]["config"].get("scheduler_system", "local") + scheduler_system = self.cfg["core"]["config"].get( + "scheduler_system", "local" + ) if not preview_path: if base_dir_location: self.cfg["core"]["config"]["preview_path"] = str( @@ -182,7 +199,9 @@ def _prep_core(self) -> None: if not scheduler_output_dir: scheduler_output_dir = str(Path(base_dir_location) / "share") scheduler_output_dir = Path(scheduler_output_dir) / scheduler_system - self.cfg["core"]["config"]["scheduler_output_dir"] = str(scheduler_output_dir) + self.cfg["core"]["config"]["scheduler_output_dir"] = str( + scheduler_output_dir + ) self.cfg["core"]["config"]["keyfile"] = self.public_key_file git_exe = self.cfg["core"]["config"].get("git_path") self.cfg["core"]["config"]["git_path"] = git_exe or "git" @@ -252,20 +271,24 @@ def _prep_web(self) -> None: except (FileNotFoundError, IOError, KeyError): pass try: - _webserver_items["IMPRINT"] = _webserver_items["IMPRINT"].split(",") + _webserver_items["IMPRINT"] = _webserver_items["IMPRINT"].split( + "," + ) except AttributeError: pass with self.web_conf_file.open("w") as f_obj: tomlkit.dump(_webserver_items, f_obj) for key in ("core", "web"): - self.cfg[key]["config"]["config_toml_file"] = str(self.web_conf_file) + self.cfg[key]["config"]["config_toml_file"] = str( + self.web_conf_file + ) if not self.master_pass: self.master_pass = get_passwd() email_user, self.email_password = get_email_credentials() self._prep_vault() - self.cfg["vault"]["config"]["ansible_python_interpreter"] = self.cfg["db"][ - "config" - ].get("ansible_python_interpreter", "/usr/bin/python3") + self.cfg["vault"]["config"]["ansible_python_interpreter"] = self.cfg[ + "db" + ]["config"].get("ansible_python_interpreter", "/usr/bin/python3") self.cfg["vault"]["config"]["email_user"] = email_user self.cfg["vault"]["config"]["email_password"] = self.email_password self.cfg["web"]["config"]["root_passwd"] = self.master_pass @@ -274,14 +297,19 @@ def _prep_web(self) -> None: self.cfg["web"]["config"]["chain_keyfile"] = ( self.chain_key_file or self.public_key_file ) - self.cfg["web"]["config"]["apache_config_file"] = str(self.apache_config) + self.cfg["web"]["config"]["apache_config_file"] = str( + self.apache_config + ) self._prep_apache_config() def _prep_apache_config(self): config = [] with (Path(asset_dir) / "web" / "freva_web.conf").open() as f_obj: for line in f_obj.readlines(): - if not self.chain_key_file and "SSLCertificateChainFile" in line: + if ( + not self.chain_key_file + and "SSLCertificateChainFile" in line + ): continue config.append(line) with open(self.apache_config, "w") as f_obj: @@ -297,7 +325,9 @@ def _read_cfg(self) -> dict[str, Any]: try: return dict(load_config(self._inv_tmpl)) except FileNotFoundError as error: - raise FileNotFoundError(f"No such file {self._inv_tmpl}") from error + raise FileNotFoundError( + f"No such file {self._inv_tmpl}" + ) from error def _check_config(self) -> None: sections = [] @@ -307,8 +337,14 @@ def _check_config(self) -> None: sections.append(section) for section in sections: for key, value in self.cfg[section]["config"].items(): - if not value and not self._empty_ok and not isinstance(value, bool): - raise ValueError(f"{key} in {section} is empty in {self._inv_tmpl}") + if ( + not value + and not self._empty_ok + and not isinstance(value, bool) + ): + raise ValueError( + f"{key} in {section} is empty in {self._inv_tmpl}" + ) @property def _empty_ok(self) -> list[str]: @@ -334,15 +370,21 @@ def db_pass(self) -> str: num_chars, num_digits, num_punctuations = 20, 4, 4 num_chars -= num_digits + num_punctuations characters = [ - "".join([random.choice(string.ascii_letters) for i in range(num_chars)]), + "".join( + [random.choice(string.ascii_letters) for i in range(num_chars)] + ), "".join([random.choice(string.digits) for i in range(num_digits)]), - "".join([random.choice(punctuations) for i in range(num_punctuations)]), + "".join( + [random.choice(punctuations) for i in range(num_punctuations)] + ), ] str_characters = "".join(characters) _db_pass = "".join(random.sample(str_characters, len(str_characters))) while _db_pass.startswith("@"): # Vault treats values starting with "@" as file names. - _db_pass = "".join(random.sample(str_characters, len(str_characters))) + _db_pass = "".join( + random.sample(str_characters, len(str_characters)) + ) self._db_pass = _db_pass return self._db_pass @@ -506,8 +548,10 @@ def play( inventory_str = inventory for passwd in (self.email_password, self.master_pass): if passwd: - inventory_str = inventory_str.replace(passwd, "*" * len(passwd)) - RichConsole.print(inventory, style="bold", markup=True) + inventory_str = inventory_str.replace( + passwd, "*" * len(passwd) + ) + RichConsole.print(inventory_str, style="bold", markup=True) logger.info("Playing the playbooks with ansible") RichConsole.print( "[b]Note:[/] The [blue]BECOME[/] password refers to the " diff --git a/src/freva_deployment/utils.py b/src/freva_deployment/utils.py index 6149705c..a50c24b4 100644 --- a/src/freva_deployment/utils.py +++ b/src/freva_deployment/utils.py @@ -18,7 +18,9 @@ from rich.prompt import Prompt import toml -logging.basicConfig(format="%(name)s - %(levelname)s - %(message)s", level=logging.INFO) +logging.basicConfig( + format="%(name)s - %(levelname)s - %(message)s", level=logging.INFO +) logger = logging.getLogger("freva-deployment") RichConsole = Console(markup=True, force_terminal=True) @@ -41,6 +43,7 @@ "[green]Choose[/] a [b]master password[/], this password will be used to:\n" "- create the [magenta]mysql root[/] password\n" "- set the [magenta]django admin[/] web password\n" + "- set the [magenta]mongo db[/] user password\n" "[b]Note:[/] Ideally this password can be shared amongst other [i]admins[/].\n" "[b][green]choose[/] master password[/]" ) @@ -54,8 +57,12 @@ class AssetDir: this_module = "freva_deployment" def __init__(self): - self._user_asset_dir = Path(appdirs.user_data_dir()) / "freva" / "deployment" - self._user_config_dir = Path(appdirs.user_config_dir()) / "freva" / "deployment" + self._user_asset_dir = ( + Path(appdirs.user_data_dir()) / "freva" / "deployment" + ) + self._user_config_dir = ( + Path(appdirs.user_config_dir()) / "freva" / "deployment" + ) @property def _central_asset_dir(self): @@ -63,15 +70,17 @@ def _central_asset_dir(self): try: records = distribution.get_metadata("RECORD").splitlines() except FileNotFoundError: - asset_dir = Path(distribution.module_path).parent / "freva" / "deployment" + asset_dir = ( + Path(distribution.module_path).parent / "freva" / "deployment" + ) if asset_dir.is_dir(): return asset_dir warnings.warn("Guessing asset dir location, this might fail") return Path(sys.exec_prefix) / "freva" / "deployment" try: - inventory = [f.partition(",")[0] for f in records if "inventory.toml" in f][ - 0 - ] + inventory = [ + f.partition(",")[0] for f in records if "inventory.toml" in f + ][0] except IndexError: warnings.warn("Guessing asset dir location, this might fail") return Path(sys.exec_prefix) / "freva" / "deployment" @@ -90,7 +99,9 @@ def config_dir(self): if inventory_file.exists(): return self._user_config_dir self._user_config_dir.mkdir(exist_ok=True, parents=True) - shutil.copy(self.asset_dir / "config" / "inventory.toml", inventory_file) + shutil.copy( + self.asset_dir / "config" / "inventory.toml", inventory_file + ) return self._user_config_dir @property @@ -137,7 +148,9 @@ def _convert_dict( def load_config(inp_file: str | Path) -> dict[str, Any]: """Load the inventory toml file and replace all environment variables.""" inp_file = Path(inp_file).expanduser().absolute() - variables = cast(dict[str, str], toml.loads(config_file.read_text())["variables"]) + variables = cast( + dict[str, str], toml.loads(config_file.read_text())["variables"] + ) config = toml.loads(inp_file.read_text()) _convert_dict(config, variables, inp_file.parent) return config @@ -199,7 +212,9 @@ def set_log_level(verbosity: int) -> None: logger.setLevel(max(logging.INFO - 10 * verbosity, logging.DEBUG)) -def get_setup_for_service(service: str, setups: list[ServiceInfo]) -> tuple[str, str]: +def get_setup_for_service( + service: str, setups: list[ServiceInfo] +) -> tuple[str, str]: """Get the setup of a service configuration.""" for setup in setups: if setup.name == service: @@ -212,7 +227,9 @@ def read_db_credentials( ) -> dict[str, str]: """Read database config.""" with cert_file.open() as f_obj: - key = "".join([k.strip() for k in f_obj.readlines() if not k.startswith("-")]) + key = "".join( + [k.strip() for k in f_obj.readlines() if not k.startswith("-")] + ) sha = hashlib.sha512(key.encode()).hexdigest() url = f"http://{db_host}:{port}/vault/data/{sha}" return requests.get(url).json() @@ -292,7 +309,9 @@ def get_email_credentials() -> tuple[str, str]: ) RichConsole.print(msg) username = Prompt.ask("[green b]Username[/] for mail server") - password = Prompt.ask("[green b]Password[/] for mail server", password=True) + password = Prompt.ask( + "[green b]Password[/] for mail server", password=True + ) return username, password @@ -327,7 +346,9 @@ def _create_passwd(min_characters: int, msg: str = "") -> str: if not re.search(check, master_pass): is_ok = False break - is_safe: bool = len([True for c in "[_@$#$%^&*-!]" if c in master_pass]) > 0 + is_safe: bool = ( + len([True for c in "[_@$#$%^&*-!]" if c in master_pass]) > 0 + ) if is_ok is False or is_safe is False: raise ValueError( ( @@ -337,7 +358,9 @@ def _create_passwd(min_characters: int, msg: str = "") -> str: "- have at least one special special character." ) ) - master_pass_2 = Prompt.ask("[bold green]re-enter[/] master password", password=True) + master_pass_2 = Prompt.ask( + "[bold green]re-enter[/] master password", password=True + ) if master_pass != master_pass_2: raise ValueError("Passwords do not match") return master_pass From 6b0e5fb444e56c10ea738bf286e28387020a4b5c Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Fri, 8 Sep 2023 13:53:24 +0200 Subject: [PATCH 03/20] Switch to docker-compose --- assets/config/inventory.toml | 17 +- .../databrowser-server-compose-template.yml | 61 +++++++ .../playbooks/databrowser-server-playbook.yml | 141 +++++++++++++++ .../playbooks/db-server-compose-template.yml | 29 ++++ assets/playbooks/db-server-playbook.yml | 52 +++--- assets/playbooks/solr-server-playbook.yml | 161 ------------------ .../vault-server-compose-template.yml | 27 +++ assets/playbooks/vault-server-playbook.yml | 45 +++-- .../playbooks/web-server-compose-template.yml | 59 +++++++ assets/playbooks/web-server-playbook.yml | 82 +++++---- assets/scripts/create_systemd.py | 15 +- assets/scripts/docker-or-podman | 56 +++++- assets/web/freva_web.conf | 2 + src/freva_deployment/cli/__init__.py | 3 +- src/freva_deployment/cli/_deploy.py | 10 +- src/freva_deployment/cli/_migrate.py | 16 +- src/freva_deployment/cli/_server_map.py | 8 +- src/freva_deployment/cli/_service.py | 18 +- src/freva_deployment/deploy.py | 145 +++++++--------- .../ui/deployment_tui/__init__.py | 4 +- .../ui/deployment_tui/base.py | 10 +- .../ui/deployment_tui/deploy_forms.py | 48 ++++-- .../ui/deployment_tui/main_window.py | 34 ++-- src/freva_deployment/utils.py | 57 +++++-- 24 files changed, 692 insertions(+), 408 deletions(-) create mode 100644 assets/playbooks/databrowser-server-compose-template.yml create mode 100644 assets/playbooks/databrowser-server-playbook.yml create mode 100644 assets/playbooks/db-server-compose-template.yml delete mode 100644 assets/playbooks/solr-server-playbook.yml create mode 100644 assets/playbooks/vault-server-compose-template.yml create mode 100644 assets/playbooks/web-server-compose-template.yml diff --git a/assets/config/inventory.toml b/assets/config/inventory.toml index 56366cce..144082ef 100644 --- a/assets/config/inventory.toml +++ b/assets/config/inventory.toml @@ -49,8 +49,8 @@ hosts = "" ## leading 0s: ## hosts = "db-[99:101]-node.example.com" -[solr] -## Set the hostnames running the apache solr service +[databrowser] +## Set the hostnames running the databrowser service hosts = "" [db] @@ -103,12 +103,15 @@ wipe = false db_playbook = "" vault_playbook = "" -[solr.config] +[databrowser.config] ## Config variables for the solr service -port = 8983 +solr_port = 8983 # Set the memory for the solr server -mem = "4g" +solr_mem = "4g" + +# Set the port the databrowser should be running on +databrowser_port = 7777 ## Set the become (sudo) user name to change to for installing the services ansible_become_user = "root" @@ -128,7 +131,7 @@ wipe = false ## located in the user config directory. ## NOTE: only adjust this if you know what you are doing, if you leave this ## blank the system will used the default playbook (standard procedure) -solr_playbook = "" +databrowser_playbook = "" @@ -316,5 +319,3 @@ web_playbook = "" ## Set the become (sudo) user name to change to for installing the services ansible_become_user = "root" - - diff --git a/assets/playbooks/databrowser-server-compose-template.yml b/assets/playbooks/databrowser-server-compose-template.yml new file mode 100644 index 00000000..32696c7b --- /dev/null +++ b/assets/playbooks/databrowser-server-compose-template.yml @@ -0,0 +1,61 @@ +version: '3' +services: + {{solr_name}}: + image: solr:latest + container_name: {{solr_name}} + hostname: {{solr_name}} + networks: + - {{project_name}} + environment: + - CORE=files + - NUM_BACKUPS=7 + - SOLR_HEAP={{databrowser_solr_mem}} + volumes: +{% for volume in solr_volumes %} + - {{volume}} +{% endfor %} + tty: true + ports: + - {{databrowser_solr_port}}:8983 + + {{mongo_name}}: + image: mongo:latest + container_name: {{mongo_name}} + hostname: {{mongo_name}} + networks: + - {{project_name}} + environment: + - MONGO_INITDB_ROOT_USERNAME=mongo + - MONGO_INITDB_ROOT_PASSWORD={{ root_passwd}} + - MONGO_INITDB_DATABASE=search_stats + volumes: +{% for volume in mongo_volumes %} + - {{volume}} +{% endfor %} + ports: + - 27017:27017 + tty: true + + {{databrowser_name}}: + image: ghcr.io/freva-clint/databrowserapi:latest + hostname: {{databrowser_name}} + networks: + - {{project_name}} + environment: + - SOLR_CORE=files + - SOLR_HOST={{solr_name}}:8983 + - MONGO_HOST={{mongo_name}}:27017 + - MONGO_PASSWORD={{ root_passwd}} + - MONGO_DB=search_stats + - MONGO_USER=mongo + ports: + - {{databrowser_port}}:8080 + container_name: {{databrowser_name}} + depends_on: + - {{solr_name}} + - {{mongo_name}} + tty: true + +networks: + {{project_name}}: + driver: bridge diff --git a/assets/playbooks/databrowser-server-playbook.yml b/assets/playbooks/databrowser-server-playbook.yml new file mode 100644 index 00000000..d09c1d92 --- /dev/null +++ b/assets/playbooks/databrowser-server-playbook.yml @@ -0,0 +1,141 @@ +--- +- hosts: databrowser + + vars: + ansible_python_interpreter: "{{ databrowser_ansible_python_interpreter }}" + databrowser_name: "{{project_name}}-databrowser" + solr_name: "{{project_name}}-solr" + mongo_name: "{{project_name}}-mongo" + solr_volumes: + - /opt/freva/{{project_name}}/solr_service:/var/solr/data:z + - /opt/freva/freva-service-config/solr/managed-schema.xml:/opt/solr/managed-schema.xml:z + - /opt/freva/freva-service-config/solr/create_cores.sh:/docker-entrypoint-initdb.d/create_cores.sh:z + - /opt/freva/freva-service-config/solr/synonyms.txt:/opt/solr/synonyms.txt:z + - /opt/freva/freva-service-config/solr/daily_backup.sh:/usr/local/bin/daily_backup:z + mongo_volumes: + - /opt/freva/{{project_name}}/databrowser/stats:/data/db:z + ansible_become_user: "{{ databrowser_ansible_become_user | default('root') }}" + use_become: "{{ databrowser_ansible_become_user is defined and databrowser_ansible_become_user != '' }}" + tasks: + - name: Copying docker/podman wrapper script + copy: + src: "{{ asset_dir }}/scripts/docker-or-podman" + dest: /tmp/docker-or-podman + mode: "0775" + - name: Registering systemctl path + stat: + path: /usr/bin/systemctl + register: systemctl + - name: Registering anacron path + stat: + path: /etc/cron.daily + register: cron + - name: Stopping services and deleting existing containers + become: "{{use_become}}" + shell: > + systemctl stop {{solr_name}} {{mongo_name}} {{databrowser_name}}; + systemctl disable {{solr_name}} {{mongo_name}} {{databrowser_name}}; + systemctl reset-failed; + /tmp/docker-or-podman rm {{solr_name}} {{mongo_name}} {{databrowser_name}}; + /tmp/docker-or-podman rmi -f {{solr_name}} {{mongo_name}} {{databrowser_name}}; + /tmp/docker-or-podman rmi -f solr mongo ghcr.io/freva-clint/databrowserapi; + exit 0 + ignore_errors: true + - pause: seconds=5 + - name: Deleting old systemd structure + file: + state: absent + path: "/etc/systemd/system/{{item}}.service" + force: true + with_items: + - "{{solr_name}}" + - "{{mongo_name}}" + - "{{databrowser_name}}" + become: "{{use_become}}" + when: systemctl.stat.exists == true + - name: Deleting config file + file: + state: absent + force: true + path: /opt/freva/freva-service-config/ + become: "{{use_become}}" + - name: Cleaning existing directory structure + file: + path: "{item}" + state: absent + with_items: + - "/opt/freva/{{ project_name }}/solr_service" + - "/opt/freva/{{ project_name }}/databrowser" + when: databrowser_wipe == true + become: "{{use_become}}" + - name: Creating solr directory structure + file: + path: /opt/freva/{{ project_name }}/solr_service + state: directory + owner: 8983 + group: 8983 + recurse: true + become: "{{use_become}}" + - name: Creating compose directory structure + file: + path: /opt/freva/compose_services/foo + state: directory + recurse: true + become: "{{use_become}}" + - name: Creating directory structure + file: + path: "{item}" + state: directory + recurse: true + with_items: + - /opt/freva/{{ project_name }}/databrowser + - /opt/freva/compose_services/foo + become: "{{use_become}}" + - name: Getting additional configurations + git: + repo: https://github.com/FREVA-CLINT/freva-service-config.git + dest: /opt/freva/freva-service-config + update: true + become: "{{use_become}}" + - name: Copy systemd files + copy: + src: "{{ asset_dir }}/scripts/create_systemd.py" + dest: /tmp/create_systemd.py + mode: "0755" + become: "{{use_become}}" + - name: Pulling container + shell: + cmd: /tmp/docker-or-podman pull {{item}} + become: "{{use_become}}" + with_items: + - "docker.io/solr:latest" + - "docker.io/mongo:latest" + - "ghcr.io/freva-clint/databrowserapi:latest" + - name: Copy cron create script to target machine + copy: src="{{ asset_dir }}/scripts/create_cron.sh" dest=/tmp/solr_service/ + become: "{{use_become}}" + - name: Creating compose file + become: "{{use_become}}" + template: + src: "{{ asset_dir }}/playbooks/databrowser-server-compose-template.yml" + dest: "/opt/freva/compose_services/{{databrowser_name}}-compose.yml" + - name: Creating systemd services + shell: | + /tmp/create_systemd.py {{databrowser_name}} compose --enable --project-name {{project_name}} -f /opt/freva/compose_services/{{databrowser_name}}-compose.yml up + when: systemctl.stat.exists == true + become: "{{use_become}}" + - pause: seconds=3 + - name: Creating cron jobs + shell: > + sh /tmp/solr_service/create_cron.sh "{{ solr_name }}" "{{databrowser_email}}" + when: systemctl.stat.exists == true + become: "{{use_become}}" + - name: Deleting tmporary files + file: + path: "{{item}}" + state: absent + with_items: + - /tmp/docker-or-podman + - /tmp/create_systemd.py + - /tmp/solr_service + become: "{{use_become}}" diff --git a/assets/playbooks/db-server-compose-template.yml b/assets/playbooks/db-server-compose-template.yml new file mode 100644 index 00000000..52dd5314 --- /dev/null +++ b/assets/playbooks/db-server-compose-template.yml @@ -0,0 +1,29 @@ +version: '3' +services: + {{db_name}}: + image: mariadb:latest + hostname: {{db_name}} + networks: + - {{project_name}} + environment: + - ROOT_PW={{ root_passwd }} + - HOST={{ db_host }} + - NUM_BACKUPS=7 + - PROJECT={{ project_name }} + - MYSQL_USER={{db_user}} + - MYSQL_PASSWORD={{db_passwd}} + - MYSQL_DATABASE={{db}} + - MYSQL_ROOT_PASSWORD={{ root_passwd }} + - BACKUP_DIR=/var/lib/mysql/backup + volumes: +{% for volume in db_volumes %} + - {{volume}} +{% endfor %} + container_name: {{db_name}} + tty: true + ports: + - {{db_port}}:3306 + +networks: + {{project_name}}: + driver: bridge diff --git a/assets/playbooks/db-server-playbook.yml b/assets/playbooks/db-server-playbook.yml index f129bd7c..9e43b01c 100644 --- a/assets/playbooks/db-server-playbook.yml +++ b/assets/playbooks/db-server-playbook.yml @@ -3,38 +3,42 @@ vars: ansible_python_interpreter: "{{ db_ansible_python_interpreter }}" + compose_file: /opt/freva/compose_services/{{db_name}}-compose.yml + db_volumes: + - /opt/freva/freva-service-config/mysql/create_tables.sql:/docker-entrypoint-initdb.d/002_create_tables.sql:z + - /opt/freva/freva-service-config/mysql/daily_backup.sh:/usr/local/bin/daily_backup:z docker_cmd: > - --net {{ project_name }} -v + --network {{ project_name }} -v /opt/freva/{{project_name}}/db_service:/var/lib/mysql:z -e HOST={{ db_host }} -e NUM_BACKUPS=7 -e PROJECT={{ project_name }} -e MYSQL_USER={{db_user}} -e MYSQL_PASSWORD='{{db_passwd}}' - -e MYSQL_DATABASE={{db_db}} + -e MYSQL_DATABASE={{db}} -e BACKUP_DIR=/var/lib/mysql/backup -p {{ db_port }}:3306 + -v /opt/freva/freva-service-config/mysql/create_tables.sql:/docker-entrypoint-initdb.d/002_create_tables.sql:z + -v /opt/freva/freva-service-config/mysql/daily_backup.sh:/usr/local/bin/daily_backup:z --rm --name {{db_name}} - -e MYSQL_ROOT_PASSWORD='{{ root_passwd }}' - -v /root/freva-service-config/mysql/create_tables.sql:/docker-entrypoint-initdb.d/002_create_tables.sql:z - -v /root/freva-service-config/mysql/daily_backup.sh:/usr/local/bin/daily_backup:z + -e MYSQL_ROOT_PASSWORD={{ root_passwd }} -t docker.io/mariadb:latest skip_tables_cmd: > - --net {{ project_name }} -v + --network {{ project_name }} -v /opt/freva/{{project_name}}/db_service:/var/lib/mysql:z -e HOST={{ db_host }} -e NUM_BACKUPS=7 -e PROJECT={{ project_name }} -e MYSQL_USER={{db_user}} -e MYSQL_PASSWORD='{{db_passwd}}' - -e MYSQL_DATABASE={{db_db}} + -e MYSQL_DATABASE={{db}} -e BACKUP_DIR=/var/lib/mysql/backup -p {{ db_port }}:3306 - -e MYSQL_ROOT_PASSWORD='{{ root_passwd }}' + -e MYSQL_ROOT_PASSWORD={{ root_passwd }} --name {{db_name}} - -v /root/freva-service-config/mysql/create_tables.sql:/docker-entrypoint-initdb.d/002_create_tables.sql:z - -v /root/freva-service-config/mysql/daily_backup.sh:/usr/local/bin/daily_backup:z + -v /opt/freva/freva-service-config/mysql/create_tables.sql:/docker-entrypoint-initdb.d/002_create_tables.sql:z + -v /opt/freva/freva-service-config/mysql/daily_backup.sh:/usr/local/bin/daily_backup:z -v /tmp/reset_root_pw.sh:/tmp/reset_root_pw.sh:z -t docker.io/mariadb:latest mariadbd-safe --skip-grant-tables continer_name: "{{ db_name }}" @@ -58,6 +62,8 @@ shell: > /tmp/docker-or-podman stop {{db_name}}; systemctl stop {{db_name}}; + systemctl disable {{db_name}}; + systemctl reset-failed; /tmp/docker-or-podman rm {{db_name}}; /tmp/docker-or-podman rmi -f mariadb; echo 0 @@ -67,7 +73,7 @@ file: state: absent force: true - path: "/root/freva-service-config/" + path: "/opt/freva/freva-service-config/" become: "{{use_become}}" - name: Pulling container become: "{{use_become}}" @@ -106,7 +112,7 @@ become: "{{use_become}}" git: repo: https://github.com/FREVA-CLINT/freva-service-config.git - dest: /root/freva-service-config + dest: /opt/freva/freva-service-config update: true - name: Preparing the root password reset I become: "{{use_become}}" @@ -127,23 +133,31 @@ become: "{{use_become}}" shell: | /tmp/docker-or-podman rm -f {{db_name}} - - name: Creating systemd service + - name: Creating compose directory structure + file: + path: /opt/freva/compose_services + state: directory + recurse: true + become: "{{use_become}}" + - name: Creating compose file + become: "{{use_become}}" + template: + src: "{{ asset_dir }}/playbooks/db-server-compose-template.yml" + dest: "/opt/freva/compose_services/{{db_name}}-compose.yml" + - name: Creating system services become: "{{use_become}}" shell: | - /tmp/create_systemd.py {{db_name}} run {{docker_cmd | trim}} --requires {{vault_name}} --enable + /tmp/create_systemd.py {{db_name}} compose --enable --project-name {{project_name}} -f {{compose_file}} up when: systemctl.stat.exists == true - name: Creating cron jobs become: "{{use_become}}" shell: sh /tmp/create_cron.sh "{{ db_name }}" "{{db_email}}" when: cron.stat.exists == true - - name: Restarting docker container - become: "{{use_become}}" - shell: systemctl restart "{{ db_name }}" - when: systemctl.stat.exists == true + - pause: seconds=5 - name: Creating potentially missing tables become: "{{use_become}}" shell: | - /tmp/docker-or-podman exec -it {{db_name}} /bin/sh -c 'mariadb -u root -p{{root_passwd}} {{db_db}} < /docker-entrypoint-initdb.d/002_create_tables.sql' + /tmp/docker-or-podman exec -it {{db_name}} /bin/sh -c 'mariadb -u root -p{{root_passwd}} {{db}} < /docker-entrypoint-initdb.d/002_create_tables.sql' - name: Deleting auxillary files become: "{{use_become}}" file: diff --git a/assets/playbooks/solr-server-playbook.yml b/assets/playbooks/solr-server-playbook.yml deleted file mode 100644 index d4c87cb3..00000000 --- a/assets/playbooks/solr-server-playbook.yml +++ /dev/null @@ -1,161 +0,0 @@ ---- -- hosts: solr - - vars: - ansible_python_interpreter: "{{ solr_ansible_python_interpreter }}" - container_name: "{{ project_name }}-databrowser" - solr_name: "{{project_name}}-solr" - mongo_name: "{{project_name}}-mongo" - docker_solr_cmd: > - -e CORE=files - -e NUM_BACKUPS=7 - -e SOLR_HEAP={{solr_mem}} - --rm - --net {{project_name}} - -v /opt/freva/{{project_name}}/solr_service:/var/solr/data:z - -v /opt/freva/freva-service-config/solr/managed-schema.xml:/opt/solr/managed-schema.xml:z - -v /opt/freva/freva-service-config/solr/create_cores.sh:/docker-entrypoint-initdb.d/create_cores.sh:z - -v /opt/freva/freva-service-config/solr/synonyms.txt:/opt/solr/synonyms.txt:z - -v /opt/freva/freva-service-config/solr/daily_backup.sh:/usr/local/bin/daily_backup:z - -p {{solr_port}}:8983 -t - --name {{project_name}}-solr - -t solr:latest - docker_databrowser_cmd: > - --net {{project_name}} - -e SOLR_CORE=files - -e SOLR_HOST={{solr_hostname}}:{{solr_port}} - -e MONGO_HOST={{solr_hostname}}:27017 - -e MONGO_PASSWORD={{ root_passwd }} - -e MONGO_DB=search_stats - -e MONGO_USER=mongo - --rm - -p 7777:8080 -t - --name {{project_name}}-databrowser - ghcr.io/freva-clint/databrowserapi:latest - docker_mongo_cmd: > - -e MONGO_INITDB_ROOT_USERNAME=mongo - -e MONGO_INITDB_ROOT_PASSWORD={{ root_passwd }} - -e MONGO_INITDB_DATABASE=search_stats - -p 27017:27017 - --net {{project_name}} - -v /opt/freva/{{project_name}}/databrowser/stats:/data/db:z - --rm - --name {{project_name}}-mongodb -t - mongo:latest - - - - ansible_become_user: "{{ solr_ansible_become_user | default('root') }}" - use_become: "{{ solr_ansible_become_user is defined and solr_ansible_become_user != '' }}" - tasks: - - name: Copying docker/podman wrapper script - copy: - src: "{{ asset_dir }}/scripts/docker-or-podman" - dest: /tmp/docker-or-podman - mode: "0775" - - name: Registering systemctl path - stat: - path: /usr/bin/systemctl - register: systemctl - - name: Registering anacron path - stat: - path: /etc/cron.daily - register: cron - - name: Stopping services and deleting existing containers - become: "{{use_become}}" - shell: > - systemctl stop {{project_name}}-solr; - systemctl stop {{project_name}}-mongo; - systemctl stop {{project_name}}-databrowser; - /tmp/docker-or-podman rm {{project_name}}-solr; - /tmp/docker-or-podman rm {{project_name}}-mongo; - /tmp/docker-or-podman rm {{project_name}}-databrowser; - /tmp/docker-or-podman rmi -f {{project_name}}-solr; - /tmp/docker-or-podman rmi -f {{project_name}}-mongo; - /tmp/docker-or-podman rmi -f {{project_name}}-databrowser; - /tmp/docker-or-podman rmi -f solr; - /tmp/docker-or-podman rmi -f solr; - /tmp/docker-or-podman rmi -f mongo; - /tmp/docker-or-podman rmi -f ghcr.io/freva-clint/databrowserapi - exit 0 - ignore_errors: true - - pause: seconds=5 - - name: Deleting config file - file: - state: absent - force: true - path: "/opt/freva/freva-service-config/" - become: "{{use_become}}" - - name: Creating docker network - shell: > - /tmp/docker-or-podman network create "{{ project_name }}"; - exit 0 - ignore_errors: true - become: "{{use_become}}" - - name: Cleaning existing directory structure - file: - path: "{items}" - state: absent - with_items: - - "/opt/freva/{{ project_name }}/solr_service" - - "/opt/freva/{{ project_name }}/databrowser" - when: solr_wipe == true - become: "{{use_become}}" - - name: Creating solr directory structure - file: - path: /opt/freva/{{ project_name }}/solr_service - state: directory - owner: 8983 - group: 8983 - recurse: true - become: "{{use_become}}" - - name: Creating mongo directory structure - file: - path: /opt/freva/{{ project_name }}/databrowser/stats - state: directory - recurse: true - become: "{{use_become}}" - - name: Getting additional configurations - git: - repo: https://github.com/FREVA-CLINT/freva-service-config.git - dest: /opt/freva/freva-service-config - update: true - become: "{{use_become}}" - - name: Copy systemd files - copy: - src: "{{ asset_dir }}/scripts/create_systemd.py" - dest: /tmp/create_systemd.py - mode: "0755" - become: "{{use_become}}" - - name: Pulling container - shell: - cmd: /tmp/docker-or-podman pull {{item}} - become: "{{use_become}}" - with_items: - - "docker.io/solr:latest" - - "docker.io/mongo:latest" - - "ghcr.io/freva-clint/databrowserapi:latest" - - name: Copy cron create script to target machine - copy: src="{{ asset_dir }}/scripts/create_cron.sh" dest=/tmp/solr_service/ - become: "{{use_become}}" - - name: Creating systemd services - shell: | - /tmp/create_systemd.py {{solr_name}} run {{docker_solr_cmd | trim}} - /tmp/create_systemd.py {{mongo_name}} run {{docker_mongo_cmd | trim}} - /tmp/create_systemd.py {{container_name}} run --enable {{docker_databrowser_cmd | trim}} --requires {{solr_name}} {{mongo_name}} - when: systemctl.stat.exists == true - become: "{{use_become}}" - - pause: seconds=3 - - name: Creating cron jobs - shell: > - sh /tmp/solr_service/create_cron.sh "{{ solr_name }}" "{{solr_email}}" - when: systemctl.stat.exists == true - become: "{{use_become}}" - - name: Deleting tmporary files - file: - path: "{{item}}" - state: absent - with_items: - - /tmp/docker-or-podman - - /tmp/create_systemd.py - become: "{{use_become}}" diff --git a/assets/playbooks/vault-server-compose-template.yml b/assets/playbooks/vault-server-compose-template.yml new file mode 100644 index 00000000..cd476db4 --- /dev/null +++ b/assets/playbooks/vault-server-compose-template.yml @@ -0,0 +1,27 @@ +version: '3' +services: + {{vault_name}}: + image: ghcr.io/freva-clint/freva-deployment/vault:latest + hostname: {{vault_name}} + cap_add: + - IPC_LOCK + - SYS_NICE + dns: + - 8.8.8.8 + - 8.8.4.4 + networks: + - {{project_name}} + environment: + - ROOT_PW={{ root_passwd }} + volumes: +{% for volume in vault_volumes %} + - {{volume}} +{% endfor %} + container_name: {{vault_name}} + tty: true + ports: + - 5002:5002 + +networks: + {{project_name}}: + driver: bridge diff --git a/assets/playbooks/vault-server-playbook.yml b/assets/playbooks/vault-server-playbook.yml index 1143fefc..ff155694 100644 --- a/assets/playbooks/vault-server-playbook.yml +++ b/assets/playbooks/vault-server-playbook.yml @@ -4,16 +4,10 @@ vars: ansible_python_interpreter: "{{ db_ansible_python_interpreter }}" continer_name: "{{ vault_name }}" - docker_cmd: > - --net {{ project_name }} --cap-add=IPC_LOCK - --dns 8.8.8.8 - -e ROOT_PW='{{ root_passwd }}' - --rm - --name {{vault_name}} - -v /opt/freva/{{project_name}}/vault_service/config:/data:z - -v /opt/freva/{{project_name}}/vault_service/files:/vault/file:z - -p 5002:5002 - -t ghcr.io/freva-clint/freva-deployment/vault:latest + compose_file: /opt/freva/compose_services/{{vault_name}}-compose.yml + vault_volumes: + - /opt/freva/{{project_name}}/vault_service/config:/data:z + - /opt/freva/{{project_name}}/vault_service/files:/vault/file:z use_become: "{{ vault_ansible_become_user is defined and vault_ansible_become_user != '' }}" tasks: - name: Copying docker/podman wrapper script @@ -35,21 +29,15 @@ - name: Stopping services and deleting existing containers shell: > systemctl stop {{vault_name}}; + systemctl disable {{vault_name}}; /tmp/docker-or-podman rm {{vault_name}}; - /tmp/docker-or-podman rmi -f {{vault_name}}; - /tmp/docker-or-podman rmi -f ghcr.io/freva-clint/freva-deployment/vault; + /tmp/docker-or-podman rmi -f {{vault_name}} ghcr.io/freva-clint/freva-deployment/vault; exit 0 ignore_errors: true become: "{{use_become}}" - name: Pulling container become: "{{use_become}}" shell: /tmp/docker-or-podman pull ghcr.io/freva-clint/freva-deployment/vault:latest - - name: Creating docker network - shell: > - /tmp/docker-or-podman network create "{{ project_name }}"; - exit 0 - ignore_errors: true - become: "{{use_become}}" - name: Copying auxillary files to target machine copy: src="{{ asset_dir }}/vault" dest=/tmp - name: Copying systemd files @@ -70,11 +58,22 @@ src: "{{ vault_keyfile }}" dest: "/opt/freva/{{project_name}}/vault_service/config/freva.crt" become: "{{use_become}}" - - name: Creating system service + - name: Creating compose directory structure + file: + path: /opt/freva/compose_services/foo + state: directory + recurse: true + become: "{{use_become}}" + - name: Creating compose file become: "{{use_become}}" - shell: > - /tmp/create_systemd.py {{vault_name}} run {{docker_cmd}} - ignore_errors: true + template: + src: "{{ asset_dir }}/playbooks/vault-server-compose-template.yml" + dest: "/opt/freva/compose_services/{{vault_name}}-compose.yml" + - name: Creating system services + become: "{{use_become}}" + shell: | + /tmp/create_systemd.py {{vault_name}} compose --enable --project-name {{project_name}} -f {{compose_file}} up; + when: systemctl.stat.exists == true - pause: seconds=20 - name: Deleting existing infrastructure shell: > @@ -90,7 +89,7 @@ db.port={{ db_port }} db.passwd='{{ vault_passwd }}' db.user={{ db_user }} - db.db={{ db_db }} + db.db={{ vault_db }} become: "{{use_become}}" - name: Deleting auxillary files file: diff --git a/assets/playbooks/web-server-compose-template.yml b/assets/playbooks/web-server-compose-template.yml new file mode 100644 index 00000000..aa4e752d --- /dev/null +++ b/assets/playbooks/web-server-compose-template.yml @@ -0,0 +1,59 @@ +version: '3' +services: + + {{apache_name}}: + image: httpd:latest + container_name: {{apache_name}} + hostname: {{apache_name}} + networks: + - {{project_name}} + environment: + - SCHEDULER_DIR={{core_scheduler_output_dir}} + - UID={{uid}} + - GID={{gid}} + - DATABROWSER_HOST={{web_databrowser_host}} + - FREVA_HOST={{web_server_name}} + volumes: +{% for volume in apache_volumes %} + - {{volume}} +{% endfor %} + command: /usr/local/bin/prepare-httpd + ports: + - 80:80 + - 443:443 + + {{redis_name}}: + image: redis + networks: + - {{project_name}} + ports: + - 6379:6379 + tty: true + + {{web_name}}: + image: ghcr.io/freva-clint/freva-web + container_name: {{web_name}} + hostname: {{web_name}} + networks: + - {{project_name}} + environment: + - EVALUATION_SYSTEM_CONFIG_FILE={{core_root_dir}}/freva/evaluation_system.conf + - LDAP_USER_DN={{web_ldap_user_dn}} + - LDAP_USER_PW={{web_ldap_user_pw}} + - DJANGO_SUPERUSER_PASSWORD={{ root_passwd }} + volumes: + container_name: {{redis_name}} + hostname: {{redis_name}} +{% for volume in web_volumes %} + - {{volume}} +{% endfor %} + tty: true + ports: + - 8000:8000 + depends_on: + - {{redis_name}} + - {{apache_name}} + +networks: + {{project_name}}: + driver: bridge diff --git a/assets/playbooks/web-server-playbook.yml b/assets/playbooks/web-server-playbook.yml index a31573e1..4ec9625b 100644 --- a/assets/playbooks/web-server-playbook.yml +++ b/assets/playbooks/web-server-playbook.yml @@ -61,39 +61,45 @@ vars: ansible_python_interpreter: "{{ web_ansible_python_interpreter }}" service_dir: /opt/freva/{{project_name}}/web_service - docker_web_cmd: > - -p 8000:8000 - --rm - -v {{ core_root_dir }}:{{ core_root_dir }}:ro - -v {{ core_scheduler_output_dir}}:{{ core_scheduler_output_dir }}:ro - -v /opt/freva/{{project_name}}/web_service/static:/opt/freva_web/static:z - -v {{core_base_dir_location}}:{{core_base_dir_location}}:ro - -e EVALUATION_SYSTEM_CONFIG_FILE={{core_root_dir}}/freva/evaluation_system.conf - -e LDAP_USER_DN={{web_ldap_user_dn}} - -e LDAP_USER_PW={{web_ldap_user_pw}} - -e DJANGO_SUPERUSER_PASSWORD={{ root_passwd }} - --name {{web_name}} - -t ghcr.io/freva-clint/freva-web - docker_apache_cmd: > - -v /opt/freva/{{project_name}}/web_service/freva_web.conf:/usr/local/apache2/conf/httpd.conf:z - -v /opt/freva/{{project_name}}/web_service/server-cert.crt:/etc/ssl/certs/server-cert.crt:z - -v /opt/freva/{{project_name}}/web_service/server-key.key:/etc/ssl/private/server-key.key:z - -v /opt/freva/{{project_name}}/web_service/cacert.pem:/etc/ssl/certs/cacert.pem:z - -v /opt/freva/{{project_name}}/web_service/static:/srv/static:z - -v /opt/freva/{{project_name}}/web_service/prepare-httpd:/usr/local/bin/prepare-httpd:z - -v {{ core_preview_path }}:/srv/static/preview:z + compose_file: /opt/freva/compose_services/{{databrowser_name}}-compose.yml + web_volumes: + - "{{ core_root_dir }}:{{ core_root_dir }}:ro" + - "{{ core_scheduler_output_dir}}:{{ core_scheduler_output_dir }}:ro" + - "/opt/freva/{{project_name}}/web_service/static:/opt/freva_web/static:z" + - "{{core_base_dir_location}}:{{core_base_dir_location}}:ro" + apache_volumes: + - "/opt/freva/{{project_name}}/web_service/freva_web.conf:/usr/local/apache2/conf/httpd.conf:z" + - "/opt/freva/{{project_name}}/web_service/server-cert.crt:/etc/ssl/certs/server-cert.crt:z" + - "/opt/freva/{{project_name}}/web_service/server-key.key:/etc/ssl/private/server-key.key:z" + - "/opt/freva/{{project_name}}/web_service/cacert.pem:/etc/ssl/certs/cacert.pem:z" + - "/opt/freva/{{project_name}}/web_service/static:/srv/static:z" + - "/opt/freva/{{project_name}}/web_service/prepare-httpd:/usr/local/bin/prepare-httpd:z" + - "{{ core_preview_path }}:/srv/static/preview:z" + docker_apache_cmd: > --rm -e SCHEDULER_DIR={{core_scheduler_output_dir}} -e UID=$(id -u {{ansible_user}}) -e GID=$(id -g {{ansible_user}}) --name "{{project_name}}-httpd" --security-opt label=disable + -e DATABROWSER_HOST={{web_databrowser_host}} -e FREVA_HOST={{web_server_name}} -p 80:80 -p 443:443 httpd:latest /usr/local/bin/prepare-httpd redis_name: "{{ project_name }}-redis" apache_name: "{{project_name}}-httpd" + web_name: "{{project_name}}-web" use_become: "{{ web_ansible_become_user is defined and web_ansible_become_user != '' }}" tasks: + - name: Get UID + command: id -u {{ ansible_user }} + register: uid_result + - name: Get GID + command: id -g {{ ansible_user }} + register: gid_result + - name: Set UID and GID as facts + set_fact: + uid: "{{ uid_result.stdout }}" + gid: "{{ gid_result.stdout }}" - name: Copying docker/podman wrapper script copy: src: "{{ asset_dir }}/scripts/docker-or-podman" @@ -116,18 +122,13 @@ when: core.stat.exists == false - name: Stopping services shell: > - systemctl stop {{ web_name }}; - systemctl stop {{ redis_name }}; - systemctl stop {{ apache_name }}; - /tmp/docker-or-podman stop {{web_name}}; - /tmp/docker-or-podman rm {{web_name}}; - /tmp/docker-or-podman stop {{redis_name}}; - /tmp/docker-or-podman rm {{redis_name}}; - /tmp/docker-or-podman stop {{apache_name}}; - /tmp/docker-or-podman rm {{apache_name}}; - /tmp/docker-or-podman rmi -f docker.io/redis; - /tmp/docker-or-podman rmi -f ghcr.io/freva-clint/freva-web; - /tmp/docker-or-podman rmi -f docker.io/httpd; echo 0 + systemctl stop {{ web_name }} {{ redis_name }} {{ apache_name }}; + systemctl disable {{ web_name }} {{ redis_name }} {{ apache_name }}; + systemctl reset-failed; + /tmp/docker-or-podman stop {{web_name}} {{web_name}} {{redis_name}}; + /tmp/docker-or-podman rm {{web_name}} {{redis_name}} {{apache_name}}; + /tmp/docker-or-podman rmi -f docker.io/redis ghcr.io/freva-clint/freva-web docker.io/httpd; + echo 0 ignore_errors: true become: "{{use_become}}" - name: Deleting existing web-directory @@ -146,6 +147,12 @@ src: "{{ asset_dir }}/scripts/prepare-httpd" dest: "/opt/freva/{{project_name}}/web_service/prepare-httpd" mode: "0775" + - name: Creating compose directory structure + file: + path: /opt/freva/compose_services/foo + state: directory + recurse: true + become: "{{use_become}}" - name: Copying key files files become: "{{use_become}}" copy: @@ -173,12 +180,15 @@ src: "{{ asset_dir }}/scripts/create_systemd.py" dest: /tmp/create_systemd.py mode: "0755" + - name: Creating compose file + become: "{{use_become}}" + template: + src: "{{ asset_dir }}/playbooks/web-server-compose-template.yml" + dest: "/opt/freva/compose_services/{{databrowser_name}}-compose.yml" - name: Creating system services become: "{{use_become}}" shell: | - /tmp/create_systemd.py {{redis_name}} run --name {{redis_name}} --rm -p 6379:6379 redis:latest - /tmp/create_systemd.py {{apache_name}} run "{{docker_apache_cmd}}" - /tmp/create_systemd.py {{web_name}} run "{{docker_web_cmd}}" --requires {{redis_name}} {{apache_name}} --enable + /tmp/create_systemd.py {{web_name}} compose --enable --project-name {{project_name}} -f {{compose_file}} up when: systemctl.stat.exists == true - pause: seconds=15 - name: Restarting services diff --git a/assets/scripts/create_systemd.py b/assets/scripts/create_systemd.py index ca213c7e..dc9a1c15 100644 --- a/assets/scripts/create_systemd.py +++ b/assets/scripts/create_systemd.py @@ -15,7 +15,7 @@ Service=dict( TimeoutStartSec="35s", TimeoutStopSec="35s", - ExecStartPre="{delete_command}", + ExecStartPre='/bin/sh -c "{delete_command}"', ExecStart='/bin/sh -c "{container_cmd} {container_args}"', Restart="no", ), @@ -86,7 +86,6 @@ def get_container_cmd(args: str) -> Tuple[str, str]: cmd, check=True, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, env=env, ) out = res.stdout.decode().split() @@ -100,7 +99,17 @@ def create_unit( ) -> None: """Create the systemd unit.""" container_cmd, container_args = get_container_cmd(args) - _, delete_command = get_container_cmd("rm -f {}".format(unit)) + cmd = args.split() + if "compose" in cmd and "up" in cmd: + new_cmd = [] + for word in cmd: + if word == "up": + new_cmd.append("down") + elif word not in ("-d", "--detach"): + new_cmd.append(word) + _, delete_command = get_container_cmd(" ".join(new_cmd)) + else: + _, delete_command = get_container_cmd("rm -f {}".format(unit)) if delete_command: delete_command = "{} {}".format(container_cmd, delete_command) else: diff --git a/assets/scripts/docker-or-podman b/assets/scripts/docker-or-podman index 6e2d2dba..4e1fb152 100755 --- a/assets/scripts/docker-or-podman +++ b/assets/scripts/docker-or-podman @@ -45,12 +45,35 @@ class CommandFactory(metaclass=abc.ABCMeta): @staticmethod def get_container_cmd() -> str: """Get the command of the container.""" - for cmd in ("apptainer", "podman", "docker"): + for cmd in ("podman", "docker", "apptainer"): cont_cmd = shutil.which(cmd) if cont_cmd: return cont_cmd raise ValueError("Docker, Podman or Apptainer must be installed") + def compose(self, *args, **kwargs) -> List[str]: + """User the compose command. + + Parameters + ---------- + **kwargs: Any + Additional command line arguments + Returns + ------- + list[str]: Constructed command line arguemnts. + """ + container_cmd = Path(self.get_container_cmd()).name + command = shutil.which(f"{container_cmd}-compose") + if not command: + txt = ( + f"{container_cmd} is available but not {container_cmd}-compose" + " which should be installed on the system." + ) + print(txt, file=sys.stderr) + raise ValueError(txt) + _args = self._kwargs_to_list(**kwargs) + return [command] + _args + list(args) + def network(self, sub_command: str, *flags: str) -> List[str]: """Interact with the network sub commands. @@ -369,6 +392,30 @@ def parse_args() -> List[str]: action="store_true", ) + # -------------------------- Compose commands ----------------------------- + parser_compose = subparsers.add_parser( + "compose", + help="Define and run multi-container applications", + ) + parser_compose.add_argument("-p", "--project-name", help="Project name") + parser_compose.add_argument( + "-f", + "--file", + help="Compose configuration files", + ) + parser_compose.add_argument( + "--project-directory", + help=( + "Specify an alternate working directory " + "(default: the path of the, first specified, Compose file)" + ), + ) + parser_compose.add_argument( + "command", + nargs=argparse.REMAINDER, + type=str, + metavar="command", + ) # -------------------------- Exec commands -------------------------------- _ = subparsers.add_parser( @@ -401,6 +448,9 @@ def parse_args() -> List[str]: parser_run.add_argument( "--network", help="Connect a container to a network" ) + parser_run.add_argument( + "--network-alias", help="Add network-scoped alias for the container" + ) parser_run.add_argument( "-v", "--volume", help="Bind mount a volume", action="append" ) @@ -513,7 +563,7 @@ def parse_args() -> List[str]: "docker": Docker, } container_inst = container_cls[container_cmd]( - args.container, print_only=args.print_only + getattr(args, "container", None), print_only=args.print_only ) kwargs = { k.replace("_", "-"): v @@ -530,6 +580,8 @@ def parse_args() -> List[str]: cmd = container_inst.network( args.subcommand, args.command, *args.flags ) + elif args.subcommand in ("compose",): + cmd = container_inst.compose(*args.command, **kwargs) else: cmd = [] if args.print_only: diff --git a/assets/web/freva_web.conf b/assets/web/freva_web.conf index bd18a44b..683104f0 100644 --- a/assets/web/freva_web.conf +++ b/assets/web/freva_web.conf @@ -187,6 +187,8 @@ SSLCryptoDevice builtin ProxyPass /static/ ! + ProxyPass /api/databrowser/ http://${DATABROWSER_HOST}/api/databrowser/ + ProxyPassReverse /api/databrowser/ http://${DATABROWSER_HOST}/api/databrowser/ ProxyPass / http://${FREVA_HOST}:8000/ ProxyPassReverse / http://${FREVA_HOST}:8000/ RequestHeader set X-Forwarded-Proto 'https' env=HTTPS diff --git a/src/freva_deployment/cli/__init__.py b/src/freva_deployment/cli/__init__.py index 695f8a84..1e77f923 100644 --- a/src/freva_deployment/cli/__init__.py +++ b/src/freva_deployment/cli/__init__.py @@ -1,11 +1,12 @@ """Command line interfaces.""" from __future__ import annotations + import importlib from ._deploy import cli as deploy -from ._service import cli as service from ._migrate import cli as migrate from ._server_map import cli as server_map +from ._service import cli as service __all__ = ["deploy", "service", "migrate", "server_map"] diff --git a/src/freva_deployment/cli/_deploy.py b/src/freva_deployment/cli/_deploy.py index f64cea63..ea5340c6 100644 --- a/src/freva_deployment/cli/_deploy.py +++ b/src/freva_deployment/cli/_deploy.py @@ -1,12 +1,14 @@ """Command line interface for the deployment.""" from __future__ import annotations + import argparse -from pathlib import Path import sys +from pathlib import Path from freva_deployment import __version__ + from ..deploy import DeployFactory -from ..utils import config_dir, set_log_level, guess_map_server +from ..utils import config_dir, guess_map_server, set_log_level def parse_args(argv: list[str] | None) -> argparse.Namespace: @@ -38,8 +40,8 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: "--steps", type=str, nargs="+", - default=["db", "solr", "web", "core"], - choices=["web", "core", "db", "solr"], + default=["db", "databrowser", "web", "core"], + choices=["web", "core", "db", "databrowser"], help="The services/code stack to be deployed", ) ap.add_argument( diff --git a/src/freva_deployment/cli/_migrate.py b/src/freva_deployment/cli/_migrate.py index 6792459b..806e4d8a 100644 --- a/src/freva_deployment/cli/_migrate.py +++ b/src/freva_deployment/cli/_migrate.py @@ -1,27 +1,29 @@ """CLI to assist with migrating the system.""" from __future__ import annotations + import argparse import json -from pathlib import Path import os import shlex import shutil -from subprocess import run, PIPE, CalledProcessError import sys +from pathlib import Path +from subprocess import PIPE, CalledProcessError, run from tempfile import NamedTemporaryFile, TemporaryDirectory -from typing import cast, TextIO +from typing import TextIO, cast -import toml import pymysql +import toml from freva_deployment import __version__ + from ..utils import ( - logger, - read_db_credentials, download_server_map, get_setup_for_service, - set_log_level, guess_map_server, + logger, + read_db_credentials, + set_log_level, ) DUMP_SCRIPT = """#!{python_bin} diff --git a/src/freva_deployment/cli/_server_map.py b/src/freva_deployment/cli/_server_map.py index ee7f91df..7cb5595b 100644 --- a/src/freva_deployment/cli/_server_map.py +++ b/src/freva_deployment/cli/_server_map.py @@ -1,19 +1,21 @@ """CLI to interact with running services.""" from __future__ import annotations + import argparse -from getpass import getuser import json import logging -from pathlib import Path import shlex -from subprocess import run import sys +from getpass import getuser +from pathlib import Path +from subprocess import run from tempfile import NamedTemporaryFile import appdirs from numpy import sign from freva_deployment import __version__ + from ..utils import asset_dir, logger diff --git a/src/freva_deployment/cli/_service.py b/src/freva_deployment/cli/_service.py index 75d7264b..cd9c97f0 100644 --- a/src/freva_deployment/cli/_service.py +++ b/src/freva_deployment/cli/_service.py @@ -1,26 +1,28 @@ """CLI to interact with running services.""" from __future__ import annotations + import argparse -from getpass import getuser -from pathlib import Path import shlex -from subprocess import run, PIPE import sys +from getpass import getuser +from pathlib import Path +from subprocess import PIPE, run from tempfile import TemporaryDirectory -from numpy import sign import requests -from rich.console import Console import yaml +from numpy import sign +from rich.console import Console from freva_deployment import __version__ + from ..utils import ( - download_server_map, - logger, - set_log_level, ServiceInfo, + download_server_map, get_setup_for_service, guess_map_server, + logger, + set_log_level, ) diff --git a/src/freva_deployment/deploy.py b/src/freva_deployment/deploy.py index 46ae130b..3dc0c9b1 100644 --- a/src/freva_deployment/deploy.py +++ b/src/freva_deployment/deploy.py @@ -1,30 +1,31 @@ """Module to run the freva deployment.""" from __future__ import annotations -from getpass import getuser +import json import os -from pathlib import Path import random import shlex import string -from subprocess import run import sys -from urllib.parse import urlparse +from getpass import getuser +from pathlib import Path +from subprocess import run from tempfile import TemporaryDirectory from typing import Any +from urllib.parse import urlparse -from numpy import sign import tomlkit import yaml +from numpy import sign from .utils import ( + RichConsole, asset_dir, config_dir, - load_config, - get_passwd, get_email_credentials, + get_passwd, + load_config, logger, upload_server_map, - RichConsole, ) @@ -44,11 +45,11 @@ class DeployFactory: -------- >>> from freva_deployment import DeployFactory as DF - >>> deploy = DF(steps=["solr"]) + >>> deploy = DF(steps=["databrowser"]) >>> deploy.play(ask_pass=True) """ - step_order: tuple[str, ...] = ("vault", "db", "solr", "core", "web") + step_order: tuple[str, ...] = ("vault", "db", "databrowser", "core", "web") _steps_with_cert: tuple[str, ...] = ("db", "vault", "core", "web") def __init__( @@ -57,13 +58,11 @@ def __init__( config_file: Path | str | None = None, ) -> None: self._config_keys: list[str] = [] - self.master_pass: str = "" + self.master_pass: str = "Freva4all!" self.email_password: str = "" self._td: TemporaryDirectory = TemporaryDirectory(prefix="inventory") self.inventory_file: Path = Path(self._td.name) / "inventory.yaml" - self.eval_conf_file: Path = ( - Path(self._td.name) / "evaluation_system.conf" - ) + self.eval_conf_file: Path = Path(self._td.name) / "evaluation_system.conf" self.web_conf_file: Path = Path(self._td.name) / "freva_web.toml" self.apache_config: Path = Path(self._td.name) / "freva_web.conf" self._db_pass: str = "" @@ -108,9 +107,7 @@ def _prep_vault(self) -> None: self._config_keys.append("vault") self.cfg["vault"] = self.cfg["db"].copy() self.cfg["vault"]["config"].setdefault("ansible_become_user", "root") - self.playbooks["vault"] = self.cfg["db"]["config"].get( - "vault_playbook" - ) + self.playbooks["vault"] = self.cfg["db"]["config"].get("vault_playbook") if not self.master_pass: self.master_pass = get_passwd() self.cfg["vault"]["config"]["root_passwd"] = self.master_pass @@ -131,9 +128,7 @@ def _prep_db(self) -> None: self.cfg["db"]["config"]["passwd"] = self.db_pass self.cfg["db"]["config"]["keyfile"] = self.public_key_file for key in ("name", "user", "db"): - self.cfg["db"]["config"][key] = ( - self.cfg["db"]["config"].get(key) or "freva" - ) + self.cfg["db"]["config"][key] = self.cfg["db"]["config"].get(key) or "freva" db_host = self.cfg["db"]["config"].get("host", "") if not db_host: self.cfg["db"]["config"]["host"] = host @@ -144,22 +139,24 @@ def _prep_db(self) -> None: self.playbooks["db"] = self.cfg["db"]["config"].get("db_playbook") self._prep_vault() - def _prep_solr(self) -> None: - """prepare the apache solr service.""" + def _prep_databrowser(self) -> None: + """prepare the databrowser service.""" if not self.master_pass: self.master_pass = get_passwd() - self._config_keys.append("solr") - self.cfg["solr"]["config"].setdefault("ansible_become_user", "root") - self.cfg["solr"]["config"]["root_passwd"] = self.master_pass - self.cfg["solr"]["config"].pop("core", None) - self.playbooks["solr"] = self.cfg["solr"]["config"].get( - "solr_playbook" + self._config_keys.append("databrowser") + self.cfg["databrowser"]["config"].setdefault("ansible_become_user", "root") + self.cfg["databrowser"]["config"]["root_passwd"] = self.master_pass + self.cfg["databrowser"]["config"].pop("core", None) + self.playbooks["databrowser"] = self.cfg["databrowser"]["config"].get( + "databrowser_playbook" ) - for key, default in dict(mem="4g", port=8983).items(): - self.cfg["solr"]["config"][key] = ( - self.cfg["solr"]["config"].get(key) or default + for key, default in dict( + solr_mem="4g", solr_port=8983, databrowser_port=7777 + ).items(): + self.cfg["databrowser"]["config"][key] = ( + self.cfg["databrowser"]["config"].get(key) or default ) - self.cfg["solr"]["config"]["email"] = self.cfg["web"]["config"].get( + self.cfg["databrowser"]["config"]["email"] = self.cfg["web"]["config"].get( "contacts", "" ) @@ -167,9 +164,7 @@ def _prep_core(self) -> None: """prepare the core deployment.""" self._config_keys.append("core") self.cfg["core"]["config"].setdefault("ansible_become_user", "") - self.playbooks["core"] = self.cfg["core"]["config"].get( - "core_playbook" - ) + self.playbooks["core"] = self.cfg["core"]["config"].get("core_playbook") self.cfg["core"]["config"]["admins"] = ( self.cfg["core"]["config"].get("admins") or getuser() ) @@ -180,15 +175,11 @@ def _prep_core(self) -> None: if not root_dir: self.cfg["core"]["config"]["root_dir"] = install_dir preview_path = self.cfg["core"]["config"].get("preview_path", "") - base_dir_location = self.cfg["core"]["config"].get( - "base_dir_location", "" - ) + base_dir_location = self.cfg["core"]["config"].get("base_dir_location", "") scheduler_output_dir = self.cfg["core"]["config"].get( "scheduler_output_dir", "" ) - scheduler_system = self.cfg["core"]["config"].get( - "scheduler_system", "local" - ) + scheduler_system = self.cfg["core"]["config"].get("scheduler_system", "local") if not preview_path: if base_dir_location: self.cfg["core"]["config"]["preview_path"] = str( @@ -199,9 +190,7 @@ def _prep_core(self) -> None: if not scheduler_output_dir: scheduler_output_dir = str(Path(base_dir_location) / "share") scheduler_output_dir = Path(scheduler_output_dir) / scheduler_system - self.cfg["core"]["config"]["scheduler_output_dir"] = str( - scheduler_output_dir - ) + self.cfg["core"]["config"]["scheduler_output_dir"] = str(scheduler_output_dir) self.cfg["core"]["config"]["keyfile"] = self.public_key_file git_exe = self.cfg["core"]["config"].get("git_path") self.cfg["core"]["config"]["git_path"] = git_exe or "git" @@ -215,6 +204,12 @@ def _prep_web(self) -> None: self.playbooks["web"] = self.cfg["web"]["config"].get("web_playbook") self.cfg["web"]["config"].setdefault("ansible_become_user", "root") self._prep_core() + self._prep_databrowser() + databrowser_host = ( + f'{self.cfg["databrowser"]["hosts"]}:' + f'{self.cfg["databrowser"]["config"]["databrowser_port"]}' + ) + self.cfg["web"]["config"]["databrowser_host"] = databrowser_host admin = self.cfg["core"]["config"]["admins"] if not isinstance(admin, str): self.cfg["web"]["config"]["admin"] = admin[0] @@ -271,24 +266,20 @@ def _prep_web(self) -> None: except (FileNotFoundError, IOError, KeyError): pass try: - _webserver_items["IMPRINT"] = _webserver_items["IMPRINT"].split( - "," - ) + _webserver_items["IMPRINT"] = _webserver_items["IMPRINT"].split(",") except AttributeError: pass with self.web_conf_file.open("w") as f_obj: tomlkit.dump(_webserver_items, f_obj) for key in ("core", "web"): - self.cfg[key]["config"]["config_toml_file"] = str( - self.web_conf_file - ) + self.cfg[key]["config"]["config_toml_file"] = str(self.web_conf_file) if not self.master_pass: self.master_pass = get_passwd() email_user, self.email_password = get_email_credentials() self._prep_vault() - self.cfg["vault"]["config"]["ansible_python_interpreter"] = self.cfg[ - "db" - ]["config"].get("ansible_python_interpreter", "/usr/bin/python3") + self.cfg["vault"]["config"]["ansible_python_interpreter"] = self.cfg["db"][ + "config" + ].get("ansible_python_interpreter", "/usr/bin/python3") self.cfg["vault"]["config"]["email_user"] = email_user self.cfg["vault"]["config"]["email_password"] = self.email_password self.cfg["web"]["config"]["root_passwd"] = self.master_pass @@ -297,19 +288,14 @@ def _prep_web(self) -> None: self.cfg["web"]["config"]["chain_keyfile"] = ( self.chain_key_file or self.public_key_file ) - self.cfg["web"]["config"]["apache_config_file"] = str( - self.apache_config - ) + self.cfg["web"]["config"]["apache_config_file"] = str(self.apache_config) self._prep_apache_config() def _prep_apache_config(self): config = [] with (Path(asset_dir) / "web" / "freva_web.conf").open() as f_obj: for line in f_obj.readlines(): - if ( - not self.chain_key_file - and "SSLCertificateChainFile" in line - ): + if not self.chain_key_file and "SSLCertificateChainFile" in line: continue config.append(line) with open(self.apache_config, "w") as f_obj: @@ -323,11 +309,9 @@ def __exit__(self, *args): def _read_cfg(self) -> dict[str, Any]: try: - return dict(load_config(self._inv_tmpl)) + return dict(load_config(self._inv_tmpl).items()) except FileNotFoundError as error: - raise FileNotFoundError( - f"No such file {self._inv_tmpl}" - ) from error + raise FileNotFoundError(f"No such file {self._inv_tmpl}") from error def _check_config(self) -> None: sections = [] @@ -337,14 +321,8 @@ def _check_config(self) -> None: sections.append(section) for section in sections: for key, value in self.cfg[section]["config"].items(): - if ( - not value - and not self._empty_ok - and not isinstance(value, bool) - ): - raise ValueError( - f"{key} in {section} is empty in {self._inv_tmpl}" - ) + if not value and not self._empty_ok and not isinstance(value, bool): + raise ValueError(f"{key} in {section} is empty in {self._inv_tmpl}") @property def _empty_ok(self) -> list[str]: @@ -370,21 +348,15 @@ def db_pass(self) -> str: num_chars, num_digits, num_punctuations = 20, 4, 4 num_chars -= num_digits + num_punctuations characters = [ - "".join( - [random.choice(string.ascii_letters) for i in range(num_chars)] - ), + "".join([random.choice(string.ascii_letters) for i in range(num_chars)]), "".join([random.choice(string.digits) for i in range(num_digits)]), - "".join( - [random.choice(punctuations) for i in range(num_punctuations)] - ), + "".join([random.choice(punctuations) for i in range(num_punctuations)]), ] str_characters = "".join(characters) _db_pass = "".join(random.sample(str_characters, len(str_characters))) while _db_pass.startswith("@"): # Vault treats values starting with "@" as file names. - _db_pass = "".join( - random.sample(str_characters, len(str_characters)) - ) + _db_pass = "".join(random.sample(str_characters, len(str_characters))) self._db_pass = _db_pass return self._db_pass @@ -432,7 +404,7 @@ def parse_config(self) -> str: config[step]["hosts"] = self.cfg[step]["hosts"] config[step]["vars"] = {} for key, value in self.cfg[step]["config"].items(): - if key in ("root_passwd",): + if key in ("root_passwd",) or key.startswith(step): new_key = key else: new_key = f"{step}_{key}" @@ -440,7 +412,7 @@ def parse_config(self) -> str: config[step]["vars"]["project_name"] = self.project_name # Add additional keys self._set_additional_config_values(step, config) - return yaml.dump(config) + return yaml.dump(json.loads(json.dumps(config.copy()))) @property def _playbook_file(self) -> Path: @@ -509,7 +481,7 @@ def create_eval_config(self) -> None: cfg = self.cfg[key]["config"].get(value, "") if cfg: lines[num] = f"{value}={cfg}\n" - for step in ("solr", "db"): + for step in ("databrowser", "db"): cfg = self.cfg[step]["config"].get("port", "") if line.startswith(f"{step}.port"): lines[num] = f"{step}.port={cfg}\n" @@ -543,14 +515,13 @@ def play( self.create_playbooks() inventory = self.parse_config() self.create_eval_config() + print(self.inventory_file) with self.inventory_file.open("w") as f_obj: f_obj.write(inventory) inventory_str = inventory for passwd in (self.email_password, self.master_pass): if passwd: - inventory_str = inventory_str.replace( - passwd, "*" * len(passwd) - ) + inventory_str = inventory_str.replace(passwd, "*" * len(passwd)) RichConsole.print(inventory_str, style="bold", markup=True) logger.info("Playing the playbooks with ansible") RichConsole.print( diff --git a/src/freva_deployment/ui/deployment_tui/__init__.py b/src/freva_deployment/ui/deployment_tui/__init__.py index f9e5f6ae..17ab65de 100644 --- a/src/freva_deployment/ui/deployment_tui/__init__.py +++ b/src/freva_deployment/ui/deployment_tui/__init__.py @@ -1,10 +1,12 @@ """Call the freva tui.""" from __future__ import annotations + import argparse from freva_deployment import __version__ from freva_deployment.deploy import DeployFactory -from freva_deployment.utils import set_log_level, RichConsole +from freva_deployment.utils import RichConsole, set_log_level + from .main_window import MainApp diff --git a/src/freva_deployment/ui/deployment_tui/base.py b/src/freva_deployment/ui/deployment_tui/base.py index ba35af71..061cc726 100644 --- a/src/freva_deployment/ui/deployment_tui/base.py +++ b/src/freva_deployment/ui/deployment_tui/base.py @@ -1,15 +1,17 @@ from __future__ import annotations -import os + +import curses import json -from pathlib import Path import logging +import os +from pathlib import Path from typing import Any, cast -import curses -from freva_deployment.utils import asset_dir, config_file import npyscreen import tomlkit +from freva_deployment.utils import asset_dir, config_file + logging.basicConfig(level=logging.DEBUG) logger: logging.Logger = logging.getLogger("deploy-freva-tui") diff --git a/src/freva_deployment/ui/deployment_tui/deploy_forms.py b/src/freva_deployment/ui/deployment_tui/deploy_forms.py index a8e1f2fd..41bd1e64 100644 --- a/src/freva_deployment/ui/deployment_tui/deploy_forms.py +++ b/src/freva_deployment/ui/deployment_tui/deploy_forms.py @@ -1,13 +1,16 @@ from __future__ import annotations + from getpass import getuser -import npyscreen from pathlib import Path -from typing import cast, List, Dict +from typing import Dict, List, cast -from .base import BaseForm, logger -from freva_deployment import AVAILABLE_PYTHON_VERSIONS, AVAILABLE_CONDA_ARCHS +import npyscreen + +from freva_deployment import AVAILABLE_CONDA_ARCHS, AVAILABLE_PYTHON_VERSIONS from freva_deployment.utils import get_current_file_dir +from .base import BaseForm, logger + def get_index(values: list[str], target: str, default: int = 0) -> int: """Get the target index of item in list. @@ -696,25 +699,31 @@ def _add_widgets(self) -> None: ) -class SolrScreen(BaseForm): - """Form for the solr deployment configuration.""" +class DatabrowserScreen(BaseForm): + """Form for the databrowser deployment configuration.""" - step: str = "solr" + step: str = "databrowser" def _add_widgets(self) -> None: """Add widgets to the screen.""" self.list_keys: list[str] = [] cfg = self.get_config(self.step) solr_ports: list[int] = list(range(8980, 9000)) - port_idx = get_index( - [str(p) for p in solr_ports], str(cfg.get("port", 8983)), 3 + databrowser_ports: list[int] = list(range(7770, 7780)) + solr_port_idx = get_index( + [str(p) for p in solr_ports], str(cfg.get("solr_port", 8983)), 3 + ) + databrowser_port_idx = get_index( + [str(p) for p in databrowser_ports], + str(cfg.get("databrowser_port", 7777)), + 7, ) self.input_fields: dict[str, tuple[npyscreen.TitleText, bool]] = dict( hosts=( self.add_widget_intelligent( npyscreen.TitleText, name=f"{self.num}Server Name(s) where the solr service is deployed:", - value=self.get_host("solr"), + value=self.get_host("databrowser"), ), True, ), @@ -729,7 +738,7 @@ def _add_widgets(self) -> None: ), True, ), - mem=( + solr_mem=( self.add_widget_intelligent( npyscreen.TitleCombo, name=f"{self.num}Virtual memory (in GB) for the solr server:", @@ -738,23 +747,32 @@ def _add_widgets(self) -> None: ), True, ), - port=( + solr_port=( self.add_widget_intelligent( npyscreen.TitleCombo, name=f"{self.num}Solr port:", - value=port_idx, + value=solr_port_idx, values=solr_ports, ), True, ), - solr_playbook=( + databrowser_port=( + self.add_widget_intelligent( + npyscreen.TitleCombo, + name=f"{self.num}Databrowser API port:", + value=databrowser_port_idx, + values=databrowser_ports, + ), + True, + ), + databrowser_playbook=( self.add_widget_intelligent( npyscreen.TitleFilename, name=( f"{self.num}Set the path to the playbook used for" " setting up the system." ), - value=cfg.get("solr_playbook", ""), + value=cfg.get("databrowser_playbook", ""), ), False, ), diff --git a/src/freva_deployment/ui/deployment_tui/main_window.py b/src/freva_deployment/ui/deployment_tui/main_window.py index f418a944..376fda37 100644 --- a/src/freva_deployment/ui/deployment_tui/main_window.py +++ b/src/freva_deployment/ui/deployment_tui/main_window.py @@ -1,19 +1,27 @@ """The Freva deployment Text User Interface (TUI) helps to configure a deployment setup for a new instance of freva.""" from __future__ import annotations + import json -from pathlib import Path -import time import threading +import time +from pathlib import Path from typing import Any, Dict, List, cast import appdirs import npyscreen import tomlkit -from freva_deployment.utils import asset_dir, config_dir -from .base import selectFile, BaseForm, VarForm -from .deploy_forms import WebScreen, DBScreen, SolrScreen, CoreScreen, RunForm +from freva_deployment.utils import asset_dir, config_dir, load_config + +from .base import BaseForm, VarForm, selectFile +from .deploy_forms import ( + CoreScreen, + DBScreen, + RunForm, + DatabrowserScreen, + WebScreen, +) class MainApp(npyscreen.NPSAppManaged): @@ -43,7 +51,7 @@ def init(self) -> None: self._steps_lookup = { "core": "MAIN", "web": "SECOND", - "solr": "FOURTH", + "databrowser": "FOURTH", "db": "THIRD", } self.config = cast(Dict[str, Any], self._read_cache("config", {})) @@ -70,7 +78,9 @@ def _add_froms(self) -> None: ) self._forms["web"] = self.addForm("SECOND", WebScreen, name="Web deployment") self._forms["db"] = self.addForm("THIRD", DBScreen, name="Database deployment") - self._forms["solr"] = self.addForm("FOURTH", SolrScreen, name="Solr deployment") + self._forms["databrowser"] = self.addForm( + "FOURTH", DatabrowserScreen, name="Databrowser deployment" + ) self._setup_form = self.addForm("SETUP", RunForm, name="Apply the Deployment") def exit_application(self, *args, **kwargs) -> None: @@ -206,11 +216,9 @@ def _save_config_to_file( if write_toml_file is False: return None try: - with open(self.save_file) as f: - config_tmpl = cast(Dict[str, Any], tomlkit.load(f)) + config_tmpl = load_config(self.save_file) except FileNotFoundError: - with open(asset_dir / "config" / "inventory.toml") as f: - config_tmpl = cast(Dict[str, Any], tomlkit.load(f)) + config_tmpl = load_config(asset_dir / "config" / "inventory.toml") except Exception: config_tmpl = self.config config_tmpl["certificates"] = cert_files @@ -244,7 +252,9 @@ def _read_cache( @property def _steps(self) -> list[str]: """Read the deployment-steps from the cache.""" - return cast(List[str], self._read_cache("steps", ["core", "web", "db", "solr"])) + return cast( + List[str], self._read_cache("steps", ["core", "web", "db", "databrowser"]) + ) def read_cert_file(self, key: str) -> str: """Read the certificate file from the cache.""" diff --git a/src/freva_deployment/utils.py b/src/freva_deployment/utils.py index a50c24b4..3a214110 100644 --- a/src/freva_deployment/utils.py +++ b/src/freva_deployment/utils.py @@ -1,22 +1,22 @@ """Collection of utils for deployment.""" from __future__ import annotations + import hashlib -import logging import json -from pathlib import Path -import pkg_resources +import logging import re -from subprocess import PIPE -import sys import shutil -from typing import Any, NamedTuple, cast +import sys import warnings +from pathlib import Path +from typing import Any, MutableMapping, NamedTuple, cast import appdirs +import pkg_resources import requests +import tomlkit from rich.console import Console from rich.prompt import Prompt -import toml logging.basicConfig( format="%(name)s - %(levelname)s - %(message)s", level=logging.INFO @@ -93,15 +93,17 @@ def asset_dir(self): return self._user_asset_dir return self._central_asset_dir + @property + def inventory_file(self) -> Path: + return self.asset_dir / "config" / "inventory.toml" + @property def config_dir(self): inventory_file = self._user_config_dir / "inventory.toml" if inventory_file.exists(): return self._user_config_dir self._user_config_dir.mkdir(exist_ok=True, parents=True) - shutil.copy( - self.asset_dir / "config" / "inventory.toml", inventory_file - ) + shutil.copy(self.inventory_file, inventory_file) return self._user_config_dir @property @@ -145,13 +147,40 @@ def _convert_dict( inp_dict[key] = get_current_file_dir(cfd, value) +def _update_config( + new_config: MutableMapping[str, Any], old_config: MutableMapping[str, Any] +) -> None: + for key, value in old_config.items(): + if key in new_config and isinstance(value, dict): + _update_config(new_config[key], old_config[key]) + else: + new_config[key] = value + + +def _create_new_config(inp_file: Path) -> Path: + """Update any old configuration file to a newer version.""" + config = tomlkit.loads(inp_file.read_text()) + config_tmpl = tomlkit.loads(AD.inventory_file.read_text()) + # Legacy solr: + if "solr" in config: + config["databrowser"] = config.pop("solr") + for key in ("port", "mem"): + if key in config["databrowser"]["config"]: + config["databrowser"]["config"][f"solr_{key}"] = config[ + "databrowser" + ]["config"].pop(key) + _update_config(config_tmpl, config) + inp_file.write_text(tomlkit.dumps(config_tmpl)) + return inp_file + + def load_config(inp_file: str | Path) -> dict[str, Any]: """Load the inventory toml file and replace all environment variables.""" - inp_file = Path(inp_file).expanduser().absolute() + inp_file = _create_new_config(Path(inp_file).expanduser().absolute()) variables = cast( - dict[str, str], toml.loads(config_file.read_text())["variables"] + dict[str, str], tomlkit.loads(config_file.read_text())["variables"] ) - config = toml.loads(inp_file.read_text()) + config = tomlkit.loads(inp_file.read_text()) _convert_dict(config, variables, inp_file.parent) return config @@ -285,7 +314,7 @@ def upload_server_map( _upload_data[service] = config host, _, port = server_map.partition(":") port = port or "6111" - req_data = dict(config=toml.dumps(_upload_data)) + req_data = dict(config=tomlkit.dumps(_upload_data)) logger.debug("Uploading %s", req_data) req = requests.put(f"http://{host}:{port}/{project_name}", data=req_data) if req.status_code == 201: From f35c3322261ada2f057a11b6081e52d45b476f53 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Mon, 11 Sep 2023 06:55:34 +0200 Subject: [PATCH 04/20] Use - as compose project name. --- .../databrowser-server-compose-template.yml | 8 +++---- .../playbooks/databrowser-server-playbook.yml | 24 +++++++++++++------ .../playbooks/db-server-compose-template.yml | 4 ++-- assets/playbooks/db-server-playbook.yml | 14 ++++++++++- .../vault-server-compose-template.yml | 4 ++-- assets/playbooks/vault-server-playbook.yml | 4 ++-- .../playbooks/web-server-compose-template.yml | 12 +++++----- assets/playbooks/web-server-playbook.yml | 10 ++++---- 8 files changed, 51 insertions(+), 29 deletions(-) diff --git a/assets/playbooks/databrowser-server-compose-template.yml b/assets/playbooks/databrowser-server-compose-template.yml index 32696c7b..1f241333 100644 --- a/assets/playbooks/databrowser-server-compose-template.yml +++ b/assets/playbooks/databrowser-server-compose-template.yml @@ -5,7 +5,7 @@ services: container_name: {{solr_name}} hostname: {{solr_name}} networks: - - {{project_name}} + - {{databrowser_name}} environment: - CORE=files - NUM_BACKUPS=7 @@ -23,7 +23,7 @@ services: container_name: {{mongo_name}} hostname: {{mongo_name}} networks: - - {{project_name}} + - {{databrowser_name}} environment: - MONGO_INITDB_ROOT_USERNAME=mongo - MONGO_INITDB_ROOT_PASSWORD={{ root_passwd}} @@ -40,7 +40,7 @@ services: image: ghcr.io/freva-clint/databrowserapi:latest hostname: {{databrowser_name}} networks: - - {{project_name}} + - {{databrowser_name}} environment: - SOLR_CORE=files - SOLR_HOST={{solr_name}}:8983 @@ -57,5 +57,5 @@ services: tty: true networks: - {{project_name}}: + {{databrowser_name}}: driver: bridge diff --git a/assets/playbooks/databrowser-server-playbook.yml b/assets/playbooks/databrowser-server-playbook.yml index d09c1d92..6587a81c 100644 --- a/assets/playbooks/databrowser-server-playbook.yml +++ b/assets/playbooks/databrowser-server-playbook.yml @@ -4,6 +4,7 @@ vars: ansible_python_interpreter: "{{ databrowser_ansible_python_interpreter }}" databrowser_name: "{{project_name}}-databrowser" + compose_file: /opt/freva/compose_services/{{databrowser_name}}-compose.yml solr_name: "{{project_name}}-solr" mongo_name: "{{project_name}}-mongo" solr_volumes: @@ -78,18 +79,15 @@ become: "{{use_become}}" - name: Creating compose directory structure file: - path: /opt/freva/compose_services/foo + path: /opt/freva/compose_services state: directory recurse: true become: "{{use_become}}" - name: Creating directory structure file: - path: "{item}" + path: /opt/freva/{{ project_name }}/databrowser state: directory recurse: true - with_items: - - /opt/freva/{{ project_name }}/databrowser - - /opt/freva/compose_services/foo become: "{{use_become}}" - name: Getting additional configurations git: @@ -118,10 +116,10 @@ become: "{{use_become}}" template: src: "{{ asset_dir }}/playbooks/databrowser-server-compose-template.yml" - dest: "/opt/freva/compose_services/{{databrowser_name}}-compose.yml" + dest: "{{compose_file}}" - name: Creating systemd services shell: | - /tmp/create_systemd.py {{databrowser_name}} compose --enable --project-name {{project_name}} -f /opt/freva/compose_services/{{databrowser_name}}-compose.yml up + /tmp/create_systemd.py {{databrowser_name}} compose --enable --project-name {{databrowser_name}} -f {{compose_file}} up --remove-orphans when: systemctl.stat.exists == true become: "{{use_become}}" - pause: seconds=3 @@ -139,3 +137,15 @@ - /tmp/create_systemd.py - /tmp/solr_service become: "{{use_become}}" +- hosts: web + vars: + use_become: "{{ web_ansible_become_user is defined and web_ansible_become_user != '' }}" + tasks: + - name: Registering systemctl path + stat: + path: /usr/bin/systemctl + register: systemctl + - name: Restarting web container + shell: systemctl restart {{web_name}}; echo 0 + become: "{{use_become}}" + when: systemctl.stat.exists == true diff --git a/assets/playbooks/db-server-compose-template.yml b/assets/playbooks/db-server-compose-template.yml index 52dd5314..6614caa7 100644 --- a/assets/playbooks/db-server-compose-template.yml +++ b/assets/playbooks/db-server-compose-template.yml @@ -4,7 +4,7 @@ services: image: mariadb:latest hostname: {{db_name}} networks: - - {{project_name}} + - {{db_name}} environment: - ROOT_PW={{ root_passwd }} - HOST={{ db_host }} @@ -25,5 +25,5 @@ services: - {{db_port}}:3306 networks: - {{project_name}}: + {{db_name}}: driver: bridge diff --git a/assets/playbooks/db-server-playbook.yml b/assets/playbooks/db-server-playbook.yml index 9e43b01c..8c69e495 100644 --- a/assets/playbooks/db-server-playbook.yml +++ b/assets/playbooks/db-server-playbook.yml @@ -147,7 +147,7 @@ - name: Creating system services become: "{{use_become}}" shell: | - /tmp/create_systemd.py {{db_name}} compose --enable --project-name {{project_name}} -f {{compose_file}} up + /tmp/create_systemd.py {{db_name}} compose --enable --project-name {{db_name}} -f {{compose_file}} up --remove-orphans when: systemctl.stat.exists == true - name: Creating cron jobs become: "{{use_become}}" @@ -168,3 +168,15 @@ - /tmp/create_cron.sh - /tmp/reset_root_pw.sh - /tmp/docker-or-podman +- hosts: web + vars: + use_become: "{{ web_ansible_become_user is defined and web_ansible_become_user != '' }}" + tasks: + - name: Registering systemctl path + stat: + path: /usr/bin/systemctl + register: systemctl + - name: Restarting web container + shell: systemctl restart {{web_name}}; echo 0 + become: "{{use_become}}" + when: systemctl.stat.exists == true diff --git a/assets/playbooks/vault-server-compose-template.yml b/assets/playbooks/vault-server-compose-template.yml index cd476db4..48c56268 100644 --- a/assets/playbooks/vault-server-compose-template.yml +++ b/assets/playbooks/vault-server-compose-template.yml @@ -10,7 +10,7 @@ services: - 8.8.8.8 - 8.8.4.4 networks: - - {{project_name}} + - {{vault_name}} environment: - ROOT_PW={{ root_passwd }} volumes: @@ -23,5 +23,5 @@ services: - 5002:5002 networks: - {{project_name}}: + {{vault_name}}: driver: bridge diff --git a/assets/playbooks/vault-server-playbook.yml b/assets/playbooks/vault-server-playbook.yml index ff155694..e678b679 100644 --- a/assets/playbooks/vault-server-playbook.yml +++ b/assets/playbooks/vault-server-playbook.yml @@ -60,7 +60,7 @@ become: "{{use_become}}" - name: Creating compose directory structure file: - path: /opt/freva/compose_services/foo + path: /opt/freva/compose_services state: directory recurse: true become: "{{use_become}}" @@ -72,7 +72,7 @@ - name: Creating system services become: "{{use_become}}" shell: | - /tmp/create_systemd.py {{vault_name}} compose --enable --project-name {{project_name}} -f {{compose_file}} up; + /tmp/create_systemd.py {{vault_name}} compose --enable --project-name {{vault_name}} -f {{compose_file}} up --remove-orphans when: systemctl.stat.exists == true - pause: seconds=20 - name: Deleting existing infrastructure diff --git a/assets/playbooks/web-server-compose-template.yml b/assets/playbooks/web-server-compose-template.yml index aa4e752d..ee6bee70 100644 --- a/assets/playbooks/web-server-compose-template.yml +++ b/assets/playbooks/web-server-compose-template.yml @@ -6,7 +6,7 @@ services: container_name: {{apache_name}} hostname: {{apache_name}} networks: - - {{project_name}} + - {{web_name}} environment: - SCHEDULER_DIR={{core_scheduler_output_dir}} - UID={{uid}} @@ -25,25 +25,25 @@ services: {{redis_name}}: image: redis networks: - - {{project_name}} + - {{web_name}} ports: - 6379:6379 tty: true + container_name: {{redis_name}} + hostname: {{redis_name}} {{web_name}}: image: ghcr.io/freva-clint/freva-web container_name: {{web_name}} hostname: {{web_name}} networks: - - {{project_name}} + - {{web_name}} environment: - EVALUATION_SYSTEM_CONFIG_FILE={{core_root_dir}}/freva/evaluation_system.conf - LDAP_USER_DN={{web_ldap_user_dn}} - LDAP_USER_PW={{web_ldap_user_pw}} - DJANGO_SUPERUSER_PASSWORD={{ root_passwd }} volumes: - container_name: {{redis_name}} - hostname: {{redis_name}} {% for volume in web_volumes %} - {{volume}} {% endfor %} @@ -55,5 +55,5 @@ services: - {{apache_name}} networks: - {{project_name}}: + {{web_name}}: driver: bridge diff --git a/assets/playbooks/web-server-playbook.yml b/assets/playbooks/web-server-playbook.yml index 4ec9625b..01781088 100644 --- a/assets/playbooks/web-server-playbook.yml +++ b/assets/playbooks/web-server-playbook.yml @@ -61,7 +61,7 @@ vars: ansible_python_interpreter: "{{ web_ansible_python_interpreter }}" service_dir: /opt/freva/{{project_name}}/web_service - compose_file: /opt/freva/compose_services/{{databrowser_name}}-compose.yml + compose_file: /opt/freva/compose_services/{{web_name}}-compose.yml web_volumes: - "{{ core_root_dir }}:{{ core_root_dir }}:ro" - "{{ core_scheduler_output_dir}}:{{ core_scheduler_output_dir }}:ro" @@ -125,7 +125,7 @@ systemctl stop {{ web_name }} {{ redis_name }} {{ apache_name }}; systemctl disable {{ web_name }} {{ redis_name }} {{ apache_name }}; systemctl reset-failed; - /tmp/docker-or-podman stop {{web_name}} {{web_name}} {{redis_name}}; + /tmp/docker-or-podman stop {{web_name}} {{apache_name}} {{redis_name}}; /tmp/docker-or-podman rm {{web_name}} {{redis_name}} {{apache_name}}; /tmp/docker-or-podman rmi -f docker.io/redis ghcr.io/freva-clint/freva-web docker.io/httpd; echo 0 @@ -149,7 +149,7 @@ mode: "0775" - name: Creating compose directory structure file: - path: /opt/freva/compose_services/foo + path: /opt/freva/compose_services state: directory recurse: true become: "{{use_become}}" @@ -184,11 +184,11 @@ become: "{{use_become}}" template: src: "{{ asset_dir }}/playbooks/web-server-compose-template.yml" - dest: "/opt/freva/compose_services/{{databrowser_name}}-compose.yml" + dest: "{{compose_file}}" - name: Creating system services become: "{{use_become}}" shell: | - /tmp/create_systemd.py {{web_name}} compose --enable --project-name {{project_name}} -f {{compose_file}} up + /tmp/create_systemd.py {{web_name}} compose --enable --project-name {{web_name}} -f {{compose_file}} up --remove-orphans when: systemctl.stat.exists == true - pause: seconds=15 - name: Restarting services From 713cf25a0d485c40236f89db1d8546ee40a3b784 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Mon, 11 Sep 2023 06:58:29 +0200 Subject: [PATCH 05/20] Add execstop command for systemd units. --- assets/scripts/create_systemd.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/assets/scripts/create_systemd.py b/assets/scripts/create_systemd.py index dc9a1c15..756b3603 100644 --- a/assets/scripts/create_systemd.py +++ b/assets/scripts/create_systemd.py @@ -17,6 +17,7 @@ TimeoutStopSec="35s", ExecStartPre='/bin/sh -c "{delete_command}"', ExecStart='/bin/sh -c "{container_cmd} {container_args}"', + ExecStop='/bin/sh -c "{delete_command}"', Restart="no", ), Install=dict(WantedBy="default.target"), @@ -123,9 +124,10 @@ def create_unit( container_args=container_args, unit=unit, ) - SYSTEMD_TMPL["Service"]["ExecStartPre"] = SYSTEMD_TMPL["Service"][ - "ExecStartPre" - ].format(delete_command=delete_command) + for key in ("ExecStartPre", "ExecStop"): + SYSTEMD_TMPL["Service"][key] = SYSTEMD_TMPL["Service"][key].format( + delete_command=delete_command + ) for service in requires: for key in ("After", "Requires"): try: From b7ad172082060bfb83396a51415ded6b3f5b5326 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Mon, 11 Sep 2023 06:58:52 +0200 Subject: [PATCH 06/20] Use rich logger --- src/freva_deployment/utils.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/freva_deployment/utils.py b/src/freva_deployment/utils.py index 3a214110..c49f10a8 100644 --- a/src/freva_deployment/utils.py +++ b/src/freva_deployment/utils.py @@ -17,9 +17,19 @@ import tomlkit from rich.console import Console from rich.prompt import Prompt +from rich.logging import RichHandler +logger_stream_handle = RichHandler( + rich_tracebacks=False, + show_path=False, + console=Console(soft_wrap=True, stderr=True), +) +logger_stream_handle.setLevel(logging.INFO) logging.basicConfig( - format="%(name)s - %(levelname)s - %(message)s", level=logging.INFO + format="%(message)s", + handlers=[logger_stream_handle], + datefmt="[%X]", + level=logging.INFO, ) logger = logging.getLogger("freva-deployment") From 2daa3124e54b38ca6a1dc1a01d3e61ae9efd5f76 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Mon, 11 Sep 2023 06:59:34 +0200 Subject: [PATCH 07/20] Only print relevant ansible variables. --- src/freva_deployment/deploy.py | 143 +++++++++++++++++++++++---------- 1 file changed, 99 insertions(+), 44 deletions(-) diff --git a/src/freva_deployment/deploy.py b/src/freva_deployment/deploy.py index 3dc0c9b1..5b2e960f 100644 --- a/src/freva_deployment/deploy.py +++ b/src/freva_deployment/deploy.py @@ -62,7 +62,9 @@ def __init__( self.email_password: str = "" self._td: TemporaryDirectory = TemporaryDirectory(prefix="inventory") self.inventory_file: Path = Path(self._td.name) / "inventory.yaml" - self.eval_conf_file: Path = Path(self._td.name) / "evaluation_system.conf" + self.eval_conf_file: Path = ( + Path(self._td.name) / "evaluation_system.conf" + ) self.web_conf_file: Path = Path(self._td.name) / "freva_web.toml" self.apache_config: Path = Path(self._td.name) / "freva_web.conf" self._db_pass: str = "" @@ -70,6 +72,7 @@ def __init__( self._inv_tmpl = Path(config_file or config_dir / "inventory.toml") self._cfg_tmpl = self.aux_dir / "evaluation_system.conf.tmpl" self.cfg = self._read_cfg() + self.info = {} self.project_name = self.cfg.pop("project_name", None) self.playbooks: dict[str, Path | None] = {} if not self.project_name: @@ -107,7 +110,9 @@ def _prep_vault(self) -> None: self._config_keys.append("vault") self.cfg["vault"] = self.cfg["db"].copy() self.cfg["vault"]["config"].setdefault("ansible_become_user", "root") - self.playbooks["vault"] = self.cfg["db"]["config"].get("vault_playbook") + self.playbooks["vault"] = self.cfg["db"]["config"].get( + "vault_playbook" + ) if not self.master_pass: self.master_pass = get_passwd() self.cfg["vault"]["config"]["root_passwd"] = self.master_pass @@ -116,6 +121,7 @@ def _prep_vault(self) -> None: self.cfg["vault"]["config"]["email"] = self.cfg["web"]["config"].get( "contacts", "" ) + self.info["vault"] = self.cfg["vault"]["config"] def _prep_db(self) -> None: """prepare the mariadb service.""" @@ -128,7 +134,9 @@ def _prep_db(self) -> None: self.cfg["db"]["config"]["passwd"] = self.db_pass self.cfg["db"]["config"]["keyfile"] = self.public_key_file for key in ("name", "user", "db"): - self.cfg["db"]["config"][key] = self.cfg["db"]["config"].get(key) or "freva" + self.cfg["db"]["config"][key] = ( + self.cfg["db"]["config"].get(key) or "freva" + ) db_host = self.cfg["db"]["config"].get("host", "") if not db_host: self.cfg["db"]["config"]["host"] = host @@ -138,13 +146,16 @@ def _prep_db(self) -> None: ) self.playbooks["db"] = self.cfg["db"]["config"].get("db_playbook") self._prep_vault() + self._prep_web(False) - def _prep_databrowser(self) -> None: + def _prep_databrowser(self, prep_web=True) -> None: """prepare the databrowser service.""" if not self.master_pass: self.master_pass = get_passwd() self._config_keys.append("databrowser") - self.cfg["databrowser"]["config"].setdefault("ansible_become_user", "root") + self.cfg["databrowser"]["config"].setdefault( + "ansible_become_user", "root" + ) self.cfg["databrowser"]["config"]["root_passwd"] = self.master_pass self.cfg["databrowser"]["config"].pop("core", None) self.playbooks["databrowser"] = self.cfg["databrowser"]["config"].get( @@ -156,15 +167,20 @@ def _prep_databrowser(self) -> None: self.cfg["databrowser"]["config"][key] = ( self.cfg["databrowser"]["config"].get(key) or default ) - self.cfg["databrowser"]["config"]["email"] = self.cfg["web"]["config"].get( - "contacts", "" - ) + self.cfg["databrowser"]["config"]["email"] = self.cfg["web"][ + "config" + ].get("contacts", "") + self.info["databrowser"] = self.cfg["databrowser"]["config"] + if prep_web: + self._prep_web(False) def _prep_core(self) -> None: """prepare the core deployment.""" self._config_keys.append("core") self.cfg["core"]["config"].setdefault("ansible_become_user", "") - self.playbooks["core"] = self.cfg["core"]["config"].get("core_playbook") + self.playbooks["core"] = self.cfg["core"]["config"].get( + "core_playbook" + ) self.cfg["core"]["config"]["admins"] = ( self.cfg["core"]["config"].get("admins") or getuser() ) @@ -175,11 +191,15 @@ def _prep_core(self) -> None: if not root_dir: self.cfg["core"]["config"]["root_dir"] = install_dir preview_path = self.cfg["core"]["config"].get("preview_path", "") - base_dir_location = self.cfg["core"]["config"].get("base_dir_location", "") + base_dir_location = self.cfg["core"]["config"].get( + "base_dir_location", "" + ) scheduler_output_dir = self.cfg["core"]["config"].get( "scheduler_output_dir", "" ) - scheduler_system = self.cfg["core"]["config"].get("scheduler_system", "local") + scheduler_system = self.cfg["core"]["config"].get( + "scheduler_system", "local" + ) if not preview_path: if base_dir_location: self.cfg["core"]["config"]["preview_path"] = str( @@ -190,7 +210,9 @@ def _prep_core(self) -> None: if not scheduler_output_dir: scheduler_output_dir = str(Path(base_dir_location) / "share") scheduler_output_dir = Path(scheduler_output_dir) / scheduler_system - self.cfg["core"]["config"]["scheduler_output_dir"] = str(scheduler_output_dir) + self.cfg["core"]["config"]["scheduler_output_dir"] = str( + scheduler_output_dir + ) self.cfg["core"]["config"]["keyfile"] = self.public_key_file git_exe = self.cfg["core"]["config"].get("git_path") self.cfg["core"]["config"]["git_path"] = git_exe or "git" @@ -198,13 +220,12 @@ def _prep_core(self) -> None: "git_url" ] = "https://github.com/FREVA-CLINT/freva.git" - def _prep_web(self) -> None: + def _prep_web(self, ask_pass: bool = True) -> None: """prepare the web deployment.""" self._config_keys.append("web") self.playbooks["web"] = self.cfg["web"]["config"].get("web_playbook") self.cfg["web"]["config"].setdefault("ansible_become_user", "root") self._prep_core() - self._prep_databrowser() databrowser_host = ( f'{self.cfg["databrowser"]["hosts"]}:' f'{self.cfg["databrowser"]["config"]["databrowser_port"]}' @@ -266,36 +287,47 @@ def _prep_web(self) -> None: except (FileNotFoundError, IOError, KeyError): pass try: - _webserver_items["IMPRINT"] = _webserver_items["IMPRINT"].split(",") + _webserver_items["IMPRINT"] = _webserver_items["IMPRINT"].split( + "," + ) except AttributeError: pass with self.web_conf_file.open("w") as f_obj: tomlkit.dump(_webserver_items, f_obj) for key in ("core", "web"): - self.cfg[key]["config"]["config_toml_file"] = str(self.web_conf_file) + self.cfg[key]["config"]["config_toml_file"] = str( + self.web_conf_file + ) if not self.master_pass: self.master_pass = get_passwd() - email_user, self.email_password = get_email_credentials() + if ask_pass: + email_user, self.email_password = get_email_credentials() + self.cfg["vault"]["config"]["email_user"] = email_user + self.cfg["vault"]["config"]["email_password"] = self.email_password self._prep_vault() - self.cfg["vault"]["config"]["ansible_python_interpreter"] = self.cfg["db"][ - "config" - ].get("ansible_python_interpreter", "/usr/bin/python3") - self.cfg["vault"]["config"]["email_user"] = email_user - self.cfg["vault"]["config"]["email_password"] = self.email_password + self.cfg["vault"]["config"]["ansible_python_interpreter"] = self.cfg[ + "db" + ]["config"].get("ansible_python_interpreter", "/usr/bin/python3") self.cfg["web"]["config"]["root_passwd"] = self.master_pass self.cfg["web"]["config"]["private_keyfile"] = self.private_key_file self.cfg["web"]["config"]["public_keyfile"] = self.public_key_file self.cfg["web"]["config"]["chain_keyfile"] = ( self.chain_key_file or self.public_key_file ) - self.cfg["web"]["config"]["apache_config_file"] = str(self.apache_config) - self._prep_apache_config() + self.cfg["web"]["config"]["apache_config_file"] = str( + self.apache_config + ) + if ask_pass: + self._prep_apache_config() def _prep_apache_config(self): config = [] with (Path(asset_dir) / "web" / "freva_web.conf").open() as f_obj: for line in f_obj.readlines(): - if not self.chain_key_file and "SSLCertificateChainFile" in line: + if ( + not self.chain_key_file + and "SSLCertificateChainFile" in line + ): continue config.append(line) with open(self.apache_config, "w") as f_obj: @@ -311,7 +343,9 @@ def _read_cfg(self) -> dict[str, Any]: try: return dict(load_config(self._inv_tmpl).items()) except FileNotFoundError as error: - raise FileNotFoundError(f"No such file {self._inv_tmpl}") from error + raise FileNotFoundError( + f"No such file {self._inv_tmpl}" + ) from error def _check_config(self) -> None: sections = [] @@ -321,8 +355,14 @@ def _check_config(self) -> None: sections.append(section) for section in sections: for key, value in self.cfg[section]["config"].items(): - if not value and not self._empty_ok and not isinstance(value, bool): - raise ValueError(f"{key} in {section} is empty in {self._inv_tmpl}") + if ( + not value + and not self._empty_ok + and not isinstance(value, bool) + ): + raise ValueError( + f"{key} in {section} is empty in {self._inv_tmpl}" + ) @property def _empty_ok(self) -> list[str]: @@ -348,15 +388,21 @@ def db_pass(self) -> str: num_chars, num_digits, num_punctuations = 20, 4, 4 num_chars -= num_digits + num_punctuations characters = [ - "".join([random.choice(string.ascii_letters) for i in range(num_chars)]), + "".join( + [random.choice(string.ascii_letters) for i in range(num_chars)] + ), "".join([random.choice(string.digits) for i in range(num_digits)]), - "".join([random.choice(punctuations) for i in range(num_punctuations)]), + "".join( + [random.choice(punctuations) for i in range(num_punctuations)] + ), ] str_characters = "".join(characters) _db_pass = "".join(random.sample(str_characters, len(str_characters))) while _db_pass.startswith("@"): # Vault treats values starting with "@" as file names. - _db_pass = "".join(random.sample(str_characters, len(str_characters))) + _db_pass = "".join( + random.sample(str_characters, len(str_characters)) + ) self._db_pass = _db_pass return self._db_pass @@ -396,13 +442,16 @@ def _set_additional_config_values( def parse_config(self) -> str: """Create config files for anisble and evaluation_system.conf.""" + playbook = self.create_playbooks() logger.info("Parsing configurations") self._check_config() config: dict[str, dict[str, dict[str, str | int | bool]]] = {} + info: dict[str, dict[str, dict[str, str | int | bool]]] = {} for step in set(self._config_keys): - config[step] = {} + config[step], info[step] = {}, {} config[step]["hosts"] = self.cfg[step]["hosts"] - config[step]["vars"] = {} + info[step]["hosts"] = self.cfg[step]["hosts"] + config[step]["vars"], info[step]["vars"] = {}, {} for key, value in self.cfg[step]["config"].items(): if key in ("root_passwd",) or key.startswith(step): new_key = key @@ -412,7 +461,19 @@ def parse_config(self) -> str: config[step]["vars"]["project_name"] = self.project_name # Add additional keys self._set_additional_config_values(step, config) - return yaml.dump(json.loads(json.dumps(config.copy()))) + for step in info: + for key, value in config[step]["vars"].items(): + if ( + f"{{{{{key}}}}}" in playbook + or f"{{{{ {key} }}}}" in playbook + ): + info[step]["vars"][key] = value + info_str = yaml.dump(json.loads(json.dumps(info))) + for passwd in (self.email_password, self.master_pass): + if passwd: + info_str = info_str.replace(passwd, "*" * len(passwd)) + RichConsole.print(info_str, style="bold", markup=True) + return yaml.dump(json.loads(json.dumps(config))) @property def _playbook_file(self) -> Path: @@ -443,7 +504,7 @@ def steps(self) -> list[str]: steps.append("vault") return [s for s in self.step_order if s in steps] - def create_playbooks(self): + def create_playbooks(self) -> str: """Create the ansible playbook form all steps.""" logger.info("Creating Ansible playbooks") playbook = [] @@ -455,8 +516,9 @@ def create_playbooks(self): ) with Path(playbook_file).open() as f_obj: playbook += yaml.safe_load(f_obj) - with self._playbook_file.open("w") as f_obj: - yaml.dump(playbook, f_obj) + with self._playbook_file.open("w") as stream: + yaml.dump(playbook, stream) + return self._playbook_file.read_text() def create_eval_config(self) -> None: """Create and dump the evaluation_systme.config.""" @@ -512,17 +574,10 @@ def play( verbosity: int, default: 0 Verbosity level, default 0 """ - self.create_playbooks() inventory = self.parse_config() self.create_eval_config() - print(self.inventory_file) with self.inventory_file.open("w") as f_obj: f_obj.write(inventory) - inventory_str = inventory - for passwd in (self.email_password, self.master_pass): - if passwd: - inventory_str = inventory_str.replace(passwd, "*" * len(passwd)) - RichConsole.print(inventory_str, style="bold", markup=True) logger.info("Playing the playbooks with ansible") RichConsole.print( "[b]Note:[/] The [blue]BECOME[/] password refers to the " From 32b54c7d3cb4f82f29882ea8d295cad1f16ace67 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Mon, 11 Sep 2023 07:15:17 +0200 Subject: [PATCH 08/20] Fix linting issues. --- .github/workflows/ci_job.yml | 2 +- src/freva_deployment/deploy.py | 4 +--- src/freva_deployment/ui/deployment_tui/main_window.py | 2 +- src/freva_deployment/utils.py | 9 ++++++--- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci_job.yml b/.github/workflows/ci_job.yml index 7cdf070a..0b2b044f 100644 --- a/.github/workflows/ci_job.yml +++ b/.github/workflows/ci_job.yml @@ -24,7 +24,7 @@ jobs: mkdir .mypy_cache - name: formating - run: black --check -t py311 src + run: isort --check --profile black -t py311 -l 79 src - name: type checking run: mypy --install-types --non-interactive diff --git a/src/freva_deployment/deploy.py b/src/freva_deployment/deploy.py index 5b2e960f..b27d2285 100644 --- a/src/freva_deployment/deploy.py +++ b/src/freva_deployment/deploy.py @@ -1,5 +1,6 @@ """Module to run the freva deployment.""" from __future__ import annotations + import json import os import random @@ -72,7 +73,6 @@ def __init__( self._inv_tmpl = Path(config_file or config_dir / "inventory.toml") self._cfg_tmpl = self.aux_dir / "evaluation_system.conf.tmpl" self.cfg = self._read_cfg() - self.info = {} self.project_name = self.cfg.pop("project_name", None) self.playbooks: dict[str, Path | None] = {} if not self.project_name: @@ -121,7 +121,6 @@ def _prep_vault(self) -> None: self.cfg["vault"]["config"]["email"] = self.cfg["web"]["config"].get( "contacts", "" ) - self.info["vault"] = self.cfg["vault"]["config"] def _prep_db(self) -> None: """prepare the mariadb service.""" @@ -170,7 +169,6 @@ def _prep_databrowser(self, prep_web=True) -> None: self.cfg["databrowser"]["config"]["email"] = self.cfg["web"][ "config" ].get("contacts", "") - self.info["databrowser"] = self.cfg["databrowser"]["config"] if prep_web: self._prep_web(False) diff --git a/src/freva_deployment/ui/deployment_tui/main_window.py b/src/freva_deployment/ui/deployment_tui/main_window.py index 376fda37..f76cbcde 100644 --- a/src/freva_deployment/ui/deployment_tui/main_window.py +++ b/src/freva_deployment/ui/deployment_tui/main_window.py @@ -17,9 +17,9 @@ from .base import BaseForm, VarForm, selectFile from .deploy_forms import ( CoreScreen, + DatabrowserScreen, DBScreen, RunForm, - DatabrowserScreen, WebScreen, ) diff --git a/src/freva_deployment/utils.py b/src/freva_deployment/utils.py index c49f10a8..19434ff1 100644 --- a/src/freva_deployment/utils.py +++ b/src/freva_deployment/utils.py @@ -9,15 +9,15 @@ import sys import warnings from pathlib import Path -from typing import Any, MutableMapping, NamedTuple, cast +from typing import Any, Dict, MutableMapping, NamedTuple, Union, cast import appdirs import pkg_resources import requests import tomlkit from rich.console import Console -from rich.prompt import Prompt from rich.logging import RichHandler +from rich.prompt import Prompt logger_stream_handle = RichHandler( rich_tracebacks=False, @@ -169,7 +169,10 @@ def _update_config( def _create_new_config(inp_file: Path) -> Path: """Update any old configuration file to a newer version.""" - config = tomlkit.loads(inp_file.read_text()) + config = cast( + Dict[str, Dict[str, Dict[str, Union[str, float, int, bool]]]], + tomlkit.loads(inp_file.read_text()), + ) config_tmpl = tomlkit.loads(AD.inventory_file.read_text()) # Legacy solr: if "solr" in config: From 668bf04fccd434921205db9123bcc172d7c8bcdc Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Mon, 11 Sep 2023 09:46:04 +0200 Subject: [PATCH 09/20] update tui --- .../ui/deployment_tui/deploy_forms.py | 50 +++++++++++++------ .../ui/deployment_tui/main_window.py | 23 +++++---- src/freva_deployment/utils.py | 6 +++ 3 files changed, 54 insertions(+), 25 deletions(-) diff --git a/src/freva_deployment/ui/deployment_tui/deploy_forms.py b/src/freva_deployment/ui/deployment_tui/deploy_forms.py index 41bd1e64..efabe5c0 100644 --- a/src/freva_deployment/ui/deployment_tui/deploy_forms.py +++ b/src/freva_deployment/ui/deployment_tui/deploy_forms.py @@ -85,7 +85,9 @@ def _add_widgets(self) -> None: max_height=2, value=cfg.get("install", True), editable=True, - name=(f"{self.num}Install a new Freva anaconda environment?"), + name=( + f"{self.num}Install a new Freva anaconda environment?" + ), scroll_exit=True, ), True, @@ -121,7 +123,9 @@ def _add_widgets(self) -> None: self.add_widget_intelligent( npyscreen.TitleCombo, name=f"{self.num}Workload manger", - value=self.scheduler_index(cast(str, cfg.get("scheduler_system"))), + value=self.scheduler_index( + cast(str, cfg.get("scheduler_system")) + ), values=self.scheduler_systems, ), True, @@ -204,7 +208,9 @@ def _add_widgets(self) -> None: self.add_widget_intelligent( npyscreen.TitleFilename, name=f"{self.num}Python path on remote machine:", - value=cfg.get("ansible_python_interpreter", "/usr/bin/python3"), + value=cfg.get( + "ansible_python_interpreter", "/usr/bin/python3" + ), ), False, ), @@ -376,7 +382,9 @@ def _add_widgets(self) -> None: self.add_widget_intelligent( npyscreen.TitleText, name=f"{self.num}A brief describtion of the project:", - value=cfg.get("homepage_heading", "Lorem ipsum dolor sit amet"), + value=cfg.get( + "homepage_heading", "Lorem ipsum dolor sit amet" + ), ), True, ), @@ -541,7 +549,9 @@ def _add_widgets(self) -> None: ldap_model=( self.add_widget_intelligent( npyscreen.TitleText, - name=(f"{self.num}Ldap tools class to be used for authentication."), + name=( + f"{self.num}Ldap tools class to be used for authentication." + ), value=cfg.get("ldap_model", "MiklipUserInformation"), ), True, @@ -572,7 +582,9 @@ def _add_widgets(self) -> None: self.add_widget_intelligent( npyscreen.TitleFilename, name=f"{self.num}Pythonpath on remote machine:", - value=cfg.get("ansible_python_interpreter", "/usr/bin/python3"), + value=cfg.get( + "ansible_python_interpreter", "/usr/bin/python3" + ), ), False, ), @@ -684,7 +696,9 @@ def _add_widgets(self) -> None: self.add_widget_intelligent( npyscreen.TitleFilename, name=f"{self.num}Pythonpath on remote machine:", - value=cfg.get("ansible_python_interpreter", "/usr/bin/python3"), + value=cfg.get( + "ansible_python_interpreter", "/usr/bin/python3" + ), ), False, ), @@ -791,7 +805,9 @@ def _add_widgets(self) -> None: self.add_widget_intelligent( npyscreen.TitleFilename, name=f"{self.num}Pythonpath on remote machine:", - value=cfg.get("ansible_python_interpreter", "/usr/bin/python3"), + value=cfg.get( + "ansible_python_interpreter", "/usr/bin/python3" + ), ), False, ), @@ -822,7 +838,9 @@ def on_ok(self) -> None: self.parentApp.thread_stop.set() if not self.project_name.value: - npyscreen.notify_confirm("You have to set a project name", title="ERROR") + npyscreen.notify_confirm( + "You have to set a project name", title="ERROR" + ) return if not self.server_map.value: value = npyscreen.notify_yes_no( @@ -845,7 +863,9 @@ def on_ok(self) -> None: self.parentApp.get_save_file(self.inventory_file.value or None) ) for key_type, keyfile in cert_files.items(): - key_file = Path(get_current_file_dir(save_file.parent, str(keyfile))) + key_file = Path( + get_current_file_dir(save_file.parent, str(keyfile)) + ) for step, deploy_form in self.parentApp._forms.items(): if not keyfile or not Path(key_file).is_file(): if ( @@ -853,11 +873,11 @@ def on_ok(self) -> None: and step in self.parentApp.steps ): if keyfile: + msg = f"{key_type} certificate file `{key_file}` must exist." + else: msg = ( - f"{key_type} certificate file `{key_file}` must exist." + f"You must give a {key_type} certificate file." ) - else: - msg = f"You must give a {key_type} certificate file." npyscreen.notify_confirm(msg, title="ERROR") return @@ -920,7 +940,9 @@ def _add_widgets(self) -> None: ) self.server_map = self.add_widget_intelligent( npyscreen.TitleText, - name=(f"{self.num}Hostname of the service mapping the freva server arch."), + name=( + f"{self.num}Hostname of the service mapping the freva server arch." + ), value=self.parentApp._read_cache("server_map", ""), ) self.public_keyfile = self.add_widget_intelligent( diff --git a/src/freva_deployment/ui/deployment_tui/main_window.py b/src/freva_deployment/ui/deployment_tui/main_window.py index f76cbcde..31acaf03 100644 --- a/src/freva_deployment/ui/deployment_tui/main_window.py +++ b/src/freva_deployment/ui/deployment_tui/main_window.py @@ -15,13 +15,7 @@ from freva_deployment.utils import asset_dir, config_dir, load_config from .base import BaseForm, VarForm, selectFile -from .deploy_forms import ( - CoreScreen, - DatabrowserScreen, - DBScreen, - RunForm, - WebScreen, -) +from .deploy_forms import CoreScreen, DatabrowserScreen, DBScreen, RunForm, WebScreen class MainApp(npyscreen.NPSAppManaged): @@ -76,12 +70,18 @@ def _add_froms(self) -> None: CoreScreen, name="Core deployment", ) - self._forms["web"] = self.addForm("SECOND", WebScreen, name="Web deployment") - self._forms["db"] = self.addForm("THIRD", DBScreen, name="Database deployment") + self._forms["web"] = self.addForm( + "SECOND", WebScreen, name="Web deployment" + ) + self._forms["db"] = self.addForm( + "THIRD", DBScreen, name="Database deployment" + ) self._forms["databrowser"] = self.addForm( "FOURTH", DatabrowserScreen, name="Databrowser deployment" ) - self._setup_form = self.addForm("SETUP", RunForm, name="Apply the Deployment") + self._setup_form = self.addForm( + "SETUP", RunForm, name="Apply the Deployment" + ) def exit_application(self, *args, **kwargs) -> None: value = npyscreen.notify_ok_cancel( @@ -253,7 +253,8 @@ def _read_cache( def _steps(self) -> list[str]: """Read the deployment-steps from the cache.""" return cast( - List[str], self._read_cache("steps", ["core", "web", "db", "databrowser"]) + List[str], + self._read_cache("steps", ["core", "web", "db", "databrowser"]), ) def read_cert_file(self, key: str) -> str: diff --git a/src/freva_deployment/utils.py b/src/freva_deployment/utils.py index 19434ff1..3f7bcce8 100644 --- a/src/freva_deployment/utils.py +++ b/src/freva_deployment/utils.py @@ -173,9 +173,11 @@ def _create_new_config(inp_file: Path) -> Path: Dict[str, Dict[str, Dict[str, Union[str, float, int, bool]]]], tomlkit.loads(inp_file.read_text()), ) + create_backup = False config_tmpl = tomlkit.loads(AD.inventory_file.read_text()) # Legacy solr: if "solr" in config: + create_backup = True config["databrowser"] = config.pop("solr") for key in ("port", "mem"): if key in config["databrowser"]["config"]: @@ -183,6 +185,10 @@ def _create_new_config(inp_file: Path) -> Path: "databrowser" ]["config"].pop(key) _update_config(config_tmpl, config) + if create_backup: + inp_file.with_suffix(inp_file.suffix + ".bck").write_text( + inp_file.read_text() + ) inp_file.write_text(tomlkit.dumps(config_tmpl)) return inp_file From 2d2d5bcede87cf502a2c438e057c50330c923ae2 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Mon, 11 Sep 2023 11:55:33 +0200 Subject: [PATCH 10/20] Update docs. --- docs/Installation.md | 26 ++++++++++++++++---------- docs/environment.yml | 2 ++ docs/index.rst | 5 +++++ docs/whatsnew.rst | 19 +++++++++++++++++++ src/freva_deployment/__init__.py | 2 +- 5 files changed, 43 insertions(+), 11 deletions(-) create mode 100644 docs/whatsnew.rst diff --git a/docs/Installation.md b/docs/Installation.md index 6512bba8..4c58ad52 100644 --- a/docs/Installation.md +++ b/docs/Installation.md @@ -4,9 +4,13 @@ The freva-deployment is used to deploy Freva in different computing environments The general strategy is to split the deployment into different steps, these are : - Deploy MariaDB service via docker - Deploy a HashiCorp Vault service for storing and retrieving passwords and other sensitive data via docker (this step get automatically activated once the MariaDB service is set) -- Deploy Apache Solr service via docker -- Deploy command line interface backend ([evaluation_system](https://github.com/FREVA-CLINT/freva)) -- Deploy web front end ([freva_web](https://gitlab.dkrz.de/freva/freva_web)) +- Deploy the [DatabrowserAPI](https://github.com/FREVA-CLINT/databrowserAPI) + The databrowser API deployment consists of three parts: + - The actual databrowser rest API + - Apache solr search backend + - Mongodb to store search statistics +- Deploy command line interface and python library ([freva](https://github.com/FREVA-CLINT/freva)) +- Deploy web front end ([freva_web](https://github.com/FREVA-CLINT/freva-web)) The web front end deployment is sub divided into three parts: - Deployment of the django web application - Deployment of a redis instance acting as database cache @@ -80,13 +84,15 @@ export PATH=$PATH:$HOME/.local/bin ``` -## Installing docker/podman and sudo access to the service servers -Since the services of MariaDB, Apache Solr and Apache httpd will be deployed on -docker container images, docker needs to be available on the target servers. -Usually installing and running docker requires *root* privileges. -Hence, on the servers that will be running docker you will need root access. -There exists an option to install and run docker without root, -information on a root-less docker option +## Installing docker-compose/podman-compose and sudo access to the service servers +Because the services of MariaDB, DatabrowserAPI and Apache httpd will be deployed +on docker container images, docker needs to be available on the target servers. +Since version *v2309.0.0* of the deployment the containers are set up +using `doker-compose`. Hence `docker-compose` (or `podman-compose`) has to be +installed on the host systems. Usually installing and running docker +requires *root* privileges. Hence, on the servers that will be running docker +you will need root access. There exists an option to install and run docker +without root, information on a root-less docker option can be found [on the docker docs](https://docs.docker.com/engine/security/rootless/) > **_Note:_** Some systems use `podman` instead of `docker`. The deployment routine is able to distinguish and use the right service. diff --git a/docs/environment.yml b/docs/environment.yml index 6ee35bb7..aebdedbd 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -9,6 +9,8 @@ dependencies: - freva-deployment[docs] - sphinx-execute-code-python3 - sphinx-rtd-theme + - sphinx-copybutton + - pydata-sphinx-theme - sphinxcontrib_github_alt - nbsphinx - furo diff --git a/docs/index.rst b/docs/index.rst index 6f865609..034b49a4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,6 +10,10 @@ Welcome to freva admin documentation! :width: 320 :align: center +.. note:: + + Please check :ref:`whatsnew` for any update announcments. + Freva, the free evaluation system framework, is a data search and analysis @@ -62,6 +66,7 @@ basic deployment setup and servicing Freva will be introduced. Transition LegalNotes modules + whatsnew diff --git a/docs/whatsnew.rst b/docs/whatsnew.rst new file mode 100644 index 00000000..b3c508fc --- /dev/null +++ b/docs/whatsnew.rst @@ -0,0 +1,19 @@ +.. _whatsnew: +What's new +=========== + +.. toctree:: + :maxdepth: 0 + :titlesonly: + +v2309.0.0 +~~~~~~~~~ + +* All containers are created with `docker-compose` or `podman-compose` in order to be able to successfully deploy + the containers you will have to install `docker-compose` or `podman-compose` on + the host machines running the containers. +* With v2309 comes a new configuration file. If you are using the tui please + just load your old config file. The code will update this configuration file. + If you are using the `deploy-freva-cmd` command you will not have to do anything. + The code automatically update your config file to the new config file. A backup + with the suffix (`.bck`) will be created. diff --git a/src/freva_deployment/__init__.py b/src/freva_deployment/__init__.py index faf5c0dd..8e8f4440 100644 --- a/src/freva_deployment/__init__.py +++ b/src/freva_deployment/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2308.2.2" +__version__ = "2309.0.0" AVAILABLE_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] AVAILABLE_CONDA_ARCHS = [ "Linux-x86_64", From c09c6cb0d97f511204e6f013d511ffe45963552f Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Mon, 11 Sep 2023 12:03:48 +0200 Subject: [PATCH 11/20] Update README --- README.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 88f3b1c1..0f600805 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,9 @@ environments. The general strategy is to split the deployment into - Deploy a Hashicorp Vault service for storing and retrieving passwords and other sensitive data via docker (this step get automatically activated once the MariaDB service is set) -- Deploy Apache Solr service via docker +- Deploy [Databrowser API](https://github.com/FREVA-CLINT/databrowserAPI) service via docker - Deploy command line interface backend ([evaluation_system](https://github.com/FREVA-CLINT/freva)) -- Deploy web front end ([freva_web](https://gitlab.dkrz.de/freva/freva_web)) +- Deploy web front end ([freva_web](https://github.com/FREVA-CLINT/freva-web)) > **_Note:_** A vault server is auto deployed once the mariadb server is deployed. @@ -96,13 +96,15 @@ export PATH=$PATH:$HOME/.local/bin ``` -## Installing docker/podman and sudo access to the service servers -Since the services of MariaDB, Apache Solr and Apache web will be deployed on -docker container images, docker needs to be available on the target servers. -Usually installing and running docker requires *root* privileges. -Hence, on the servers that will be running docker you will need root access. -There exist an option to install and run docker without root, -information on a root-less docker option +## Installing docker-compose/podman-compose and sudo access to the service servers +Because the services of MariaDB, DatabrowserAPI and Apache httpd will be deployed +on docker container images, docker needs to be available on the target servers. +Since version *v2309.0.0* of the deployment the containers are set up +using `doker-compose`. Hence `docker-compose` (or `podman-compose`) has to be +installed on the host systems. Usually installing and running docker +requires *root* privileges. Hence, on the servers that will be running docker +you will need root access. There exists an option to install and run docker +without root, information on a root-less docker option can be found [on the docker docs](https://docs.docker.com/engine/security/rootless/) > **_Note:_** Some systems use `podman` instead of `docker`. The deployment routine is able to distinguish and use the right service. From 6a246148bcf2428c799b3fa952b5d53ab7ed7d3e Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Mon, 11 Sep 2023 12:11:25 +0200 Subject: [PATCH 12/20] Fix linting issues. --- src/freva_deployment/ui/deployment_tui/main_window.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/freva_deployment/ui/deployment_tui/main_window.py b/src/freva_deployment/ui/deployment_tui/main_window.py index 31acaf03..6136c7cd 100644 --- a/src/freva_deployment/ui/deployment_tui/main_window.py +++ b/src/freva_deployment/ui/deployment_tui/main_window.py @@ -15,7 +15,13 @@ from freva_deployment.utils import asset_dir, config_dir, load_config from .base import BaseForm, VarForm, selectFile -from .deploy_forms import CoreScreen, DatabrowserScreen, DBScreen, RunForm, WebScreen +from .deploy_forms import ( + CoreScreen, + DatabrowserScreen, + DBScreen, + RunForm, + WebScreen, +) class MainApp(npyscreen.NPSAppManaged): From 318d946cde0b88bb67eabe2cf295ddbb24ed2d0c Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Mon, 11 Sep 2023 12:14:12 +0200 Subject: [PATCH 13/20] Fix linting issues. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2678a475..b2dd59e3 100644 --- a/setup.py +++ b/setup.py @@ -180,7 +180,7 @@ def get_data_files() -> List[Tuple[str, List[str]]]: "ipython", # For nbsphinx syntax highlighting "sphinxcontrib_github_alt", ], - "test": ["mypy", "black", "types-toml", "types-PyMySQL"], + "test": ["mypy", "isort", "black", "types-toml", "types-PyMySQL"], }, python_requires=">=3.8", classifiers=[ From 06674502873a5ab92d85fdfc7f0c1396365e1481 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Mon, 11 Sep 2023 14:34:21 +0200 Subject: [PATCH 14/20] update tui --- assets/playbooks/db-server-playbook.yml | 1 + assets/playbooks/web-server-playbook.yml | 2 +- assets/scripts/create_systemd.py | 2 +- src/freva_deployment/deploy.py | 98 +++++++----------------- 4 files changed, 32 insertions(+), 71 deletions(-) diff --git a/assets/playbooks/db-server-playbook.yml b/assets/playbooks/db-server-playbook.yml index 8c69e495..034f46d1 100644 --- a/assets/playbooks/db-server-playbook.yml +++ b/assets/playbooks/db-server-playbook.yml @@ -7,6 +7,7 @@ db_volumes: - /opt/freva/freva-service-config/mysql/create_tables.sql:/docker-entrypoint-initdb.d/002_create_tables.sql:z - /opt/freva/freva-service-config/mysql/daily_backup.sh:/usr/local/bin/daily_backup:z + - /opt/freva/{{project_name}}/db_service:/var/lib/mysql:z docker_cmd: > --network {{ project_name }} -v /opt/freva/{{project_name}}/db_service:/var/lib/mysql:z diff --git a/assets/playbooks/web-server-playbook.yml b/assets/playbooks/web-server-playbook.yml index 01781088..431e4d46 100644 --- a/assets/playbooks/web-server-playbook.yml +++ b/assets/playbooks/web-server-playbook.yml @@ -75,7 +75,7 @@ - "/opt/freva/{{project_name}}/web_service/static:/srv/static:z" - "/opt/freva/{{project_name}}/web_service/prepare-httpd:/usr/local/bin/prepare-httpd:z" - "{{ core_preview_path }}:/srv/static/preview:z" - docker_apache_cmd: > + docker_apache_cmd: > --rm -e SCHEDULER_DIR={{core_scheduler_output_dir}} -e UID=$(id -u {{ansible_user}}) diff --git a/assets/scripts/create_systemd.py b/assets/scripts/create_systemd.py index 756b3603..a212a440 100644 --- a/assets/scripts/create_systemd.py +++ b/assets/scripts/create_systemd.py @@ -91,7 +91,7 @@ def get_container_cmd(args: str) -> Tuple[str, str]: ) out = res.stdout.decode().split() if out: - return out[0], " ".join(out[1:]).replace("%", "%%") + return out[0], " ".join(out[1:]) return "", "" diff --git a/src/freva_deployment/deploy.py b/src/freva_deployment/deploy.py index b27d2285..11dc44e1 100644 --- a/src/freva_deployment/deploy.py +++ b/src/freva_deployment/deploy.py @@ -59,13 +59,11 @@ def __init__( config_file: Path | str | None = None, ) -> None: self._config_keys: list[str] = [] - self.master_pass: str = "Freva4all!" + self.master_pass: str = "" self.email_password: str = "" self._td: TemporaryDirectory = TemporaryDirectory(prefix="inventory") self.inventory_file: Path = Path(self._td.name) / "inventory.yaml" - self.eval_conf_file: Path = ( - Path(self._td.name) / "evaluation_system.conf" - ) + self.eval_conf_file: Path = Path(self._td.name) / "evaluation_system.conf" self.web_conf_file: Path = Path(self._td.name) / "freva_web.toml" self.apache_config: Path = Path(self._td.name) / "freva_web.conf" self._db_pass: str = "" @@ -110,9 +108,7 @@ def _prep_vault(self) -> None: self._config_keys.append("vault") self.cfg["vault"] = self.cfg["db"].copy() self.cfg["vault"]["config"].setdefault("ansible_become_user", "root") - self.playbooks["vault"] = self.cfg["db"]["config"].get( - "vault_playbook" - ) + self.playbooks["vault"] = self.cfg["db"]["config"].get("vault_playbook") if not self.master_pass: self.master_pass = get_passwd() self.cfg["vault"]["config"]["root_passwd"] = self.master_pass @@ -133,9 +129,7 @@ def _prep_db(self) -> None: self.cfg["db"]["config"]["passwd"] = self.db_pass self.cfg["db"]["config"]["keyfile"] = self.public_key_file for key in ("name", "user", "db"): - self.cfg["db"]["config"][key] = ( - self.cfg["db"]["config"].get(key) or "freva" - ) + self.cfg["db"]["config"][key] = self.cfg["db"]["config"].get(key) or "freva" db_host = self.cfg["db"]["config"].get("host", "") if not db_host: self.cfg["db"]["config"]["host"] = host @@ -152,9 +146,7 @@ def _prep_databrowser(self, prep_web=True) -> None: if not self.master_pass: self.master_pass = get_passwd() self._config_keys.append("databrowser") - self.cfg["databrowser"]["config"].setdefault( - "ansible_become_user", "root" - ) + self.cfg["databrowser"]["config"].setdefault("ansible_become_user", "root") self.cfg["databrowser"]["config"]["root_passwd"] = self.master_pass self.cfg["databrowser"]["config"].pop("core", None) self.playbooks["databrowser"] = self.cfg["databrowser"]["config"].get( @@ -166,9 +158,9 @@ def _prep_databrowser(self, prep_web=True) -> None: self.cfg["databrowser"]["config"][key] = ( self.cfg["databrowser"]["config"].get(key) or default ) - self.cfg["databrowser"]["config"]["email"] = self.cfg["web"][ - "config" - ].get("contacts", "") + self.cfg["databrowser"]["config"]["email"] = self.cfg["web"]["config"].get( + "contacts", "" + ) if prep_web: self._prep_web(False) @@ -176,9 +168,7 @@ def _prep_core(self) -> None: """prepare the core deployment.""" self._config_keys.append("core") self.cfg["core"]["config"].setdefault("ansible_become_user", "") - self.playbooks["core"] = self.cfg["core"]["config"].get( - "core_playbook" - ) + self.playbooks["core"] = self.cfg["core"]["config"].get("core_playbook") self.cfg["core"]["config"]["admins"] = ( self.cfg["core"]["config"].get("admins") or getuser() ) @@ -189,15 +179,11 @@ def _prep_core(self) -> None: if not root_dir: self.cfg["core"]["config"]["root_dir"] = install_dir preview_path = self.cfg["core"]["config"].get("preview_path", "") - base_dir_location = self.cfg["core"]["config"].get( - "base_dir_location", "" - ) + base_dir_location = self.cfg["core"]["config"].get("base_dir_location", "") scheduler_output_dir = self.cfg["core"]["config"].get( "scheduler_output_dir", "" ) - scheduler_system = self.cfg["core"]["config"].get( - "scheduler_system", "local" - ) + scheduler_system = self.cfg["core"]["config"].get("scheduler_system", "local") if not preview_path: if base_dir_location: self.cfg["core"]["config"]["preview_path"] = str( @@ -208,9 +194,7 @@ def _prep_core(self) -> None: if not scheduler_output_dir: scheduler_output_dir = str(Path(base_dir_location) / "share") scheduler_output_dir = Path(scheduler_output_dir) / scheduler_system - self.cfg["core"]["config"]["scheduler_output_dir"] = str( - scheduler_output_dir - ) + self.cfg["core"]["config"]["scheduler_output_dir"] = str(scheduler_output_dir) self.cfg["core"]["config"]["keyfile"] = self.public_key_file git_exe = self.cfg["core"]["config"].get("git_path") self.cfg["core"]["config"]["git_path"] = git_exe or "git" @@ -221,6 +205,8 @@ def _prep_core(self) -> None: def _prep_web(self, ask_pass: bool = True) -> None: """prepare the web deployment.""" self._config_keys.append("web") + # if ask_pass: + # self._prep_vault() self.playbooks["web"] = self.cfg["web"]["config"].get("web_playbook") self.cfg["web"]["config"].setdefault("ansible_become_user", "root") self._prep_core() @@ -285,36 +271,30 @@ def _prep_web(self, ask_pass: bool = True) -> None: except (FileNotFoundError, IOError, KeyError): pass try: - _webserver_items["IMPRINT"] = _webserver_items["IMPRINT"].split( - "," - ) + _webserver_items["IMPRINT"] = _webserver_items["IMPRINT"].split(",") except AttributeError: pass with self.web_conf_file.open("w") as f_obj: tomlkit.dump(_webserver_items, f_obj) for key in ("core", "web"): - self.cfg[key]["config"]["config_toml_file"] = str( - self.web_conf_file - ) + self.cfg[key]["config"]["config_toml_file"] = str(self.web_conf_file) if not self.master_pass: self.master_pass = get_passwd() + self._prep_vault() if ask_pass: email_user, self.email_password = get_email_credentials() self.cfg["vault"]["config"]["email_user"] = email_user self.cfg["vault"]["config"]["email_password"] = self.email_password - self._prep_vault() - self.cfg["vault"]["config"]["ansible_python_interpreter"] = self.cfg[ - "db" - ]["config"].get("ansible_python_interpreter", "/usr/bin/python3") + self.cfg["vault"]["config"]["ansible_python_interpreter"] = self.cfg["db"][ + "config" + ].get("ansible_python_interpreter", "/usr/bin/python3") self.cfg["web"]["config"]["root_passwd"] = self.master_pass self.cfg["web"]["config"]["private_keyfile"] = self.private_key_file self.cfg["web"]["config"]["public_keyfile"] = self.public_key_file self.cfg["web"]["config"]["chain_keyfile"] = ( self.chain_key_file or self.public_key_file ) - self.cfg["web"]["config"]["apache_config_file"] = str( - self.apache_config - ) + self.cfg["web"]["config"]["apache_config_file"] = str(self.apache_config) if ask_pass: self._prep_apache_config() @@ -322,10 +302,7 @@ def _prep_apache_config(self): config = [] with (Path(asset_dir) / "web" / "freva_web.conf").open() as f_obj: for line in f_obj.readlines(): - if ( - not self.chain_key_file - and "SSLCertificateChainFile" in line - ): + if not self.chain_key_file and "SSLCertificateChainFile" in line: continue config.append(line) with open(self.apache_config, "w") as f_obj: @@ -341,9 +318,7 @@ def _read_cfg(self) -> dict[str, Any]: try: return dict(load_config(self._inv_tmpl).items()) except FileNotFoundError as error: - raise FileNotFoundError( - f"No such file {self._inv_tmpl}" - ) from error + raise FileNotFoundError(f"No such file {self._inv_tmpl}") from error def _check_config(self) -> None: sections = [] @@ -353,14 +328,8 @@ def _check_config(self) -> None: sections.append(section) for section in sections: for key, value in self.cfg[section]["config"].items(): - if ( - not value - and not self._empty_ok - and not isinstance(value, bool) - ): - raise ValueError( - f"{key} in {section} is empty in {self._inv_tmpl}" - ) + if not value and not self._empty_ok and not isinstance(value, bool): + raise ValueError(f"{key} in {section} is empty in {self._inv_tmpl}") @property def _empty_ok(self) -> list[str]: @@ -386,21 +355,15 @@ def db_pass(self) -> str: num_chars, num_digits, num_punctuations = 20, 4, 4 num_chars -= num_digits + num_punctuations characters = [ - "".join( - [random.choice(string.ascii_letters) for i in range(num_chars)] - ), + "".join([random.choice(string.ascii_letters) for i in range(num_chars)]), "".join([random.choice(string.digits) for i in range(num_digits)]), - "".join( - [random.choice(punctuations) for i in range(num_punctuations)] - ), + "".join([random.choice(punctuations) for i in range(num_punctuations)]), ] str_characters = "".join(characters) _db_pass = "".join(random.sample(str_characters, len(str_characters))) while _db_pass.startswith("@"): # Vault treats values starting with "@" as file names. - _db_pass = "".join( - random.sample(str_characters, len(str_characters)) - ) + _db_pass = "".join(random.sample(str_characters, len(str_characters))) self._db_pass = _db_pass return self._db_pass @@ -461,10 +424,7 @@ def parse_config(self) -> str: self._set_additional_config_values(step, config) for step in info: for key, value in config[step]["vars"].items(): - if ( - f"{{{{{key}}}}}" in playbook - or f"{{{{ {key} }}}}" in playbook - ): + if f"{{{{{key}}}}}" in playbook or f"{{{{ {key} }}}}" in playbook: info[step]["vars"][key] = value info_str = yaml.dump(json.loads(json.dumps(info))) for passwd in (self.email_password, self.master_pass): From 61e863554652a29ae976981f7505d58bfdfeacc3 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Thu, 14 Sep 2023 12:12:49 +0200 Subject: [PATCH 15/20] delete setup.py --- setup.py | 195 ------------------------------------------------------- 1 file changed, 195 deletions(-) delete mode 100644 setup.py diff --git a/setup.py b/setup.py deleted file mode 100644 index b2dd59e3..00000000 --- a/setup.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python3 -"""Setup script for packaging freva deployment.""" - -from pathlib import Path -import re -import urllib.request -from setuptools import setup, find_packages -from setuptools.command.develop import develop -from setuptools.command.install import install -import sys -from typing import List, Tuple -import urllib.request - - -THIS_DIR = Path(__file__).parent -CONFIG_DIR = Path("freva") / "deployment" -ASSET_DIR = THIS_DIR / "assets" - -if not Path("appdirs.py").is_file(): - url = "https://raw.githubusercontent.com/ActiveState/appdirs/master/appdirs.py" - urllib.request.urlretrieve(url, "appdirs.py") - -import appdirs - -Path("appdirs.py").unlink() - -INSTALL_REQUIRES = [ - "appdirs", - "npyscreen", - "numpy", - "PyMySQL", - "PyYAML", - "rich", - "toml", - "tomlkit", - "requests", -] - - -def find_version(*parts): - vers_file = read(*parts) - match = re.search(r'^__version__ = "(\d+.\d+.\d+)"', vers_file, re.M) - if match is not None: - return match.group(1) - raise RuntimeError("Unable to find version string.") - - -def download_assets() -> None: - """Download additional assets.""" - assets = [ - ( - ASSET_DIR / "config" / "evaluation_system.conf.tmpl", - "https://raw.githubusercontent.com/FREVA-CLINT/freva/main/assets/evaluation_system.conf", - ) - ] - for path, url in assets: - urllib.request.urlretrieve(url, filename=str(path)) - - -def prepare_config(develop_cmd: bool = False) -> None: - """Create the freva config dir in loacal user directory.""" - import appdirs - - download_assets() - user_config_dir = Path(appdirs.user_config_dir()) / CONFIG_DIR - user_data_dir = Path(appdirs.user_data_dir()) / CONFIG_DIR - for cfg_path in (user_config_dir, user_data_dir): - cfg_path.mkdir(exist_ok=True, parents=True) - inventory_file = user_config_dir / "inventory.toml" - inventory_asset = ASSET_DIR / "config" / "inventory.toml" - with inventory_asset.open() as f: - if inventory_file.is_symlink(): - inventory_file.unlink() - if not inventory_file.exists() and develop_cmd is False: - with inventory_file.open("w") as w: - w.write(f.read()) - elif develop_cmd is True: - inventory_file.unlink(missing_ok=True) - inventory_file.symlink_to(inventory_asset) - for datafile in ASSET_DIR.rglob("*"): - new_path = user_data_dir / datafile.relative_to(ASSET_DIR) - if datafile.is_dir(): - continue - new_path.parent.mkdir(exist_ok=True, parents=True) - with datafile.open() as f: - new_path.unlink(missing_ok=True) - if develop_cmd is False: - with new_path.open("w") as w: - w.write(f.read()) - else: - new_path.symlink_to(datafile) - - -class InstallCommand(install): - """Customized setuptools install command.""" - - def run(self): - install.run(self) - prepare_config(develop_cmd=False) - - -class DevelopCommand(develop): - """Customized setuptools install command.""" - - def run(self): - develop.run(self) - prepare_config(develop_cmd=True) - - -def read(*parts: str) -> str: - """Read the content of a file.""" - with THIS_DIR.joinpath(*parts).open() as f: - return f.read() - - -def get_packages() -> List[str]: - """Get the packages needed to install.""" - plf = sys.platform - if plf.startswith("win") or plf.startswith("cygwin") or plf.startswith("ms"): - return INSTALL_REQUIRES - INSTALL_REQUIRES.append("ansible>=2.10") - return INSTALL_REQUIRES - - -def get_data_files() -> List[Tuple[str, List[str]]]: - dirs = [d for d in ASSET_DIR.rglob("*") if d.is_dir()] - data_files = [] - for d in dirs: - target_dir = Path("freva") / "deployment" / d.relative_to(ASSET_DIR) - add_files = [str(f.relative_to(THIS_DIR)) for f in d.rglob("*") if f.is_file()] - if add_files: - data_files.append((str(target_dir), add_files)) - prepare_config() - return data_files - - -setup( - name="freva_deployment", - version=find_version("src", "freva_deployment", "__init__.py"), - author="Martin Bergemann", - author_email="martin.bergemann@dkrz.de", - maintainer="Martin Bergemann", - url="https://github.com/FREVA-CLINT/freva.git", - description="Deploy freva and its services on different machines.", - long_description=read("README.md"), - long_description_content_type="text/markdown", - include_package_data=True, - data_files=get_data_files(), - license="GPLv3", - project_urls={ - "Documentation": "https://freva-deployment.readthedocs.io/en/latest/", - "Issues": "https://github.com/FREVA-CLINT/freva/issues", - "Source": "https://github.com/FREVA-CLINT/freva", - }, - packages=find_packages("src"), - package_dir={"": "src"}, - cmdclass={ - "develop": DevelopCommand, - "install": InstallCommand, - }, - entry_points={ - "console_scripts": [ - "deploy-freva-cmd = freva_deployment.cli:deploy", - "deploy-freva-map = freva_deployment.cli:server_map", - "freva-service = freva_deployment.cli:service", - "freva-migrate = freva_deployment.cli:migrate", - "deploy-freva = freva_deployment.ui.deployment_tui:tui", - ] - }, - setup_requires=["appdirs"], - install_requires=get_packages(), - extras_require={ - "docs": [ - "pydata-sphinx-theme", - "sphinx", - "sphinx-copybutton", - "nbsphinx", - "recommonmark", - "sphinx_rtd_theme", - "ipython", # For nbsphinx syntax highlighting - "sphinxcontrib_github_alt", - ], - "test": ["mypy", "isort", "black", "types-toml", "types-PyMySQL"], - }, - python_requires=">=3.8", - classifiers=[ - "Development Status :: 3 - Alpha", - "Environment :: Console", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: BSD License", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3", - ], -) From 3365a8728478506bde6d7f7b520c6ceb7fca6953 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Fri, 15 Sep 2023 09:39:32 +0200 Subject: [PATCH 16/20] Rename extras --- docs/environment.yml | 2 +- pyproject.toml | 2 +- src/freva_deployment/utils.py | 91 ++++++++++++++--------------------- 3 files changed, 38 insertions(+), 57 deletions(-) diff --git a/docs/environment.yml b/docs/environment.yml index aebdedbd..ba9f85e7 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -6,7 +6,7 @@ dependencies: - ipython - pip - pip: - - freva-deployment[docs] + - freva-deployment[doc] - sphinx-execute-code-python3 - sphinx-rtd-theme - sphinx-copybutton diff --git a/pyproject.toml b/pyproject.toml index 6fa99664..752b09df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "appdirs", "requests", ] [project.optional-dependencies] -docs = [ +doc = [ "pydata-sphinx-theme", "sphinx", "sphinx-copybutton", diff --git a/src/freva_deployment/utils.py b/src/freva_deployment/utils.py index f1a498b3..a7189210 100644 --- a/src/freva_deployment/utils.py +++ b/src/freva_deployment/utils.py @@ -55,33 +55,26 @@ class AssetDir: this_module = "freva_deployment" def __init__(self): - self._user_asset_dir = ( - Path(appdirs.user_data_dir()) / "freva" / "deployment" - ) - self._user_config_dir = ( - Path(appdirs.user_config_dir()) / "freva" / "deployment" - ) - - @property - def _central_asset_dir(self): - distribution = pkg_resources.get_distribution(self.this_module) - try: - records = distribution.get_metadata("RECORD").splitlines() - except FileNotFoundError: - asset_dir = Path(distribution.module_path).parent / "freva" / "deployment" - if asset_dir.is_dir(): - return asset_dir - warnings.warn("Guessing asset dir location, this might fail") - return Path(sys.exec_prefix) / "freva" / "deployment" - try: - inventory = [f.partition(",")[0] for f in records if "inventory.toml" in f][ - 0 - ] - except IndexError: - warnings.warn("Guessing asset dir location, this might fail") - return Path(sys.exec_prefix) / "freva" / "deployment" - asset_path = (Path(distribution.module_path) / inventory).parent.parent - return Path.resolve(asset_path) + self._user_asset_dir = Path(appdirs.user_data_dir()) / "freva" / "deployment" + self._user_config_dir = Path(appdirs.user_config_dir()) / "freva" / "deployment" + + @staticmethod + def get_dirs(user: bool = True, key: str = "data") -> Path: + """Get the 'scripts' and 'purelib' directories we'll install into. + + This is now a thin wrapper around sysconfig.get_paths(). It's not inlined, + because some tests mock it out to install to a different location. + """ + + if user: + if (sys.platform == "darwin") and sysconfig.get_config_var( + "PYTHONFRAMEWORK" + ): + return Path(sysconfig.get_paths("osx_framework_user")[key]) + return Path(sysconfig.get_paths(os.name + "_user")[key]) + # The default scheme is 'posix_prefix' or 'nt', and should work for e.g. + # installing into a virtualenv + return Path(sysconfig.get_paths()[key]) @property def asset_dir(self) -> Path: @@ -94,17 +87,17 @@ def asset_dir(self) -> Path: "Could not find asset dir, please consider reinstalling the package." ) - @property - def inventory_file(self) -> Path: - return self.asset_dir / "config" / "inventory.toml" - @property def config_dir(self): inventory_file = self._user_config_dir / "inventory.toml" if inventory_file.exists(): - return self._user_config_dir - self._user_config_dir.mkdir(exist_ok=True, parents=True) - shutil.copy(self.asset_dir / "config" / "inventory.toml", inventory_file) + return self.asset_dir + inventory_file.parent.mkdir(exist_ok=True, parents=True) + try: + inventory_file.unlink() + except FileNotFoundError: + pass + shutil.copy2(self.asset_dir / "config" / "inventory.toml", inventory_file) return self._user_config_dir @property @@ -172,14 +165,12 @@ def _create_new_config(inp_file: Path) -> Path: config["databrowser"] = config.pop("solr") for key in ("port", "mem"): if key in config["databrowser"]["config"]: - config["databrowser"]["config"][f"solr_{key}"] = config[ - "databrowser" - ]["config"].pop(key) + config["databrowser"]["config"][f"solr_{key}"] = config["databrowser"][ + "config" + ].pop(key) _update_config(config_tmpl, config) if create_backup: - inp_file.with_suffix(inp_file.suffix + ".bck").write_text( - inp_file.read_text() - ) + inp_file.with_suffix(inp_file.suffix + ".bck").write_text(inp_file.read_text()) inp_file.write_text(tomlkit.dumps(config_tmpl)) return inp_file @@ -249,9 +240,7 @@ def set_log_level(verbosity: int) -> None: logger.setLevel(max(logging.INFO - 10 * verbosity, logging.DEBUG)) -def get_setup_for_service( - service: str, setups: list[ServiceInfo] -) -> tuple[str, str]: +def get_setup_for_service(service: str, setups: list[ServiceInfo]) -> tuple[str, str]: """Get the setup of a service configuration.""" for setup in setups: if setup.name == service: @@ -264,9 +253,7 @@ def read_db_credentials( ) -> dict[str, str]: """Read database config.""" with cert_file.open() as f_obj: - key = "".join( - [k.strip() for k in f_obj.readlines() if not k.startswith("-")] - ) + key = "".join([k.strip() for k in f_obj.readlines() if not k.startswith("-")]) sha = hashlib.sha512(key.encode()).hexdigest() url = f"http://{db_host}:{port}/vault/data/{sha}" return requests.get(url).json() @@ -346,9 +333,7 @@ def get_email_credentials() -> tuple[str, str]: ) RichConsole.print(msg) username = Prompt.ask("[green b]Username[/] for mail server") - password = Prompt.ask( - "[green b]Password[/] for mail server", password=True - ) + password = Prompt.ask("[green b]Password[/] for mail server", password=True) return username, password @@ -383,9 +368,7 @@ def _create_passwd(min_characters: int, msg: str = "") -> str: if not re.search(check, master_pass): is_ok = False break - is_safe: bool = ( - len([True for c in "[_@$#$%^&*-!]" if c in master_pass]) > 0 - ) + is_safe: bool = len([True for c in "[_@$#$%^&*-!]" if c in master_pass]) > 0 if is_ok is False or is_safe is False: raise ValueError( ( @@ -395,9 +378,7 @@ def _create_passwd(min_characters: int, msg: str = "") -> str: "- have at least one special special character." ) ) - master_pass_2 = Prompt.ask( - "[bold green]re-enter[/] master password", password=True - ) + master_pass_2 = Prompt.ask("[bold green]re-enter[/] master password", password=True) if master_pass != master_pass_2: raise ValueError("Passwords do not match") return master_pass From 988ebff585a196019ebb4634ebe2eeac1d52e7de Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Tue, 19 Sep 2023 05:48:28 +0200 Subject: [PATCH 17/20] Fix broken merge from main --- src/freva_deployment/utils.py | 69 ++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 17 deletions(-) diff --git a/src/freva_deployment/utils.py b/src/freva_deployment/utils.py index a7189210..935addfb 100644 --- a/src/freva_deployment/utils.py +++ b/src/freva_deployment/utils.py @@ -4,21 +4,32 @@ import hashlib import json import logging +import os import re import shutil import sys -import warnings from pathlib import Path +import sysconfig from typing import Any, Dict, MutableMapping, NamedTuple, Union, cast import appdirs -import pkg_resources import requests from rich.console import Console from rich.logging import RichHandler from rich.prompt import Prompt +import tomlkit -logging.basicConfig(format="%(name)s - %(levelname)s - %(message)s", level=logging.INFO) +logger_stream_handle = RichHandler( + rich_tracebacks=True, + show_path=False, + console=Console(soft_wrap=True, stderr=True), +) +logging.basicConfig( + format="%(name)s - %(levelname)s - %(message)s", + level=logging.INFO, + handlers=[logger_stream_handle], + datefmt="[%X]", +) logger = logging.getLogger("freva-deployment") RichConsole = Console(markup=True, force_terminal=True) @@ -55,8 +66,12 @@ class AssetDir: this_module = "freva_deployment" def __init__(self): - self._user_asset_dir = Path(appdirs.user_data_dir()) / "freva" / "deployment" - self._user_config_dir = Path(appdirs.user_config_dir()) / "freva" / "deployment" + self._user_asset_dir = ( + Path(appdirs.user_data_dir()) / "freva" / "deployment" + ) + self._user_config_dir = ( + Path(appdirs.user_config_dir()) / "freva" / "deployment" + ) @staticmethod def get_dirs(user: bool = True, key: str = "data") -> Path: @@ -87,6 +102,10 @@ def asset_dir(self) -> Path: "Could not find asset dir, please consider reinstalling the package." ) + @property + def inventory_file(self) -> Path: + return self.asset_dir / "config" / "inventory.toml" + @property def config_dir(self): inventory_file = self._user_config_dir / "inventory.toml" @@ -97,7 +116,9 @@ def config_dir(self): inventory_file.unlink() except FileNotFoundError: pass - shutil.copy2(self.asset_dir / "config" / "inventory.toml", inventory_file) + shutil.copy2( + self.asset_dir / "config" / "inventory.toml", inventory_file + ) return self._user_config_dir @property @@ -165,12 +186,14 @@ def _create_new_config(inp_file: Path) -> Path: config["databrowser"] = config.pop("solr") for key in ("port", "mem"): if key in config["databrowser"]["config"]: - config["databrowser"]["config"][f"solr_{key}"] = config["databrowser"][ - "config" - ].pop(key) + config["databrowser"]["config"][f"solr_{key}"] = config[ + "databrowser" + ]["config"].pop(key) _update_config(config_tmpl, config) if create_backup: - inp_file.with_suffix(inp_file.suffix + ".bck").write_text(inp_file.read_text()) + inp_file.with_suffix(inp_file.suffix + ".bck").write_text( + inp_file.read_text() + ) inp_file.write_text(tomlkit.dumps(config_tmpl)) return inp_file @@ -178,8 +201,10 @@ def _create_new_config(inp_file: Path) -> Path: def load_config(inp_file: str | Path) -> dict[str, Any]: """Load the inventory toml file and replace all environment variables.""" inp_file = Path(inp_file).expanduser().absolute() - variables = cast(dict[str, str], toml.loads(config_file.read_text())["variables"]) - config = toml.loads(inp_file.read_text()) + variables = cast( + dict[str, str], tomlkit.loads(config_file.read_text())["variables"] + ) + config = tomlkit.loads(inp_file.read_text()) _convert_dict(config, variables, inp_file.parent) return config @@ -240,7 +265,9 @@ def set_log_level(verbosity: int) -> None: logger.setLevel(max(logging.INFO - 10 * verbosity, logging.DEBUG)) -def get_setup_for_service(service: str, setups: list[ServiceInfo]) -> tuple[str, str]: +def get_setup_for_service( + service: str, setups: list[ServiceInfo] +) -> tuple[str, str]: """Get the setup of a service configuration.""" for setup in setups: if setup.name == service: @@ -253,7 +280,9 @@ def read_db_credentials( ) -> dict[str, str]: """Read database config.""" with cert_file.open() as f_obj: - key = "".join([k.strip() for k in f_obj.readlines() if not k.startswith("-")]) + key = "".join( + [k.strip() for k in f_obj.readlines() if not k.startswith("-")] + ) sha = hashlib.sha512(key.encode()).hexdigest() url = f"http://{db_host}:{port}/vault/data/{sha}" return requests.get(url).json() @@ -333,7 +362,9 @@ def get_email_credentials() -> tuple[str, str]: ) RichConsole.print(msg) username = Prompt.ask("[green b]Username[/] for mail server") - password = Prompt.ask("[green b]Password[/] for mail server", password=True) + password = Prompt.ask( + "[green b]Password[/] for mail server", password=True + ) return username, password @@ -368,7 +399,9 @@ def _create_passwd(min_characters: int, msg: str = "") -> str: if not re.search(check, master_pass): is_ok = False break - is_safe: bool = len([True for c in "[_@$#$%^&*-!]" if c in master_pass]) > 0 + is_safe: bool = ( + len([True for c in "[_@$#$%^&*-!]" if c in master_pass]) > 0 + ) if is_ok is False or is_safe is False: raise ValueError( ( @@ -378,7 +411,9 @@ def _create_passwd(min_characters: int, msg: str = "") -> str: "- have at least one special special character." ) ) - master_pass_2 = Prompt.ask("[bold green]re-enter[/] master password", password=True) + master_pass_2 = Prompt.ask( + "[bold green]re-enter[/] master password", password=True + ) if master_pass != master_pass_2: raise ValueError("Passwords do not match") return master_pass From c84242662ced2662bbe4903a8756fd0a25a33f90 Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Tue, 19 Sep 2023 05:50:24 +0200 Subject: [PATCH 18/20] Fix linting issues. --- src/freva_deployment/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/freva_deployment/utils.py b/src/freva_deployment/utils.py index 935addfb..d626cfe7 100644 --- a/src/freva_deployment/utils.py +++ b/src/freva_deployment/utils.py @@ -8,16 +8,16 @@ import re import shutil import sys -from pathlib import Path import sysconfig +from pathlib import Path from typing import Any, Dict, MutableMapping, NamedTuple, Union, cast import appdirs import requests +import tomlkit from rich.console import Console from rich.logging import RichHandler from rich.prompt import Prompt -import tomlkit logger_stream_handle = RichHandler( rich_tracebacks=True, From ad3eec0e442b162e40ab5f4897c9dca01bdff58c Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Tue, 19 Sep 2023 21:51:36 +0200 Subject: [PATCH 19/20] Fix linting issues --- pyproject.toml | 2 +- src/freva_deployment/deploy.py | 71 ++++++++++++------- .../ui/deployment_tui/deploy_forms.py | 60 ++++------------ .../ui/deployment_tui/main_window.py | 8 +-- src/freva_deployment/utils.py | 2 +- 5 files changed, 65 insertions(+), 78 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 752b09df..0351dafe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ doc = [ "ipython", "sphinxcontrib_github_alt", ] -test = ["mypy", "black", "isort", "types-toml", "types-PyMySQL"] +test = ["mypy", "black", "flit", "isort", "types-toml", "types-PyMySQL"] [project.urls] Documentation = "https://freva-deployment.readthedocs.io/en/latest/" diff --git a/src/freva_deployment/deploy.py b/src/freva_deployment/deploy.py index e7d3b570..14e53a47 100644 --- a/src/freva_deployment/deploy.py +++ b/src/freva_deployment/deploy.py @@ -64,7 +64,9 @@ def __init__( self.email_password: str = "" self._td: TemporaryDirectory = TemporaryDirectory(prefix="inventory") self.inventory_file: Path = Path(self._td.name) / "inventory.yaml" - self.eval_conf_file: Path = Path(self._td.name) / "evaluation_system.conf" + self.eval_conf_file: Path = ( + Path(self._td.name) / "evaluation_system.conf" + ) self.web_conf_file: Path = Path(self._td.name) / "freva_web.toml" self.apache_config: Path = Path(self._td.name) / "freva_web.conf" self._db_pass: str = "" @@ -109,7 +111,9 @@ def _prep_vault(self) -> None: self._config_keys.append("vault") self.cfg["vault"] = self.cfg["db"].copy() self.cfg["vault"]["config"].setdefault("ansible_become_user", "root") - self.playbooks["vault"] = self.cfg["db"]["config"].get("vault_playbook") + self.playbooks["vault"] = self.cfg["db"]["config"].get( + "vault_playbook" + ) if not self.master_pass: self.master_pass = get_passwd() self.cfg["vault"]["config"]["root_passwd"] = self.master_pass @@ -130,7 +134,9 @@ def _prep_db(self) -> None: self.cfg["db"]["config"]["passwd"] = self.db_pass self.cfg["db"]["config"]["keyfile"] = self.public_key_file for key in ("name", "user", "db"): - self.cfg["db"]["config"][key] = self.cfg["db"]["config"].get(key) or "freva" + self.cfg["db"]["config"][key] = ( + self.cfg["db"]["config"].get(key) or "freva" + ) db_host = self.cfg["db"]["config"].get("host", "") if not db_host: self.cfg["db"]["config"]["host"] = host @@ -147,7 +153,9 @@ def _prep_databrowser(self, prep_web=True) -> None: if not self.master_pass: self.master_pass = get_passwd() self._config_keys.append("databrowser") - self.cfg["databrowser"]["config"].setdefault("ansible_become_user", "root") + self.cfg["databrowser"]["config"].setdefault( + "ansible_become_user", "root" + ) self.cfg["databrowser"]["config"]["root_passwd"] = self.master_pass self.cfg["databrowser"]["config"].pop("core", None) self.playbooks["databrowser"] = self.cfg["databrowser"]["config"].get( @@ -159,9 +167,9 @@ def _prep_databrowser(self, prep_web=True) -> None: self.cfg["databrowser"]["config"][key] = ( self.cfg["databrowser"]["config"].get(key) or default ) - self.cfg["databrowser"]["config"]["email"] = self.cfg["web"]["config"].get( - "contacts", "" - ) + self.cfg["databrowser"]["config"]["email"] = self.cfg["web"][ + "config" + ].get("contacts", "") if prep_web: self._prep_web(False) @@ -169,7 +177,9 @@ def _prep_core(self) -> None: """prepare the core deployment.""" self._config_keys.append("core") self.cfg["core"]["config"].setdefault("ansible_become_user", "") - self.playbooks["core"] = self.cfg["core"]["config"].get("core_playbook") + self.playbooks["core"] = self.cfg["core"]["config"].get( + "core_playbook" + ) self.cfg["core"]["config"]["admins"] = ( self.cfg["core"]["config"].get("admins") or getuser() ) @@ -180,11 +190,15 @@ def _prep_core(self) -> None: if not root_dir: self.cfg["core"]["config"]["root_dir"] = install_dir preview_path = self.cfg["core"]["config"].get("preview_path", "") - base_dir_location = self.cfg["core"]["config"].get("base_dir_location", "") + base_dir_location = self.cfg["core"]["config"].get( + "base_dir_location", "" + ) scheduler_output_dir = self.cfg["core"]["config"].get( "scheduler_output_dir", "" ) - scheduler_system = self.cfg["core"]["config"].get("scheduler_system", "local") + scheduler_system = self.cfg["core"]["config"].get( + "scheduler_system", "local" + ) if not preview_path: if base_dir_location: self.cfg["core"]["config"]["preview_path"] = str( @@ -195,7 +209,9 @@ def _prep_core(self) -> None: if not scheduler_output_dir: scheduler_output_dir = str(Path(base_dir_location) / "share") scheduler_output_dir = Path(scheduler_output_dir) / scheduler_system - self.cfg["core"]["config"]["scheduler_output_dir"] = str(scheduler_output_dir) + self.cfg["core"]["config"]["scheduler_output_dir"] = str( + scheduler_output_dir + ) self.cfg["core"]["config"]["keyfile"] = self.public_key_file git_exe = self.cfg["core"]["config"].get("git_path") self.cfg["core"]["config"]["git_path"] = git_exe or "git" @@ -272,13 +288,17 @@ def _prep_web(self, ask_pass: bool = True) -> None: except (FileNotFoundError, IOError, KeyError): pass try: - _webserver_items["IMPRINT"] = _webserver_items["IMPRINT"].split(",") + _webserver_items["IMPRINT"] = _webserver_items["IMPRINT"].split( + "," + ) except AttributeError: pass with self.web_conf_file.open("w") as f_obj: tomlkit.dump(_webserver_items, f_obj) for key in ("core", "web"): - self.cfg[key]["config"]["config_toml_file"] = str(self.web_conf_file) + self.cfg[key]["config"]["config_toml_file"] = str( + self.web_conf_file + ) if not self.master_pass: self.master_pass = get_passwd() self._prep_vault() @@ -286,16 +306,18 @@ def _prep_web(self, ask_pass: bool = True) -> None: email_user, self.email_password = get_email_credentials() self.cfg["vault"]["config"]["email_user"] = email_user self.cfg["vault"]["config"]["email_password"] = self.email_password - self.cfg["vault"]["config"]["ansible_python_interpreter"] = self.cfg["db"][ - "config" - ].get("ansible_python_interpreter", "/usr/bin/python3") + self.cfg["vault"]["config"]["ansible_python_interpreter"] = self.cfg[ + "db" + ]["config"].get("ansible_python_interpreter", "/usr/bin/python3") self.cfg["web"]["config"]["root_passwd"] = self.master_pass self.cfg["web"]["config"]["private_keyfile"] = self.private_key_file self.cfg["web"]["config"]["public_keyfile"] = self.public_key_file self.cfg["web"]["config"]["chain_keyfile"] = ( self.chain_key_file or self.public_key_file ) - self.cfg["web"]["config"]["apache_config_file"] = str(self.apache_config) + self.cfg["web"]["config"]["apache_config_file"] = str( + self.apache_config + ) if ask_pass: self._prep_apache_config() @@ -303,7 +325,10 @@ def _prep_apache_config(self): config = [] with (Path(asset_dir) / "web" / "freva_web.conf").open() as f_obj: for line in f_obj.readlines(): - if not self.chain_key_file and "SSLCertificateChainFile" in line: + if ( + not self.chain_key_file + and "SSLCertificateChainFile" in line + ): continue config.append(line) with open(self.apache_config, "w") as f_obj: @@ -439,7 +464,10 @@ def parse_config(self) -> str: self._set_additional_config_values(step, config) for step in info: for key, value in config[step]["vars"].items(): - if f"{{{{{key}}}}}" in playbook or f"{{{{ {key} }}}}" in playbook: + if ( + f"{{{{{key}}}}}" in playbook + or f"{{{{ {key} }}}}" in playbook + ): info[step]["vars"][key] = value info_str = yaml.dump(json.loads(json.dumps(info))) for passwd in (self.email_password, self.master_pass): @@ -580,11 +608,6 @@ def play( self.create_eval_config() with self.inventory_file.open("w") as f_obj: f_obj.write(inventory) - inventory_str = inventory - for passwd in (self.email_password, self.master_pass): - if passwd: - inventory_str = inventory_str.replace(passwd, "*" * len(passwd)) - RichConsole.print(inventory, style="bold", markup=True) logger.info("Playing the playbooks with ansible") RichConsole.print( "[b]Note:[/] The [blue]BECOME[/] password refers to the " diff --git a/src/freva_deployment/ui/deployment_tui/deploy_forms.py b/src/freva_deployment/ui/deployment_tui/deploy_forms.py index efabe5c0..1f1a0ecb 100644 --- a/src/freva_deployment/ui/deployment_tui/deploy_forms.py +++ b/src/freva_deployment/ui/deployment_tui/deploy_forms.py @@ -85,9 +85,7 @@ def _add_widgets(self) -> None: max_height=2, value=cfg.get("install", True), editable=True, - name=( - f"{self.num}Install a new Freva anaconda environment?" - ), + name=(f"{self.num}Install a new Freva anaconda environment?"), scroll_exit=True, ), True, @@ -123,9 +121,7 @@ def _add_widgets(self) -> None: self.add_widget_intelligent( npyscreen.TitleCombo, name=f"{self.num}Workload manger", - value=self.scheduler_index( - cast(str, cfg.get("scheduler_system")) - ), + value=self.scheduler_index(cast(str, cfg.get("scheduler_system"))), values=self.scheduler_systems, ), True, @@ -208,9 +204,7 @@ def _add_widgets(self) -> None: self.add_widget_intelligent( npyscreen.TitleFilename, name=f"{self.num}Python path on remote machine:", - value=cfg.get( - "ansible_python_interpreter", "/usr/bin/python3" - ), + value=cfg.get("ansible_python_interpreter", "/usr/bin/python3"), ), False, ), @@ -382,9 +376,7 @@ def _add_widgets(self) -> None: self.add_widget_intelligent( npyscreen.TitleText, name=f"{self.num}A brief describtion of the project:", - value=cfg.get( - "homepage_heading", "Lorem ipsum dolor sit amet" - ), + value=cfg.get("homepage_heading", "Lorem ipsum dolor sit amet"), ), True, ), @@ -549,9 +541,7 @@ def _add_widgets(self) -> None: ldap_model=( self.add_widget_intelligent( npyscreen.TitleText, - name=( - f"{self.num}Ldap tools class to be used for authentication." - ), + name=(f"{self.num}Ldap tools class to be used for authentication."), value=cfg.get("ldap_model", "MiklipUserInformation"), ), True, @@ -582,9 +572,7 @@ def _add_widgets(self) -> None: self.add_widget_intelligent( npyscreen.TitleFilename, name=f"{self.num}Pythonpath on remote machine:", - value=cfg.get( - "ansible_python_interpreter", "/usr/bin/python3" - ), + value=cfg.get("ansible_python_interpreter", "/usr/bin/python3"), ), False, ), @@ -696,9 +684,7 @@ def _add_widgets(self) -> None: self.add_widget_intelligent( npyscreen.TitleFilename, name=f"{self.num}Pythonpath on remote machine:", - value=cfg.get( - "ansible_python_interpreter", "/usr/bin/python3" - ), + value=cfg.get("ansible_python_interpreter", "/usr/bin/python3"), ), False, ), @@ -805,9 +791,7 @@ def _add_widgets(self) -> None: self.add_widget_intelligent( npyscreen.TitleFilename, name=f"{self.num}Pythonpath on remote machine:", - value=cfg.get( - "ansible_python_interpreter", "/usr/bin/python3" - ), + value=cfg.get("ansible_python_interpreter", "/usr/bin/python3"), ), False, ), @@ -838,18 +822,8 @@ def on_ok(self) -> None: self.parentApp.thread_stop.set() if not self.project_name.value: - npyscreen.notify_confirm( - "You have to set a project name", title="ERROR" - ) + npyscreen.notify_confirm("You have to set a project name", title="ERROR") return - if not self.server_map.value: - value = npyscreen.notify_yes_no( - "If you don't set a map server value you wont be able " - "to start|stop the services. Continue anyway?", - title="WARNING", - ) - if not value: - return missing_form: None | str = self.parentApp.check_missing_config() if missing_form: self.parentApp.change_form(missing_form) @@ -863,9 +837,7 @@ def on_ok(self) -> None: self.parentApp.get_save_file(self.inventory_file.value or None) ) for key_type, keyfile in cert_files.items(): - key_file = Path( - get_current_file_dir(save_file.parent, str(keyfile)) - ) + key_file = Path(get_current_file_dir(save_file.parent, str(keyfile))) for step, deploy_form in self.parentApp._forms.items(): if not keyfile or not Path(key_file).is_file(): if ( @@ -873,11 +845,11 @@ def on_ok(self) -> None: and step in self.parentApp.steps ): if keyfile: - msg = f"{key_type} certificate file `{key_file}` must exist." - else: msg = ( - f"You must give a {key_type} certificate file." + f"{key_type} certificate file `{key_file}` must exist." ) + else: + msg = f"You must give a {key_type} certificate file." npyscreen.notify_confirm(msg, title="ERROR") return @@ -894,7 +866,7 @@ def on_ok(self) -> None: save_file=save_file, write_toml_file=True ) self.parentApp.setup = { - "server_map": self.server_map.value, + "server_map": self.server_map.value or None, "steps": list(set(self.parentApp.steps)), "ask_pass": bool(self.use_ssh_pw.value), "config_file": str(save_file) or None, @@ -940,9 +912,7 @@ def _add_widgets(self) -> None: ) self.server_map = self.add_widget_intelligent( npyscreen.TitleText, - name=( - f"{self.num}Hostname of the service mapping the freva server arch." - ), + name=(f"{self.num}Hostname of the service mapping the freva server arch."), value=self.parentApp._read_cache("server_map", ""), ) self.public_keyfile = self.add_widget_intelligent( diff --git a/src/freva_deployment/ui/deployment_tui/main_window.py b/src/freva_deployment/ui/deployment_tui/main_window.py index 6136c7cd..31acaf03 100644 --- a/src/freva_deployment/ui/deployment_tui/main_window.py +++ b/src/freva_deployment/ui/deployment_tui/main_window.py @@ -15,13 +15,7 @@ from freva_deployment.utils import asset_dir, config_dir, load_config from .base import BaseForm, VarForm, selectFile -from .deploy_forms import ( - CoreScreen, - DatabrowserScreen, - DBScreen, - RunForm, - WebScreen, -) +from .deploy_forms import CoreScreen, DatabrowserScreen, DBScreen, RunForm, WebScreen class MainApp(npyscreen.NPSAppManaged): diff --git a/src/freva_deployment/utils.py b/src/freva_deployment/utils.py index d626cfe7..74b3f802 100644 --- a/src/freva_deployment/utils.py +++ b/src/freva_deployment/utils.py @@ -25,7 +25,7 @@ console=Console(soft_wrap=True, stderr=True), ) logging.basicConfig( - format="%(name)s - %(levelname)s - %(message)s", + format="%(name)s - %(message)s", level=logging.INFO, handlers=[logger_stream_handle], datefmt="[%X]", From 8301fb90492e38eeb1fc631b3bb2521af5fc110c Mon Sep 17 00:00:00 2001 From: antarcticrainforest Date: Wed, 14 Feb 2024 14:49:12 +0100 Subject: [PATCH 20/20] Fix isort issues. --- src/freva_deployment/ui/deployment_tui/main_window.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/freva_deployment/ui/deployment_tui/main_window.py b/src/freva_deployment/ui/deployment_tui/main_window.py index 31acaf03..6136c7cd 100644 --- a/src/freva_deployment/ui/deployment_tui/main_window.py +++ b/src/freva_deployment/ui/deployment_tui/main_window.py @@ -15,7 +15,13 @@ from freva_deployment.utils import asset_dir, config_dir, load_config from .base import BaseForm, VarForm, selectFile -from .deploy_forms import CoreScreen, DatabrowserScreen, DBScreen, RunForm, WebScreen +from .deploy_forms import ( + CoreScreen, + DatabrowserScreen, + DBScreen, + RunForm, + WebScreen, +) class MainApp(npyscreen.NPSAppManaged):