From 5a4218d4e8801fa05a661f8d306eabb62298c445 Mon Sep 17 00:00:00 2001 From: sash19 Date: Sat, 21 Sep 2024 15:34:13 -0700 Subject: [PATCH 01/41] Sorted imports --- scalable/__init__.py | 12 ++++++------ scalable/client.py | 14 ++++---------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/scalable/__init__.py b/scalable/__init__.py index 3227333..37b9fd5 100755 --- a/scalable/__init__.py +++ b/scalable/__init__.py @@ -1,12 +1,12 @@ -# flake8: noqa -from .core import JobQueueCluster -from .slurm import SlurmCluster -from .caching import * -from .common import SEED -from .client import ScalableClient + from dask.distributed import Security from ._version import get_versions +from .caching import * +from .client import ScalableClient +from .common import SEED +from .core import JobQueueCluster +from .slurm import SlurmCluster __version__ = get_versions()["version"] del get_versions diff --git a/scalable/client.py b/scalable/client.py index ff07faf..713da28 100644 --- a/scalable/client.py +++ b/scalable/client.py @@ -1,13 +1,12 @@ from collections.abc import Awaitable -from distributed import Scheduler, Client +from distributed import Client, Scheduler from distributed.diagnostics.plugin import SchedulerPlugin - from .common import logger -from .slurm import SlurmJob, SlurmCluster +from .slurm import SlurmCluster, SlurmJob -class SlurmSchedulerPlugin(SchedulerPlugin): +class SlurmSchedulerPlugin(SchedulerPlugin): def __init__(self, cluster): self.cluster = cluster super().__init__() @@ -127,9 +126,4 @@ def submit( actors=actors, pure=False, **kwargs) - - - - - - + \ No newline at end of file From db30ee1c94e5ac9b53d7a3b1c2795e57ce5f2cf0 Mon Sep 17 00:00:00 2001 From: sash19 Date: Sat, 21 Sep 2024 15:41:00 -0700 Subject: [PATCH 02/41] Communicator port read from environment now --- scalable/core.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/scalable/core.py b/scalable/core.py index f8778f7..0d39bed 100755 --- a/scalable/core.py +++ b/scalable/core.py @@ -1,28 +1,27 @@ -from contextlib import suppress + +import abc +import asyncio +import copy import os import re import shlex import sys -import abc import tempfile import threading import time -import copy import warnings -import asyncio +from contextlib import suppress from dask.utils import parse_bytes - from distributed.core import Status from distributed.deploy.spec import ProcessInterface, SpecCluster from distributed.scheduler import Scheduler from distributed.security import Security from distributed.utils import NoOpAwaitable -from .utilities import * -from .support import * - from .common import logger +from .support import * +from .utilities import * DEFAULT_WORKER_COMMAND = "distributed.cli.dask_worker" @@ -439,6 +438,8 @@ def __init__( **job_kwargs ): + if comm_port is None: + comm_port = os.getenv("COMM_PORT") if comm_port is None: raise ValueError( "Communicator port not given. You must specify the communicator port " From b8b316a06f67ceb3525a7b8a221440f19c85cd91 Mon Sep 17 00:00:00 2001 From: sash19 Date: Sat, 21 Sep 2024 15:41:42 -0700 Subject: [PATCH 03/41] Cleaned up older code --- scalable/utilities.py | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/scalable/utilities.py b/scalable/utilities.py index 879f501..23f0feb 100755 --- a/scalable/utilities.py +++ b/scalable/utilities.py @@ -1,12 +1,12 @@ -import subprocess -import yaml -import os import asyncio -from dask.utils import parse_bytes -from importlib.resources import files +import os import re +import subprocess import sys +from importlib.resources import files +import yaml +from dask.utils import parse_bytes from .common import logger comm_port_regex = r'0\.0\.0\.0:(\d{1,5})' @@ -44,20 +44,6 @@ async def get_cmd_comm(port, communicator_path=None): ) return proc -def get_comm_port(logpath=None): - if logpath is None: - logpath = "./communicator.log" - ret = -1 - with open(logpath, 'r') as file: - for line in file: - match = re.search(comm_port_regex, line) - if match: - port = int(match.group(1)) - if 0 <= port <= 65535: - ret = port - break - return ret - def run_bootstrap(): bootstrap_location = files('scalable').joinpath('scalable_bootstrap.sh') result = subprocess.run(["/bin/bash", bootstrap_location], stdin=sys.stdin, From 0b19ffa6ed14884fe388f1ed19d88834ff4b53d0 Mon Sep 17 00:00:00 2001 From: sash19 Date: Sat, 21 Sep 2024 16:06:35 -0700 Subject: [PATCH 04/41] Enabled port checking and random generation --- scalable/scalable_bootstrap.sh | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/scalable/scalable_bootstrap.sh b/scalable/scalable_bootstrap.sh index 68756f5..21842aa 100755 --- a/scalable/scalable_bootstrap.sh +++ b/scalable/scalable_bootstrap.sh @@ -4,6 +4,7 @@ GO_VERSION_LINK="https://go.dev/VERSION?m=text" GO_DOWNLOAD_LINK="https://go.dev/dl/*.linux-amd64.tar.gz" SCALABLE_REPO="https://github.com/JGCRI/scalable.git" APPTAINER_VERSION="1.3.2" +DEFAULT_PORT="1919" # set -x @@ -112,6 +113,8 @@ if [[ "$transfer" =~ [Yy]|^[Yy][Ee]|^[Yy][Ee][Ss]$ ]]; then done fi +exist=$(ssh $user@$host "[[ -f $work_dir/containers/scalable_container.sif ]]") + echo -e "${YELLOW}To reinstall any directory or file already on remote, \ please delete it from remote and run this script again${NC}" @@ -179,6 +182,17 @@ mkdir -p tmp-apptainer/cache APPTAINER_TMPDIR="/tmp-apptainer/tmp" APPTAINER_CACHEDIR="/tmp-apptainer/cache" +ssh $user@$host "[[ -f $work_dir/containers/scalable_container.sif ]]" +exist=$(echo $?) +if [[ "$exist" -eq 0 ]]; then + docker images | grep scalable_container + exist=$(echo $?) +fi +if [[ "$exist" -ne 0 ]]; then + transfer=Y + build+=("scalable") +fi + if [[ "$transfer" =~ [Yy]|^[Yy][Ee]|^[Yy][Ee][Ss]$ ]]; then flush @@ -239,7 +253,7 @@ if [[ "$transfer" =~ [Yy]|^[Yy][Ee]|^[Yy][Ee][Ss]$ ]]; then IMAGE_TAG=$(docker images | grep $target\_container | sed -E 's/[\t ][\t ]*/ /g' | cut -d ' ' -f 2) flush docker run --rm -v //var/run/docker.sock:/var/run/docker.sock -v /$(pwd):/work -v /$(pwd)/tmp-apptainer:/tmp-apptainer \ - apptainer_container build --userns --force containers/$target\_container.sif docker-daemon://$IMAGE_NAME:$IMAGE_TAG + apptainer_container build --userns --force //work/containers/$target\_container.sif docker-daemon://$IMAGE_NAME:$IMAGE_TAG check_exit_code $? done @@ -257,15 +271,24 @@ docker run --rm -v /$(pwd):/host -v /$HOME/.ssh:/root/.ssh scalable_container \ && rsync -aP Dockerfile $user@$host:~/$work_dir" check_exit_code $? +COMM_PORT=$DEFAULT_PORT +ssh $user@$host "netstat -tuln | grep :$COMM_PORT" +while [ $? -eq 0 ] +do + COMM_PORT=$(awk -v min=1024 -v max=49151 'BEGIN{srand(); print int(min+rand()*(max-min+1))}') + check_exit_code $? + ssh $user@$host "netstat -tuln | grep :$COMM_PORT" +done + ssh -L 8787:deception.pnl.gov:8787 -t $user@$host \ "{ module load apptainer/$APPTAINER_VERSION && cd $work_dir && $SHELL --rcfile <(echo \". $RC_FILE; python3() { - ./communicator -s >> logs/communicator.log & + ./communicator -s $COMM_PORT >> logs/communicator.log & COMMUNICATOR_PID=\\\$! - apptainer exec --userns --compat --home ~/$work_dir --cwd ~/$work_dir ~/$work_dir/containers/scalable_container.sif python3 \\\$@ + apptainer exec --userns --compat --env COMM_PORT=$COMM_PORT --home ~/$work_dir --cwd ~/$work_dir ~/$work_dir/containers/scalable_container.sif python3 \\\$@ kill -9 \\\$COMMUNICATOR_PID } \" ); }" From 30a7a32ea459e77c22b082d269a63a0d7912eb2f Mon Sep 17 00:00:00 2001 From: sash19 Date: Sat, 21 Sep 2024 16:12:26 -0700 Subject: [PATCH 05/41] Cleaned up old code --- scalable/support.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scalable/support.py b/scalable/support.py index 404215c..736c32f 100755 --- a/scalable/support.py +++ b/scalable/support.py @@ -1,7 +1,8 @@ -from datetime import datetime import os import re import shlex +from datetime import datetime + def salloc_command(account=None, chdir=None, clusters=None, exclusive=True, gpus=None, name=None, memory=None, nodes=None, partition=None, time=None, extras=None): @@ -85,7 +86,6 @@ def core_command(): """ return ["nproc", "--all"] -# Handle what to do if name is null or invalid def jobid_command(name): """Make the command to get the job id of a job with a given name. From 69c314aa18cdf64d8690e54556f2225b8c291f84 Mon Sep 17 00:00:00 2001 From: sash19 Date: Sat, 21 Sep 2024 16:24:10 -0700 Subject: [PATCH 06/41] Added dependency --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 0a6b57d..3559332 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "joblib >= 1.3.2", "xxhash >= 3.4.1", "versioneer >= 0.29", + "numpy >= 1.26.4" ] classifiers = [ "Development Status :: 4 - Beta", From 5a8b6c3828779ca5e847338f8b300e63f5c781d0 Mon Sep 17 00:00:00 2001 From: sash19 Date: Mon, 23 Sep 2024 08:38:26 -0700 Subject: [PATCH 07/41] Removed deprecated warnings --- scalable/caching.py | 15 ++++++++------- scalable/utilities.py | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/scalable/caching.py b/scalable/caching.py index 905674c..70243b3 100644 --- a/scalable/caching.py +++ b/scalable/caching.py @@ -1,13 +1,14 @@ import os import pickle -from diskcache import Cache -from xxhash import xxh32 import types +import dill import numpy as np import pandas as pd -import dill -from .common import logger, cachedir, SEED +from diskcache import Cache +from xxhash import xxh32 +from .common import SEED, cachedir, logger + class GenericType: """The GenericType class is a base class for all types that can be hashed. @@ -189,7 +190,7 @@ def convert_to_type(arg): elif isinstance(arg, (np.ndarray, pd.DataFrame)): ret = UtilityType(arg) else: - logger.warn(f"Could not identify type for argument: {arg}. Using default hash function. " + logger.warning(f"Could not identify type for argument: {arg}. Using default hash function. " "For more reliable performance, either wrap the argument in a class with a defined" " __hash__() function or open an issue on the scalable Github: github.com/JGCRI/scalable.") ret = ObjectType(arg) @@ -306,7 +307,7 @@ def inner(*args, **kwargs): ret = value[1] else: if not disk.delete(key, True): - logger.warn(f"{func.__name__} could not be deleted from cache after hash" + logger.warning(f"{func.__name__} could not be deleted from cache after hash" " mismatch.") if ret is None: ret = func(*args, **kwargs) @@ -318,7 +319,7 @@ def inner(*args, **kwargs): new_digest = hash(return_type(ret)) value = [new_digest, ret] if not disk.add(key=key, value=value, retry=True): - logger.warn(f"{func.__name__} could not be added to cache.") + logger.warning(f"{func.__name__} could not be added to cache.") disk.close() return ret ret = inner diff --git a/scalable/utilities.py b/scalable/utilities.py index 23f0feb..ff50de4 100755 --- a/scalable/utilities.py +++ b/scalable/utilities.py @@ -108,7 +108,7 @@ def __init__(self, path=None, path_overwrite=True): logger.error("Failed to run sed command...manual entry of container info may be required") return if not os.path.exists(self.path): - logger.warn("No resource dict found...making one") + logger.warning("No resource dict found...making one") path_overwrite = True for container in avail_containers: self.config_dict[container] = ModelConfig.default_spec() From 1c595b4699cc4dda8e48b313526be454635745f4 Mon Sep 17 00:00:00 2001 From: sash19 Date: Mon, 23 Sep 2024 08:41:02 -0700 Subject: [PATCH 08/41] Communicator port read from environment; updated old calls --- scalable/core.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scalable/core.py b/scalable/core.py index 0d39bed..9f903a7 100755 --- a/scalable/core.py +++ b/scalable/core.py @@ -619,7 +619,7 @@ def remove_workers(self, tag=None, n=0): can_remove.extend([worker_name for worker_name in list(self.worker_spec.keys()) if tag in worker_name]) current = len(can_remove) if n > current: - logger.warn(f"Cannot remove {n} workers. Only {current} workers found, removing all.") + logger.warning(f"Cannot remove {n} workers. Only {current} workers found, removing all.") n = current can_remove = can_remove[:n] if n != 0 and self.status not in (Status.closing, Status.closed): @@ -738,7 +738,6 @@ def new_worker_spec(self, tag): Dictionary containing the name and spec for the next worker """ if tag not in self.specifications: - lock = self.new_spec["options"]["shared_lock"] self.specifications[tag] = copy.copy(self.new_spec) if tag not in self.containers: raise ValueError(f"The tag ({tag}) given is not a recognized tag for any of the containers." @@ -776,7 +775,7 @@ def _get_worker_security(self, security): # a shared temp directory should be configured correctly elif self.shared_temp_directory is None: shared_temp_directory = os.getcwd() - warnings.warn( + logger.warning( "Using a temporary security object without explicitly setting a shared_temp_directory: \ writing temp files to current working directory ({}) instead. You can set this value by \ using dask for e.g. `dask.config.set({{'jobqueue.pbs.shared_temp_directory': '~'}})`\ @@ -842,7 +841,7 @@ def scale(self, n=None, jobs=0, memory=None, cores=None): Target number of cores """ - logger.warn("This function must only be called internally on exit. " + + logger.warning("This function must only be called internally on exit. " + "Any calls made explicity or during execution can result " + "in undefined behavior. " + "If called accidentally, an " + "immediate shutdown and restart of the cluster is recommended.") From 4b7b6f84f1ee9c226d234045824f2b601f35b97f Mon Sep 17 00:00:00 2001 From: sash19 Date: Mon, 23 Sep 2024 08:44:14 -0700 Subject: [PATCH 09/41] Updated description --- scalable/slurm.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scalable/slurm.py b/scalable/slurm.py index 6e081ee..29ce2c4 100755 --- a/scalable/slurm.py +++ b/scalable/slurm.py @@ -5,7 +5,7 @@ from distributed.deploy.spec import ProcessInterface from .common import logger -from .core import Job, JobQueueCluster, job_parameters, cluster_parameters +from .core import Job, JobQueueCluster, cluster_parameters, job_parameters from .support import * from .utilities import * @@ -172,7 +172,8 @@ class SlurmCluster(JobQueueCluster): def close(self, timeout: float | None = None) -> Awaitable[None] | None: """Close the cluster - This closes all running jobs and the scheduler.""" + This closes all running jobs and the scheduler. Pending jobs belonging + to the user are also cancelled.""" active_jobs = self.hardware.get_active_jobids() jobs_command = "squeue -o \"%i %t\" -u $(whoami) | sed '1d'" result = subprocess.run(jobs_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) From d65894fbf3651d578432e94273c90b2a867e6015 Mon Sep 17 00:00:00 2001 From: sash19 Date: Sat, 19 Oct 2024 11:32:42 -0700 Subject: [PATCH 10/41] Added documentation --- docs/Makefile | 20 +++++++++++ docs/_static/custom.css | 0 docs/caching.rst | 17 +++++++++ docs/conf.py | 45 ++++++++++++++++++++++++ docs/functions.rst | 15 ++++++++ docs/images/scalable_architecture.png | Bin 0 -> 53059 bytes docs/index.rst | 48 ++++++++++++++++++++++++++ docs/make.bat | 35 +++++++++++++++++++ docs/workers.rst | 13 +++++++ 9 files changed, 193 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/_static/custom.css create mode 100644 docs/caching.rst create mode 100644 docs/conf.py create mode 100644 docs/functions.rst create mode 100644 docs/images/scalable_architecture.png create mode 100644 docs/index.rst create mode 100755 docs/make.bat create mode 100644 docs/workers.rst diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 0000000..e69de29 diff --git a/docs/caching.rst b/docs/caching.rst new file mode 100644 index 0000000..dfde54c --- /dev/null +++ b/docs/caching.rst @@ -0,0 +1,17 @@ +Caching +======= + +.. autofunction:: scalable.cacheable + +.. autoclass:: scalable.GenericType + :exclude-members: __init__ + +.. autoclass:: scalable.FileType + +.. autoclass:: scalable.DirType + +.. autoclass:: scalable.ValueType + +.. autoclass:: scalable.ObjectType + +.. autoclass:: scalable.UtilityType \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..8c15af6 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,45 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +import os +import sys + +sys.path.insert(0, os.path.abspath('..')) +sys.path.insert(0, os.path.abspath('../scalable')) + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'Scalable' +copyright = '2024, Shashank Lamba, Pralit Patel' +author = 'Shashank Lamba, Pralit Patel' +release = '0.5.0' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = ["sphinx.ext.autodoc", "sphinx.ext.todo", "sphinx.ext.viewcode", "sphinx.ext.napoleon"] + +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +autodoc_default_options = { + 'members': True, + 'undoc-members': True, + 'private-members': False, + 'special-members': '__init__', + 'inherited-members': False, + 'show-inheritance': False, + 'no-index': True, +} + +# add_module_names = False + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'sphinx_rtd_theme' +html_static_path = ['_static'] +html_css_files = ['custom.css'] \ No newline at end of file diff --git a/docs/functions.rst b/docs/functions.rst new file mode 100644 index 0000000..6ddcc02 --- /dev/null +++ b/docs/functions.rst @@ -0,0 +1,15 @@ +Submitting Functions +==================== + +.. autoclass:: scalable.ScalableClient + :exclude-members: submit, map, get_versions, cancel, close + +.. autofunction:: scalable.ScalableClient.submit + +.. autofunction:: scalable.ScalableClient.map + +.. autofunction:: scalable.ScalableClient.get_versions + +.. autofunction:: scalable.ScalableClient.cancel + +.. autofunction:: scalable.ScalableClient.close \ No newline at end of file diff --git a/docs/images/scalable_architecture.png b/docs/images/scalable_architecture.png new file mode 100644 index 0000000000000000000000000000000000000000..2a001f73b583cd2839a65283bbd03b9578aebc54 GIT binary patch literal 53059 zcmdSB`9G9x*av)#B|>CZLM2povSn)_MMx!NCp&}e+l&?@p%StSQQ4Dy8CkPu$u@S` zcLsx5-gDG_-_P@W-ap`df9Nyxxt!N=9_O)qkK_2B6M9EW?HJ8T8VG`p-MXoA7lJ6k zk7Pfo$-u`I9(5S_fVtgOy9VWTo>~CEP}p46yb3{uk+e8-(xXRRZW_2j5Pd!AAFSE= z(<2D#9K5A+^}e^s3Lcrmx*M=|FemEHN07UTC;v<~noiSl?Q35calSOhH%a;Rr(ws& zPoIv>oie19>eSr|f4j<=&a@Pd{&_j{=DSNTH>WYQij8vo)fYUcqfV7+E$yRMhA1yd0U!U4%Rkd51#Her}!(Nc5)0PQ{oJ^{B7+Vtov~f9XjUlaRc$i1HN5o zAT0#ZK`;)|rx-Pb8R@h3^zlH_CZT6npd+MDI4n2?cpfN2^JO|qk7DIt;16~k z#@}6z5ir`t;HhM7c{wSm z;`tLzb#-+Z%s1;OCv-{qXOHtk;yeo8cM{48k~PbIi|(_n*M43w4jbGn(0(8zvBDs(#Yjn=MU zUyCbQ!N=Z8c-c&hAsoX{Z2WneIcsLf%?e)qJs4lutpb(U4!fVey|#B#I1TrtV&3h1 ze~q{3k?pgPWe>}{h4-x#W-Gsu``W}?(TZ!1^|J~Is_(2i_b035>xIP|Hxw3Cp{9%r z=X=GY>1zFUS9Q0$5!IPqM=(l5?Me#OBmb|NPNBe~>b^oH<;1=6pnYr|FpTl}aIyM4v}zj=8I zPrz&TW>tBDG`l+oi7UT_Z7~X{-RiYs9}Gvmz;Wnub}G-U@o0mU`ks?thtwZTJa4a7 z|4nnx$S>f;Kww9Sbw~WcYGv2#E-sjDMO;$vlNBDLf;10vNE>gFUoGS9mVdYWNW=36 zWp(+hVclrl&&m~ZZ?nOtq1PL&HFAUtH4P4^SFc&ttXf65ocL$Y!Mj#>Qyv7Kho0fZ>VrR zj_dW)5VOjxEeQXMZiciqnlN@2^h!g&#*QcSGwmtjT!3DM*091C?+0nZz6Rg@=Rbs8 zy52&Q__|zATHMH`dzdrEb@bzC?`0kY)*O{xg66@`s} zVN4$5`FER5+z(sgo=E&T+=n@)-PMzvg9p27rJ)GMgq7I@XL`M~n&z0E>QLtk-xd?; zUxp$&X}x-8Z!Z_D1V$gYFNhYAH!^&ewhzN<<+Z1C*nKacGWMS1%w=}I1YH@>4}DO5 z_gc}~?1oTo5Uaeg0JADOFJSK;@`AxlW(hwT!&Y{*MDc*LBIDz8;#u3g!xpZQaiw*b zdQ2>=QCT_E_S`VMMXR5GKRnZ>DCEuvYeb*!UPythskNZ?+HF}?ri0f?53l*`#`v5j zoX|k6erQiR^;uQOH#G)HuQ@9Mn?4O3?9l@g(R@pd+oG$)o*ulBulo7IsL&2RU)d$; zm*WSwo%66~GUPRGX}9|sg^9c`-^L6>sn2i=h-7!1#=CJ9l|qSC4U3j*@n(J>rO2cMMEHGJ&%fm zd3Vz!nn~I!j%ltDy!zi3%eXD^x9jlv-EJ{7pLJhaV^Vx*Eu;4#s$==pW<{O7akS}| z50H>_f~VisY=pOnSK% z)q_`e)s7mwg;aX8O3Q3>=FPJO4t#8ROQQFF|9GG8 zcis(o!=BNK{ytj0ML^k=jp9}0ANg{ZepM3!tP0n@(X|E4`bAOEpZa-sVO8y3&YHnD z=N$>Jo(E?`7s8HAv~!*Jrat*&L=m0s@&@C`$Np_juFzv+y!t4Vr28{6V@>_OYQmX~ z;`*a}w076NoA_ASleBkMEqYme>%n`u>T8h5cuQtV{Zw87(BDCN(nz&mX@Yxwce>j6 zZq4q{*9FCIqqIEnvjgv3crHbnKD$+QK2%3?796!gJ55A2#+Xtvq*Wn=f9nkSp*!o( z=;q$`H$dSds;mptFYb6S?l?=Ehn(w&q1_WSM}u$V5*~)wd+qIFt-5Q5+LpO<$Tf1* zm|*&?%U3q8pZ8gOot*po=~Rl%!SU1p&XAuWu=%p5l30u}a3Kr$cK5o%Qk%#!Y{ORlPV*J-e7efK$Gsz?}H|(dq}MPUTC- zmW))3_5w;mQWr{;cbMGjV5>Va?tz>{IE*7aIkoIP)^BaWo2lV0( zjt2Pfe17dq=@)&+$WO8q?%2gV+-OOOeQfv8{3)s_Wi$clJEA3u1W{=zzh^FxRNzvw+ptmI}&6nH#phG_C))q82ZVK zQ|mqK?Z5oy8_E1UDKxp&^@)>&g;TkA_Oz{xlZw%EDR^YzrnnYHb^qSa-1EdjC$n=T z7#%n~*j2huYw_k}Y zHvtG3;!s%PY6$101tR~n%wsFHzmk-rIP1$u zCI*?U=Xf2Li4gk#;Zjjn=H}-1_Vyj|QtTo|E>RC9l$BABXIe=}-8HrT|L*JC%Eb3) z8Zkn0{@4Oe9_<+@XY|g}&E=N%_V%wEcG{w++Z6Un`l+MLW%cN$_a8RdRr|dqbyfom{*IE znlM6l@+-mQXuc$Qu3IzT(3j)?ovu?GZ7mynK;SnW9@%I_tcbDiV&8>hhMvVz8RP{q zG;?%%Oq+s72V$se8!K5N_7fB3+%D^puk?Jk`(Nk(Q<|^d9V$7MPd@x7-Kwc! z9JYp6A$sQY=Wa1XYIlGhhjayfc7|-2VY1G#JsUs|xiH@awPW8Jx!yg>zD6;@HhA#H z#Ck8;j53K0dey3XmrU$u35{Oh%DdVKo1^Ucd{Eeqy2DsafZthYZl&jwUf>h0!pw7r z?-aiCW}8yc*hWo4QL)-`_=e?4wIxkMsAh4}HAvQhn2)$~tsW_!IT4{OHiG z9{sDObLWib>lY?j3YBVOy}S84ljLLH+aM5#Ute??6xIW!?MJWAvPrqt2Orz1_li#* zO1Wj>w)3Knb=c&i%woOMY_J5+N$Anz=%*0fH(>=qF~Z2{DnW(MH%;&SlDq3IF)@(3 z5KUg7oKG$Du2iGxZk}y=a|BoKluz5fB1J_lS1iQ@Nz>Rb_DtQ~dTt*Z#FjF%&;=>U zyK-qR+)H+Kso7EoSgzT-n!EU`X#?VM2^wo(9wZNb(lm9yJ_#qk!mW4fXzdk-kJ@5P z>_?zi|M(RANoxQW@!@TD&a)8t`T9dc3t7JVMfq4k(NmRkNA4v5b+VDCD$tIzXT*uX#?_v!6NvX& zvJkp|Mfde4Our{VWBJQ1Q7!1m9rkSFsV1sGfI=)z4AN@92DT%0j?Co@nG0<^ZEZ5! z#)ogGAps@ZcV3jTTNUsh&LVD|n=o0UM2Iexj^xBJ=-EG(UwRtPXvU_O9rGAKuQ*N` zFLWk!WjNfzEs2vSkd1sj28E@CkCeGiOA|VDe^vZroj=UVFTKh+7Fu!!ifRQ4oH)-U z;6i$=DR=mPJqJ2JKiNqcL#=z>hZCZM!OA=+%pQS!G*3&~dOG-tlr?jC|8e?>^B|Xq z@sqiLVgd?0^ki8Y5D#C4LL-;|jf)Y${zw?36rM)DJ~J?enD%q-Qo`-zD|z8}!Ya#L zvnM?RB#KwjBUpzGiyrfGLf zX>WXdTdA#3)D*cz`G5cPD$IK|wA2*)KJ2VIzrQrAs}W*l-{!;0;K58tK9wcJNQw}r zC_(sbQi0&385CxG$Mfnws7_`vL=d@nWS}%x5YGNpLPaQbtpFi`miy)RD`B(AWe7(7 zkSRg*IAZB(0(I)G|0VVS5yI_18`uiI+q{=G%95%Y8&QFzBm9~xqo-4u_+RQ!WTLIEi#z8UBzbyxeCc5#+C{}lA!Y)tMiH>}4F_gSgPqNvF zwDK)!s}EGG0k6vBzoAD_Zp2QMH)g%mq)gUph>NB+gBL2z>w*o|xXu4Ud8A*-N*wdT zWBsivIVpc0`c1m+RMpB)86@}Yn0795{sWix@p;~%4CWQxmkc5jed^8HzIkDSW*MKTXKSG+%ditSa24mR7Wn)4@WPdR6116Qq2!{ zuCwNYmW<@y`^|%k>Z+zu6);s=TCEp5&&0Do+lAVhTpKno*xu#-X*CO-6M!4hYL z(sm@AWj%X`2EmP$$=4fW^Fm(6dYJ?ng) zY|-(h8?e5P!;sM==X~!=p|+ABep6UYzwf$y^3U$aK>XjtC*tNLP1-=fQi1feD3|mr zHpZ2Yf%FU0|4E;?Rg#3<__WzBK%7V^+X<5ZmRH<$Z4Db0;?qnK_54tPiMZT>>y)NK z&rJ%}zAq-|bybzEl^=MZ_tH`OJq&Av-!<~eC0<;mBIB$0PAZIgz)GRta**F-pZ2JO z+`=D2-8{4bowq~#ehll^)y}U?jaVIM<%D=DO!C9NcXW3yNblpt$PCm+iPV&a~Vwd*PkMaYr z)s%$ld48B8Y1tHVv7fYxaz?CCbtBp4e3JoPqr~>Y>!OMK3oYg8a3%L&3$y|l^@U8m z6|^#X`!jM+l4yVNC~+c)No-rrURe{~ny}s^2p9^G%vHFwC*sLf~X=|QQ z<|s#Fwz6`-?%GKH^%vtEbs(abMu1*<4mxB>CWSEdD2w+8Z5CobXbstCGM}WppB<*h zzbL3{eCYUfKInS~Hf6Xl@uB>)0S95k*`_B-z}pX3$PUTA`x+%J`#mKxXt%F>%{c%Y ziFv%{{ku!sRCY3RCxrxrbneSKqw%p+EXLS1(^u2L-+Og1pSKwRHt+hAO^z-7p8SHd zB=UYy!PJ@6h;RtZMQDvJtsVxw^yE@uqnZlfxOdzqvy8;~?|`YxNg%04lJ-Wfe8*5QZ#?5(}ms%n6V3DBX}@R7s=f7 zB+kHOBEXPg6`SBkTzLmiA+C0#2s?r)-s>NUI4@#n?u6&AjAS^kdbjA?lCHG(!WxQ}xu#w`xH=(< zA413|nZ0x}m3LWwMqGEZlH>W!5^Nfm&Ma$*hFyMoh?5|$-<&c6Kh=RYM7V&l!?I!ecFdXPkhk_J)nd$ z4vLv$s9tP(@@3XqbWwkw6jC{PlEEevaZv-TgDmK?R3%3Y2_5Apae zpw;DDYjIPIRr!jZkr6LL5c|3^W7q%LeL{fSTBDpjy}emT6z7Rode(DmgxRoe5iR;e zivqF&i*;?}uL<@?#{&)3piVJK$s+%~P12xdqw#p4Kug9Ev!F3Q)D}!N2`Kt;R3I>t z?e?$$6!r2y#(W`q3>SlU@ zy%&DunGDm58dqTYKHr;U={_snh_v4a1uUZT7^n0pS(hsg{fVHMZJ`?+#(dGu=JuX= zLQ=ZILin)b!>&BzN=K)7#PXncg0PWW99L&z2b*?VVlV%G(uREMxpFYTpk)^s(Z>vE z2Wh|O<8d6@7gRoqu&_v)OuTDIw|WCAVW?aM{yZwXPJeJ?KA>xR`&m$4BNGV?rmJ3; zPh_cOsmHdv8Lkq$LyaT3ym*U2CW?R(HSIb)+m)z=V?=&2d_yF8^sbF1hK9^~U5Omk{^qmXf-w|D z6d*#j`b{Tq>AR;m4R)DE!YL-G5IBRNQ-lL{yRH=PuLI5zBjzMS^7&NH+i#!Z<0Hu_Y(%z3pEf+3SK{=ByCn2b*ub6~3Q2@~kruB}HFh>lTMciqxsREbdY2 z+MNIth*}k{6I>*3p6u>5ZThOEGJ=JU2a4aWmrLAHl-Jl_p|+YB&L?LEAax#c-0Q~* zy-TGib{vxUD9h>MhN_RD;l1a264T2(*`aZ}3vC;>%pQFCDErYlQE#K8K5lH!Zy_6h zG1#V!`%dZnURi$+^X?0vAGjAuPcgx+g}=Wsdylwj13%?mI;!ojuJ(h$#QSIX?n{S3 z5D2MjZM(0~XQdDpaX=fcyz_wvNcXfcA{9RX=09O&bQmqX zsB1IVaEO#cKWNnDnN$y#)idpx%HOMYFf}vixI;)Noc8ikKuK`api-`4E$~7 z7>Sxa3_GhcJ!(^wq+C5M<$T`X8p>5@pxna@cRa8=EmS%2ogXPyWXU~9&}q6?Hm5qm!_#>6{TV1H!zTci{nb4%)22Xj2bog%AVQ>(<+)E;lkf$7B! zY1bpP&(V2=a&`I{*~PgHH?)h{rN>LYazZ%wGDy3T9AT4Wd(7He#~zavSUo(p^yJZz ziK_dFUh6CcPcB6gvdroDAxj;K32rc4de4t_*&krpGw%3QX;aw4f<9K!K^^`UFX)mU=ww-BRBaDw_}-*oZN{c zjDglm1g0&|s;{`SDu7!z4*$8ND~a%S1vC<-iA(5wd^JDa8EA=Pu)<>rv_*3=Mm*6@ z42qc7m21IMCuZm7^zw}2e3n^9q;4@$g{qF5oRZsl`Vw>^sA1T3L1J1gVQkI+G~=_Q zAns&vYgKZ~6RPL1Np)i!#FvUBP&@G^NLiIPjQ@I@HA0%qWrJ`$(BZg>X>S;ZBz@Z_ z`$(Bl_g-VK(PFz^eurej>+^EgXA5P%L8oq*Di^jbe`EpH+)Y+6TE}r4x)H{|?Q%E% zZsud&`Kk>RRT#;3)_<@B0;-55YQo;830>k;R4Kt2s0d8zDn_dMh4^$q>BqWs445PU zGhGj9NRvDsxN`cIiYa^q*I^;{M}ePXQespuv5T1ECzNp`;rBtE#s%|!WDdpX)SrQs zc_XQ7H4C=%WF%adf9aiQk%!olu2R=}zV?4So+MoxZn$5vrG`|6@<#vkmhBfF35)?{ zAPL*1iPdEjH-1tP_tprb$8`T)xCGJG`Gxs}5Zz<*K)MhoS4AA3M$9s30&UTuQ!2w! zzVojR7i-MTlwc?8L!1bz197D(@ipf4!JuWFF>qqlZI-}tTXc(KYxY8sFj_ppf}yp) z1T@0i8(FO%WjYT_;f z?Y>ba%S}jI%eFD;!Kf}D+F9l(#YXj6E z?zn)Mo-K3PcC)XxP|2Q?>>8bXVQ0fvC6ryo%8ek8V9-j-;YqS_7VV`g1+`DIF-MH>I4%&O$QUA%!Q?BgRH6y((|5mCVyoQr)0!zRk%*b&KQK zXhvRzPH4W{H2HB!=wRL<_*9_o6>0d0_rS5zk;h@x6gAU^@3z%zqPD4PXBY#!87U@e zc%eukJmb9B`;X5)b}%22=6sM+>c97RoH2N7ZX=-+{UFX`Ryg539A6#2AE1>ZmMHLh zx2T=)yCbSJw*lAWi8$CKIfjVwe-*dI;wYFnj!hcKT)~`mlRg7!SIwm$$C{tDBlkNJ&umc) zuz%12-r>dzGBR`hl6F5aexad2IQR^@?P`&Sg{Y}+IHs2Wvl7P3O> zr6adv`z=3q6*yIFzmZ0|LLx<4EhE~7SxC2s43gDEX(U$P|7XWYm&8GojP_WGqjLSi zFdX~b_GP~d@GKYUt_Mpg6&%#HP6bjH9KH^S3O{-dxmHR{?oR&20@`SHiP zb2HjUZVzT(EF1F}FpgKtkH_s2W(6BvPXrtSf|DETEd8D;N zFI`w(ZsdQuo%K7-?CIUlWuI2j(#(VOZ+ywxZ?F%=ag;<{R7C!8)~X(e;F7O9K7UZP zZebVaVZEA_E4h{Mu|8m@O&vQZxqGj-ZCQIv!3Rx~g^mc)esRo!0N{{atu?dXOGg8; z^@bw$iit!~@n_+~S1lyHDZMm70*O2x$lE4F*1r^#^z}x4r>b#LCU&fM_7*$eIJteN zO6+1Rt6{>(7&*dpFT!aiU#i0R5YfDN) zIYx@1#67Mwb8UMnAK#bH<)js#6+cxaqC2gNC@Y8dm=4u==@kgS2Q~M`yT53UR{6Mj ze<=k4ZZ4LuD*$MDgqLFChdSF6F-)-eAYk5Y#YCH%!06NSNB3C+S7t3;vVh}%w+?yD zWqr~FAnWDj9F%O%v!6W1O3lNLp(mIG2i(VV>Y!8fA%bi4hd%uz7(_v;L!hH(Z9@z@ zVy29K!>iWw#4QmLT2K)#>2%pYPUdoUe+OT(CCZH#HT7*$tm`b8?%>IzDnc~%^y?TCaE8K+Q>$SDL21Sl;N)vVUeJ}jAXq!e!sP)z;w0Zy zku4AO8yw~d@?*SEI>VpnmE#ZXq}=M%NH8pg7Y}GT6m1dSoLwT|51h+q*bJYgt#Nsc zFq#~XnWQkAwsbKisW4uPaw19IYo9^%e8aH3kSBEzkv!*mcPX{ifD4#p;_3RY;3}nu zW0ZyPPo=f@$yt)FkpkQQRGvt$PnF3@WH_xsiteHVV1ns2=QmDF(1 z{0T6Glukpt05TS}nhM^ajPjNBxCo`w9hyl9pZ_yq23VOjg&78ne?ag{vLeoCPz@g^ z6TAEr8u6l-Fu%YYs3ZCBF~h%>M%HVGrmvP~um?Gk8K^(&vcIJpAY`j(n%vA!e{6Wp3(5na_COBjs zA*cP3?7#|}u4Ie~ZJM+qF6JpFb~&_T7WIj@s*PbT_hO+PQ4tY}M8^B<=EpIx{J*Xu z5%mbmx@9rhOJ$kj$ULsHcF*D3_t@?%z34%n%Jpm=6a%8-hCIP|C;Z>TZxZ_#I_zFQ zyA;1cUO>F-CwUj5XTKCL%oy01IX|{O-CKVRy1{dt6r^66pM}zS57oUI!k-5jzSu4b zccUECpfb~LqrU{|{=YtGNgAVRB(jxg&BYK#q`(9X!TKsWLl}rkg>ud(u$0MQmD#< z;MM!>pMdPv&Fi1Cr~_2~IR^zLd7mOto5D#<<14bz-a1gs_WnbuNHwRJP%6+;#9s)Y zt!3OFsN#A7MhU#2%4CLl`vaJ~K0c00B+YVkrGNX=dsTsQdtXoM8C4r=#@*|uz!dfD zO^IwByZo9qA3_&wF79tH3E$#Jpd}I~N6OtEs`yCVM#XqI>9-B@Ss6+*y5oOYcd%tn zRYA)#Z?KQj#`&C{LWySokQji+?_N#I56|D``r{lr;>4BlJgacbiM}%cId<)z6g>TQ zXkosD3C zG+Rk{$n))@{KfIm;cCL8(XP`m3{uYs_KNWxWF~`I>McVJ((!{IRZ#Cs(Jx~WeXINH zAsD&JHL z(VjWveO#&M7lE=-ItkT#vkCQDI*grU38UE}VpaBqhp2~r1OM^s$F&Nfg&;e(jG;P{ zI{`M{cB(JE(EG#gmy4xB&|h|8zCQAI9=lBgbNSIgHQ}y7V)F+d7%q;lE zmnSv0l$=jSh7<^HdqO28scL%DlcSpUjpk8ZbhY(9(yga~>GJ3#D}T!|d?RdFk^?$& z5!aVWE_Fxf8ji3XE&62oASY6T!mRMZq(bPzWh`K=Id}zeGVugTEqmSCQYK)hK?K+I z`m70dOd25!KB6>Rit3T;U>R{35XV(^E|^{Drwzti#0~j-tt!jdE?Y*K>&2I%-^yIK zpt45>n#{(xIsZLf!CFs=>iJ3h4-4W>%6oIb_(cR)RZ1A<*%D3?S4$oEJUfYc=u6zitC z;?>=R1{n%edgR9va-c+?m$R%R4o^B#9t?l(N!u%EcO+1VM|C`?Lce8~VwS&iK#q~{ zO2r-e4kOPwekq8jYGcSuD&(-{ywFc^DYAO&<7{Skf$LuMAf0+JVeBbMS|<%h1Q8da zSL%4RBNg?qDEGmZt*j(D`f`2Z>!C5HG}BvUtip&k;Zp2g+j2&(CNrz4R3tvKa3wdO zS$}Mw2Z?h{u%NY8`gXVxslmwaWA4++6;A1KRxY3!1;0DBt=Y6MLFOX2+dKd z-g9xDaunSXl1yS~YkyFsVcqSsX?=6~$_m5JHY2yaj_^njdYO3wt(N+8cQ`z7f=2}6 zUlr#UT=-l#=waFBN9mL`A#$tKNTW^NRf>Q{&MN5#P!C)vIYPXNf52pw@mq_3@rJJq zO@9hjX%TZ>Lz%n;p~;BS@q6#ng#@C=1c`p@8x;3o0|8+FM4;(Dc3Z{Osz02}*gC`^ zd$B`$(!l!RY;}znrnlE7?yfMTu^|#SZ6uhVyxrOw3jl-pP3tz%tOw+1bDEA46duYG+tax0vlyKH8# zA^m2)vM~O*1drpaAru-OqXykj1H}m#eq-me@&#T3N(dP2y5|r!>)~a5+t!@&Ubhlw zz@(&Bgr3Fd1vUExSz@+Wg43l!jWlLk6lY_laGcHS1+GuKuwY;gzyE~NClqE|xUH^( zW(U0(9Oo6K^YQEB1gV5#u9<2;7e(PV+hiWhvW!IAmQ&gdslP-GO5Kg>a$J@!%Jh@@ zh_Fy}6P)q-`Uuj>83)-KThE|;N(nug7}qv@FRAR@`@!tZza`4IQTqiR zdSmy}mBib-i+QBV8Pm@7X@j(ne?sbs`;c`1<>l54=eD@I9afX;JL64{HdocZ895Zi z>8Uuy;u2>P{KqcSmdvm^er(qo{-m9}@H?>_(Va+Xnq(yr%Ll~*Y|jPq7!VajnApA} zeo(+kd3k9{V<7UEX+URqNyd(2denz}{~5u{1BE*YHMdFe^dOVDdq{UGr8(nwi^ZOs z`Yp%%x3Gt-;I&{?FEkPTL5pwYmDrklK_1on*JJ3Yv?0%eD4CmI8$XbkIAR{m__W~hw zpbNsyb^%60dWt7tBqZ%O1I;cmRCLE+WYDKUUXG+wfE@ZZ-xJ@doM5YvU_ZL1o>kWF zWH3vYWz&RehjLi0gMYP>c-t9WEeX<2J?fW?>!Xz|l9DCe3=WINu`|#viMCNJI6(Y- zx7=bfF@emzx4nOf${tP3*8+W7P(w%z!Sv-8OVIEl(;0CZGXhPpT_c=P=?x8ogK)zVj&Z-l79s= z>zx)3yfur$8xC&m4oh0=6ikQ+zmJ>3l{67ciBFehw4fLX`OmV8DTe7;GPu~6RoiH| zgVV!uRG<*aewhN>Y!#t{XKF7auW*ozLK;dz4y%Sa=#8+XI+zS00EY(fX}&2^mX0=r#uowq|kxgi|IC+6kMO2I% zxRYhg_sDYV!1$s*ados3FPN6K->gh1+>uUS$zG39rPe7$BvlE`A#># z)>?A|yd(Do#gqSSJ?e{WjdaIXTi>PVX>S}> zH+-%9LM|ZB|Cu1{zDxRUctCpsg-3(1_L zt36pV{hgkJ*m?8p)5S5}><1SFz|}=D*ZT+5PU#3O&4CTJnqP0_KK}l?auyhL7kPp9 z$$UrTK`wV6oUSgE*k8PBMO~}Ua##<3YC`M|5>pCBe3x=7z1CJ$_gTqYuCw@#!rtr; zyQIi5UmY0x*`mTS_r-AZSSV6r;NALDH(9HUKgvv|GTTUz`ZcbhHmZ`!>TK+? z_QQQSYVWVT9F`7cXC_(AoD&6W8+r7!atv_qXxd6h)hC{z_OVre!00M1&veL}pYbxt z*|D&0*TF)f#=p8Cl~%_wNDub>h`7Alwyx#rnX1Kz83vymzG$^OW zHC3RPQjw~F<+g#M5V+{LAD-6xiLD5i2Rp*U&&rRA$eRScQf2(7w3ErW4Y4IyGz#Db zi-<0dXJzEVIvh>wt%Yu8WxX>kF$T~h;T`1I$z4H+jE%W&T~ZUUi9!Wz@OiP{scg%y z#shlaxJ%*n)u2dYqE4O`K}-t95e9S)2+;K^f6^HID_OyLMAe4k?ybJ{nB=-}!{P+( zDnjPG-y%;XPRj9rs_C#JS{X`PJN7YsAiop8=r(|gIM1lJSb@jJ{APhat5}%7yf^N* zA=xKv&3W+X{&gPx%b<3|c0`Rn>bf$eAt4XO0Dr2rKWnudt7XImTGHah-oOS4yq_9* zehC3G$E78ehHwUOW%wT{BOrw%fF*D7l-y-%<}Gje zc1PU0in^4tyWOGwVgFa%i75n37FzSi-^t(KTnwhP3qqdLP2pKuNt>wVBtobQ=yRkw zz`W-=lls?xwXRkyi6Es@_MRQI#>LNp%eLJGPt$^VK+Ce@HG5#O7QR?cIDbE7R$CBt z6HG!D7VI?X^sJa*gZ-I9j!whAzR5ohyGo?SLPEO9g=rObxkI$ynHjaVy)>}&2m_cM zm+SlI7k_0(nM!LwH?-X^LI=*y^r9)N*!Owe4OaSX9|1gv63Zgu<+=sSZ|(*3+fHd>-Z~Tw zZ9es-_-|D??{za1(NORn0KMmE^2ruP+ySx3&IPU>E z^d=d{cY7B*>RFf{8S%hVs{%=v85KhltFq)HfnFtK*c`!C1Q%SM z$Z@FD2{A%BR-;MmmhV$>jY^U;L-bgYyFENKQSCLdpdJG4l;$fW?Fm4JC3JpLqW*0y zkq@xc?y{;)47Gf_a@8SC(r@T%&#hpC9^q=3oTSDXyJ^91+gXI!w6J>YtmegZW<@4 zKUsU`bXD^xilQxFt!GF|QI*L~`a}C!@MdRev`SoYF_=;ER@2s!_*t%IbbvC3rxkjPz3Z3~ujqq1U0$XuAeP@4G>2cE}<;h&OPugmqhtJ``3xs~o)qZ8voJ6?NU zEohK=<;h$`q<$2fu(>1$VnNzHn11dpO%r3Yea`3r8$0|)hs=1Q|G$_0;N}`x0nbkr z54T3WKuMdOr1Xrn>cXdVp~g;sls?qgghL9wcqlba7EV~a53yBJT%2@XO<01;x&Lu-UK zavEXnl@!>~t)<4edxeSftHGHza59Np}Nc zAF#Qi&*4sk1>aMK7lnLQhKpO6ZAqGOfaxFJtLqnh8D{=fij6UvJDMT9BL#cqe%b5| z66onl>nQ|&sN7|vd|$?Rpy2?9nBpv0|+5)!VpG1QFB?kV!u(`wgb zS}r*75b`)vPOls>(+OER_^$G!{eb!#lGE!?M_2^hnve`(ZtSuXbiFqom#164;e}+q z2?8z{)OK14d@u1MBpw)(Ty)Heb7`dqpNQwtQ6o^=)`4bbh8heD=-PQdzk+dq2hatH zPag4+M56_|Sc1vYWxIZoVJQ71Hj%{AUq3k!_bhqUkUD6(sv$UI3n~7nm$@sRB~_lTd$wkK2$C_)>^hC$2K{c&v_6 zRsPv5E1~TP7U_qTBArI$&@6Ui8npAbiF|^kW4=oOVctR=16ckhifnz?Q!#yhv$i_2 z+||rIDuBfUL{30L0fEiN(0k4y#cOq>9DLnG4Fzp=|G48b(5zAH$%h;VR#yp02ii6M z{hDMq3SIUVs~w20PnFd`(~?v=0n6w>ZAspwV&J|H?fCqOwMYNrR4|h80mRgICfJKY z_qU$q6DuQ9LUFKjKy!cpcCR5Ds&Yb01Myb?8Ru?Ex%W&JC)M2&C7c?NZQdA4PN?=K z5q#;k05!St))l$S-uwrf`9NEw8i8WMsJFM*b7g>_sXWbtISP(k8{d|40iBpA4>+DI zez7Dzj>lPWH73;yrq4aR@m!*veHBM*Eb*|anaxQb77)@5* z$cT6isLp~iV}gH}pR+y*k(ZRQzHK(Rd0DQD^*)suyfE*9JYxbo)??AZYpA7w^_0es zt*x!>2f4$AmLZvg+RyiWzNKawLSyA_bKq7WTu|rYjxxVJYRftMV$ow_{tHCwwp2;@ z;R!g}JMZM`0m%G-bv=C*ejOwy1rN-@OBvJs$x#wni^N0aIx|if68Tpqf=@!gC*9If zq)#A`>IVP8u+8X^&8=>(qqR!ATH4x%p4-ZyY37sB@HizH}HdV;C4 zbY$m8(u|77*U2|7I&A#JM$wS%U+14q;Dd26PfSH`0dJ%skyS1$n8@L)IM-PJkjH1L zR=|A+PuDeCFT@;1{3R1&g+!z878)h-pk+LIJkCJ}kymUGujJ_R2uyI<(Hr=$2x#pzu981JE)jvLT&uz58{Mqn(%86y99mtrasXa zu^tkoJtCaMos}7r`R4)#8Zg(L#zQ}z!~Kj90951(d9nK-WfCTqh;Wp5M*0-#T1EtH z{xS5xq}ZO<(_H$EPJN^d`je@rukXVTdRs_LI*;x50N`r#huE6j*F^`R+U0=9xg$EofWsEStDJMx!vL}AD0S^)*VV81 zx^U7RnkUZ4jdA{i1jHK~_|DV^XTr$X@$MBcXhzzi*hhz;Q^Dw`Vc+8P0`co$TFClr zrLN}*In{%F6DLVMeIKvGaXJY^?_0tN;gv)m^-^%%0o)~!p?{UjKzu{Cz9o?C9<|?e z$YQrwcD~oQHyna;Xi+Qudw-@1T=Mb1u>QJk4dOtst>oK_la9Tyb#aYK7odLgD5ExV*LC8q440MY+HG=`omj3GS$> zDdICtFFhiQpgFV%+xaR@gZU?ZJVlWxes~-+_?n6m(!^F+d}DRI964vI;7g*ar<;3U z<_wg|v9U`&FQq$T;qb(Q0h>(Cc6%V4G)(>ZhFZ+B0TwL4KJ;=Aj&zu#T0)wdkf@8`%pJip`mfHG9sL+q-duo== zS^S5}Xl$S|tHe3Th>-DuQUBV%cIB(l(k!tK_O#n5h#UEFM+n?;Nb&D?`hVDa%cv^5 z{cZFH3_t-TRZ2lXKtQ@dNktYRDGe$hUDEX^As`?M(u+{Kr5g!@ZjcTs>2B7UYb`*Z z{oiN2?>ojhW1J7?(;km|anJtEUtHHUM{T#u(}ZVGXmpUAIh@ihx4wME`t+tiUc5$t zw32oREdWl*k64iIn=#&yyx3NIe-P9 zN~i=4Z7*3Bqiugyg|2 z#x1UeZ@<_Fml{0xLNy$P@RhZK@EM`<8}UwO#xyQ@1|HK+g%a20b=xZE2ImP%uX>I= zy8CFUkotkl5C?zRqR9>}FSH39tzGQR0Xfi~!?}Aa^$}qD)2a1#gc$MAR z8c|Vkak}VqmNM=K6#qL7#K&oAjLi5v4bz|)ngY`T6<>fCDqQhj3%M0r9x!oHK3FrRYFII=l+Ic}f>D z_B)0>|1+S1UxWKr|5^+jUUj%NXs-UF3lsim)Sra;K*(VHPE41oK^G^MRmlzV;Ue>B zWV%adJ#?rb4{vvfoHe=>UNKihRn0Jv@p2dBXu^4I&!X-LHA0;;(`#%{!8g=l0%YpN zui(8S*laA&Ia)<*sVn(#e-Kse_UmK}cN-bHUIes=yxG?mEPiA^LD zB9e1(7FpE>Q+=}n-S^)a~~KV#e39Y9_8twd8OM!IuiT?mHa0 z7&seQ@(SA=VTo4lr0MDi*iJ>ckV5f@KP&zLGE3vAXpb6ul7dH!$jF+Q0Qg+MH4ohL zNY<`)ciUMmj3o5qWrJpfy(;F$;a-9<{2n9C_RGk5;H`%a4t9%~W=2-FJeCnn^`SaL zAC}|xkLE8dl|r8IlU$5Kxdvvg`bz%9b0}^mxn=^SlOT#o>y# zzB{3MTg^9kY#-+eb(QVX)P=g&M-Y9~>AHf99n$Ts!>x()Sbe9xPx)PCRA||4VUM8( zv5^*FK>;~8oo+1bno3~Y%k|HW4HD=otnBa8h7a#39L@%2I8(Xr?}MNi+W7EaJ`EP> zaJ@}F4zQ$rO~9%m!V%VTyuA2OCE7LtNd)^dW5^o8FyIY!=2&1M4)XI-Y6GFWmM1LY zRDf?m7AwH2EJ*nTreM}QMKR7mG7QLno!C?bArgrG)oB+bFVr0MSsyOXehuHbJjUMA z)|R7D*ytX>hXzfY_y5;t%8fC0ka`zrNjq{&@k$JJdl1BcaDtcc0?PTuYEZ(^<3Qae z0D+UL-3$#)6zVFuKo(Q?z^(=9xc}%u15OQyvI@#>Aa(YVcjP4ksyWQ4nMU%SNb@Nc zl(5i^hdpfp3LG*Q3ZAJt0`d{HXCWNmnETI1JY0I?LG1oTRpcxTWs~o&Dnk($PmfYx zBZ=OH7?Ca}b&cAByYsZEyud^mG~h;;00lWsi_!+M%s3qzZ=UE6cwIR_K*P6bz@7dB z-u9PuX1hs6)a5sHn|rt?LyckJcVc&KIEyA}@7a(33VjHEK;D-kz6FEpV;F$hzCRL3f7WUT-{>OY1A2WW4$C@%v^{BQXZzkU7gUtF?QbDn+I%e* z@}OxJ)0A&>Zc2_p2bMJ`lf_nhEq$=bL4@lOJj%iwqA2=RuW11I>)55gWuYLDDa~Ct z2#_Y4b3mpOmawjM$w0kUj*)%3(y*tkZ)0Oke1hFKJ~k$Ta=Wm@QcU%phK=`8g+N8~ zlOMf}QF$(1f|D~B+kA0qZn$7JAoX1wFPo%-{8%o%Nud2FkHQ=ZPtJz~fS68*vJ?iN zv8?uxhU)S?wrnVXr3#efVgm-|R)G_G@Yv!9$)4VV=uD=c-?Pe!*axxF7vZ|%pbr27 zVOckKDY}@`oz8C@1k=Dy>v#>QT=8reF^a-D(*8oA`5fD*m-y@$t#r`Nxy(oqkeT*; z%ii#ztLe8RZ@pME7dBMlaRyBE`!A>?O)moZlKoS95WuH_Q)95&>F*Z@59beT zS$rqYXB0`|PZ^04@UiF8TDF9=9ZSt;ACmEIDP; z)xw%$9gnGAMzhMAsodv+3;>C${{Z&*Srr!o1`811$~!GC6(IGpxe-h?euk zEsJ|&rEDH!8;H&79(R|OU_@|z=#8P)qeW-oB+DaF52v{_AX4PsD?DsWg;DZZRRnPx zy^Y1@xDMjHR$$4m9_zkLdSjwQ2>z#>*z|C7&##2n&N)s={4
qwY%`-;|A^m|L+0jN6@GjyVnV{nnY|ISlI{((3 zIhv2PjOCyHdby!+x3pB7y8Hd|*ACzT9($vI*{uRXPqHA9k5|5Wj~2yv=Ir9kjKzBIf&su{CUVOQqXH?<`6w6qi@;P5^o?AU7>H_LzE<&GYj<0>E;ReEn+% zXlx+&`GbYXw!<6L?n^Iib&sIY(&4vAPH`KhM^|9{x?GF)E+D|pNip3(@+sXr*Krm= zDMmCQN{*CEE<>{{DC_u|!ZQakvdp;;#!+as5owFql;eG=;Wp{_o&fRd1@Ft&{S%rL zic#a>;y8}P~N}p+~PMXo*b$1`5y1R_J=_G$ane|CSQ&R2e^VHIaW)0-$ zmP>ity=luwz0j=hQB?y<-YM%#f5My7HC63@x`bu2PT}} z`UNhVwCxU6W>xCcFmU+h!OTo82WzQrbuG%rmGKpLoO567ER;W7dzUndm`qu>V1hPK z+CGXGd7w6M@9I?Ub*C}Kxn$-pBnB)1S9`FiUzb1%vS_U`GFnD-;PAf07hDP8)D^t5 z)bZ-@%^VY6_CJbnt7368aAuYRcPS0s?p=T7);XSoK<4R9a3%j~g z8C>~;Ar3>95VfGQFQAcJ{W>IJNL}9%rIHnT$oi&20@N7pRl{ADD|tCT6M8CsovC)& z_z}Kq-zn$v%>D?F`hH1$y$fVc?qUIbFtmfv@brwHVy#xG%@h!^GOK~Nfot6h=O;a@ zcA4_MZiYwI@mUNPYvA+)pNqsax2pD+^8MT&d#guJuu1Qq?aR`f;hlMJ#Yb^&tloJc z#A2Y}fzofhA7{p-dPuU{W6DN8p|uH)6|+7@L_oJe=F-n~wkz`-VO&Vjiqo&}A$Jb1 zfkd}%*fhYPtCZY;bv!B(%lb*U2}o80g{Cb0rg)?chc6oo{Km}ZADkcCaK2p`ymxh( z1C$!e0Y`mHxTz2B&~5u|6wn~6C#1dt0s?@x9m}Ojv+HB@CuUZ^WFqxs`K)oKFw)%L z@ct*=Cy`y*+k3=|OD+ypn|0$-lFxax+S@*qX8O2+JAvWw{ygAt4g3tafaLk*XjM*s z?Vv+559fE!0ldgx5;^{UI5oKM?Mq((v0{*^`@mF}k;W4o{Do8m!0*VDC(e|A(h`9f zSUXigbi4^7^F^1o%Dnl!NsBYFt1I$4h_6Hp!{7VS+6id+t+8A5^bm8V#=ylVDRIW@ zueApX+Zgu8uC;ztET>6Gp-}`?3xEIDkZho zQWS$O94Q`979xNmW}K+SDQ*Zr`^$yzR?^dKa$s7;7Q>*TPGUjAX71k~qyM^)xssQ( zQe;@<+TsjxwIdL+EHdMZiC~GbX%pTy8!G8ARFWw6136io@Xh_V>I4{$nB8ldf~gI_ zD2f2)a~=$>0q(p_nIuFy+g22HYQ8{f0g~%mC*~7qu_{&B8EQ# zcHfIhG->@rsSR-339hn9GXwM4%s}OeB&81VUE2co#p7^M4KUqtct_Cs6Ms~XR^G1K z6ioF;J_K42ib)*LIC*jK__!S;+t!OLTZ9}AV7zH)p2@M|9uPMgyD>Rs9R4Ws^H0$E z-4R<$BquQ0zzoSZJhrf+TYw*K#mV4;9-u2o{C56bjBCgJg9D4_{6fBX0b`DcKw&>5D| zhZLw#fO>UVF;jrJ!nb@Z=x*F4&uBxO-o)gyY60cA86UKb3=LH?)kD4~f^ij`F%3|6 z?Wwmj*|aqg22OdQki8d=ta4v)dA!q{!gAa2bp@~O)UN5>!CkVg?vvjLP*dU|U zw32d=7K*-AB#B@&-uHMS$`ncF`w>*a6u-4<4eVQ{EF{m)`*IJrU|?FEUpZOm3D?a~ zn!IYvp;)%^MseutePYkyOCzBAApoa@pgq9hy5~4m#6M0l#_HOl!j5@IN(^d6+TJ#N z2yz1?3T8+CJAkZ4^=E8%G`=+b6LQHD$c3M-8QA;yZ@i!w5aFME_>Jm{4Upg*6Z*67 z^a@^vV*wK64ANUNIuCh@+<=0gH(NSp=k5R@y~8%jz_brJDZMfazORQJ`OoP6$0uW+ zJ3E*CTAuZ}`q?q9*@fXc6hJkA)NQ=~pTB*&hPk3?6jYkvFo|oWp^b{R=|+NBOIN^s!0x2|cFeKik?X7zn zK##;Ja(C3~U(xCc@O|7(V5&FO7{)X6j_x5owMeE*-KN)m8o&8AxI}uhwOxQBel4*1INF#;CpgdMx;-iw15>H)c(nE9K!UiCK&~No%E_||6X;fgn zRJ3AeR&AK?()GWlEAl&(X_cf&b50GvM#hKqY#0N}+Vcs39&x4EbU|iY9zns`fGpixNNt_o*)5XYo~w zpJaig=)$ultGtz|+*#9{YZzC6HE=pL2=qX^ujQK_-r&V&Ui#dz58W1Io8JbN=)*U* z4O!u(H#8e{u0J@hCz^sJLpN2uX?$r)lQtkZcjs*YX)$y22Y0%mm|rXtoYfr?)N>C% zhts~PvZsY;ftn=XHla8FgDFs<`LHzFde|qp*|+G7Bl6S-%rV8P2_vkOIYd~jrQ5NDo(CH^nXKE!G<30NAzzU*;su%9|pD4F(aI%H($&;D3 zZb-*Vt|N)|;4H3pbc{xqmPpE!6V5aEC_cvF0b~NM;39)Q_ZNLzi-WwDBbjsKss|t@ z?e!^$&(gA`KZ@q%gaVS~VX&Y0Dd8HSIzh#)6b~!Z%dF2{i7dg4NfEoDB5o@lkcSd~ zEjU<6T?danOdW1nB8p;1jr%u~r$R-&8@ueFt&YCr;DS1lr-tYBqQAh4U6x9X05Hlu zF*x}HftzdC$_OBoKQo{Bz@mBJ6Y64a5MfgXnf>^geCsui^1XQ!5X# zA!PFn{sIIDfr2M0aH>zr<+Q)eIqj1kgw0Fm6$ z3lJo=y3r+HK2i|VdTva}@}LFb4&qM`J5*4g7?i#+XyGdcbeGea%(LxiaW5mMdI3Zz zrt7xIpD#&y5~BAcQtt2$Ho)rGmW#Shh5HU+YrNRp3a@uaQvnudvZ6dDqRFU^$p7b$rqiD|SaDN<($vYwYOZgvLQT8pli84V;od{iAjH8? zxagRar_JA;aJyIo!d-h1DeG#MEPU-a$BIY>>I(p}oXa-PrXL{k>4r*Roq=~qsoVI^ z1wdM~m>>&X1Ff)v9Ut==5B#6aEtFHQy`eW>z~8-wwAjFsR$O4ngno+&e*rx7L{b}GY_+n82?*fpC# z+_k3Msunk%wVUW7;Fn0D`#h%2r+-#~>^IK8Iu(Xc(CQXyY+PGq@`RlZ1W~NP#t~8>j3Z8 zJQ;*klQK0CNgR2xy`_0v)O^-t0wu zjWjLi(_bvP7Vs5#Kv@JDWp;CEN-0C#^v?qT>(G`q;IX=TL{yZ<=cj%-=YS8am2cSM zeSyrZ0DAU`HkDyEwk-WTgClxf-+WNX4vr~Jyb+UYucghPV$xCdAl+#MBrSL<6-gRj zr^k%)Hr`)Wkf$eMxB)QE$Gb*DHA}>1+JD-rcEGwYMtQ36Gg>6>@wWnis)BXI04g&Q zLjqI2te{ymUUBt6LXf3bKrK@}VH`L_(b8r=H>Sl@rHU*^tAKdNU{CIwr>yha<^XJP zgh6U~HZTz`ds9vf@ZL@vOh7g$WsPp|8T>-}Vvbg6=d~!q+zWy{9fn0xCJdqOc;gpM za(jPqHof+#Q|h;}9T$R!U2cirdtchotX=4*QO|mGyNra(-)_Sr7WZJig1=9HRqkTAM!}n>7OhJhb8agE zPg%+3WZ4K?o!k~*zXV^c<(W!rnA%*ZYQbN;>GkL9+87{~7BTIvx9aa(wK6CnHp&+N z3iby#nP<<}McIzo+X^s|Ou}(1!>gC8l_NPO%4!^bg26YOs~T-kG~ZWLZ?>j}C=#_+6~W!7d+w6%G)@MU_uX-=#wxzK+kp(iml@ zf>TbZ_7GC-+KRil!O^xQ#Y8*)h&Hy?5Vz^IY6&P811L!o>5rkF3XVq==o#V|BxblF zg|Q>=Ae_P!TT;m(xP1E6mt=A}L#PPi^DHugP<+j)#^q1ci!Z?K(4hN3p<|_PQcI=% zd7P!L$sBkcrzqK^MK!Cv}nJ%Lvn!iwkx z)AR}KVc1+i4P|Ni-K+O7e&gWEnr!~-$FQAqcvf!jgYM+@-$J~1fEbSi1K9t{h69{SLd^tPD;;~9LRA#%sGaR?W zrt7BRLDW@h&0JNwHS~sFPMv7KEo*I&t>~x_Gxvx-*>;k2W;)95Io{c^dL5(=wF7To zL=&o8%~@0A44;RXhO}HP0Fi657_p2voT?|b7)l)0de0gNH3yOxH%FT`PhIky=WYj8 zQ)mO`ML(nk0SDhXRfE}r3o5@1RX$2MBt%^`?F^TrfSA)=jY3sf=-gX{|EK@m0GYX{ zblyZ7Qa-862{=Ta>>;T_Pttq+acUxjh~KlT7+0e<+drQ{+0|)(v3M{y9RZ#(16q~S z(m+V)#mEPB_8D5J1GC)j9FGl)fZoW>iA}8*aHb2nY%N$%ia*LC0TKd~ z!;JLy9GuNEG$aFTs2e3d6IfADY}YBV`r-php3{PaVQU~K(ytwmKErv$LPk<9&qoQ}oWH_o;pZsYK`5_8S55M->WPAp7!xO&5Np_hM96b(@Ls zH=s}AT70>R(NtT^AfzBw~}UU=2aFb__@piw`<-lM<#u-ap^`xDuNT=!=Y!qQ&!KtZgp zK*y=p9B-?jP=lEs`luvea}5w<{&ct3Xkoa#uk5@_lP%)tu&Vg}HbwgK9K23-qPkW_ zSh^&Xc$;FN5fCIBC5&r<76xoSOnTm-a#PUm<@^%)8`K=g&3%9V=91?yNFg9yvVSh8 z#)D987%-{q*HT3E06GsTUd1-?`yq#@VCWKmw^3?cZZNy49AY|y5&Wn5#g&Dw`obxRWA0gztmKd zPaR@={JFl*PGzOOp?~F(AIWwFT)a-C(WJ2Gbyh|E%zX#WSA@vTTprv=RK4c-O4vLL zxuYqi+5L273tqP{j>9iXdv85N`F%;LPj)=w_|lH_<~_VffQdi=7zH448DQ`dKfT=@2W8T}56pcVQov#rb(v~5nwG2ZP%Zvgy*a>K zZT)8O%xKWTw6#mtXvxgb!IPd*=V0W1&ic4!l`*fXHWh+g$styMoZ>QzMN*LSL-w>& zq0M?VWOs5|I4Rm0pXBoJh`|1basr~?14*V^8gcu-oi?efHQWrAmPhTj18KyFK!<`F zUzK&MIDI!8gfY&6C>gkJN!Xn9r&iiX?Zgx_H$8Xcq8nRFJt9rbI5gNkq5+rGVDBzyzozB zumGi##sY=Er3uRG?zC2rV_=5ED_kcI^972nCcSC6^g(2rD)d_KQzPxkYDrv50J{~0 z0a77l&iI>GfAbGvzg_z(Jk#p7SwYT7jNEm|4pK}p2c!0dy`t8=Lg6oIhAzq`j) zdFMRN#m1iZYP#CCiOuCHNUKotLYb;wvFcC%thuyL`o|*t?chl`Smxe9h~H4uX8Hhr z!Q!Uw$*?*?Glfu=bx@N19#`PWqCR@qW^K4X^pmZ*^z!v@(F8iKhp#M4LR?Q;lTiBn zr?n0)jlWv++iq(Wwf_NHBMs6)?7SuaL3T1KF1o))1^ak9s!rQfiZ>gJD_4?D!?P)O zTPFXvI+$Riu%?M|$13OGD?4u}+sVd?X#BIb*ORAD*EVV9)?<7iuX-`;Ja{O9;Y!09 zc5b!feq#%tH(;_&MVc4n>AIw3JCg}2T3}oK|Gz=HLD<65#^}dIkDm^T?luA-s3CxA zB%kx)@Rd2YrCE*P}Eefu`JC4XLi^rRAwMobQWzf~}{lcY*u z0^V^s%2OVRe22?JwncHaykmpPr;&o`c)!8ll73nPPjcC+f@D~L0b%D$;`dUgt9O51 z*f+-+8MC2-x`k}jRG%fFyB(+;c54d8+^;1+DiC;V$-Gj9V5UnkD%i-g?{$kFW}1C+ zB9}HOkPHX49M!h*^PNP?<;x<1F_~3Ha>Tmq>xK#h`btmHuPk9Rm^3adnEi0OwCTk2 z3`w`Zr(Cl=3>){^VN_i5aUc-_ZH!M~idm$;}C|(?hd^RSSXE=yPi8ju=&WP6iXc+un0m(OGX( zat%-d-bwx_^Hjb4;gYxk-S{iQYR-WrKt3=pqi)qoIcFv#Hmx&do~@@|JDh!Br=l>e z;CC_)Fuz95&MuXEr^{tVN&G$po{byDOpm&v6+dmlX;)k0=nZ?ntr2lke5l>HIJrSF z=a-)A=RmYBy zn1l#r_cA8q(MZ(P->mjUl(xPZ#$^?;j~?UZvk zB#+Id$NyPF97U&2LEs{s+(KZ>KUkL4J+V1+a&0%5hEzW(XD#G{hj{~fa+(uA@S{eW zWDEYG$bI$2W7oDDbldmp24Y&~JTSi`_iLuR8`bA1nTbcvZ!m|h2+KWK4x^to2YaZ} z#zSqol~_odB$QwIC?8<$mh+++TJsRj6S? z?erp4koDh8Iynm%TNL9i|7@+2JDmUAy>A!xSA%ea)jwwZ7Jb9sqhl-j9df}hNc}TA z_hegQ^FbvVcq#SWTI}f1i)Mr3b=v=zep5NiUrYURy;j#nT=#Td%XFP?3BLbyY;d}+ z{MG-;d45pUUF01yS>QhqczEr_ks_VDx~LM%emouw1}mlCgKvK-pBXrvGN4kPIm23k>nsp&h!t)Y5mEZw{EqPO0eZo1Lc{p>l~|`H{IJFlM`JQ|ald_F1a*sV0Lj zd^UuN&Q9RrCxV z*|PXv-cu1yo(}E~<+eVrlPgKv1MAK+X{;w7V(Wdl^0SKgk9TCu*YA4semnf@RA!Zb z?A@|u?4Hcx{FB6K6L$yf5l4nj=b#NC4I{o3V}FvVT+us``)~W>avP&1o)_6xHFMwG zx^sBR%;1a2C+dfWmZAK4il@_ZbZ#s+Qpvt{eB%k@WBY~YcC;h1{b$hrZZ#q*4Jo@TJ6ZW2HHRM61C*`PQUw#cjRPo|MXJc zD4S8}$=454M*K|374ezU3>)xScMi&xVlWO#8}oSRpRpDA*uC;057Rg?Fp54_1f0|_ zeFC;`9IA7;1U35wo*I))UI>gf2|3;$nRopB#`mJT&_=)E`J0PHa4%;0V4j3h1MrAE z$dya(9rZq+E|&J%RP5=p&X{J=a zmr%jo|5^207r__9TO+>py`2A}2N`~f_Wpg>8=grsOz=cM)yYlQKEB;j*eCiTjbFJd z(|?8=w-gb6Y=}ORc<^T)5@%}azY8jl2&~V~o%p!*%2ugwbX`%7!u8~LzA~qQ)-3-e zr9Fkxjqc)B*ChfrfH~x)N;>x3Wg~rMv>QNu9)w8-OiroF#~BbsFE|#vx37P!Tl&Pj z6hw86iNHXHf@za9j{diw!Xuf)h~qXitKOA7m4sM<9>is5{Q9T9GLBo%c`;Yo6`U@; z7gdhBB?9XvO868i!G2Gcjvx$ovDlT)`iDQH^%{DEG_G^A9E+9Ysl z#KouX$rT3zt1n>wvP&<7>mRPRj5QFt18K#FIe!ED?@P zmvj@XTW7+%9bVjW>$4@G)QTDy>X2FOS|kfGwcQ6nUD~e%5H|5D%6q+gL~4tOxY{Gns=`yfioFqOMa zQTbcS)Afwr6|gyIAo)9>D~-45{Va*zp5%A@yE|i=0NHtb=`{gZV>0-Osl(o zYv6zW%>?WDvEAKWKs$&7@6H7FrId09`y0G8zQ%ImM}B}$f@FZS$2|8TVAFqihj>bI z8WK(+D)Tj%gD!c7@c>76PA+9+9X@DTJ)F!2wltX8XMYEmzA=ZjLy?O~BP4ZqS`H7R zQ$#^6m)QSg3_)wiF--7bID&K%C*nJ5~40n{k05*xaauieul;|jy~0On*d zSOf!Fv*o7X0t0syczktb{&Kk~c#RdrGDyiRVwPFJ{VM5w9?;MIyPhU$Vk!&f;B6|m zTtONW74?}Buu^)=h>8B45-443^6qm-yy(H^#jil+Kz6=YkUu_6LtK8Nj7*yZ5qWY> zW-F>|=Nkq-$}ntd{y{3u9Ze@y-o%#FUT!fga3NJDH|ntxNGe*)uhxAKI!wu7iKXJV zxk(A<77`ZD^7E4vi!^-*0HUp?*Ym{uacEvEpxwCQw(2&4^u`Mx)Sk56NXP#z@Q3{*Lp(ue<;bSjPZSrE@_8JoO+8 z>rQ*Eoxi!S2VN*C+F%|(y6+sK;Ea4U)i$w+oZiVF=Nd^C+V#v1tkjQmT<6vf=yDB$ zDZ9R4id=r<3;+l=yFZz~jk^3^Ae-Ldz^T|BYj{%avR3OL<=CYhWm*e>{2&X$m7;;n zQk8^3LTK)c;Oa`ajx=tKfi*y}EcZwp3fug8-2w!a?fH2r9|Oqge4B^e*LQqL&uO61 zh{L^9(M^yN;%5&)rG&-?)GJ58leSl6dgO=fx?ok*jiF?oWIf-Y+m^9z^UYhOYJD+| zlW^PhU~L;E_A3G|eCv_?n^$u@Rsz=$cP6>}yo6Cnpw!_A)Lf}zT1C7dQ8@ICJd_TH z-SZ*NdJ3MWwqI16As;NZ9IbH5Pxb}t643w3}C$=X?lWHP|7Hi+HD&$?{DjugIv*`9hqfxgi2ZAhNM{)vSMsrOH?+ zo!%M$<2&o6j2Za~kFBs#zOUvA#mJl;e6o>^Y$)MQ;&fCl5+E7otGQUrj&kq;V#jTy zzyG^ zxrL7+(J^K%k%G&-H@1WnWJ`M7O^<_%?7*IR@QTjQ=%A=X{3qb(w2ADSMM4XGW6V`? z*^zDxpY#z@IdNZjo(@O5U3q|2Xq-I*r9aQc0gA21sZVPlH;nmL?z4nPq>9RQX7tJJ zfW6>Dpc;PIJVBcscMogHsp1QL?@KtAc9#~>7Qk42T#J8oV&-P!kFBbexvOhoKmmA& zJ-7QB1Jt+w!zlQF`6snI>2K1{Kbr%x+;f)Jx#Qq{fE^FMqb7!fi|$0%vdM(cmnE0d zFCbsDn!0yh1)1ap$-MS^$90Rb0k2i=H);9;Fpq#&^m`Phb!Ejn4|5Nycaq6MH-g84 zU;VFoKyXt8ladHDLw}O`!bcTekU7m)Cy8WX@UUscNy7BMrGr80>pws6F{*Smy6(lH zD9C?NDjye(kk^_Ws5YUqp_IzV>O)fm9KI!Ln4ph1q#xffxkZ0EqsW_`E1{vSK>j2b zEl+e(82Y$Yy{Ue|{{hi0`X{GZX=H}UXvL)|{phGT&CCwiZ#vN&Zo{_m>W5kf_!m)i z3a~jDy1g6oQ7nO}->ohqUF~db4qAn(Lun9&JvAxEV(NWFvsd0^%aV*MJ-^V+1tUtL z3>k!sx2V~H#mo9JlBbvSYvYXDlS+KlsbF6XWFML_%H-%n_f6 z?PsRucld;*`z5`rbGa!_(oqUzOu0v~!&b&S};=f2^nJ z>vIt=A@~gv@~~Mo2ww4-RvC`$Wo-JMI)^OqJsf~K`)4L2A8GI8r=&z4UiYuOUATpd zWouyFls7(+$YOXdYvn$ml+iZiUuwAdsV{ZYfDo5g;MU1@6`)H;oB_3wT+VZNX2WHh z?Fr)cQpPGHzMUJI3-(L1Z9)rt{J^#@^VIS_%UYJdP59ZtyvGE0Bt1?qkVn7kC$ymJ zYzcB@;FMh{T=0@M<5hGb)W{Ly8K$&PBWT2z05b`mBkyAuvV0I%H;cV$neQVVkRs`N zQSP?Iyjtx83Lx`tU#KH16#b>T_ti;}Gh3-_eOAyp=#1b$D{KXuE#r)0dl$tyy|e@R zufD?H&kZj2(d!KG;>vI3p#rSjmAZ#@9mYgFv3NB)j-S;N7KkadDep}^m_^naSw`HL za|AK?IQzZM-qe=e(Ae)!d&R{0@^?hul_%#T!sx4yR&H^RTR7^&@^_xrxjRokdZr*t zSf(Uf@EvN?mp$G^JKC;kg+=CgnlHYL^gdFwD%Tt}63d*)z1o>V>>jWcMo=}e!x;6A zCu$du)TunByjHtKQ)`x=aA9r$pg$_3?9CQ{+!@)bU0Xs5R|nos*p8aeFy2kl)dF`9 zT!ZuS6LiszBxQk$o9s|Zv{mAoqkc8QHRJp+iKz$1 zm1|xFa1%t}PY2XK$o({!MoHpF^oo34K;VPf5mGv}+G4;?_fmZjjb=qWskg3E7Uj>M`U^U>Pdlfv8<=oGs$yCxkPXUu#t*a~)AM}I`0 zx;GAZ@PrFL%<|T}I)xkVg0e|0ZcX6Y`3UmmbN&y+$o%~TLQJ&l>Qj%s?dpg9p8(tc znWg=}Pk7luZmCJ4iF6swY`r4Ex4z3+->Bk$Xx?0ZO7liQr$ns0Z#jPn&zCMo&%5qy zzy)FHD4mxpi$;_qc1U>$`OhLLSk@TByA2AlIwuY+66YCk>< zxJhG?h_8+T`W(5&N5~|vQ647Uq_Iq2CtH0|e|cb99FdxXQ{8M3vH3}CpAK1(j`{Q< zv)6sOay$Lr?`2RF&J0OuTjA0)luTWTyd+qBpDbPi!XMRZ64Hd(iJm>%L_l_%b%SCy z3U~NxDm^VkR#6R4y;#Ik-6}zutD#m=%Q0nrQ<=(6p-@#Q1{#9>b%*jjFtRzK*0yUc z23%ht^J`j$Y4kA;r(+|{0U>HsB1t4=5Nn`Xf|(g%9^Fo*F~X`2_G~*hawEd-LUL+w zg+vbH=|PKf*o(Gg^@QBbeJ16K+FbSB@d^#Kb7AMvR|c}obp8Yt@Sc#t1Av#-6O?zm z7A@T3a8|cv_AnnBJcf0cnET7mMW$5qerUxWJi4(%?ZN!7_Ij<#oV^xwT!>)a1|}wr z)uo2ajO#F-1j(jJ5T4D(pHwOZQjaGR85_?=VZ%!HF-q*W5}uLERbM8h{;%CahKSlk z2{;5S;5GCSfSlm?AGfz2@$-V2l7odX&bUay)uYG5K}Wr&$U>9_k*nZZ^tZ)A_hYZ{ z{Fhl8uJMGkWB7^x?^RLZo{-iwVmGCHuPK|Ds5@!+B-}As=2sTcL5z_zAa*vUjB-jI zT=|s2kOM#3`H74&@Jb#`ofN&=-!`IL)PK&5wE_>~MhMyk2367G8FfBqm9Pn9>}Y*) zB#D8I+G9B_M2_|(2W_nEjVy73J0=&AZjX>$nq#w+_rR%edsq>e{W#yy-rgRG-oNI4 zqXJ>FCs{2u!b+@}3LNYA$vevh+qHoM@;oL6&X!H_|A*Dsp%5rjtIVV)eTB(ly3N6q zXG*gv6*q=dWIY=)J%?C-4kY|??#=6hSfu6oD8J=p-esjOjXTzA(<1}g4Y%lxHDb^+ z<(RlbmCDM!MUk>3FGJ8DThVa*3N!!-^n08Y0wU)-Tx}plBd8%;_J360Q4!ktRqQ-$` z<%~B@lgPg=R@m`UrsHTJ=jl=wIU~sDoVkpy9E>fcuY?hTE9y6Aw{6Mg>r;U)t+Akx z&im+=7}1gqv}F|7+aDlL!?~;Us2qPXnN%ju)MpT{&0${bv~(AMxyo&}%=slJ z`FX!?pEei7&}!7!&C0U$#A1Sr|v!$#Sr8fpI`J!0SX@LwS-mxta#2~Gi+(#GAY5}p# z`a$g@PEo(5o0S@y!aS0hM7UVFK8x9kY_z^04KYn!ZU<`X7H7V*?}u>CY#;3XmVjT2 zwt|E7gMvD*GR9g}488U(Gv{Qx$Lj(6&$<4hb_Q zTsNCEF0XLwgV+mrUu+;=xK>{T$F32X9Y~FP?27SV+59Sq%)5-ylcS|LqM0(c>|Vm| z@@@&QO@cd0MEYX+`wf#q#r@fkrOi>ZakABI6LmM0-E5i!knI40VZLsJEl8U1QdjKk zogcNL-TiG+dv=e|lg^rOZWs6c<6CP_KXBuvnBFfLp1@4$vwMB3_n3Jk9*DH`Jj{w-4S%J8($gAsG=)C&r%JP-BYSS zUHr73uNEqzq`cTFbh+TFgi&IoF6H?1kUQu0xuQ!oqJP4LJ#6O4I_{RZg!TNl0=_n^ z z{$T?!?990zEUbC8^*Js`{$uy}y&Xhat-Vi}X*W!sat6Q#z`HLmk(t%wyt0+7JM&iPcYXF{rE|h>gpUH7ebSYz{R_;q5EZ7g^i_v*Xzzwm+ zm0Uz-9ZxFYFC$K?7RbEv>F*M#Y&<+~pUtJ%?}H+;j2<1a)98@Zg}CnBYq;H7vHoy!ZaJ`c^!_073NXG$?OE4A zv>&N1p;WPvolLE`dZhGc7ST!K|7@)TGnXLgnjFF|^4+IH^{5FXIjdHeK41CyW959h zDwa03)Xf~t;#SOb!p!Nu&Ot^&Q=Ha~@N@x=aXiS$_apT@e5Nw_KAqfSrmxn`q9Ey}5bymsh8;$Y9w5lOaQ-Vx ziHVB7Ng=g%ep9JPp@u+DuiJI`WpqpL+$9?J<^Ng^%6EO|yXl76Z~Mb(xojuNp4Ae( z+l-QESVb`FTT1T~J&Z14{$C{;;D-gu*FExC6ezGMf#UQzM zA`Za2m%bmkS+<5#MoLlLO|LVh(JA)N%NB28=`R*smA(n66%YdQIE>1F;22;&>a2`w zp|qJUA0UP_^qCR=ZQ^DonE33&-&2ECZu>wEo?BQej9RW@!`)k>efh3|=sK>lvl=IQ znZ-2T8g!-}@4Yd})PVu2y~e4O*KVJr&#F<@A>U1gJub6LJIo32$$G58zopxDN$%qO zX5N!;u3-gH**@kU=j@}^QxH!Tokwf^e%BmGT~MVmshn9+kq@R>4HjNa^Vu^h{9FOA zx^Xu8f=f`)E!$M(SstmLpODsP3OfH+LI30yf%lp<{&~0J)l5y`G)1S#wa_l(VHv0^ zUoxyn&K7opY9VBS0xUmOJ)a-TV-)$4Z1gbh?M-|~m~uz18Z5yM)bo!#Wy9m^A5v#4 z2tHwcuyA*^gRnJ?aqfT4xuo&~BHfy|yY5E)ys0JXkz32wKHGaFKHLKYzJXc+WUkTP zSzU7CZVy@pc@dYC84qI4G84Q=nOSLjG_7QW<)^gO>^H$|IC8PdBW$I`P*n1A;xCcc zqN45pSkJa6Yz$IVxu&$;0 zu3H6WL3{+q%W#G5@s60$hwyi!nnY9&=Ug5puI==BH@@G7Xi+GEC+GA&DaziW?}2BV zoS2`0R1!Lkm&sl!b|#!A%K8B+ocUK>doC=Kuq4ulTl~6)e<7)N5*|hFdVx1Gz-UQx z=h8t;0~pKiEG3ugk-^$nbI7q=!+)s%#jk1!_6yHtk!!4&9ALy*D&MW#^Bav^)0e{H zp}S$5v&N%#=k~aT8UT=-tYHh3Ipf-Gqil7hac<-*GCT*CnW4&?72hx4%ig+e#g`i}OvF?I(t0f`p1kaQEH+J0MJwSTTD# zs0MQsO0qrP(|pRMbs#oBBGGZ5LPKXlz4ICeL7#y#e8n$m?NcB1A0&yF!J}Pdr_t_d z$!iU758fs(r30(}#O~(I!+p{Wm!8(FMehLP#IPGI1}fLhenoMvvRqEE3UKGM1t-B> z968OuvXt;rFEFO+PBEC2ln|TTWecSIP~{x;U07NtBJV0!)T}6nQR|@A7b^pKCQ%fy zu9<+Wx_vG&gs-4tMV+J9J!>&kVyZ)%DvY7heK4>&CFAAE zMdr7(c8h)vJe%=UM4RLU&BP|w@^!1-1LM^|7%aq&vg4rlw`)M5%Y~y90}%d=Ped6M z(pNrb%f~b2tF)Zxvqftsun)ccq(=4oep`aR$B}tbBULPwcUWsP!_DQ#osZ5YDYWV9 zoW1B0^ltwxK8sUQg=<7Cp>Z?H z+?pTP*Zgq?i@-bckoxTj;)-^6JOfR63Jth!aGY+fZ3suI&*u7c~0dTRzT64+9xQ+d7e2xH(X$d^CaIvsf;6D7^Yv_BnD(o9NbRN&tmM4XP zg{PsqWt=%jH>KZ)qi;^_mY-J02s5mCP;1)CdFm*;IhN58wXH!k2K@fIdjKAWgz?b3 z{{h`L)Q^Uha=J01-Dh&LqT-R~#WTj<%W)Fke2P&a3d>z8fAC!jPW}ylgMx_Xk30W{ z25W)8b{lhXl6MLJY1)4W1ToM;gG5Vg&=z(B*om2^V4{n;Gj{|2(|?eag79TQ#?|gBnYBxqB@S>cz*(Q8O^?uESt~9+TobP_J%FMX?T;+e+$OR1b zgbt7X{*+!Tbu_X+RI=-5r!EF-bqRg8Mj=zhNOTG20@jLbYun7HzO;W7MY&;r-Qu*^ z^X}p_W(GIW&?XK!BG%`%m{#7FIgGa?tWAiI9V9?0(Oi0eRd^%~|GV={%0R_~80BWwF7=7|=&$y&S?mP&ACGq!bLGjRr6RBB4nF;AhK z!?%$SR`Ll*^XhNr=06bCN=_%{ic{27EfkPKyMeeEJ9}%qp#(L@Xu1!1gnpD>*Z~ zrEdv{kU6h?c=RHd@tug|$~^I?<+JP*$THSmArk&k%GnvITW8SVk1cN)DV`)oM5V%v z!MBi7XWJ;5;lJ~BNpxm`+&hPG&N7x!U0(gQ>|S=R->$wO-(lb4ENgW`#oqhJ-%^Dr ziSyUIW~v0XI@@e3kweQqtGC+X1Ps;LmY;?^&XS!*w*y$00%X;rxT094-D3&ic{h8( zkv_s^lwzNYH*UB&)ZW}I>%|%Q$E|^OEv*&2KNBexCIYI;fM3)mZj?F5i5v;Fk^YEo zNiS-w6m5Z7*7_Xcc0Jq;^9^u067II*vlZqwZ0jk9)4Fzc+up1dpI8`zUyANZ$byn_Z0Wl=1+E22c1=+)5bW!xIP}45vNl@)j~?u8R_smQIvnZ&fe9Bpx#n( zJv8TuGTbWmT5ja|M^1MCS9@m}7UkCV{b4IAD4-080)mnPDw0E^(hX8dgLJd$Q3Mf0 zutk)R?(U9JLKKj01O%iNq(kCa*A(o%y`TF&-uHNq_rvq}Z4QT7eO+s<^Y{Os8nvGF zpCO1=hwR+}87`upFngb1eMQ^2VZ<(-jWd2IPw=0z@nb(;)pfd%7i8NsV|`VZzcJFJ zWGT?L4qnTV0@Rru?7nR~-|Pm>KX%MUNicUkSkdd#g=x9fSbSK@zJ{dJwGg_@6lKbZ z#=n{sC_bRMuLn6X0b3HSjtfxhhIqy%n{MZ<02T?ov6F`tX|p>$_j|&do#vNaK1l}4 zCA8~R;WSV&=?HW8Us@koet()SXnJFtAc+kQ_em9@{-}q+$3}?9u#{F@=trYRfl4mh zddD9}bYPjaiibITX(#&SBErq$o@|uccIw<(um!JMI5jDaHC2|ZFee4B3)#pVQ-`!X7UWe zp78Ug)-iFZm4Az#e1PA0D1UBpTa3Cr*H_{c&v#9{ZRghee$)4;^M3iv;Z6d&CUDR+ zGZP5KeYJiOnI<_`<-W@*jhrP!4mu~)NW@V9CeEx4$HbjbxWKQoRXrFw-cxQGPp%C1 z)iy`gE-#(4^k}=!x^ScYnGX4&u1NurXF!9l#a+=#%roc)XP8aa{FY2`rNJYukP0?8 z*uzNwUPpmSBT?gX++pq;3}dLHtl1luXXjO$GHz+m<+>*6rYB3WQ)%vDr^bQw)0zi?lLE0>tUa>LNu$=a@?DS#tGf%Q1qVcB@ zykkX8CH`{`$c_pU?NBY#@*DB!8vTqX7uyZ=lGXS@ldt#YMnuw-YXO@HpO6DdQN{{< zI>3mBVqU^zHcV8;<8k0ghfVd?X^HXuo+%!$I;sHBptFutN~-{`oD(3*V6zg>{USt$ zS3h}--@?Nv`| zF(7o#BJGjVOO(Q1Wq2-L=u;Kr4fZ#fCx!#r^~og zM5^W+g@)y2mm!oH<(IDGU?Fg?us!j7#E$uNsDP~fY{oAq$pHo+o@o%#aN$M*I|T$ev)t=> z^M-V@;&i?{-XCt36aTtp`0Kkj4XO2= z6I^YtM=%GBETj&8U7QkBy$-ey(2e39gu21_Ah?40zT1}#gQ_p00r)ZiY-(I$3nQ1q zde|9vbiV@TxC6)7*>)IP8`8Vuwk2t@PWn*h+i+@u{nj_K5XoaKFn1Dy65_69;7vZT z1oWxk&LPb$8P#~ze<+D*NG>FY?B{mCE-))iBg!q$pjsGa8oB*Qa_!cm-U;$Ymhyzy z#wx%sb=MNv6b{m%I%VVfd*ALze5?xftKok>^qfoxU)#oKd+zT7GjJcxwAz}<$%x1Z-4GHy{%5MLHbLF1Hb z46tWR7pPaRd;P4FXEtkspOOF@VF1)#G(3L=0*6cDYHwO?9&1NwTa&ZKjM#$A7CNZA zmCcerIxnj&XNn&|u(}PUc5SMES_Yk_A%VH1XEU74%_$; zo`tOeKgep(^9mYTfIoWjyr=y7lcdlI>IU3Ju9R7`{G;KD0{DAy!=}wM3q@jW-!BKo zn}h7;pDTb@@~}h#q9+nKD~H36%47q?lAb`oZtMEkfgI&a+q)|pvme4{o8Ah5kop%I z9+0S#6KqD@L<;+LiZcn;)fWf5-6 zq{1U+m0@>(tQn$~znLNa_*Q~koXOUT#)TIseW)(-0S{mzt*f1e1gpDpv4Y#lkvxcR z3c;VSz5Vnw*zlZ>H{Cl|##t3$(K}_^r0i{G4qa|l zs7|I9#q`!zz%q;@Rr}@1(RBg3K)~Wjfx3ixe{Tl?v$7X=`1nUeKn$MSJhiP-i$QX{ zdSZFL$|{aiV8_O`I)uvhyaRu5)-7^$KS+Ptr;fx0TplwNF4NBNcY1>0SAEzIXA9Kj zN$EuZ9_k-`cR+Id>c18e>Xp0vm~yC0LfO-PyuKKnfeTSSOxi-ZYhfX*%_o@|ho@VzV)Rs}OFcY1nybUwF5{m-lWaLagEebE5D zjqz$5)NYxzAI;$*r^RO~hIdSdo?O6#T}PzT@cJHnln|{~J+67!m9F9*v5Eyb6uP|I zlI^YIriMemoyD_NiEF^GZKgj*Ju|3yxVLW zecSbOAYx7svD?dxpu8WO9{@t~>d^}z=yraWjK|=Il0c>mfxzAAhxtXO9*Ov-36dJXC(NL4QI)EWUFCF|^>)I*Fu}V;4v{!bLwn>49`LWfb=JvB6F7&AF z=O_0H*J~lrPzMD7&I8>#i0=dI8*3)8b7v7~f~xt`wK8O_rrS7Q3Pw15zgx1!67*&T zptaex`QC@L65&f3ye-*+P9{0Q}J zatc$mZ-56c5I}~xT9fdrObDmO$4y@8=^XOwD5LfHA=%o$9?oZ@ak70yhIr=6)fc6X zBfgySChp8ZkHt~Ps8nn(>A#T%fhq`!1hZrPg9p|uH7+LFl9jAG=;(~JfUeMu*Ta(U z^LBWp)$fGJewmd?EO}*S4^73Zl-Mx2Q!2sa_53p&1x)5gu!Wu%3Ni6+dU4uaFR3A#k4zmR7h3Tl()_>dZA{z zI&rBH>(poG94N&egQi7CKN6HYc9DN;w2zkY&@+}HCk>d_+sv6!(XESqVBO0(b+}+1 z1cAD<4YN;gv3-x5J!bLTy2s!e1fKLqIO?DP# zI=v%8;lg!--{L6sA)bE-!M8Xbp+v5;z-)S(B|((O;0YQBYRFc%h#S*$m5av?0Lxhz zRnNT&hG5cn;TO;N@GT8h-ri2uZ;xPv-C_X!edgG^``7j%IM9z%KhTYoG&ji9e5m9d z?DwGXJM<39dz=K`Kwi+vqq0e8sY)V_8UjF}DfTsGk-n?~=gBVz4a`qVuT> zFI`ObBGSV3maXm4Af=|G@2It;@#uN%bt{UxbVhdJ#I8TZbaYj;5nyB<5eRHKY%OwbuJ6`FtD5@@#RZ|ttGI*Y zlAclok7lQb8ewNQR2$g{&q+jztp|AhR#6e5Qg5q`xzLQP%~{94z26OzXbQ2bBweDQ zx4M+>0w`ETSeBPRV{E7?8Y^A(%AzJcuyI`DJXVI`7saT=Jb)5Rds%7Lc+zm2Z)R9x zMca~ji+>QxKpJGeH2C0<77!aRt-DKPvdUig)2kBUr!Q3^E^!Y{+ zOzE>i3__}XHZv%$9mwpOZ8dIyz({>dyCgutpY0>P_Ku1Erfhx<;}>}(f?T&Ecg~Rh z-8A3&XXMxiN*Ed}^9=ErRW|?gu8OAf zgZG+)A0T6t=o3|p=}NsqHBurga)1-~e9PvPFZPA464m|a*+j)kw=VF4G52yuJ|NP{ zqy&{^7P)*Ox_6dx^=uT2&M3*7#R8Af$nGY_i_RQj926)ik^lmM$_I<}7Swf2DzkNx zyXiz&?4thbZBF-b|Nv&irqr@ zO9zsR$SSp#fWHq{f!tJIKf*+-ap6G4W!kWenqCMLeYJXb64+dAb41MXe^+XiEH>s? zn6FJgAWKVQW*|0vyL3SgH<8g5X?a%H?N9^H1}My*Fj7M$;nyIqkPta{F?rTy=SS2< z%aoTWNwsHbN$QxMiB=8@29HFS2BA&sq8za$FK2)ehfG_lYP3eG0KB;s7$fwuiHtO{ zoMg8DM=gU0&t*{didGRw$mn(s_^WV=qhcwHLe z=pUxr=?GHedf!P>Q(U~EYo8nHN1$6Hc8%<+KbFvP>UOE|9@Z8v9wd#(Qt|}3?4> z*9+FD>)x^tU0FQ`A3P}Z7(MlKSmtG)UCRB_6T4sK{8>cU(W(u?{)=Md-SpmBA_vhc z_r`a#E+QWPtk8gj2Y6`I_9DB#d+J) zgc^hjyV9vGWoCakE?C1AZn~`~P=Z5VO{QfW`J${t{y%@|Z}Nn?#9-@DA!}g4mfK%@ zeHx~B8AY#bE9IiSaY_+9Xj5DLWTeo>D8J!4sos2i4vwY#?BSy*)+?o*-UoFAJ9f;f zRSHI$U~diVz(7Z#fT=Kxa>%do8_;V7-{rp{H!Dh99%#@O4z3!~eNDFQ%d1OP-5n z_Ka()GqZHdG5cc2&%hH??~dsXu$q}pijt6$EG}-Y8lS82D+87?mEDt7NQ3Qqa8?Wb z^$RiR6>nKa1QHMd$x>%x*4_!|{|ep2i1#dIzv1Z5nIDh`i)Nz7(tD8i>Aw5)r$(Tl zvQFrDh&$a-YsTWu3l2UB(!whFWuJ)F7b^60+WmaWM*jWIHSfW76WmG4vOfiTpaDuj zJV1F$3drFhjaaq&#Nd(RCClH&;q%d5rT#RWcvBm!v<~gMa~8y&_DpS3T2xy`q$tpGAMi!C z3fJveBJpnooLa)`D2Z{S1c!k7%lhCP`_X~U{XeN7X~_MT!^A%KpvhYri}&cM)kxYK z+dUr}pf$z7Y{uYHH4rB99=b}!3$r0<^PYRh{ZJ49gA5D)_d8hO`jjA~4+6nJT})QN z!341ZkX#iw^nZ~>;O!&O{6Gi3@6H9ub>MKwNo6o$BuJQBeG0`TBicTGS|-Q2=LVeWoPOFkRq4dgQQ+?%JjoA$MX zcrt~^LD7qMnT;K0)@sSBMd{5`#Ut1a6UAEH-L zH;^sfzGK^QLWs{uot%4(-NBi#hJ4O#9Plc8JRz&CJP?6HzBjn3Zafsc2-R+YZ{I6Qoc5%-PPld^_Kl!(VN^|# zeN71PpxK8?u&Y4j;v#WGbqSDB4xXrV0i@3CHR2iVjX>&4m;9+!^^0n_FC>wdPPP_t z_gL{*^0s-KHtT^S!FRd8)?daQh}iL-H@G&GHO7Y~^f;8i*x28WGt+{+SYfPTT83|W z%FXE}4(sW_B8{tT8iP&gKE^F-f6^BW?7*+GWWNe<-W+LiH8A{sXjat@!Q`AR{X=kT z$+n{o;!p?yLA5uADi0fihr-tPUJPK$BQ5aGFtYAQQnisHhD2+#fX`s)$C8O@1uvOlLAK|YXy1pXJcsJ67ra>>+FxGG0NgkQa%jE#( z&~(VJe|qu!J75M{c=41~-sF^{l2+2!Dpj=io^ul8Y$yN1Uj2o5IWl~-;Ms7A@bko> zl`Ds*E=|d7AQ0KIg zQtkB9Y{u<1^Y<=3|H=jD$*&mAn~a6SG4ARSYI`)&M&CN!bC84AJKMOfFW`XNNAYx~|aw5mQO+S!F1Qk5uI68HAZ1i5}|xob}m0HPVL&8Ty1&bVbNwO5xBz$VGrs_ldXX9JolxhDIjKK6k;Blq8Rl& zY4;Y*Gw_3%U6vBTI-b1Z1RiQ=7ZDSvW8+DjsMX)#dG%@6mRMi=nlzuCdqZPKE#fC>ih0G^M}%MOkFBTBGmS>SXa*9$|W$iFe9-hUJuX#`Pf*XT!3H%bSM<-T{96Rw;@0J5}de>1AiZ|jSN1+ zZXEq$P7tWy_8G0<1JGC_6z95x-`Ll*Srm9EY}Gf1rid^?g-LU?&_v&yXb$=36r*&T#lE%YX1)@_?Xu@go?&n*Z)MxydM3=(WwGTo^0us zLxwX#T|sPUGa@CKAiM`rYT~47acpx9a1aII9pPX8f{2&d$x5LOJml;cEth&`D7wjQ zE(!Q1LkBe|5VxJ#vkVf?-<018*2HzEr25XVu<*Yq>p2Kw3FlXKmYbYPDn)rOE!Tq( z<~32=kjZT98?t!ey&(f$tWdDAcRuQy0>z8z%zTIGpC!mVZz|)4cyy;jLA&-;Ek!xD zKoj31LYj)WdHUZ({E~3(WH^<&D4VRqh4v$2n9|_FTRZ`gV%$tj^L>Qs$h6{UXJm5A zeCgK@3RAeTmex$!D^}VyXRn8lw@FKZWEHr6xUL>`z7uv3I?m<}i+YX-Gger!G9)3#$ILmT`Tq~^Kbn~qWF!`R!bmP$`hd9!;ZNh za3^_6g|AUN6awq$U{OT-Kb`5sbvLPX!FCzWMQerj4b^(Jgo`Y0KDemjdR8c7=*a=-FLrY}u*VVSFYk{A)Dbp-kB4^)hb0A9lq)LSpVy+o7<_GPfkU!b3~6 zzDU2td&*fQJMyj~f+R&am#)r!Mw>SR6&Y3j{p2C*E~uWVJ`MMk(QEs$pNX%s)m}X* zIjc64R*qLqs-jH10ta_I2H@TG7myuI_K8=B(1W9+>e(8nTxcTS5DKnd)Z%~iV12ZF z#L!C}=G6mr0KsPZUaPJ|qwDTdH48mYVB1}mzg#hHVaH-r(2McPFuF}XJ9~Y0sFf8e zkS=B(y+W^e^Y!1=$2?^ZOa((DL)PjJJ;*3xP43#~FC< zCnI_i2444a0B(3?Zi3o*5W7U7|MN!lMEzpS-ThMHG&}v7Z@P$iH3%d|sVp!3poc&= z>kt*lECYiXErQ2x5xXSFG6aM93^{)avekSD(#wEY$ zhvg^*Ez@*hxdwfh5`B~LQ44seK_6OgU%L4slNicc1B1n^fg0&jf)p{}14tMaYD^8a;L^>|OObvCFoRqR;)-@yF{{lPz+|2+0 literal 0 HcmV?d00001 diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..8e02f07 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,48 @@ +.. Scalable documentation master file, created by + sphinx-quickstart on Thu Aug 22 10:55:42 2024. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Scalable Documentation +====================== + +Scalable is a Python library for running complex workflows on HPC systems +efficiently and with minimal manual intervention. It uses a dask backend and a +range of custom programs to achieve this. The figure below shows the general +architecture of Scalable. + +.. image:: images/scalable_architecture.png + :align: center + +These questions can help answering if Scalable would be useful for you: + +* Is your workflow ran on a HPC system and takes a significant amount of time? +* Does your workflow involve pipelines, where outputs from certain functions or + models are passed as inputs to other functions or models? +* Do you want the hardware allocation to be done automatically? + + +Scalable could be useful if one of more of the above questions are affirmative. +To incorporate the ability to run functions under different environments, +docker containers can be used. A Dockerfile with multiple targets can be used +to make multiple containers, each with different installed libraries and models. +When adding workers to cluster, it can be specified how many workers of +each type should be added. + +Contents: +~~~~~~~~~ + +.. toctree:: + :maxdepth: 1 + + workers + +.. toctree:: + :maxdepth: 1 + + caching + +.. toctree:: + :maxdepth: 1 + + functions diff --git a/docs/make.bat b/docs/make.bat new file mode 100755 index 0000000..32bb245 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/workers.rst b/docs/workers.rst new file mode 100644 index 0000000..15d091d --- /dev/null +++ b/docs/workers.rst @@ -0,0 +1,13 @@ +Worker Management +================= + +.. autoclass:: scalable.SlurmCluster + :exclude-members: close, job_cls, set_default_request_quantity + +.. autofunction:: scalable.SlurmCluster.add_workers + +.. autofunction:: scalable.SlurmCluster.remove_workers + +.. autofunction:: scalable.SlurmCluster.close + +.. autofunction:: scalable.SlurmCluster.set_default_request_quantity From 237dcae4c6e00a5103c3bd186186d38f04c6f2f4 Mon Sep 17 00:00:00 2001 From: sash19 Date: Sat, 19 Oct 2024 11:34:16 -0700 Subject: [PATCH 11/41] Built docs not included --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index bc3e6af..78e731e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,6 @@ communicator/src/communicator containers/ config_dict.yaml cache/ +docs/_build/ logs/ .vscode/ \ No newline at end of file From b9f372f76adca775a8ddf5b0d82116569dffcffd Mon Sep 17 00:00:00 2001 From: sash19 Date: Sat, 19 Oct 2024 11:36:35 -0700 Subject: [PATCH 12/41] Changed copyright to JGCRI --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 8c15af6..57313e5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,7 +13,7 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information project = 'Scalable' -copyright = '2024, Shashank Lamba, Pralit Patel' +copyright = '2024, Joint Global Change Research Institute' author = 'Shashank Lamba, Pralit Patel' release = '0.5.0' From 62c40f881ec3df9d7b81f6a874de8d5093e9ad55 Mon Sep 17 00:00:00 2001 From: sash19 Date: Sat, 19 Oct 2024 11:37:49 -0700 Subject: [PATCH 13/41] Added pandas --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3559332..8f999e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,8 @@ dependencies = [ "joblib >= 1.3.2", "xxhash >= 3.4.1", "versioneer >= 0.29", - "numpy >= 1.26.4" + "numpy >= 1.26.4", + "pandas >= 2.2.3" ] classifiers = [ "Development Status :: 4 - Beta", @@ -49,7 +50,6 @@ test = [ [project.urls] "Github" = "https://github.com/JGCRI/scalable/tree/master/scalable" -"Homepage" = "https://www.pnnl.gov" [project.scripts] scalable_bootstrap = "scalable.utilities:run_bootstrap" From 5b7f2ca586bd6650c7f948279d3edc540d1573b2 Mon Sep 17 00:00:00 2001 From: sash19 Date: Thu, 31 Oct 2024 14:09:57 -0700 Subject: [PATCH 14/41] Installs latest pip in base environment --- scalable/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) mode change 100644 => 100755 scalable/Dockerfile diff --git a/scalable/Dockerfile b/scalable/Dockerfile old mode 100644 new mode 100755 index a560961..d7a5f9f --- a/scalable/Dockerfile +++ b/scalable/Dockerfile @@ -16,6 +16,7 @@ RUN apt-get install -y --no-install-recommends python3-rpy2 ENV R_LIBS_USER usr/lib/R/site-library RUN chmod a+w /usr/lib/R/site-library RUN cp -r /usr/lib/R/site-library /usr/local/lib/R/site-library +RUN python3 -m pip install -U pip FROM build_env AS conda RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh @@ -30,7 +31,7 @@ RUN eval "$(/root/miniconda3/bin/conda shell.bash hook)" \ ENV PATH /root/miniconda3/bin:$PATH RUN conda init -FROM build_env as scalable +FROM build_env AS scalable ADD "https://api.github.com/repos/JGCRI/scalable/commits?per_page=1" latest_commit RUN git clone https://github.com/JGCRI/scalable.git /scalable RUN pip3 install /scalable/. @@ -172,7 +173,8 @@ RUN cd /gcamwrapper && sed -i "s/python_requires='>=3.6.*, <4'/python_requires=' RUN cd /gcamwrapper && pip3 install . RUN pip install gcamreader RUN git clone https://github.com/JGCRI/gcam_config.git /gcam_config -RUN pip3 install /gcam_config/. +RUN pip3 install /gcam_config/. +RUN pip3 install dtaidistance scipy COPY --from=scalable /scalable /scalable RUN pip3 install /scalable/. RUN pip3 install --force-reinstall numpy==1.26.4 From 71eaaeb71d02dd7c45fefa9258f0c88ad83dd4e3 Mon Sep 17 00:00:00 2001 From: sash19 Date: Thu, 31 Oct 2024 14:11:41 -0700 Subject: [PATCH 15/41] Enabled username/hostname cache; improved efficiency --- scalable/scalable_bootstrap.sh | 245 +++++++++++++++++++-------------- 1 file changed, 139 insertions(+), 106 deletions(-) diff --git a/scalable/scalable_bootstrap.sh b/scalable/scalable_bootstrap.sh index 21842aa..33ed995 100755 --- a/scalable/scalable_bootstrap.sh +++ b/scalable/scalable_bootstrap.sh @@ -1,10 +1,13 @@ -#!/bin/bash +#!/usr/bin/env bash + +### EDITABLE CONSTANTS ### GO_VERSION_LINK="https://go.dev/VERSION?m=text" GO_DOWNLOAD_LINK="https://go.dev/dl/*.linux-amd64.tar.gz" SCALABLE_REPO="https://github.com/JGCRI/scalable.git" APPTAINER_VERSION="1.3.2" DEFAULT_PORT="1919" +CONFIG_FILE="/tmp/.scalable_config" # set -x @@ -15,24 +18,6 @@ GREEN='\033[0;32m' YELLOW='\033[0;33m' NC='\033[0m' -prompt() { - local color="$1" - local prompt_text="$2" - echo -e -n "${color}${prompt_text}${NC}" # Print prompt in specified color - read input -} - -flush() { - read -t 0.1 -n 10000 discard -} - -echo -e "${RED}Connection to HPC/Cloud...${NC}" -flush -prompt "$RED" "Hostname: " -host=$input -flush -prompt "$RED" "Username: " -user=$input if [[ $* == *"-i"* ]]; then while getopts ":i:" flag; do case $flag in @@ -44,6 +29,10 @@ if [[ $* == *"-i"* ]]; then done fi +### FUNCTIONS ### + +### check_exit_code: checks the exit code of the last command and exits if it is non-zero + check_exit_code() { if [ $1 -ne 0 ]; then echo -e "${RED}Command failed with exit code $1${NC}" @@ -52,6 +41,65 @@ check_exit_code() { fi } +### prompt: prompts the user for input + +prompt() { + local color="$1" + local prompt_text="$2" + echo -e -n "${color}${prompt_text}${NC}" + read input +} + +### flush: flushes the input buffer + +flush() { + read -t 0.1 -n 10000 discard +} + +# comments, headlining sections and methods + +# add documentation for copying scalable target + + +echo -e "${RED}Connection to HPC/Cloud...${NC}" + +choice="N" + +if [[ -f $CONFIG_FILE ]]; then + echo -e "${YELLOW}Found saved configuration file${NC}" + flush + prompt "$RED" "Do you want to use the saved configuration? (Y/n): " + choice=$input +fi + +if [[ "$choice" =~ [Yy]|^[Yy][Ee]|^[Yy][Ee][Ss]$ ]]; then + source $CONFIG_FILE + check_exit_code $? +else + flush + prompt "$RED" "Hostname: " + host=$input + flush + prompt "$RED" "Username: " + user=$input + + flush + prompt "$RED" "Enter Remote Work Directory Name (created in home directory of remote system/if one exists): " + work_dir=$input + + flush + prompt "$RED" "Do you want to save the username, hostname, and work directory for future use? (Y/n): " + save=$input + + if [[ "$save" =~ [Yy]|^[Yy][Ee]|^[Yy][Ee][Ss]$ ]]; then + rm -f $CONFIG_FILE + check_exit_code $? + echo -e "host=$host\nuser=$user\nwork_dir=$work_dir" > $CONFIG_FILE + check_exit_code $? + fi +fi + +### Go version is set to latest ### GO_VERSION=$(ssh $user@$host "curl -s $GO_VERSION_LINK | head -n 1 | tr -d '\n'") check_exit_code $? @@ -60,24 +108,20 @@ DOWNLOAD_LINK="${GO_DOWNLOAD_LINK//\*/$GO_VERSION}" FILENAME=$(basename $DOWNLOAD_LINK) check_exit_code $? -flush -prompt "$RED" "Enter Work Directory Name \ -(created in home directory of remote system or if it already exists): " -work_dir=$input - echo -e "${GREEN}To prevent local environment setup every time on launch, please run the \ scalable_bootstrap script from the same directory each time.${NC}" +if [[ ! -f "Dockerfile" ]]; then + flush + echo -e "${YELLOW}Dockefile not found in current directory. Downloading default Dockerfile from remote...${NC}" + curl -O "https://raw.githubusercontent.com/JGCRI/scalable/master/scalable/Dockerfile" + check_exit_code $? +fi + prompt "$RED" "Do you want to build and transfer containers? (Y/n): " transfer=$input build=() if [[ "$transfer" =~ [Yy]|^[Yy][Ee]|^[Yy][Ee][Ss]$ ]]; then - if [[ ! -f "Dockerfile" ]]; then - flush - echo -e "${YELLOW}Dockefile not found in current directory. Downloading from remote...${NC}" - wget "https://raw.githubusercontent.com/JGCRI/scalable/master/Dockerfile" - check_exit_code $? - fi echo -e "${YELLOW}Available container targets: ${NC}" avail=$(sed -n -E 's/^FROM[[:space:]]{1,}[^ ]{1,}[[:space:]]{1,}AS[[:space:]]{1,}([^ ]{1,})$/\1/p' Dockerfile) check_exit_code $? @@ -113,63 +157,46 @@ if [[ "$transfer" =~ [Yy]|^[Yy][Ee]|^[Yy][Ee][Ss]$ ]]; then done fi -exist=$(ssh $user@$host "[[ -f $work_dir/containers/scalable_container.sif ]]") - echo -e "${YELLOW}To reinstall any directory or file already on remote, \ please delete it from remote and run this script again${NC}" flush ssh -t $user@$host \ "{ - [[ -d \"$work_dir\" ]] && - [[ -d \"$work_dir/logs\" ]] && - echo '$work_dir already exists on remote' -} || -{ - mkdir -p $work_dir - mkdir -p $work_dir/logs -}" -check_exit_code $? - -flush -ssh -t $user@$host \ -"{ - [[ -d \"$work_dir/go\" ]] && - echo '$work_dir/go already exists on remote' -} || -{ - wget $DOWNLOAD_LINK -P $work_dir && - tar -C $work_dir -xzf $work_dir/$FILENAME -}" -check_exit_code $? - -flush -ssh -t $user@$host \ -"{ - [[ -d \"$work_dir/scalable\" ]] && - echo '$work_dir/scalable already exists on remote' -} || + if [[ -d \"$work_dir\" && -d \"$work_dir/logs\" ]]; then + echo '$work_dir already exists on remote' + else + mkdir -p $work_dir + mkdir -p $work_dir/logs + fi +} && { - git clone $SCALABLE_REPO $work_dir/scalable -}" -check_exit_code $? - -GO_PATH=$(ssh $user@$host "cd $work_dir/go/bin/ && pwd") -GO_PATH="$GO_PATH/go" -flush -ssh -t $user@$host \ -"{ - [[ -f \"$work_dir/communicator\" ]] && - echo '$work_dir/communicator file already exists on remote' && - [[ -f \"$work_dir/scalable/communicator/communicator\" ]] && - cp $work_dir/scalable/communicator/communicator $work_dir/. -} || + if [[ -d \"$work_dir/go\" ]]; then + echo '$work_dir/go already exists on remote' + else + echo 'go directory not found on remote...installing version $GO_VERSION (likely latest)' && + wget $DOWNLOAD_LINK -P $work_dir && + tar -C $work_dir -xzf $work_dir/$FILENAME + fi +} && { - cd $work_dir/scalable/communicator && - $GO_PATH mod init communicator && - $GO_PATH build src/communicator.go && - cd && - cp $work_dir/scalable/communicator/communicator $work_dir/. + if [[ -d \"$work_dir/scalable\" ]]; then + echo '$work_dir/scalable already exists on remote' + else + git clone $SCALABLE_REPO $work_dir/scalable + fi +} && +{ + if [[ -f \"$work_dir/communicator\" ]]; then + echo '$work_dir/communicator file already exists on remote' + elif [[ -f \"$work_dir/scalable/communicator/communicator\" ]]; then + cp $work_dir/scalable/communicator/communicator $work_dir/. + else + cd $work_dir/scalable/communicator && + ../../go/bin/go mod init communicator && + ../../go/bin/go build src/communicator.go && + cp communicator ../../. + fi }" check_exit_code $? @@ -189,6 +216,7 @@ if [[ "$exist" -eq 0 ]]; then exist=$(echo $?) fi if [[ "$exist" -ne 0 ]]; then + echo -e "${YELLOW}Scalable container not found locally or on remote. Building and transferring...${NC}" transfer=Y build+=("scalable") fi @@ -203,6 +231,24 @@ if [[ "$transfer" =~ [Yy]|^[Yy][Ee]|^[Yy][Ee][Ss]$ ]]; then mkdir -p run_scripts check_exit_code $? + rebuild="false" + docker images | grep apptainer_container + if [ "$?" -ne 0 ]; then + rebuild="true" + fi + current_version=$(docker run --rm apptainer_container version) + if [ "$current_version" != "$APPTAINER_VERSION" ]; then + rebuild="true" + fi + if [ "$rebuild" == "true" ]; then + flush + APPTAINER_COMMITISH="v$APPTAINER_VERSION" + docker build --target apptainer --build-arg APPTAINER_COMMITISH=$APPTAINER_COMMITISH \ + --build-arg APPTAINER_TMPDIR=$APPTAINER_TMPDIR --build-arg APPTAINER_CACHEDIR=$APPTAINER_CACHEDIR \ + -t apptainer_container . + check_exit_code $? + fi + for target in "${build[@]}" do flush @@ -226,31 +272,10 @@ if [[ "$transfer" =~ [Yy]|^[Yy][Ee]|^[Yy][Ee][Ss]$ ]]; then chmod +x run_scripts/$target\_script.sh check_exit_code $? - done - - rebuild="false" - docker images | grep apptainer_container - if [ "$?" -ne 0 ]; then - rebuild="true" - fi - current_version=$(docker run --rm apptainer_container version) - if [ "$current_version" != "$APPTAINER_VERSION" ]; then - rebuild="true" - fi - if [ "$rebuild" == "true" ]; then - flush - APPTAINER_COMMITISH="v$APPTAINER_VERSION" - docker build --target apptainer --build-arg APPTAINER_COMMITISH=$APPTAINER_COMMITISH \ - --build-arg APPTAINER_TMPDIR=$APPTAINER_TMPDIR --build-arg APPTAINER_CACHEDIR=$APPTAINER_CACHEDIR \ - -t apptainer_container . - check_exit_code $? - fi - - for target in "${build[@]}" - do flush IMAGE_NAME=$(docker images | grep $target\_container | sed -E 's/[\t ][\t ]*/ /g' | cut -d ' ' -f 1) IMAGE_TAG=$(docker images | grep $target\_container | sed -E 's/[\t ][\t ]*/ /g' | cut -d ' ' -f 2) + flush docker run --rm -v //var/run/docker.sock:/var/run/docker.sock -v /$(pwd):/work -v /$(pwd)/tmp-apptainer:/tmp-apptainer \ apptainer_container build --userns --force //work/containers/$target\_container.sif docker-daemon://$IMAGE_NAME:$IMAGE_TAG @@ -266,21 +291,29 @@ flush docker run --rm -v /$(pwd):/host -v /$HOME/.ssh:/root/.ssh scalable_container \ bash -c "chmod 700 /root/.ssh && chmod 600 ~/.ssh/* \ && cd /host \ - && rsync -aP --include '*.sif' containers $user@$host:~/$work_dir \ - && rsync -aP --include '*.sh' run_scripts $user@$host:~/$work_dir \ + && (rsync -aP --include '*.sif' containers $user@$host:~/$work_dir || true) \ + && (rsync -aP --include '*.sh' run_scripts $user@$host:~/$work_dir || true) \ && rsync -aP Dockerfile $user@$host:~/$work_dir" check_exit_code $? COMM_PORT=$DEFAULT_PORT ssh $user@$host "netstat -tuln | grep :$COMM_PORT" -while [ $? -eq 0 ] -do +while [[ $? -eq 0 && "$COMM_PORT" != "8787" ]]; do COMM_PORT=$(awk -v min=1024 -v max=49151 'BEGIN{srand(); print int(min+rand()*(max-min+1))}') check_exit_code $? ssh $user@$host "netstat -tuln | grep :$COMM_PORT" done -ssh -L 8787:deception.pnl.gov:8787 -t $user@$host \ +ssh $user@$host "netstat -tuln | grep :8787" +if [ $? -eq 0 ]; then + echo -e "${RED}Port 8787 is already in use on remote system${NC}" + echo -e "${YELLOW}Not forwarding port 8787, dask dashboard may be unavailable...${NC}" + connect="ssh" +else + connect="ssh -L 8787:$host:8787" +fi + +$connect -t $user@$host \ "{ module load apptainer/$APPTAINER_VERSION && cd $work_dir && From 8381972db6bf3151db866f3b738d57abc4aa9513 Mon Sep 17 00:00:00 2001 From: sash19 Date: Thu, 31 Oct 2024 14:14:00 -0700 Subject: [PATCH 16/41] Added sphinx documentation --- docs/Makefile | 0 docs/_static/custom.css | 0 docs/caching.rst | 0 docs/conf.py | 0 docs/functions.rst | 0 docs/images/scalable_architecture.png | Bin docs/index.rst | 2 +- docs/workers.rst | 0 8 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 docs/Makefile mode change 100644 => 100755 docs/_static/custom.css mode change 100644 => 100755 docs/caching.rst mode change 100644 => 100755 docs/conf.py mode change 100644 => 100755 docs/functions.rst mode change 100644 => 100755 docs/images/scalable_architecture.png mode change 100644 => 100755 docs/index.rst mode change 100644 => 100755 docs/workers.rst diff --git a/docs/Makefile b/docs/Makefile old mode 100644 new mode 100755 diff --git a/docs/_static/custom.css b/docs/_static/custom.css old mode 100644 new mode 100755 diff --git a/docs/caching.rst b/docs/caching.rst old mode 100644 new mode 100755 diff --git a/docs/conf.py b/docs/conf.py old mode 100644 new mode 100755 diff --git a/docs/functions.rst b/docs/functions.rst old mode 100644 new mode 100755 diff --git a/docs/images/scalable_architecture.png b/docs/images/scalable_architecture.png old mode 100644 new mode 100755 diff --git a/docs/index.rst b/docs/index.rst old mode 100644 new mode 100755 index 8e02f07..2615ff6 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,7 +30,7 @@ When adding workers to cluster, it can be specified how many workers of each type should be added. Contents: -~~~~~~~~~ +--------- .. toctree:: :maxdepth: 1 diff --git a/docs/workers.rst b/docs/workers.rst old mode 100644 new mode 100755 From 346012227d6db6e899a901593c2ea468c834601b Mon Sep 17 00:00:00 2001 From: sash19 Date: Thu, 31 Oct 2024 14:17:48 -0700 Subject: [PATCH 17/41] Added -h help option --- communicator/src/communicator.go | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) mode change 100644 => 100755 communicator/src/communicator.go diff --git a/communicator/src/communicator.go b/communicator/src/communicator.go old mode 100644 new mode 100755 index 5afe589..73c2dfb --- a/communicator/src/communicator.go +++ b/communicator/src/communicator.go @@ -17,9 +17,10 @@ import ( // Changing CONNECTION_TYPE is not recommended const ( - DEFAULT_HOST = "0.0.0.0" - DEFAULT_PORT = "1919" - CONNECTION_TYPE = "tcp" + DEFAULT_HOST = "0.0.0.0" + DEFAULT_PORT = "1919" + CONNECTION_TYPE = "tcp" + NUM_PORT_RETRIES = 5 ) var BUFFER_LEN = 5120 @@ -27,16 +28,17 @@ var BUFFER_LEN = 5120 func main() { arguments := os.Args[1:] listen_port := DEFAULT_PORT - if len(arguments) > 1 { - listen_port = arguments[1] - } else if len(arguments) == 0 { - fmt.Println("Either -s or -c option needed") + argslen := len(arguments) + if argslen == 0 { + fmt.Println("Either -s or -c option needed. Use -h for help.") gracefulExit() + } else if argslen > 1 { + listen_port = arguments[1] } if arguments[0] == "-s" { loop := 0 server, err := net.Listen(CONNECTION_TYPE, DEFAULT_HOST+":"+listen_port) - for err != nil && loop < 5 && len(arguments) <= 1 { + for err != nil && loop < NUM_PORT_RETRIES && argslen <= 1 { listen_port = strconv.Itoa(rand.Intn(40000-2000) + 2000) server, err = net.Listen(CONNECTION_TYPE, DEFAULT_HOST+":"+listen_port) loop++ @@ -119,6 +121,13 @@ func main() { received += read } fmt.Print(output.String()) + } else if arguments[0] == "-h" { + fmt.Println("Usage: communicator [OPTION] [PORT]") + fmt.Println("Options:") + fmt.Println(" -s\t\tStart server") + fmt.Println(" -c\t\tStart client") + fmt.Println(" -h\t\tShow help") + fmt.Println("PORT is optional and defaults to 1919 for server use.") } else { fmt.Println("Invalid option") gracefulExit() @@ -141,7 +150,7 @@ func handleRequest(client net.Conn) { buffer := make([]byte, BUFFER_LEN) received := 0 flag := 0 - for received < len(lenBuffer) { + for received < len(lenBuffer) && received < BUFFER_LEN { read, err := client.Read(lenBuffer[received:]) if err != nil { clientClose(err, client) @@ -230,7 +239,7 @@ func handleRequest(client net.Conn) { return } sent = 0 - for sent < len(lastOutput.String()) { + for sent < len(lastOutput.String()) && sent < BUFFER_LEN { wrote, err := client.Write(([]byte(lastOutput.String()))[sent:]) if err != nil { clientClose(err, client) From d03956b5e7342c9d474313949ea7e406f3e2f431 Mon Sep 17 00:00:00 2001 From: sash19 Date: Thu, 31 Oct 2024 14:19:12 -0700 Subject: [PATCH 18/41] Added docs _build --- .gitignore | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .gitignore diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 From 7864ec6481a944bfc3affd4eff4df311c90ab717 Mon Sep 17 00:00:00 2001 From: sash19 Date: Thu, 31 Oct 2024 18:14:32 -0700 Subject: [PATCH 19/41] Initialization --- .gitattributes | 0 scalable/_version.py | 0 versioneer.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .gitattributes mode change 100644 => 100755 scalable/_version.py mode change 100644 => 100755 versioneer.py diff --git a/.gitattributes b/.gitattributes old mode 100644 new mode 100755 diff --git a/scalable/_version.py b/scalable/_version.py old mode 100644 new mode 100755 diff --git a/versioneer.py b/versioneer.py old mode 100644 new mode 100755 From 41e15cd9ed13e7794d19fa656fd584a19cea2a36 Mon Sep 17 00:00:00 2001 From: sash19 Date: Thu, 31 Oct 2024 18:14:46 -0700 Subject: [PATCH 20/41] Added pandas --- pyproject.toml | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml old mode 100644 new mode 100755 From dea97647be02fe7c325662f9b8b3dda6dd5b4160 Mon Sep 17 00:00:00 2001 From: sash19 Date: Thu, 31 Oct 2024 18:15:00 -0700 Subject: [PATCH 21/41] Name change --- LICENSE.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 LICENSE.md diff --git a/LICENSE.md b/LICENSE.md old mode 100644 new mode 100755 From 61eb59d02b5c92e38ab668901bd406e4f07586d5 Mon Sep 17 00:00:00 2001 From: sash19 Date: Thu, 31 Oct 2024 18:26:07 -0700 Subject: [PATCH 22/41] Moved constants around --- scalable/common.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scalable/common.py diff --git a/scalable/common.py b/scalable/common.py old mode 100644 new mode 100755 From 02ebe5ed873048a31da035d7af4a183fe6390a4e Mon Sep 17 00:00:00 2001 From: sash19 Date: Thu, 31 Oct 2024 18:38:05 -0700 Subject: [PATCH 23/41] Made file/directory hash checking optional --- scalable/caching.py | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) mode change 100644 => 100755 scalable/caching.py diff --git a/scalable/caching.py b/scalable/caching.py old mode 100644 new mode 100755 index 70243b3..d61febd --- a/scalable/caching.py +++ b/scalable/caching.py @@ -1,12 +1,14 @@ +import dill import os import pickle import types -import dill + import numpy as np import pandas as pd from diskcache import Cache from xxhash import xxh32 + from .common import SEED, cachedir, logger @@ -32,8 +34,8 @@ class FileType(GenericType): """ def __hash__(self) -> int: - digest = 0 if os.path.exists(self.value): + digest = 0 with open(self.value, 'rb') as file: x = xxh32(seed=SEED) x.update(str(os.path.basename(self.value)).encode('utf-8')) @@ -53,10 +55,10 @@ class DirType(GenericType): """ def __hash__(self) -> int: - digest = 0 - x = xxh32(seed=SEED) - x.update(str(os.path.basename(self.value)).encode('utf-8')) if os.path.exists(self.value): + digest = 0 + x = xxh32(seed=SEED) + x.update(str(os.path.basename(self.value)).encode('utf-8')) filenames = os.listdir(self.value) filenames = sorted(filenames) for filename in filenames: @@ -104,10 +106,6 @@ def __hash__(self) -> int: x = xxh32(seed=SEED) if isinstance(self.value, list): value_list = self.value - try: - value_list = sorted(self.value) - except: - pass for element in value_list: x.update(hash_to_bytes(hash(convert_to_type(element)))) elif isinstance(self.value, dict): @@ -142,7 +140,7 @@ def __hash__(self) -> int: if isinstance(self.value, np.ndarray): x.update(self.value.tobytes()) elif isinstance(self.value, pd.DataFrame): - x.update(self.value.to_string().encode('utf-8')) + x.update(pickle.dumps(self.value)) digest = x.intdigest() return digest @@ -196,7 +194,7 @@ def convert_to_type(arg): ret = ObjectType(arg) return ret -def cacheable(return_type=None, void=False, recompute=False, store=True, **arg_types): +def cacheable(return_type=None, void=False, check_output=False, recompute=False, store=True, **arg_types): """Decorator function to cache the output of a function. This function is used to cache other functions' outputs for certain @@ -220,6 +218,10 @@ def cacheable(return_type=None, void=False, recompute=False, store=True, **arg_t void : bool, optional Whether the function returns a value or not. A function is void if it does not return a value. The default is False. + check_output : bool, optional + Whether to check the output of a function has the same hash as when + it was stored. Useful to ensure entities like files haven't been + modified since initially stored. The default is False. recompute : bool, optional Whether to recompute the value or not. The default is False. store : bool, optional @@ -299,16 +301,18 @@ def inner(*args, **kwargs): raise KeyError(f"Key for function {func.__name__} could not be found.") stored_digest = value[0] new_digest = 0 - if return_type is None: - new_digest = hash(convert_to_type(value[1])) - else: - new_digest = hash(return_type(value[1])) - if new_digest == stored_digest: - ret = value[1] - else: - if not disk.delete(key, True): + if check_output: + if return_type is None: + new_digest = hash(convert_to_type(value[1])) + else: + new_digest = hash(return_type(value[1])) + if new_digest == stored_digest: + ret = value[1] + elif not disk.delete(key, True): logger.warning(f"{func.__name__} could not be deleted from cache after hash" " mismatch.") + else: + ret = value[1] if ret is None: ret = func(*args, **kwargs) if store: From 20184898e95863abd26c6972ed8cb00c1cedf9db Mon Sep 17 00:00:00 2001 From: sash19 Date: Thu, 31 Oct 2024 18:45:24 -0700 Subject: [PATCH 24/41] Updated function parameters and documentation --- scalable/client.py | 192 ++++++++++++++++++++++++++------------------- 1 file changed, 112 insertions(+), 80 deletions(-) mode change 100644 => 100755 scalable/client.py diff --git a/scalable/client.py b/scalable/client.py old mode 100644 new mode 100755 index 713da28..f27e3fe --- a/scalable/client.py +++ b/scalable/client.py @@ -1,57 +1,40 @@ -from collections.abc import Awaitable -from distributed import Client, Scheduler +from dask.typing import no_default +from distributed import Client from distributed.diagnostics.plugin import SchedulerPlugin from .common import logger -from .slurm import SlurmCluster, SlurmJob +from .slurm import SlurmCluster class SlurmSchedulerPlugin(SchedulerPlugin): def __init__(self, cluster): self.cluster = cluster super().__init__() - class ScalableClient(Client): + """Client for submitting tasks to a Dask cluster. Inherits the dask + client object. + + Parameters + ---------- + cluster : Cluster + The cluster object to connect to for submitting tasks. + """ def __init__(self, cluster, *args, **kwargs): super().__init__(address = cluster, *args, **kwargs) if isinstance(cluster, SlurmCluster): self.register_scheduler_plugin(SlurmSchedulerPlugin(None)) - def submit( - self, - func, - *args, - key=None, - workers=None, - tag=None, - n=1, - retries=None, - priority=0, - fifo_timeout="100 ms", - allow_other_workers=False, - actor=False, - actors=False, - pure=True, - **kwargs, - ): - """Submit a function application to the scheduler + def submit(self, func, *args, tag=None, n=1, **kwargs): + """Submit a function to be ran by workers in the cluster. Parameters ---------- - func : callable - Callable to be scheduled as ``func(*args,**kwargs)``. If ``func`` - returns a coroutine, it will be run on the main event loop of a - worker. Otherwise ``func`` will be run in a worker's task executor - pool (see ``Worker.executors`` for more information.) - \*args : tuple - Optional positional arguments - key : str - Unique identifier for the task. Defaults to function-name and hash - workers : string or iterable of strings - A set of worker addresses or hostnames on which computations may be - performed. Leave empty to default to all workers (common case) + func : function + Function to be scheduled for execution. + *args : tuple + Optional positional arguments to pass to the function. tag : str (optional) User-defined tag for the container that can run func. If not provided, func is assigned to be ran on a random container. @@ -59,48 +42,17 @@ def submit( Number of workers needed to run this task. Meant to be used with tag. Multiple workers can be useful for application level distributed computing. - retries : int (default to 0) - Number of allowed automatic retries if the task fails - priority : Number - Optional prioritization of task. Zero is default. - Higher priorities take precedence - fifo_timeout : str timedelta (default '100ms') - Allowed amount of time between calls to consider the same priority - allow_other_workers : bool (defaults to False) - Used with ``workers``. Indicates whether or not the computations - may be performed on workers that are not in the `workers` set(s). - actor : bool (default False) - Whether this task should exist on the worker as a stateful actor. - actors : bool (default False) - Alias for `actor` - pure : bool (defaults to True) - Whether or not the function is pure. Set ``pure=False`` for - impure functions like ``np.random.random``. Note that if both - ``actor`` and ``pure`` kwargs are set to True, then the value - of ``pure`` will be reverted to False, since an actor is stateful. - \*\*kwargs : dict + **kwargs : dict (optional) Optional key-value pairs to be passed to the function. Examples -------- >>> c = client.submit(add, a, b) - Notes - ----- - The current implementation of a task graph resolution searches for - occurrences of ``key`` and replaces it with a corresponding ``Future`` - result. That can lead to unwanted substitution of strings passed as - arguments to a task if these strings match some ``key`` that already - exists on a cluster. To avoid these situations it is required to use - unique values if a ``key`` is set manually. See - https://github.com/dask/dask/issues/9969 to track progress on resolving - this issue. - Returns ------- Future - If running in asynchronous mode, returns the future. Otherwise - returns the concrete value + Returns the future object that runs the function. Raises ------ @@ -113,17 +65,97 @@ def submit( resources = None if tag is not None: resources = {tag: n} - return super().submit(func, - *args, - key=key, - workers=workers, - resources=resources, - retries=retries, - priority=priority, - fifo_timeout=fifo_timeout, - allow_other_workers=allow_other_workers, - actor=actor, - actors=actors, - pure=False, - **kwargs) + return super().submit(func, resources=resources, *args, **kwargs) + + def cancel(self, futures, *args, **kwargs): + """ + Cancel running futures + This stops future tasks from being scheduled if they have not yet run + and deletes them if they have already run. After calling, this result + and all dependent results will no longer be accessible + + Parameters + ---------- + futures : future | future, list + One or more futures to cancel (as a list). + *args : tuple + Positional arguments to pass to dask client's cancel method. + **kwargs : dict + Keyword arguments to pass to dask client's cancel method. + """ + return super().cancel(futures, *args, **kwargs) + + def close(self, timeout=no_default): + """Close this client + + Clients will also close automatically when your Python session ends + + Parameters + ---------- + timeout : number + Time in seconds after which to raise a + ``dask.distributed.TimeoutError`` + + """ + return super().close(timeout) + + def map(self, func, parameters, tag, n, *args, **kwargs): + """Map a function on multiple sets of arguments to run the function + multiple times with different inputs. + + Parameters + ---------- + func : function + Function to be scheduled for execution. + parameters : list of lists + Lists of parameters to be passed to the function. The first list + should have the first parameter values, the second list should have + the second parameter values, and so on. The lists should be of the + same length. + tag : str (optional) + User-defined tag for the container that can run func. If not + provided, func is assigned to be ran on a random container. + n : int (default 1) + Number of workers needed to run this task. Meant to be used with + tag. Multiple workers can be useful for application level + distributed computing. + *args : tuple + Positional arguments to pass to dask client's map method. + **kwargs : dict + Keyword arguments to pass to dask client's map method. + + Examples + -------- + >>> def add(a, b): ... + >>> L = client.map(add, [[1, 2, 3], [4, 5, 6]]) + + Returns + ------- + List of futures + Returns a list of future objects, each for a separate run of the + function with the given parameters. + """ + resources = None + if tag is not None: + resources = {tag: n} + return super().map(func=func, iterables=parameters, resources=resources, *args, **kwargs) + + def get_versions(self, check=False, packages = None): + """Return version info for the scheduler, all workers and myself + + Parameters + ---------- + check : bool + Raise ValueError if all required & optional packages do not match. + Default is False. + packages : list + Extra package names to check. + + Examples + -------- + >>> c.get_versions() + + >>> c.get_versions(packages=['sklearn', 'geopandas']) + """ + return super().get_versions(check, packages) \ No newline at end of file From 53146f0aa40cdced75b198ffd82504faafe2e755 Mon Sep 17 00:00:00 2001 From: sash19 Date: Thu, 31 Oct 2024 19:40:29 -0700 Subject: [PATCH 25/41] Fixed documentation and added ability to suppress logs --- scalable/core.py | 159 +++++++++++----------------------------------- scalable/slurm.py | 39 +++++++----- 2 files changed, 61 insertions(+), 137 deletions(-) diff --git a/scalable/core.py b/scalable/core.py index 9f903a7..97875b6 100755 --- a/scalable/core.py +++ b/scalable/core.py @@ -7,11 +7,10 @@ import shlex import sys import tempfile -import threading import time -import warnings from contextlib import suppress + from dask.utils import parse_bytes from distributed.core import Status from distributed.deploy.spec import ProcessInterface, SpecCluster @@ -25,80 +24,13 @@ DEFAULT_WORKER_COMMAND = "distributed.cli.dask_worker" -# The length of the period (in mins) to check and account for dead workers -CHECK_DEAD_WORKER_PERIOD = 1 - -job_parameters = """ - cpus : int - Akin to the number of cores the current job should have available. - memory : str - Amount of memory the current job should have available. - nanny : bool - Whether or not to start a nanny process. - interface : str - Network interface like 'eth0' or 'ib0'. This will be used both for the - Dask scheduler and the Dask workers interface. If you need a different - interface for the Dask scheduler you can pass it through - the ``scheduler_options`` argument: ``interface=your_worker_interface, - scheduler_options={'interface': your_scheduler_interface}``. - death_timeout : float - Seconds to wait for a scheduler before closing workers - local_directory : str - Dask worker local directory for file spilling. - worker_command : list - Command to run when launching a worker. Defaults to - "distributed.cli.dask_worker" - worker_extra_args : list - Additional arguments to pass to `dask-worker` - python : str - Python executable used to launch Dask workers. - Defaults to the Python that is submitting these jobs - hardware : HardwareResources - A shared object containing the hardware resources available to the - cluster. - tag : str - The tag or the container type of the worker to be launched. - container : Container - The container object containing the information about launching the - worker. - launched : list - A list of launched workers which is shared across all workers and the - cluster object to keep track of the workers that have been launched. - security : Security - A security object containing the TLS configuration for the worker. If - True then a temporary security object with a self signed certificate - is created. - run_scripts_path : str - The path where the run scripts are located. Defaults to ./run_scripts. - The run scripts should be in the format _script.sh. - use_run_scripts : bool - Whether or not to use the run scripts. Defaults to True. +job_parameters = """ """.strip() cluster_parameters = """ - silence_logs : str - Log level like "debug", "info", or "error" to emit here if the - scheduler is started locally - asynchronous : bool - Whether or not to run this cluster object with the async/await syntax - name : str - The name of the cluster, which would also be used to name workers. - Defaults to class name. - scheduler_options : dict - Used to pass additional arguments to Dask Scheduler. For example use - ``scheduler_options={'dashboard_address': ':12435'}`` to specify which - port the web dashboard should use or - ``scheduler_options={'host': 'your-host'}`` to specify the host the Dask - scheduler should run on. See :class:`distributed.Scheduler` for more - details. - scheduler_cls : type - Changes the class of the used Dask Scheduler. Defaults to Dask's - :class:`distributed.Scheduler`. - shared_temp_directory : str - Shared directory between scheduler and worker (used for example by - temporary security certificates) defaults to current working directory - if not set. + account : str + Accounting string associated with each worker job. comm_port : int The network port on which the cluster can contact the host config_overwrite : bool @@ -107,7 +39,24 @@ logs_location : str The location to store worker logs. Default to the logs folder in the current directory. - + suppress_logs : bool + Whether or not to suppress logs. Defaults to False. + name : str + The name of the cluster, which would also be used to name workers. + Defaults to class name. + queue : str + Destination queue for each worker job. + run_scripts_path : str + The path where the run scripts are located. Defaults to ./run_scripts. + The run scripts should be in the format _script.sh. + security : Security + A security object containing the TLS configuration for the worker. If + True then a temporary security object with a self signed certificate + is created. + use_run_scripts : bool + Whether or not to use the run scripts. Defaults to True. + walltime : str + Walltime for each worker job. """.strip() @@ -137,7 +86,7 @@ class Job(ProcessInterface, abc.ABC): SLURMCluster """.format(job_parameters=job_parameters) - # Following class attributes should be overridden by extending classes. + # Following class attributes should be overridden by extending classes if necessary. cancel_command = None job_id_regexp = r"(?P\d+)" @@ -268,7 +217,6 @@ def __init__( if protocol and "--protocol" not in worker_extra_args: worker_extra_args.extend(["--protocol", protocol]) - # Keep information on process, cores, and memory, for use in subclasses self.worker_memory = parse_bytes(self.memory) if self.memory is not None else None # dask-worker command line build @@ -435,6 +383,7 @@ def __init__( config_overwrite=True, comm_port=None, logs_location=None, + suppress_logs=False, **job_kwargs ): @@ -516,12 +465,14 @@ def __init__( "cls": scheduler_cls, "options": scheduler_options, } - - self.logs_location = logs_location - if self.logs_location is None: - directory_name = self.job_cls.__name__.replace("Job", "") + "Cluster" - self.logs_location = create_logs_folder("logs", directory_name) + if not suppress_logs: + if logs_location is None: + directory_name = self.job_cls.__name__.replace("Job", "") + "Cluster" + logs_location = create_logs_folder("logs", directory_name) + self.logs_location = logs_location + else: + self.logs_location = None self.shared_temp_directory = shared_temp_directory @@ -550,7 +501,7 @@ def __init__( name=name, ) - def add_worker(self, tag=None, n=0): + def add_workers(self, tag=None, n=0): """Add workers to the cluster. Parameters @@ -565,7 +516,7 @@ def add_worker(self, tag=None, n=0): Examples -------- - >>> cluster.add_worker("gcam", 4) + >>> cluster.add_workers("gcam", 4) """ if self.exited or self.status in (Status.closing, Status.closed): @@ -633,42 +584,6 @@ def remove_workers(self, tag=None, n=0): if self.asynchronous: return NoOpAwaitable() - def _check_dead_workers(self): - """Periodically check for dead workers. - - This function essentially calls self.add_worker() with default - parameters which only syncs the cluster to the current state of - the workers. Any dead workers may be relaunched. - """ - next_call = time.time() - while not self.exited: - self.add_worker() - temp_file = tempfile.NamedTemporaryFile(mode='w+', delete=True) - temp_file_name = temp_file.name - temp_file.close() - with open(temp_file_name, 'w+') as temp_file: - temp_file.write("scalable") - temp_file.flush() - temp_file.seek(0) - temp_file.read() - os.remove(temp_file_name) - next_call = next_call + (60 * CHECK_DEAD_WORKER_PERIOD) - time.sleep(next_call - time.time()) - - def _get_dead_worker_tags(self): - """Get the list of dead workers. - - This function returns the list of workers that are dead. - """ - dead_workers = [] - for worker in self.launched: - if worker[0] not in self.workers: - dead_workers.append(worker) - for worker in dead_workers: - del self.worker_spec[worker[0]] - self.launched.remove(worker) - return [worker[1] for worker in dead_workers] - def add_container(self, tag, dirs, path=None, cpus=None, memory=None): """Add containers to enable them launching as workers. @@ -776,10 +691,10 @@ def _get_worker_security(self, security): elif self.shared_temp_directory is None: shared_temp_directory = os.getcwd() logger.warning( - "Using a temporary security object without explicitly setting a shared_temp_directory: \ -writing temp files to current working directory ({}) instead. You can set this value by \ -using dask for e.g. `dask.config.set({{'jobqueue.pbs.shared_temp_directory': '~'}})`\ -or by setting this value in the config file found in `~/.config/dask/jobqueue.yaml` ".format( + "Using a temporary security object without explicitly setting a shared_temp_directory: " + "writing temp files to current working directory ({}) instead. You can set this value by " + "using dask for e.g. `dask.config.set({{'jobqueue.pbs.shared_temp_directory': '~'}})` " + "or by setting this value in the config file found in `~/.config/dask/jobqueue.yaml` ".format( shared_temp_directory ), category=UserWarning, diff --git a/scalable/slurm.py b/scalable/slurm.py index 29ce2c4..2f9da31 100755 --- a/scalable/slurm.py +++ b/scalable/slurm.py @@ -29,7 +29,6 @@ def __init__( tag=None, hardware=None, logs_location=None, - log=True, shared_lock=None, worker_env_vars=None, **base_class_kwargs @@ -47,11 +46,11 @@ def __init__( self.job_name = job_name self.job_id = None self.job_node = None + self.log_file = None - if log: + if logs_location is not None: self.log_file = os.path.abspath(os.path.join(logs_location, f"{self.name}-{self.tag}.log")) - # All the wanted commands should be set here self.send_command = self.container.get_command(worker_env_vars) self.send_command.extend(self.command_args) @@ -149,23 +148,18 @@ async def close(self): cluster.loop.call_later(RECOVERY_DELAY, cluster._correct_state) class SlurmCluster(JobQueueCluster): - __doc__ = """ Launch Dask on a SLURM cluster + __doc__ = """Launch Dask on a SLURM cluster. Inherits the JobQueueCluster + class. Parameters ---------- - queue : str - Destination queue for each worker job. - project : str - Deprecated: use ``account`` instead. This parameter will be removed in a future version. - account : str - Accounting string associated with each worker job. - {job} {cluster} - walltime : str - Walltime for each worker job. - + *args : tuple + Positional arguments to pass to JobQueueCluster. + **kwargs : dict + Keyword arguments to pass to JobQueueCluster. """.format( - job=job_parameters, cluster=cluster_parameters + cluster=cluster_parameters ) job_cls = SlurmJob @@ -196,6 +190,21 @@ def close(self, timeout: float | None = None) -> Awaitable[None] | None: @staticmethod def set_default_request_quantity(nodes): + """Set the default number of nodes to request when scaling the cluster. + + Static Function. Does not require an instance of the class. + + If set to 1 (the original default), the cluster will request one + hardware node at a time when scaling. If set to a higher number, like 5, + the cluster will request 5 hardware nodes at a time when scaling. This + is helpful when each worker may need almost all the resources of a + node and it is more efficient to request multiple nodes at once. + + Parameters + ---------- + nodes : int + Number of nodes to request when scaling the cluster. + """ global DEFAULT_REQUEST_QUANTITY DEFAULT_REQUEST_QUANTITY = nodes \ No newline at end of file From 7ec0cd280fef249520dff244891a8e81f3fee38e Mon Sep 17 00:00:00 2001 From: sash19 Date: Thu, 31 Oct 2024 19:41:53 -0700 Subject: [PATCH 26/41] Updated docstrings to follow sphinx rules better --- scalable/utilities.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/scalable/utilities.py b/scalable/utilities.py index ff50de4..9d39396 100755 --- a/scalable/utilities.py +++ b/scalable/utilities.py @@ -3,10 +3,11 @@ import re import subprocess import sys -from importlib.resources import files - import yaml + +from importlib.resources import files from dask.utils import parse_bytes + from .common import logger comm_port_regex = r'0\.0\.0\.0:(\d{1,5})' @@ -46,8 +47,8 @@ async def get_cmd_comm(port, communicator_path=None): def run_bootstrap(): bootstrap_location = files('scalable').joinpath('scalable_bootstrap.sh') - result = subprocess.run(["/bin/bash", bootstrap_location], stdin=sys.stdin, - stdout=sys.stdout, stderr=sys.stderr) + result = subprocess.run([os.environ.get("SHELL"), bootstrap_location.as_posix()], stdin=sys.stdin, + stdout=sys.stdout, stderr=sys.stdout) if result.returncode != 0: sys.exit(result.returncode) @@ -521,7 +522,6 @@ class Container: def __init__(self, name, spec_dict): """ - Parameters ---------- name : str @@ -532,15 +532,12 @@ def __init__(self, name, spec_dict): be in gigabytes, megabytes, or bytes. '500MB' or '2GB' are valid. A valid spec_dict can look like: { - 'CPUs': 4, - 'Memory': '8G', - 'Path': '/home/user/work/containers/container.sif', - 'Dirs': { - '/home/work/inputs': '/inputs' - '/home/work/shared': '/shared' - } - } - + 'CPUs': 4, + 'Memory': '8G', + 'Path': '/home/user/work/containers/container.sif', + 'Dirs': { + '/home/work/inputs': '/inputs' + '/home/work/shared': '/shared'}} """ self.name = name self.cpus = spec_dict['CPUs'] From 80d7f346250300153d3cf3dde8fb2ff41857eefa Mon Sep 17 00:00:00 2001 From: sash19 Date: Thu, 31 Oct 2024 19:43:37 -0700 Subject: [PATCH 27/41] Cleaned up old code --- scalable/support.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scalable/support.py b/scalable/support.py index 736c32f..0e8b66b 100755 --- a/scalable/support.py +++ b/scalable/support.py @@ -73,6 +73,8 @@ def memory_command(): list The command to get the memory available on the node. """ + + # sanity check results command = "free -g | grep 'Mem' | sed 's/[\t ][\t ]*/ /g' | cut -d ' ' -f 7" return shlex.split(command, posix=False) @@ -84,6 +86,7 @@ def core_command(): list The command to get the number of cores available on the node. """ + # sanity check return ["nproc", "--all"] def jobid_command(name): @@ -173,7 +176,7 @@ def parse_nodelist(nodelist): def create_logs_folder(folder, worker_name): """Create a folder for logs. Uses the current date and time along with the given worker name to create a unique folder name. - + Parameters ---------- folder : str @@ -186,6 +189,7 @@ def create_logs_folder(folder, worker_name): str The path to the newly created logs folder. """ + # maybe add a way to suppress logs current_datetime = datetime.now() formatted_datetime = current_datetime.strftime("%Y%m%d_%H%M%S") folder_name = f"{worker_name}_{formatted_datetime}_logs" From 55ac70ca7cffa7b706714495c13cf6c957be8c08 Mon Sep 17 00:00:00 2001 From: sash19 Date: Thu, 31 Oct 2024 19:44:50 -0700 Subject: [PATCH 28/41] Updated version to v0.6 --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) mode change 100644 => 100755 README.md diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 9ab39b6..e062f9f --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Scalable -[v0.5.7](https://github.com/JGCRI/scalable/tree/0.5.7) +[v0.6.0](https://github.com/JGCRI/scalable/tree/0.6.0) Scalable is a Python library which aids in running complex workflows on HPCs by orchestrating multiple containers, requesting appropriate HPC jobs to the scheduler, and providing a python environment for distributed computing. It's designed to be primarily used with JGCRI Climate Models but can be easily adapted for any arbitrary uses. @@ -80,9 +80,9 @@ cluster.add_container(tag="osiris", cpus=8, memory="20G", dirs={"/rcfs/projects/ Before launching the workers, the configuration of worker or container targets needs to be specified. The containers to be launched as workers need to be first added by specifying their tag, number of cpu cores they need, the memory they would need, and the directory on the HPC Host to bind to the containers so that these directories are accessible by the container. ```python -cluster.add_worker(n=3, tag="gcam") -cluster.add_worker(n=2, tag="stitches") -cluster.add_worker(n=3, tag="osiris") +cluster.add_workers(n=3, tag="gcam") +cluster.add_workers(n=2, tag="stitches") +cluster.add_workers(n=3, tag="osiris") ``` Launching workers on the cluster can be done by just adding workers to the cluster. This call will only be successful if the tags used have also had containers with the same tag added beforehand. Removing workers is similarly as easy. From bfc9ef802ef8c43d285cd56286b9e46dd702f8c4 Mon Sep 17 00:00:00 2001 From: sash19 Date: Fri, 1 Nov 2024 10:29:24 -0700 Subject: [PATCH 29/41] Added more functions --- docs/workers.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/workers.rst b/docs/workers.rst index 15d091d..c881d4f 100755 --- a/docs/workers.rst +++ b/docs/workers.rst @@ -1,9 +1,12 @@ Worker Management ================= + .. autoclass:: scalable.SlurmCluster :exclude-members: close, job_cls, set_default_request_quantity +.. autofunction:: scalable.SlurmCluster.add_container + .. autofunction:: scalable.SlurmCluster.add_workers .. autofunction:: scalable.SlurmCluster.remove_workers From 15ae59e7072409f8e89d5a88659042d8c908960b Mon Sep 17 00:00:00 2001 From: sash19 Date: Fri, 1 Nov 2024 10:33:50 -0700 Subject: [PATCH 30/41] Switched lookup directory for config dict --- scalable/utilities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scalable/utilities.py b/scalable/utilities.py index 9d39396..9076293 100755 --- a/scalable/utilities.py +++ b/scalable/utilities.py @@ -92,8 +92,8 @@ def __init__(self, path=None, path_overwrite=True): self.config_dict = {} cwd = os.getcwd() if path is None: - self.path = os.path.abspath(os.path.join(cwd, "scalable", "config_dict.yaml")) - dockerfile_path = os.path.abspath(os.path.join(cwd, "scalable", "Dockerfile")) + self.path = os.path.abspath(os.path.join(cwd, "config_dict.yaml")) + dockerfile_path = os.path.abspath(os.path.join(cwd, "Dockerfile")) list_avial_command = \ f"sed -n 's/^FROM[[:space:]]\+[^ ]\+[[:space:]]\+AS[[:space:]]\+\([^ ]\+\)$/\\1/p' {dockerfile_path}" result = subprocess.run(list_avial_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) From 2dbe31076ea1139c19f13bebc7bdb49756b74249 Mon Sep 17 00:00:00 2001 From: sash19 Date: Fri, 1 Nov 2024 10:34:33 -0700 Subject: [PATCH 31/41] Cleaned up old code --- scalable/support.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scalable/support.py b/scalable/support.py index 0e8b66b..d97806d 100755 --- a/scalable/support.py +++ b/scalable/support.py @@ -73,8 +73,6 @@ def memory_command(): list The command to get the memory available on the node. """ - - # sanity check results command = "free -g | grep 'Mem' | sed 's/[\t ][\t ]*/ /g' | cut -d ' ' -f 7" return shlex.split(command, posix=False) @@ -86,7 +84,6 @@ def core_command(): list The command to get the number of cores available on the node. """ - # sanity check return ["nproc", "--all"] def jobid_command(name): @@ -189,7 +186,6 @@ def create_logs_folder(folder, worker_name): str The path to the newly created logs folder. """ - # maybe add a way to suppress logs current_datetime = datetime.now() formatted_datetime = current_datetime.strftime("%Y%m%d_%H%M%S") folder_name = f"{worker_name}_{formatted_datetime}_logs" From 6a0dad86617759dd1713d893ed5a9a35b644c5da Mon Sep 17 00:00:00 2001 From: sash19 Date: Fri, 1 Nov 2024 10:35:31 -0700 Subject: [PATCH 32/41] Enabled username/hostname cache; improved efficiency --- scalable/scalable_bootstrap.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scalable/scalable_bootstrap.sh b/scalable/scalable_bootstrap.sh index 33ed995..526828b 100755 --- a/scalable/scalable_bootstrap.sh +++ b/scalable/scalable_bootstrap.sh @@ -56,11 +56,6 @@ flush() { read -t 0.1 -n 10000 discard } -# comments, headlining sections and methods - -# add documentation for copying scalable target - - echo -e "${RED}Connection to HPC/Cloud...${NC}" choice="N" From 7458c1d36e00a3b4e0ee696b8e54101c4764c3be Mon Sep 17 00:00:00 2001 From: sash19 Date: Sat, 9 Nov 2024 15:02:02 -0800 Subject: [PATCH 33/41] Added default interface --- scalable/core.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scalable/core.py b/scalable/core.py index 97875b6..310323b 100755 --- a/scalable/core.py +++ b/scalable/core.py @@ -408,6 +408,9 @@ def __init__( type(self) ) ) + + if interface is None: + interface = "ib0" if dashboard_address is not None: raise ValueError( From 72511a3ed489dc2f33fefb7ba53107ff33cee7b1 Mon Sep 17 00:00:00 2001 From: sash19 Date: Sat, 9 Nov 2024 15:02:39 -0800 Subject: [PATCH 34/41] Fixed command typo --- scalable/slurm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scalable/slurm.py b/scalable/slurm.py index 2f9da31..0a30eab 100755 --- a/scalable/slurm.py +++ b/scalable/slurm.py @@ -63,7 +63,7 @@ async def _srun_command(self, command): async def _ssh_command(self, command): prefix = ["ssh", self.job_node] if self.log_file: - suffix = [f">>{self.log_file}", "2>&1", "&"] + suffix = [f">> {self.log_file}", "2>&1", "&"] command = command + suffix command = list(map(str, command)) command_str = " ".join(command) From f39e7e59862b2355ac339297845508ddc4807742 Mon Sep 17 00:00:00 2001 From: sash19 Date: Tue, 12 Nov 2024 18:20:24 -0800 Subject: [PATCH 35/41] Updated version --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e062f9f..973fa60 100755 --- a/README.md +++ b/README.md @@ -156,12 +156,12 @@ def func3(param): ``` -In the example above, the functions will wait 5, 3, and 10 seconds for the first time they are computed. However, their results will be cached due to the decorator and so, if the functions are ran again with the same arguments, their results are going to be returned from memory instead and they wouldn't sleep. There are arguments which directly can be given to the cacheable decorator. **It is always recommended to specify the return type and the type of arguments for each use.** This ensures expected functioning of the module and for correct caching. --TODO-- +In the example above, the functions will wait 5, 3, and 10 seconds for the first time they are computed. However, their results will be cached due to the decorator and so, if the functions are ran again with the same arguments, their results are going to be returned from memory instead and they wouldn't sleep. There are arguments which directly can be given to the cacheable decorator. **It is always recommended to specify the return type and the type of arguments for each use.** This ensures expected functioning of the module and for correct caching. ## Contact For any contribution, questions, or requests, please feel free to [open an issue](https://github.com/JGCRI/scalable/issues) or contact us directly: -**Shashank Lamba** [shashank.lamba@pnnl.gov](mailto:shashank.lamba@pnnl.gov) +**Shashank Lamba** [shashank.lamba@pnnl.gov](mailto:shashank.lamba@pnnl.gov)\ **Pralit Patel** [pralit.patel@pnnl.gov](mailto:pralit.patel@pnnl.gov) ## [License](https://github.com/JGCRI/scalable/blob/master/LICENSE.md) \ No newline at end of file From f50b1cfa1f5aa92bd4b6c33dda48f33a740fdad8 Mon Sep 17 00:00:00 2001 From: sash19 Date: Tue, 12 Nov 2024 18:21:09 -0800 Subject: [PATCH 36/41] Added documentation target --- .github/workflows/documentation.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/documentation.yml diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 0000000..54f003c --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,27 @@ +name: documentation + +on: [push, pull_request, workflow_dispatch] + +permissions: + contents: write + +jobs: + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - name: Install dependencies + run: | + pip install sphinx sphinx_rtd_theme myst_parser + - name: Sphinx build + run: | + sphinx-build doc _build + - name: Deploy to GitHub Pages + uses: peaceiris/actions-gh-pages@v3 + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + with: + publish_branch: gh-pages + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: _build/ + force_orphan: true From 3df0b9ec5460d49efb6aa43be2a20fbedee0e446 Mon Sep 17 00:00:00 2001 From: sash19 Date: Tue, 12 Nov 2024 18:25:50 -0800 Subject: [PATCH 37/41] Updated version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 973fa60..cc8480d 100755 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ In the example above, the functions will wait 5, 3, and 10 seconds for the first ## Contact -For any contribution, questions, or requests, please feel free to [open an issue](https://github.com/JGCRI/scalable/issues) or contact us directly: +For any contribution, questions, or requests, please feel free to [open an issue](https://github.com/JGCRI/scalable/issues) or contact us directly:\ **Shashank Lamba** [shashank.lamba@pnnl.gov](mailto:shashank.lamba@pnnl.gov)\ **Pralit Patel** [pralit.patel@pnnl.gov](mailto:pralit.patel@pnnl.gov) From 7b73ed93d56d22ce17dd01ad9080bafec792275c Mon Sep 17 00:00:00 2001 From: sash19 Date: Tue, 12 Nov 2024 18:31:57 -0800 Subject: [PATCH 38/41] Added documentation target --- .github/workflows/documentation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 54f003c..b0e5366 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -16,7 +16,7 @@ jobs: pip install sphinx sphinx_rtd_theme myst_parser - name: Sphinx build run: | - sphinx-build doc _build + sphinx-build docs _build - name: Deploy to GitHub Pages uses: peaceiris/actions-gh-pages@v3 if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} From e7049ac2123207473ce47294302574603e745d15 Mon Sep 17 00:00:00 2001 From: sash19 Date: Tue, 12 Nov 2024 18:49:18 -0800 Subject: [PATCH 39/41] Adding readthedocs config --- .readthedocs.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..be3eb31 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,22 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version, and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.12" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Optionally, but recommended, +# declare the Python requirements required to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +# python: +# install: +# - requirements: docs/requirements.txt \ No newline at end of file From 0e7eeab730d01052a2765d73a3eb3efbe39375bd Mon Sep 17 00:00:00 2001 From: sash19 Date: Tue, 12 Nov 2024 18:55:06 -0800 Subject: [PATCH 40/41] Updating readthedocs config --- .readthedocs.yaml | 6 +++--- docs/requirements.txt | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 docs/requirements.txt diff --git a/.readthedocs.yaml b/.readthedocs.yaml index be3eb31..8e6dc7f 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -17,6 +17,6 @@ sphinx: # Optionally, but recommended, # declare the Python requirements required to build your documentation # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html -# python: -# install: -# - requirements: docs/requirements.txt \ No newline at end of file +python: + install: + - requirements: docs/requirements.txt \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..4170c03 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1 @@ +sphinx-rtd-theme \ No newline at end of file From 74f96f0d482da345209bde647e5b254c0df9163c Mon Sep 17 00:00:00 2001 From: sash19 Date: Tue, 12 Nov 2024 19:20:22 -0800 Subject: [PATCH 41/41] Updated for latest version --- docs/conf.py | 2 +- docs/requirements.txt | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 57313e5..33d36ae 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,7 +15,7 @@ project = 'Scalable' copyright = '2024, Joint Global Change Research Institute' author = 'Shashank Lamba, Pralit Patel' -release = '0.5.0' +release = '0.6.0' # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/docs/requirements.txt b/docs/requirements.txt index 4170c03..90b7cf8 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1 +1,2 @@ -sphinx-rtd-theme \ No newline at end of file +sphinx-rtd-theme +scalable \ No newline at end of file