From b419f19e3b457b901449f692dc57bf2244ae9f42 Mon Sep 17 00:00:00 2001 From: Mateo Espinosa Zarlenga Date: Mon, 19 Sep 2022 20:59:29 +0100 Subject: [PATCH] Adding experiments and updating README with current details. --- README.md | 142 ++- experiments/__init__.py | 5 + experiments/celeba_emb_size_ablation.py | 606 +++++++++++++ experiments/celeba_experiments.py | 826 ++++++++++++++++++ experiments/cub_emb_size_ablation.py | 441 ++++++++++ experiments/cub_experiments.py | 574 ++++++++++++ experiments/cub_randint_ablation.py | 361 ++++++++ experiments/cub_subsample_experiment.py | 460 ++++++++++ experiments/synthetic_datasets_experiments.py | 501 +++++++++++ figures/cem.png | Bin 0 -> 78124 bytes 10 files changed, 3914 insertions(+), 2 deletions(-) create mode 100644 experiments/__init__.py create mode 100644 experiments/celeba_emb_size_ablation.py create mode 100644 experiments/celeba_experiments.py create mode 100644 experiments/cub_emb_size_ablation.py create mode 100644 experiments/cub_experiments.py create mode 100644 experiments/cub_randint_ablation.py create mode 100644 experiments/cub_subsample_experiment.py create mode 100644 experiments/synthetic_datasets_experiments.py create mode 100644 figures/cem.png diff --git a/README.md b/README.md index 2e46f11..f55162c 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,140 @@ -# cem -Concept Embedding Models Pytorch Implementation +# Concept Embedding Models + +This repository contains the official Pytorch implementation of our work +*"Concept Embedding Models"* accepted at **NeurIPS 2022**. For details on our +model and motivation, please refer to our official [paper](TODO). + +# Model + +![CEM Architecture](figures/cem.png) + +[Concept Bottleneck Models (CBMs)](https://arxiv.org/abs/2007.04612) have recently gained attention as +high-performing and interpretable neural architectures that can explain their +predictions using a set of human-understandable high-level concepts. +Nevertheless, the need for a strict activation bottleneck as part of the +architecture, as well as the fact that one requires the set of concept +annotations used during training to be fully descriptive of the downstream +task of interest, are constraints that force CBMs to trade downstream +performance for interpretability purposes. This severely limits their +applicability in real-world applications, where data rarely comes with +concept annotations that are fully descriptive of any task of interest. + + +In our work, we propose Concept Embedding Models (CEMs) to tackle these two big +challenges. Our neural architecture expands a CBM's bottleneck and allows the +information related to unseen concepts to be flow as part of the model's +bottleneck. We achieve this by learning a high-dimensional representation +(i.e., a *concept embedding*) for each concept provided during training. Naively +extending the bottleneck, however, may directly impede the use of test-time +*concept interventions* where one can correct a mispredicted concept in order +to improve the end model's downstream performance. This is a crucial element +motivating the creation of traditional CBMs and therefore is a highly desirable +feature. Therefore, in order to use concept embeddings in the bottleneck while +still permitting effective test-time interventions, CEM +construct each concept's representation as a linear combination of two +concept embeddings, where each embedding has fixed semantics. Specifically, +we learn an embedding to represent the "active" space of a concept and one +to represent the "inactive" state of a concept, allowing us to selecting +between these two produced embeddings at test-time to then intervene in a +concept and improve downstream performance. Our entire architecture is +visualized in the figure above and formally described in our paper. + +# Usage + +In this repository, we include a standalone Pytorch implementation of CEM +which can be easily trained from scratch given a set of samples annotated with +a downstream task and a set of binary concepts. In order to use our implementation, +however, you first need to install all our code's requirements (listed in +`requirements.txt`). We provide an automatic mechanism for this installation using +Python's setup process with our standalone `setup.py`. To install our package, +therefore, you only need to run: +```bash +$ python setup.py install +``` + +After this command has terminated successfully, you should be able to import +`cem` as a package and use it to train a CEM object as follows: +```python +import pytorch_lightning as pl +from cem.models.cem import ConceptEmbeddingModel + +##### +# Define your dataset +##### + +train_dl = ... +val_dl = ... + +##### +# Construct the model +##### + +cem_model = ConceptEmbeddingModel( + n_concepts=n_concepts, # Number of training-time concepts + n_tasks=n_tasks, # Number of output labels + emb_size=16, + concept_loss_weight=0.1, + learning_rate=1e-3, + optimizer="adam", + c_extractor_arch=latent_code_generator_model, # Replace this appropriately + training_intervention_prob=0.25, # RandInt probability +) + +##### +# Train it +##### + +trainer = pl.Trainer( + gpus=1, + max_epochs=100, + check_val_every_n_epoch=5, +) +# train_dl and val_dl are datasets previously built... +trainer.fit(cem_model, train_dl, val_dl) +``` + +# Experiment Reproducibility + +To reproduce the experiments discussed in our paper, please use the scripts +in the `experiments` directory after installing the `cem` package as indicated +above. For example, to run our experiments on the DOT dataset (see our paper), +you can execute the following command: + +```bash +$ python experiments/synthetic_datasets_experiments.py dot -o dot_results/ +``` +This should generate a summary of all the results after execution has +terminated and dump all results/trained models/logs into the given +output directory (`dot_results/` in this case). + + +# Citation +If you would like to cite this repository, or the accompanying paper, please +use the following citation: + +``` +@article{DBLP:journals/corr/abs-2111-12628, + author = {Mateo Espinosa Zarlenga and + Pietro Barbiero and + Gabriele Ciravegna and + Giuseppe Marra and + Francesco Giannini and + Michelangelo Diligenti and + Zohreh Shams and + Frederic Precioso and + Stefano Melacci and + Adrian Weller and + Pietro Lio and + Mateja Jamnik}, + title = {Concept Embedding Models}, + journal = {CoRR}, + volume = {abs/TODO}, + year = {2021}, + url = {https://arxiv.org/abs/TODO}, + eprinttype = {arXiv}, + eprint = {TODO}, + timestamp = {TODO}, + biburl = {https://dblp.org/rec/journals/corr/abs-TODO.bib}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} +``` diff --git a/experiments/__init__.py b/experiments/__init__.py new file mode 100644 index 0000000..6c808df --- /dev/null +++ b/experiments/__init__.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# @Author: Mateo Espinosa Zarlenga +# @Date: 2022-09-19 18:28:17 +# @Last Modified by: Mateo Espinosa Zarlenga +# @Last Modified time: 2022-09-19 18:28:17 diff --git a/experiments/celeba_emb_size_ablation.py b/experiments/celeba_emb_size_ablation.py new file mode 100644 index 0000000..dec1905 --- /dev/null +++ b/experiments/celeba_emb_size_ablation.py @@ -0,0 +1,606 @@ +import argparse +import copy +import joblib +import numpy as np +import os +import torch +import torchvision + +from pathlib import Path +from pytorch_lightning import seed_everything +from torchvision import transforms + +import cem.experiments.celeba_experiments as celeba_experiments +import cem.train.training as training +import cem.train.utils as utils + + +def main( + rerun=False, + result_dir='results/cub_emb_size_ablation/', + project_name='', + activation_freq=0, + num_workers=8, + single_frequency_epochs=0, + global_params=None, + data_root=celeba_experiments.CELEBA_ROOT, +): + seed_everything(42) + # parameters for data, model, and training + og_config = dict( + cv=5, + max_epochs=200, + patience=15, + batch_size=512, + num_workers=num_workers, + emb_size=16, + extra_dims=0, + concept_loss_weight=1, + normalize_loss=False, + learning_rate=0.005, + weight_decay=4e-05, + weight_loss=False, + pretrain_model=True, + c_extractor_arch="resnet34", + optimizer="sgd", + bool=False, + early_stopping_monitor="val_loss", + early_stopping_mode="min", + early_stopping_delta=0.0, + image_size=64, + num_classes=1000, + top_k_accuracy=[3, 5, 10], + save_model=True, + use_imbalance=True, + use_binary_vector_class=True, + num_concepts=6, + label_binary_width=1, + label_dataset_subsample=12, + num_hidden_concepts=2, + selected_concepts=False, + + momentum=0.9, + shared_prob_gen=False, + sigmoidal_prob=False, + sigmoidal_embedding=False, + training_intervention_prob=0.0, + embeding_activation=None, + concat_prob=False, + ) + + utils.extend_with_global_params(og_config, global_params or []) + use_binary_vector_class = og_config.get('use_binary_vector_class', False) + if use_binary_vector_class: + # Now reload by transform the labels accordingly + width = og_config.get('label_binary_width', 5) + def _binarize(concepts, selected, width): + result = [] + binary_repr = [] + concepts = concepts[selected] + for i in range(0, concepts.shape[-1], width): + binary_repr.append( + str(int(np.sum(concepts[i : i + width]) > 0)) + ) + return int("".join(binary_repr), 2) + + celeba_train_data = torchvision.datasets.CelebA( + root=data_root, + split='all', + download=True, + target_transform=lambda x: x[0].long() - 1, + target_type=['attr'], + ) + + concept_freq = np.sum( + celeba_train_data.attr.cpu().detach().numpy(), + axis=0 + ) / celeba_train_data.attr.shape[0] + print("Concept frequency is:", concept_freq) + sorted_concepts = list(map( + lambda x: x[0], + sorted(enumerate(np.abs(concept_freq - 0.5)), key=lambda x: x[1]), + )) + num_concepts = og_config.get( + 'num_concepts', + celeba_train_data.attr.shape[-1], + ) + concept_idxs = sorted_concepts[:num_concepts] + concept_idxs = sorted(concept_idxs) + if og_config.get('num_hidden_concepts', 0): + num_hidden = og_config.get('num_hidden_concepts', 0) + hidden_concepts = sorted( + sorted_concepts[ + num_concepts:min( + (num_concepts + num_hidden), + len(sorted_concepts) + ) + ] + ) + else: + hidden_concepts = [] + print("Selecting concepts:", concept_idxs) + print("\tAnd hidden concepts:", hidden_concepts) + celeba_train_data = torchvision.datasets.CelebA( + root=data_root, + split='all', + download=True, + transform=transforms.Compose([ + transforms.Resize(og_config['image_size']), + transforms.CenterCrop(og_config['image_size']), + transforms.ToTensor(), + transforms.ConvertImageDtype(torch.float32), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), + ]), + target_transform=lambda x: [ + torch.tensor( + _binarize( + x[1].cpu().detach().numpy(), + selected=(concept_idxs + hidden_concepts), + width=width, + ), + dtype=torch.long, + ), + x[1][concept_idxs].float(), + ], + target_type=['identity', 'attr'], + ) + label_remap = {} + vals, counts = np.unique( + list(map( + lambda x: _binarize( + x.cpu().detach().numpy(), + selected=(concept_idxs + hidden_concepts), + width=width, + ), + celeba_train_data.attr + )), + return_counts=True, + ) + for i, label in enumerate(vals): + label_remap[label] = i + + celeba_train_data = torchvision.datasets.CelebA( + root=data_root, + split='all', + download=True, + transform=transforms.Compose([ + transforms.Resize(og_config['image_size']), + transforms.CenterCrop(og_config['image_size']), + transforms.ToTensor(), + transforms.ConvertImageDtype(torch.float32), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), + ]), + target_transform=lambda x: [ + torch.tensor( + label_remap[_binarize( + x[1].cpu().detach().numpy(), + selected=(concept_idxs + hidden_concepts), + width=width, + )], + dtype=torch.long, + ), + x[1][concept_idxs].float(), + ], + target_type=['identity', 'attr'], + ) + num_classes = len(label_remap) + + # And subsample to reduce its massive size + factor = og_config.get('label_dataset_subsample', 1) + if factor != 1: + train_idxs = np.random.choice( + np.arange(0, len(celeba_train_data)), + replace=False, + size=len(celeba_train_data)//factor, + ) + print("Subsampling to", len(train_idxs), "elements.") + celeba_train_data = torch.utils.data.Subset( + celeba_train_data, + train_idxs, + ) + else: + concept_selection = list(range(0, len(CONCEPT_SEMANTICS))) + if og_config.get('selected_concepts', False): + concept_selection = SELECTED_CONCEPTS + celeba_train_data = torchvision.datasets.CelebA( + root=data_root, + split='all', + download=True, + target_transform=lambda x: x[0].long() - 1, + target_type=['identity'], + ) + vals, counts = np.unique( + celeba_train_data.identity, + return_counts=True, + ) + sorted_labels = list(map( + lambda x: x[0], + sorted(zip(vals, counts), key=lambda x: -x[1]) + )) + print( + "Selecting", + og_config['num_classes'], + "out of", + len(vals), + "classes", + ) + if result_dir: + Path(result_dir).mkdir(parents=True, exist_ok=True) + np.save( + os.path.join( + result_dir, + f"selected_top_{og_config['num_classes']}_labels.npy", + ), + sorted_labels[:og_config['num_classes']], + ) + label_remap = {} + for i, label in enumerate(sorted_labels[:og_config['num_classes']]): + label_remap[label] = i + print("len(label_remap) =", len(label_remap)) + + # Now reload by transform the labels accordingly + celeba_train_data = torchvision.datasets.CelebA( + root=data_root, + split='all', + download=True, + transform=transforms.Compose([ + transforms.Resize(og_config['image_size']), + transforms.CenterCrop(og_config['image_size']), + transforms.ToTensor(), + transforms.ConvertImageDtype(torch.float32), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), + ]), + target_transform=lambda x: [ + torch.tensor( + # If it is not in our map, then we make it be the token label + # og_config['num_classes'] which will be removed afterwards + label_remap.get( + x[0].cpu().detach().item() - 1, + og_config['num_classes'] + ), + dtype=torch.long, + ), + x[1][concept_selection].float(), + ], + target_type=['identity', 'attr'], + ) + num_classes = og_config['num_classes'] + + train_idxs = np.where( + list(map( + lambda x: x.cpu().detach().item() - 1 in label_remap, + celeba_train_data.identity + )) + )[0] + celeba_train_data = torch.utils.data.Subset( + celeba_train_data, + train_idxs, + ) + total_samples = len(celeba_train_data) + train_samples = int(0.7 * total_samples) + test_samples = int(0.2 * total_samples) + val_samples = total_samples - test_samples - train_samples + print( + f"Data split is: {total_samples} = {train_samples} (train) + " + f"{test_samples} (test) + {val_samples} (validation)" + ) + celeba_train_data, celeba_test_data, celeba_val_data = \ + torch.utils.data.random_split( + celeba_train_data, + [train_samples, test_samples, val_samples], + ) + train_dl = torch.utils.data.DataLoader( + celeba_train_data, + batch_size=og_config['batch_size'], + shuffle=True, + num_workers=og_config['num_workers'], + ) + test_dl = torch.utils.data.DataLoader( + celeba_test_data, + batch_size=og_config['batch_size'], + shuffle=False, + num_workers=og_config['num_workers'], + ) + val_dl = torch.utils.data.DataLoader( + celeba_val_data, + batch_size=og_config['batch_size'], + shuffle=False, + num_workers=og_config['num_workers'], + ) + + if result_dir and activation_freq: + # Then let's save the testing data for further analysis later on + out_acts_save_dir = os.path.join(result_dir, "test_embedding_acts") + Path(out_acts_save_dir).mkdir(parents=True, exist_ok=True) + + for (ds, name) in [ + (test_dl, "test"), + (val_dl, "val"), + ]: + x_total = [] + y_total = [] + c_total = [] + for x, (y, c) in ds: + x_total.append(x.cpu().detach()) + y_total.append(y.cpu().detach()) + c_total.append(c.cpu().detach()) + x_inputs = np.concatenate(x_total, axis=0) + print(f"x_{name}.shape =", x_inputs.shape) + np.save(os.path.join(out_acts_save_dir, f"x_{name}.npy"), x_inputs) + + y_inputs = np.concatenate(y_total, axis=0) + print(f"y_{name}.shape =", y_inputs.shape) + np.save(os.path.join(out_acts_save_dir, f"y_{name}.npy"), y_inputs) + + c_inputs = np.concatenate(c_total, axis=0) + print(f"c_{name}.shape =", c_inputs.shape) + np.save(os.path.join(out_acts_save_dir, f"c_{name}.npy"), c_inputs) + + label_set = set() + sample = next(iter(train_dl)) + real_sample = [] + for derp in sample: + if isinstance(derp, list): + real_sample += derp + else: + real_sample.append(derp) + sample = real_sample + print("Sample has", len(sample), "elements.") + for i, derp in enumerate(sample): + print("Element", i, "has shape", derp.shape, "and type", derp.dtype) + + print("Training sample shape is:", sample[0].shape) + print("Training label shape is:", sample[1].shape) + print("Training concept shape is:", sample[2].shape) + + n_concepts, n_tasks = sample[2].shape[-1], num_classes + + attribute_count = np.zeros((n_concepts,)) + samples_seen = 0 + for i, (_, (y, c)) in enumerate(train_dl): + print("\rIn batch", i, "we have seen", len(label_set), "classes") + c = c.cpu().detach().numpy() + attribute_count += np.sum(c, axis=0) + samples_seen += c.shape[0] + for l in y.reshape(-1).cpu().detach(): + label_set.add(l.item()) + + print("Found a total of", len(label_set), "classes") + if og_config.get("use_imbalance", False): + imbalance = samples_seen / attribute_count - 1 + else: + imbalance = None + print("Imbalance:", imbalance) + + os.makedirs(result_dir, exist_ok=True) + results = {} + for split in range(og_config["cv"]): + for emb_size in [1, 2, 4, 6, 8, 16, 32, 64]: + if emb_size not in results: + results[emb_size] = {} + if f'{split}' not in results[emb_size]: + results[emb_size][f'{split}'] = {} + print( + f'Experiment {split+1}/{og_config["cv"]} with emb_size', + emb_size, + ) + + # Trial period for mixture embedding model + config = copy.deepcopy(og_config) + config["architecture"] = "MixtureEmbModel" + config["extra_name"] = ( + f"SharedProb_AdaptiveDropout_NoProbConcat_" + f"emb_size_{emb_size}" + ) + config["shared_prob_gen"] = True + config["sigmoidal_prob"] = False + config["sigmoidal_embedding"] = False + config['training_intervention_prob'] = 0.25 + config['concat_prob'] = False + config['emb_size'] = emb_size + config["embeding_activation"] = "leakyrelu" + mixed_emb_shared_prob_model, mixed_emb_shared_prob_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[emb_size][f'{split}'], + config, + mixed_emb_shared_prob_model, + mixed_emb_shared_prob_test_results, + ) + + # Train fuzzy CBM with extra capacity + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["bool"] = False + config["extra_dims"] = (emb_size - 1) * n_concepts + config["extra_name"] = ( + f"FuzzyExtraCapacity_Logit_emb_size_{emb_size}" + ) + config["bottleneck_nonlinear"] = "leakyrelu" + config["sigmoidal_extra_capacity"] = False + config["sigmoidal_prob"] = False + config['emb_size'] = emb_size + extra_fuzzy_logit_model, extra_fuzzy_logit_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[emb_size][f'{split}'], + config, + extra_fuzzy_logit_model, + extra_fuzzy_logit_test_results, + ) + + # train vanilla model with more capacity (i.e., no concept + # supervision) but with ReLU activation + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["extra_name"] = ( + f"NoConceptSupervisionReLU_ExtraCapacity_emb_size_{emb_size}" + ) + config["bool"] = False + config["extra_dims"] = (emb_size - 1) * n_concepts + config["bottleneck_nonlinear"] = "leakyrelu" + config["concept_loss_weight"] = 0 + config['emb_size'] = emb_size + extra_vanilla_relu_model, extra_vanilla_relu_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[emb_size][f'{split}'], + config, + extra_vanilla_relu_model, + extra_vanilla_relu_test_results, + ) + + # save results + joblib.dump(results, os.path.join(result_dir, f'results.joblib')) + return results + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=( + 'Runs embedding ablation study in CelebA dataset.' + ), + ) + parser.add_argument( + '--project_name', + default='', + help=( + "Project name used for Weights & Biases monitoring. If not " + "provided, we assume no W&B logging is done." + ), + metavar="name", + + ) + + parser.add_argument( + '--output_dir', + '-o', + default='results/celeba_emb_size_ablation/', + help=( + "directory where we will dump our experiment's results. If not " + "given, then we will use results/celeba_emb_size_ablation/." + ), + metavar="path", + + ) + parser.add_argument( + '--rerun', + '-r', + default=False, + action="store_true", + help=( + "If set, then we will force a rerun of the entire experiment even if " + "valid results are found in the provided output directory. Note that " + "this may overwrite and previous results, so use with care." + ), + + ) + parser.add_argument( + '--activation_freq', + default=0, + help=( + 'how frequently, in terms of epochs, should we store the ' + 'embedding activations for our validation set. By default we will ' + 'not store any activations.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + '--single_frequency_epochs', + default=0, + help=( + 'how frequently, in terms of epochs, should we store the ' + 'embedding activations for our validation set. By default we ' + 'will not store any activations.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + '--num_workers', + default=8, + help=( + 'number of workers used for data feeders. Do not use more workers ' + 'than cores in the machine.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + "-d", + "--debug", + action="store_true", + default=False, + help="starts debug mode in our program.", + ) + parser.add_argument( + '-p', + '--param', + action='append', + nargs=2, + metavar=('param_name=value'), + help=( + 'Allows the passing of a config param that will overwrite ' + 'anything passed as part of the config file itself.' + ), + default=[], + ) + args = parser.parse_args() + if args.project_name: + # Lazy import to avoid importing unless necessary + import wandb + main( + rerun=args.rerun, + result_dir=args.output_dir, + project_name=args.project_name, + activation_freq=args.activation_freq, + num_workers=args.num_workers, + single_frequency_epochs=args.single_frequency_epochs, + global_params=args.param, + ) diff --git a/experiments/celeba_experiments.py b/experiments/celeba_experiments.py new file mode 100644 index 0000000..a443383 --- /dev/null +++ b/experiments/celeba_experiments.py @@ -0,0 +1,826 @@ +import argparse +import copy +import joblib +import numpy as np +import os +import torch +import torchvision + +from pathlib import Path +from pytorch_lightning import seed_everything +from torchvision import transforms + +import cem.train.training as training +import cem.train.utils as utils + +############################################################################### +## GLOBAL VARIABLES +############################################################################### + +SELECTED_CONCEPTS = [ + 2, + 4, + 6, + 7, + 8, + 9, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 32, + 33, + 39, +] + +CONCEPT_SEMANTICS = [ + '5_o_Clock_Shadow', + 'Arched_Eyebrows', + 'Attractive', + 'Bags_Under_Eyes', + 'Bald', + 'Bangs', + 'Big_Lips', + 'Big_Nose', + 'Black_Hair', + 'Blond_Hair', + 'Blurry', + 'Brown_Hair', + 'Bushy_Eyebrows', + 'Chubby', + 'Double_Chin', + 'Eyeglasses', + 'Goatee', + 'Gray_Hair', + 'Heavy_Makeup', + 'High_Cheekbones', + 'Male', + 'Mouth_Slightly_Open', + 'Mustache', + 'Narrow_Eyes', + 'No_Beard', + 'Oval_Face', + 'Pale_Skin', + 'Pointy_Nose', + 'Receding_Hairline', + 'Rosy_Cheeks', + 'Sideburns', + 'Smiling', + 'Straight_Hair', + 'Wavy_Hair', + 'Wearing_Earrings', + 'Wearing_Hat', + 'Wearing_Lipstick', + 'Wearing_Necklace', + 'Wearing_Necktie', + 'Young', +] + +# IMPORANT NOTE: THIS DATASET NEEDS TO BE DOWNLOADED FIRST BEFORE BEING ABLE +# TO RUN ANY CUB EXPERIMENTS!! +# Instructions on how to download it can be found +# in https://mmlab.ie.cuhk.edu.hk/projects/CelebA.html +CELEBA_ROOT = 'data/celeba' + + +############################################################################### +## MAIN EXPERIMENT LOOP +############################################################################### + + +def main( + rerun=False, + result_dir='results/celeba/', + project_name='', + activation_freq=0, + num_workers=8, + single_frequency_epochs=0, + global_params=None, + save_model=True, + data_root=CELEBA_ROOT, +): + seed_everything(42) + # parameters for data, model, and training + og_config = dict( + cv=5, + max_epochs=200, + patience=15, + batch_size=512, + num_workers=num_workers, + emb_size=16, + extra_dims=0, + concept_loss_weight=1, + normalize_loss=False, + learning_rate=0.005, + weight_decay=4e-05, + weight_loss=False, + pretrain_model=True, + c_extractor_arch="resnet34", + optimizer="sgd", + bool=False, + early_stopping_monitor="val_loss", + early_stopping_mode="min", + early_stopping_delta=0.0, + image_size=64, + num_classes=1000, + top_k_accuracy=[3, 5, 10], + save_model=True, + use_imbalance=True, + use_binary_vector_class=True, + num_concepts=6, + label_binary_width=1, + label_dataset_subsample=12, + num_hidden_concepts=2, + selected_concepts=False, + + momentum=0.9, + shared_prob_gen=False, + sigmoidal_prob=False, + sigmoidal_embedding=False, + training_intervention_prob=0.0, + embeding_activation=None, + concat_prob=False, + ) + + utils.extend_with_global_params(og_config, global_params or []) + use_binary_vector_class = og_config.get('use_binary_vector_class', False) + if use_binary_vector_class: + # Now reload by transform the labels accordingly + width = og_config.get('label_binary_width', 5) + def _binarize(concepts, selected, width): + result = [] + binary_repr = [] + concepts = concepts[selected] + for i in range(0, concepts.shape[-1], width): + binary_repr.append( + str(int(np.sum(concepts[i : i + width]) > 0)) + ) + return int("".join(binary_repr), 2) + + celeba_train_data = torchvision.datasets.CelebA( + root=data_root, + split='all', + download=True, + target_transform=lambda x: x[0].long() - 1, + target_type=['attr'], + ) + + concept_freq = np.sum( + celeba_train_data.attr.cpu().detach().numpy(), + axis=0 + ) / celeba_train_data.attr.shape[0] + print("Concept frequency is:", concept_freq) + sorted_concepts = list(map( + lambda x: x[0], + sorted(enumerate(np.abs(concept_freq - 0.5)), key=lambda x: x[1]), + )) + num_concepts = og_config.get( + 'num_concepts', + celeba_train_data.attr.shape[-1], + ) + concept_idxs = sorted_concepts[:num_concepts] + concept_idxs = sorted(concept_idxs) + if og_config.get('num_hidden_concepts', 0): + num_hidden = og_config.get('num_hidden_concepts', 0) + hidden_concepts = sorted( + sorted_concepts[ + num_concepts:min( + (num_concepts + num_hidden), + len(sorted_concepts) + ) + ] + ) + else: + hidden_concepts = [] + print("Selecting concepts:", concept_idxs) + print("\tAnd hidden concepts:", hidden_concepts) + celeba_train_data = torchvision.datasets.CelebA( + root=data_root, + split='all', + download=True, + transform=transforms.Compose([ + transforms.Resize(og_config['image_size']), + transforms.CenterCrop(og_config['image_size']), + transforms.ToTensor(), + transforms.ConvertImageDtype(torch.float32), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), + ]), + target_transform=lambda x: [ + torch.tensor( + _binarize( + x[1].cpu().detach().numpy(), + selected=(concept_idxs + hidden_concepts), + width=width, + ), + dtype=torch.long, + ), + x[1][concept_idxs].float(), + ], + target_type=['identity', 'attr'], + ) + label_remap = {} + vals, counts = np.unique( + list(map( + lambda x: _binarize( + x.cpu().detach().numpy(), + selected=(concept_idxs + hidden_concepts), + width=width, + ), + celeba_train_data.attr + )), + return_counts=True, + ) + for i, label in enumerate(vals): + label_remap[label] = i + + celeba_train_data = torchvision.datasets.CelebA( + root=data_root, + split='all', + download=True, + transform=transforms.Compose([ + transforms.Resize(og_config['image_size']), + transforms.CenterCrop(og_config['image_size']), + transforms.ToTensor(), + transforms.ConvertImageDtype(torch.float32), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), + ]), + target_transform=lambda x: [ + torch.tensor( + label_remap[_binarize( + x[1].cpu().detach().numpy(), + selected=(concept_idxs + hidden_concepts), + width=width, + )], + dtype=torch.long, + ), + x[1][concept_idxs].float(), + ], + target_type=['identity', 'attr'], + ) + num_classes = len(label_remap) + + # And subsample to reduce its massive size + factor = og_config.get('label_dataset_subsample', 1) + if factor != 1: + train_idxs = np.random.choice( + np.arange(0, len(celeba_train_data)), + replace=False, + size=len(celeba_train_data)//factor, + ) + print("Subsampling to", len(train_idxs), "elements.") + celeba_train_data = torch.utils.data.Subset( + celeba_train_data, + train_idxs, + ) + else: + concept_selection = list(range(0, len(CONCEPT_SEMANTICS))) + if og_config.get('selected_concepts', False): + concept_selection = SELECTED_CONCEPTS + celeba_train_data = torchvision.datasets.CelebA( + root=data_root, + split='all', + download=True, + target_transform=lambda x: x[0].long() - 1, + target_type=['identity'], + ) + vals, counts = np.unique( + celeba_train_data.identity, + return_counts=True, + ) + sorted_labels = list(map( + lambda x: x[0], + sorted(zip(vals, counts), key=lambda x: -x[1]) + )) + print( + "Selecting", + og_config['num_classes'], + "out of", + len(vals), + "classes", + ) + if result_dir: + Path(result_dir).mkdir(parents=True, exist_ok=True) + np.save( + os.path.join( + result_dir, + f"selected_top_{og_config['num_classes']}_labels.npy", + ), + sorted_labels[:og_config['num_classes']], + ) + label_remap = {} + for i, label in enumerate(sorted_labels[:og_config['num_classes']]): + label_remap[label] = i + print("len(label_remap) =", len(label_remap)) + + # Now reload by transform the labels accordingly + celeba_train_data = torchvision.datasets.CelebA( + root=data_root, + split='all', + download=True, + transform=transforms.Compose([ + transforms.Resize(og_config['image_size']), + transforms.CenterCrop(og_config['image_size']), + transforms.ToTensor(), + transforms.ConvertImageDtype(torch.float32), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), + ]), + target_transform=lambda x: [ + torch.tensor( + # If it is not in our map, then we make it be the token label + # og_config['num_classes'] which will be removed afterwards + label_remap.get( + x[0].cpu().detach().item() - 1, + og_config['num_classes'] + ), + dtype=torch.long, + ), + x[1][concept_selection].float(), + ], + target_type=['identity', 'attr'], + ) + num_classes = og_config['num_classes'] + + train_idxs = np.where( + list(map( + lambda x: x.cpu().detach().item() - 1 in label_remap, + celeba_train_data.identity + )) + )[0] + celeba_train_data = torch.utils.data.Subset( + celeba_train_data, + train_idxs, + ) + total_samples = len(celeba_train_data) + train_samples = int(0.7 * total_samples) + test_samples = int(0.2 * total_samples) + val_samples = total_samples - test_samples - train_samples + print( + f"Data split is: {total_samples} = {train_samples} (train) + " + f"{test_samples} (test) + {val_samples} (validation)" + ) + celeba_train_data, celeba_test_data, celeba_val_data = \ + torch.utils.data.random_split( + celeba_train_data, + [train_samples, test_samples, val_samples], + ) + train_dl = torch.utils.data.DataLoader( + celeba_train_data, + batch_size=og_config['batch_size'], + shuffle=True, + num_workers=og_config['num_workers'], + ) + test_dl = torch.utils.data.DataLoader( + celeba_test_data, + batch_size=og_config['batch_size'], + shuffle=False, + num_workers=og_config['num_workers'], + ) + val_dl = torch.utils.data.DataLoader( + celeba_val_data, + batch_size=og_config['batch_size'], + shuffle=False, + num_workers=og_config['num_workers'], + ) + + if result_dir and activation_freq: + # Then let's save the testing data for further analysis later on + out_acts_save_dir = os.path.join(result_dir, "test_embedding_acts") + Path(out_acts_save_dir).mkdir(parents=True, exist_ok=True) + + for (ds, name) in [ + (test_dl, "test"), + (val_dl, "val"), + ]: + x_total = [] + y_total = [] + c_total = [] + for x, (y, c) in ds: + x_total.append(x.cpu().detach()) + y_total.append(y.cpu().detach()) + c_total.append(c.cpu().detach()) + x_inputs = np.concatenate(x_total, axis=0) + print(f"x_{name}.shape =", x_inputs.shape) + np.save(os.path.join(out_acts_save_dir, f"x_{name}.npy"), x_inputs) + + y_inputs = np.concatenate(y_total, axis=0) + print(f"y_{name}.shape =", y_inputs.shape) + np.save(os.path.join(out_acts_save_dir, f"y_{name}.npy"), y_inputs) + + c_inputs = np.concatenate(c_total, axis=0) + print(f"c_{name}.shape =", c_inputs.shape) + np.save(os.path.join(out_acts_save_dir, f"c_{name}.npy"), c_inputs) + + label_set = set() + sample = next(iter(train_dl)) + real_sample = [] + for derp in sample: + if isinstance(derp, list): + real_sample += derp + else: + real_sample.append(derp) + sample = real_sample + print("Sample has", len(sample), "elements.") + for i, derp in enumerate(sample): + print("Element", i, "has shape", derp.shape, "and type", derp.dtype) + + print("Training sample shape is:", sample[0].shape) + print("Training label shape is:", sample[1].shape) + print("Training concept shape is:", sample[2].shape) + + n_concepts, n_tasks = sample[2].shape[-1], num_classes + + attribute_count = np.zeros((n_concepts,)) + samples_seen = 0 + for i, (_, (y, c)) in enumerate(train_dl): + print("\rIn batch", i, "we have seen", len(label_set), "classes") + c = c.cpu().detach().numpy() + attribute_count += np.sum(c, axis=0) + samples_seen += c.shape[0] + for l in y.reshape(-1).cpu().detach(): + label_set.add(l.item()) + + print("Found a total of", len(label_set), "classes") + if og_config.get("use_imbalance", False): + imbalance = samples_seen / attribute_count - 1 + else: + imbalance = None + print("Imbalance:", imbalance) + + os.makedirs(result_dir, exist_ok=True) + + results = {} + for split in range(og_config["cv"]): + print(f'Experiment {split+1}/{og_config["cv"]}') + results[f'{split}'] = {} + + # Trial period for mixture embedding model + config = copy.deepcopy(og_config) + config["architecture"] = "MixtureEmbModel" + config["extra_name"] = f"SharedProb_AdaptiveDropout_NoProbConcat" + config["shared_prob_gen"] = True + config["sigmoidal_prob"] = True + config["sigmoidal_embedding"] = False + config['training_intervention_prob'] = 0.25 + config['concat_prob'] = False + config['emb_size'] = config['emb_size'] + config["embeding_activation"] = "leakyrelu" + mixed_emb_shared_prob_model, mixed_emb_shared_prob_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[f'{split}'], + config, + mixed_emb_shared_prob_model, + mixed_emb_shared_prob_test_results, + ) + + # Trial period for mixture embedding model + config = copy.deepcopy(og_config) + config["architecture"] = "MixtureEmbModel" + config["extra_name"] = f"SharedProb_Adaptive_NoProbConcat" + config["shared_prob_gen"] = True + config["sigmoidal_prob"] = True + config["sigmoidal_embedding"] = False + config['training_intervention_prob'] = 0.0 + config['concat_prob'] = False + config['emb_size'] = config['emb_size'] + config["embeding_activation"] = "leakyrelu" + mixed_emb_shared_prob_model, mixed_emb_shared_prob_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[f'{split}'], + config, + mixed_emb_shared_prob_model, + mixed_emb_shared_prob_test_results, + ) + + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["bool"] = False + config["extra_dims"] = (config['emb_size'] - 1) * n_concepts + config["extra_name"] = f"FuzzyExtraCapacity_Logit" + config["bottleneck_nonlinear"] = "leakyrelu" + config["sigmoidal_extra_capacity"] = False + config["sigmoidal_prob"] = False + extra_fuzzy_logit_model, extra_fuzzy_logit_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[f'{split}'], + config, + extra_fuzzy_logit_model, + extra_fuzzy_logit_test_results, + ) + + # fuzzy model + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["extra_name"] = f"Fuzzy" + config["bool"] = False + config["extra_dims"] = 0 + config["sigmoidal_extra_capacity"] = False + config["sigmoidal_prob"] = True + extra_fuzzy_logit_model, extra_fuzzy_logit_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[f'{split}'], + config, + extra_fuzzy_logit_model, + extra_fuzzy_logit_test_results, + ) + + # sequential and independent models + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["extra_name"] = f"" + config["sigmoidal_prob"] = True + ind_model, ind_test_results, seq_model, seq_test_results = \ + training.train_independent_and_sequential_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + config["architecture"] = "IndependentConceptBottleneckModel" + training.update_statistics( + results[f'{split}'], + config, + ind_model, + ind_test_results, + ) + + config["architecture"] = "SequentialConceptBottleneckModel" + training.update_statistics( + results[f'{split}'], + config, + seq_model, + seq_test_results, + ) + + # train model *without* embeddings (concepts are just *Boolean* scalars) + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["extra_name"] = f"Bool" + config["bool"] = True + trainingl_model, bool_test_results = training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + imbalance=imbalance, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + ) + training.update_statistics( + results[f'{split}'], + config, + bool_model, + bool_test_results, + ) + + # train vanilla model with more capacity (i.e., no concept supervision) + # but with ReLU activation + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["extra_name"] = f"NoConceptSupervisionReLU_ExtraCapacity" + config["bool"] = False + config["extra_dims"] = (config['emb_size'] - 1) * n_concepts + config["bottleneck_nonlinear"] = "leakyrelu" + config["concept_loss_weight"] = 0 + extra_vanilla_relu_model, extra_vanilla_relu_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[f'{split}'], + config, + extra_vanilla_relu_model, + extra_vanilla_relu_test_results, + ) + + # save results + joblib.dump(results, os.path.join(result_dir, f'results.joblib')) + + return results + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=( + 'Runs concept embedding experiment in CelebA dataset.' + ), + ) + parser.add_argument( + '--project_name', + default='', + help=( + "Project name used for Weights & Biases monitoring. If not " + "provided, then we will assume we do not run a w&b project." + ), + metavar="name", + + ) + + parser.add_argument( + '--output_dir', + '-o', + default='results/celeba/', + help=( + "directory where we will dump our experiment's results. If not " + "given, then we will use ./results/celeba/." + ), + metavar="path", + + ) + parser.add_argument( + '--rerun', + '-r', + default=False, + action="store_true", + help=( + "If set, then we will force a rerun of the entire experiment even if " + "valid results are found in the provided output directory. Note that " + "this may overwrite and previous results, so use with care." + ), + + ) + parser.add_argument( + '--activation_freq', + default=0, + help=( + 'how frequently, in terms of epochs, should we store the embedding activations for our ' + 'validation set. By default we will not store any activations.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + '--single_frequency_epochs', + default=0, + help=( + 'how frequently, in terms of epochs, should we store the embedding activations for our ' + 'validation set. By default we will not store any activations.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + '--num_workers', + default=12, + help=( + 'number of workers used for data feeders. Do not use more workers ' + 'than cores in the machine.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + '--data_root', + default=CELEBA_ROOT, + help=( + 'directory containing the CelebA dataset.' + ), + metavar='path', + type=str, + ) + parser.add_argument( + "-d", + "--debug", + action="store_true", + default=False, + help="starts debug mode in our program.", + ) + parser.add_argument( + "--no_save_model", + action="store_true", + default=False, + help="whether or not we will save the fully trained models.", + ) + parser.add_argument( + '-p', + '--param', + action='append', + nargs=2, + metavar=('param_name=value'), + help=( + 'Allows the passing of a config param that will overwrite ' + 'anything passed as part of the config file itself.' + ), + default=[], + ) + args = parser.parse_args() + main( + data_root=args.data_root, + rerun=args.rerun, + result_dir=args.output_dir, + project_name=args.project_name, + activation_freq=args.activation_freq, + num_workers=args.num_workers, + single_frequency_epochs=args.single_frequency_epochs, + global_params=args.param, + save_model=(not args.no_save_model), + ) + diff --git a/experiments/cub_emb_size_ablation.py b/experiments/cub_emb_size_ablation.py new file mode 100644 index 0000000..8055aca --- /dev/null +++ b/experiments/cub_emb_size_ablation.py @@ -0,0 +1,441 @@ +import argparse +import copy +import joblib +import numpy as np +import os +import torch + +from CUB200.cub_loader import load_data, find_class_imbalance +from pathlib import Path +from pytorch_lightning import seed_everything + +import cem.experiments.cub_experiments as cub +import cem.train.training as training +import cem.train.utils as utils + +def main( + rerun=False, + result_dir='results/cub_emb_size_ablation/', + project_name='', + activation_freq=0, + num_workers=8, + single_frequency_epochs=0, + global_params=None, + data_root=celeba_final.CELEBA_ROOT, +): + seed_everything(42) + # parameters for data, model, and training + og_config = dict( + cv=5, + max_epochs=300, + patience=15, + batch_size=128, + num_workers=num_workers, + emb_size=16, + extra_dims=0, + concept_loss_weight=5, + normalize_loss=False, + learning_rate=0.01, + weight_decay=4e-05, + scheduler_step=20, + weight_loss=True, + pretrain_model=True, + c_extractor_arch="resnet34", + optimizer="sgd", + bool=False, + early_stopping_monitor="val_loss", + early_stopping_mode="min", + early_stopping_delta=0.0, + # By default we start with 25% of the concepts in the bottleneck + sampling_percent=0.25, + + momentum=0.9, + shared_prob_gen=False, + sigmoidal_prob=False, + sigmoidal_embedding=False, + training_intervention_prob=0.0, + embeding_activation=None, + concat_prob=False, + ) + + utils.extend_with_global_params(og_config, global_params or []) + train_data_path = os.path.join(cub.BASE_DIR, 'train.pkl') + if og_config['weight_loss']: + imbalance = find_class_imbalance(train_data_path, True) + else: + imbalance = None + + val_data_path = train_data_path.replace('train.pkl', 'val.pkl') + test_data_path = train_data_path.replace('train.pkl', 'test.pkl') + sampling_percent = og_config.get("sampling_percent", 1) + n_concepts, n_tasks = 112, 200 + + if sampling_percent != 1: + # Do the subsampling + new_n_concepts = int(np.ceil(n_concepts * sampling_percent)) + selected_concepts_file = os.path.join( + result_dir, + f"selected_concepts_sampling_{sampling_percent}.npy", + ) + if (not rerun) and os.path.exists(selected_concepts_file): + selected_concepts = np.load(selected_concepts_file) + else: + selected_concepts = sorted( + np.random.permutation(n_concepts)[:new_n_concepts] + ) + np.save(selected_concepts_file, selected_concepts) + print("\t\tSelected concepts:", selected_concepts) + def subsample_transform(sample): + if isinstance(sample, list): + sample = np.array(sample) + return sample[selected_concepts] + + if og_config['weight_loss']: + imbalance = np.array(imbalance)[selected_concepts] + + train_dl = load_data( + pkl_paths=[train_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=cub.CUB_DIR, + num_workers=og_config['num_workers'], + concept_transform=subsample_transform, + ) + val_dl = load_data( + pkl_paths=[val_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=cub.CUB_DIR, + num_workers=og_config['num_workers'], + concept_transform=subsample_transform, + ) + test_dl = load_data( + pkl_paths=[test_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=cub.CUB_DIR, + num_workers=og_config['num_workers'], + concept_transform=subsample_transform, + ) + # And set the right number of concepts to be used + n_concepts = new_n_concepts + else: + train_dl = load_data( + pkl_paths=[train_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=cub.CUB_DIR, + num_workers=og_config['num_workers'], + ) + val_dl = load_data( + pkl_paths=[val_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=cub.CUB_DIR, + num_workers=og_config['num_workers'], + ) + test_dl = load_data( + pkl_paths=[test_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=cub.CUB_DIR, + num_workers=og_config['num_workers'], + ) + + if result_dir and activation_freq: + # Then let's save the testing data for furter analysis later on + out_acts_save_dir = os.path.join(result_dir, "test_embedding_acts") + Path(out_acts_save_dir).mkdir(parents=True, exist_ok=True) + for (ds, name) in [ + (test_dl, "test"), + (val_dl, "val"), + ]: + x_total = [] + y_total = [] + c_total = [] + for x, y, c in ds: + x_total.append(x.cpu().detach()) + y_total.append(y.cpu().detach()) + c_total.append(c.cpu().detach()) + x_inputs = np.concatenate(x_total, axis=0) + print(f"x_{name}.shape =", x_inputs.shape) + np.save(os.path.join(out_acts_save_dir, f"x_{name}.npy"), x_inputs) + + y_inputs = np.concatenate(y_total, axis=0) + print(f"y_{name}.shape =", y_inputs.shape) + np.save(os.path.join(out_acts_save_dir, f"y_{name}.npy"), y_inputs) + + c_inputs = np.concatenate(c_total, axis=0) + print(f"c_{name}.shape =", c_inputs.shape) + np.save(os.path.join(out_acts_save_dir, f"c_{name}.npy"), c_inputs) + + sample = next(iter(train_dl)) + n_concepts, n_tasks = sample[2].shape[-1], 200 + + print("Training sample shape is:", sample[0].shape) + print("Training label shape is:", sample[1].shape) + print("Training concept shape is:", sample[2].shape) + os.makedirs(result_dir, exist_ok=True) + results = {} + + for split in range(og_config["cv"]): + for emb_size in [1, 2, 4, 6, 8, 16, 32, 64]: + if emb_size not in results: + results[emb_size] = {} + if f'{split}' not in results[emb_size]: + results[emb_size][f'{split}'] = {} + print( + f'Experiment {split+1}/{og_config["cv"]} with emb_size', + emb_size, + ) + + # Trial period for mixture embedding model + config = copy.deepcopy(og_config) + config["architecture"] = "MixtureEmbModel" + config["extra_name"] = ( + f"SharedProb_AdaptiveDropout_NoProbConcat_emb_size_{emb_size}" + ) + config["shared_prob_gen"] = True + config["sigmoidal_prob"] = False + config["sigmoidal_embedding"] = False + config['training_intervention_prob'] = 0.25 + config['concat_prob'] = False + config['emb_size'] = emb_size + config["embeding_activation"] = "leakyrelu" + mixed_emb_shared_prob_model, mixed_emb_shared_prob_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[emb_size][f'{split}'], + config, + mixed_emb_shared_prob_model, + mixed_emb_shared_prob_test_results, + ) + + # Train fuzzy CBM with extra capacity + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["bool"] = False + config["extra_dims"] = (emb_size - 1) * n_concepts + config["extra_name"] = ( + f"FuzzyExtraCapacity_Logit_emb_size_{emb_size}" + ) + config["bottleneck_nonlinear"] = "leakyrelu" + config["sigmoidal_extra_capacity"] = False + config["sigmoidal_prob"] = False + config['emb_size'] = emb_size + extra_fuzzy_logit_model, extra_fuzzy_logit_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[emb_size][f'{split}'], + config, + extra_fuzzy_logit_model, + extra_fuzzy_logit_test_results, + ) + + # train vanilla model with more capacity (i.e., no concept + # supervision) but with ReLU activation + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["extra_name"] = ( + f"NoConceptSupervisionReLU_ExtraCapacity_emb_size_{emb_size}" + ) + config["bool"] = False + config["extra_dims"] = (emb_size - 1) * n_concepts + config["bottleneck_nonlinear"] = "leakyrelu" + config["concept_loss_weight"] = 0 + config['emb_size'] = emb_size + extra_vanilla_relu_model, extra_vanilla_relu_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[emb_size][f'{split}'], + config, + extra_vanilla_relu_model, + extra_vanilla_relu_test_results, + ) + + # save results + joblib.dump(results, os.path.join(result_dir, f'results.joblib')) + return results + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=( + 'Runs concept embedding experiment in CUB dataset.' + ), + ) + parser.add_argument( + '--project_name', + default='', + help=( + "Project name used for Weights & Biases monitoring. If not " + "provided, then we will assume no W&B is used for logging." + ), + metavar="name", + + ) + + parser.add_argument( + '--output_dir', + '-o', + default='results/cub_emb_size_ablation/', + help=( + "directory where we will dump our experiment's results. If not " + "given, then we will use ./results/cub_emb_size_ablation/." + ), + metavar="path", + + ) + parser.add_argument( + '--rerun', + '-r', + default=False, + action="store_true", + help=( + "If set, then we will force a rerun of the entire experiment " + "even if valid results are found in the provided output " + "directory. Note that this may overwrite and previous results, " + "so use with care." + ), + + ) + parser.add_argument( + '--activation_freq', + default=0, + help=( + 'how frequently, in terms of epochs, should we store the ' + 'embedding activations for our validation set. By default we will ' + 'not store any activations.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + '--single_frequency_epochs', + default=0, + help=( + 'how frequently, in terms of epochs, should we store the ' + 'embedding activations for our validation set. By default we will ' + 'not store any activations.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + '--num_workers', + default=8, + help=( + 'number of workers used for data feeders. Do not use more workers ' + 'than cores in the machine.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + "-d", + "--debug", + action="store_true", + default=False, + help="starts debug mode in our program.", + ) + parser.add_argument( + '-p', + '--param', + action='append', + nargs=2, + metavar=('param_name=value'), + help=( + 'Allows the passing of a config param that will overwrite ' + 'anything passed as part of the config file itself.' + ), + default=[], + ) + args = parser.parse_args() + if args.project_name: + # Lazy import to avoid importing unless necessary + import wandb + main( + rerun=args.rerun, + result_dir=args.output_dir, + project_name=args.project_name, + activation_freq=args.activation_freq, + num_workers=args.num_workers, + single_frequency_epochs=args.single_frequency_epochs, + global_params=args.param, + ) +# hyperparameter_sweep() diff --git a/experiments/cub_experiments.py b/experiments/cub_experiments.py new file mode 100644 index 0000000..cc08770 --- /dev/null +++ b/experiments/cub_experiments.py @@ -0,0 +1,574 @@ +import argparse +import copy +import joblib +import numpy as np +import os +import torch + +from CUB200.cub_loader import load_data, find_class_imbalance +from pathlib import Path +from pytorch_lightning import seed_everything + +import cem.train.training as training +import cem.train.utils as utils + +################################################################################ +## GLOBAL CUB VARIABLES +################################################################################ + +# IMPORANT NOTE: THIS DATASET NEEDS TO BE DOWNLOADED FIRST BEFORE BEING ABLE +# TO RUN ANY CUB EXPERIMENTS!! +# Instructions on how to download it can be found +# in the original CBM paper's repository +# found here: https://github.com/yewsiang/ConceptBottleneck +CUB_DIR = 'cem/data/CUB200/' +BASE_DIR = os.path.join(CUB_DIR, 'class_attr_data_10') + +################################################################################ +## MAIN FUNCTION +################################################################################ + + +def main( + rerun=False, + result_dir='results/cub/', + project_name='', + activation_freq=0, + num_workers=8, + single_frequency_epochs=0, + global_params=None, +): + seed_everything(42) + # parameters for data, model, and training + og_config = dict( + cv=5, + max_epochs=300, + patience=15, + batch_size=128, + num_workers=num_workers, + emb_size=16, + extra_dims=0, + concept_loss_weight=5, + normalize_loss=False, + learning_rate=0.01, + weight_decay=4e-05, + weight_loss=True, + pretrain_model=True, + c_extractor_arch="resnet34", + optimizer="sgd", + bool=False, + early_stopping_monitor="val_loss", + early_stopping_mode="min", + early_stopping_delta=0.0, + sampling_percent=1, + + momentum=0.9, + shared_prob_gen=False, + sigmoidal_prob=False, + sigmoidal_embedding=False, + training_intervention_prob=0.0, + embeding_activation=None, + concat_prob=False, + ) + utils.extend_with_global_params(og_config, global_params or []) + + train_data_path = os.path.join(BASE_DIR, 'train.pkl') + if og_config['weight_loss']: + imbalance = find_class_imbalance(train_data_path, True) + else: + imbalance = None + + val_data_path = train_data_path.replace('train.pkl', 'val.pkl') + test_data_path = train_data_path.replace('train.pkl', 'test.pkl') + sampling_percent = og_config.get("sampling_percent", 1) + n_concepts, n_tasks = 112, 200 + if sampling_percent != 1: + # Do the subsampling + new_n_concepts = int(np.ceil(n_concepts * sampling_percent)) + selected_concepts_file = os.path.join( + result_dir, + f"selected_concepts_sampling_{sampling_percent}.npy", + ) + if (not rerun) and os.path.exists(selected_concepts_file): + selected_concepts = np.load(selected_concepts_file) + else: + selected_concepts = sorted( + np.random.permutation(n_concepts)[:new_n_concepts] + ) + np.save(selected_concepts_file, selected_concepts) + print("\t\tSelected concepts:", selected_concepts) + def subsample_transform(sample): + if isinstance(sample, list): + sample = np.array(sample) + return sample[selected_concepts] + + if og_config['weight_loss']: + imbalance = np.array(imbalance)[selected_concepts] + + train_dl = load_data( + pkl_paths=[train_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=CUB_DIR, + num_workers=og_config['num_workers'], + concept_transform=subsample_transform, + ) + val_dl = load_data( + pkl_paths=[val_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=CUB_DIR, + num_workers=og_config['num_workers'], + concept_transform=subsample_transform, + ) + test_dl = load_data( + pkl_paths=[test_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=CUB_DIR, + num_workers=og_config['num_workers'], + concept_transform=subsample_transform, + ) + + # And set the right number of concepts to be used + n_concepts = new_n_concepts + else: + train_dl = load_data( + pkl_paths=[train_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=CUB_DIR, + num_workers=og_config['num_workers'], + ) + val_dl = load_data( + pkl_paths=[val_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=CUB_DIR, + num_workers=og_config['num_workers'], + ) + test_dl = load_data( + pkl_paths=[test_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=CUB_DIR, + num_workers=og_config['num_workers'], + ) + + if result_dir and activation_freq: + # Then let's save the testing data for further analysis later on + out_acts_save_dir = os.path.join(result_dir, "test_embedding_acts") + Path(out_acts_save_dir).mkdir(parents=True, exist_ok=True) + for (ds, name) in [ + (test_dl, "test"), + (val_dl, "val"), + ]: + x_total = [] + y_total = [] + c_total = [] + for x, y, c in ds: + x_total.append(x.cpu().detach()) + y_total.append(y.cpu().detach()) + c_total.append(c.cpu().detach()) + x_inputs = np.concatenate(x_total, axis=0) + print(f"x_{name}.shape =", x_inputs.shape) + np.save(os.path.join(out_acts_save_dir, f"x_{name}.npy"), x_inputs) + + y_inputs = np.concatenate(y_total, axis=0) + print(f"y_{name}.shape =", y_inputs.shape) + np.save(os.path.join(out_acts_save_dir, f"y_{name}.npy"), y_inputs) + + c_inputs = np.concatenate(c_total, axis=0) + print(f"c_{name}.shape =", c_inputs.shape) + np.save(os.path.join(out_acts_save_dir, f"c_{name}.npy"), c_inputs) + + sample = next(iter(train_dl)) + n_concepts, n_tasks = sample[2].shape[-1], 200 + + print("Training sample shape is:", sample[0].shape) + print("Training label shape is:", sample[1].shape) + print("Training concept shape is:", sample[2].shape) + + os.makedirs(result_dir, exist_ok=True) + + results = {} + for split in range(og_config["cv"]): + print(f'Experiment {split+1}/{og_config["cv"]}') + results[f'{split}'] = {} + + config = copy.deepcopy(og_config) + config["architecture"] = "MixtureEmbModel" + config["extra_name"] = f"SharedProb_AdaptiveDropout_NoProbConcat" + config["shared_prob_gen"] = True + config["sigmoidal_prob"] = True + config["sigmoidal_embedding"] = False + config['training_intervention_prob'] = 0.25 + config['concat_prob'] = False + config['emb_size'] = config['emb_size'] + config["embeding_activation"] = "leakyrelu" + mixed_emb_shared_prob_model, mixed_emb_shared_prob_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[f'{split}'], + config, + mixed_emb_shared_prob_model, + mixed_emb_shared_prob_test_results, + ) + + config = copy.deepcopy(og_config) + config["architecture"] = "MixtureEmbModel" + config["extra_name"] = f"SharedProb_Adaptive_NoProbConcat" + config["shared_prob_gen"] = True + config["sigmoidal_prob"] = True + config["sigmoidal_embedding"] = False + config['training_intervention_prob'] = 0.0 + config['concat_prob'] = False + config['emb_size'] = config['emb_size'] + config["embeding_activation"] = "leakyrelu" + mixed_emb_shared_prob_model, mixed_emb_shared_prob_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[f'{split}'], + config, + mixed_emb_shared_prob_model, + mixed_emb_shared_prob_test_results, + ) + + # train model *without* embeddings but with extra capacity (concepts + # are just *fuzzy* scalars and the model also has some extra capacity). + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["bool"] = False + config["extra_dims"] = (config['emb_size'] - 1) * n_concepts + config["extra_name"] = f"FuzzyExtraCapacity_Logit" + config["bottleneck_nonlinear"] = "leakyrelu" + config["sigmoidal_extra_capacity"] = False + config["sigmoidal_prob"] = False + extra_fuzzy_logit_model, extra_fuzzy_logit_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[f'{split}'], + config, + extra_fuzzy_logit_model, + extra_fuzzy_logit_test_results, + ) + + # fuzzy model + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["extra_name"] = f"Fuzzy" + config["bool"] = False + config["extra_dims"] = 0 + config["sigmoidal_extra_capacity"] = False + config["sigmoidal_prob"] = True + extra_fuzzy_logit_model, extra_fuzzy_logit_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[f'{split}'], + config, + extra_fuzzy_logit_model, + extra_fuzzy_logit_test_results, + ) + + # Bool model + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["extra_name"] = f"Bool" + config["bool"] = True + bool_model, bool_test_results = training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + imbalance=imbalance, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + ) + training.update_statistics( + results[f'{split}'], + config, + bool_model, + bool_test_results, + ) + + + # sequential and independent models + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["extra_name"] = f"" + config["sigmoidal_prob"] = True + ind_model, ind_test_results, seq_model, seq_test_results = \ + training.train_independent_and_sequential_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + config["architecture"] = "IndependentConceptBottleneckModel" + training.update_statistics( + results[f'{split}'], + config, + ind_model, + ind_test_results, + ) + + config["architecture"] = "SequentialConceptBottleneckModel" + training.update_statistics( + results[f'{split}'], + config, + seq_model, + seq_test_results, + ) + + # train vanilla model with more capacity (i.e., no concept supervision) + # but with ReLU activation + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["extra_name"] = f"NoConceptSupervisionReLU_ExtraCapacity" + config["bool"] = False + config["extra_dims"] = (config['emb_size'] - 1) * n_concepts + config["bottleneck_nonlinear"] = "leakyrelu" + config["concept_loss_weight"] = 0 + extra_vanilla_relu_model, extra_vanilla_relu_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[f'{split}'], + config, + extra_vanilla_relu_model, + extra_vanilla_relu_test_results, + ) + + # save results + joblib.dump(results, os.path.join(result_dir, f'results.joblib')) + + return results + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=( + 'Runs concept embedding experiment in CUB dataset.' + ), + ) + parser.add_argument( + '--project_name', + default='cub_rerun_concept_training', + help=( + "Project name used for Weights & Biases monitoring. If not " + "provided, then we will assume it is 'cub_rerun_concept_training'." + ), + metavar="name", + + ) + + parser.add_argument( + '--output_dir', + '-o', + default='results/cub_rerun/', + help=( + "directory where we will dump our experiment's results. If not " + "given, then we will use ./results/cub/." + ), + metavar="path", + + ) + + parser.add_argument( + '--rerun', + '-r', + default=False, + action="store_true", + help=( + "If set, then we will force a rerun of the entire experiment even " + "if valid results are found in the provided output directory. " + "Note that this may overwrite and previous results, so use " + "with care." + ), + + ) + parser.add_argument( + '--activation_freq', + default=0, + help=( + 'how frequently, in terms of epochs, should we store the ' + 'embedding activations for our validation set. By default we will ' + 'not store any activations.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + '--single_frequency_epochs', + default=0, + help=( + 'how frequently, in terms of epochs, should we store the ' + 'embedding activations for our validation set. By default we will ' + 'not store any activations.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + '--num_workers', + default=8, + help=( + 'number of workers used for data feeders. Do not use more workers ' + 'than cores in the machine.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + "-d", + "--debug", + action="store_true", + default=False, + help="starts debug mode in our program.", + ) + parser.add_argument( + '-p', + '--param', + action='append', + nargs=2, + metavar=('param_name=value'), + help=( + 'Allows the passing of a config param that will overwrite ' + 'anything passed as part of the config file itself.' + ), + default=[], + ) + args = parser.parse_args() + if args.project_name: + # Lazy import to avoid importing unless necessary + import wandb + main( + rerun=args.rerun, + result_dir=args.output_dir, + project_name=args.project_name, + activation_freq=args.activation_freq, + num_workers=args.num_workers, + single_frequency_epochs=args.single_frequency_epochs, + global_params=args.param, + ) diff --git a/experiments/cub_randint_ablation.py b/experiments/cub_randint_ablation.py new file mode 100644 index 0000000..59d7887 --- /dev/null +++ b/experiments/cub_randint_ablation.py @@ -0,0 +1,361 @@ +import argparse +import copy +import joblib +import numpy as np +import os +import torch + +from CUB200.cub_loader import load_data, find_class_imbalance +from pathlib import Path +from pytorch_lightning import seed_everything + +import cem.experiments.cub_experiments as cub +import cem.train.training as training +import cem.train.utils as utils + +def main( + rerun=False, + result_dir='results/cub_randint_ablation/', + project_name='', + activation_freq=0, + num_workers=8, + single_frequency_epochs=0, + global_params=None, +): + seed_everything(42) + # parameters for data, model, and training + og_config = dict( + cv=5, + max_epochs=300, + patience=15, + batch_size=128, + num_workers=num_workers, + emb_size=16, + extra_dims=0, + concept_loss_weight=5, + normalize_loss=False, + learning_rate=0.01, + weight_decay=4e-05, + scheduler_step=20, + weight_loss=True, + pretrain_model=True, + c_extractor_arch="resnet34", + optimizer="sgd", + bool=False, + early_stopping_monitor="val_loss", + early_stopping_mode="min", + early_stopping_delta=0.0, + sampling_percent=1, + + momentum=0.9, + shared_prob_gen=False, + sigmoidal_prob=False, + sigmoidal_embedding=False, + training_intervention_prob=0.0, + embeding_activation=None, + concat_prob=False, + ) + + utils.extend_with_global_params(og_config, global_params or []) + train_data_path = os.path.join(cub.BASE_DIR, 'train.pkl') + if og_config['weight_loss']: + imbalance = find_class_imbalance(train_data_path, True) + else: + imbalance = None + + val_data_path = train_data_path.replace('train.pkl', 'val.pkl') + test_data_path = train_data_path.replace('train.pkl', 'test.pkl') + sampling_percent = og_config.get("sampling_percent", 1) + n_concepts, n_tasks = 112, 200 + + if sampling_percent != 1: + # Do the subsampling + new_n_concepts = int(np.ceil(n_concepts * sampling_percent)) + selected_concepts_file = os.path.join( + result_dir, + f"selected_concepts_sampling_{sampling_percent}.npy", + ) + if (not rerun) and os.path.exists(selected_concepts_file): + selected_concepts = np.load(selected_concepts_file) + else: + selected_concepts = sorted( + np.random.permutation(n_concepts)[:new_n_concepts] + ) + np.save(selected_concepts_file, selected_concepts) + print("\t\tSelected concepts:", selected_concepts) + def subsample_transform(sample): + if isinstance(sample, list): + sample = np.array(sample) + return sample[selected_concepts] + + if og_config['weight_loss']: + imbalance = np.array(imbalance)[selected_concepts] + + train_dl = load_data( + pkl_paths=[train_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=cub.CUB_DIR, + num_workers=og_config['num_workers'], + concept_transform=subsample_transform, + ) + val_dl = load_data( + pkl_paths=[val_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=cub.CUB_DIR, + num_workers=og_config['num_workers'], + concept_transform=subsample_transform, + ) + test_dl = load_data( + pkl_paths=[test_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=cub.CUB_DIR, + num_workers=og_config['num_workers'], + concept_transform=subsample_transform, + ) + # And set the right number of concepts to be used + n_concepts = new_n_concepts + else: + train_dl = load_data( + pkl_paths=[train_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=cub.CUB_DIR, + num_workers=og_config['num_workers'], + ) + val_dl = load_data( + pkl_paths=[val_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=cub.CUB_DIR, + num_workers=og_config['num_workers'], + ) + test_dl = load_data( + pkl_paths=[test_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=cub.CUB_DIR, + num_workers=og_config['num_workers'], + ) + + if result_dir and activation_freq: + # Then let's save the testing data for furter analysis later on + out_acts_save_dir = os.path.join(result_dir, "test_embedding_acts") + Path(out_acts_save_dir).mkdir(parents=True, exist_ok=True) + for (ds, name) in [ + (test_dl, "test"), + (val_dl, "val"), + ]: + x_total = [] + y_total = [] + c_total = [] + for x, y, c in ds: + x_total.append(x.cpu().detach()) + y_total.append(y.cpu().detach()) + c_total.append(c.cpu().detach()) + x_inputs = np.concatenate(x_total, axis=0) + print(f"x_{name}.shape =", x_inputs.shape) + np.save(os.path.join(out_acts_save_dir, f"x_{name}.npy"), x_inputs) + + y_inputs = np.concatenate(y_total, axis=0) + print(f"y_{name}.shape =", y_inputs.shape) + np.save(os.path.join(out_acts_save_dir, f"y_{name}.npy"), y_inputs) + + c_inputs = np.concatenate(c_total, axis=0) + print(f"c_{name}.shape =", c_inputs.shape) + np.save(os.path.join(out_acts_save_dir, f"c_{name}.npy"), c_inputs) + + sample = next(iter(train_dl)) + n_concepts, n_tasks = sample[2].shape[-1], 200 + + print("Training sample shape is:", sample[0].shape) + print("Training label shape is:", sample[1].shape) + print("Training concept shape is:", sample[2].shape) + os.makedirs(result_dir, exist_ok=True) + results = {} + + for prob in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.99, 1.0]: + results[prob] = {} + for split in range(og_config["cv"]): + print(f'Experiment {split+1}/{og_config["cv"]} with prob', prob) + results[prob][f'{split}'] = {} + + # Trial period for mixture embedding model + config = copy.deepcopy(og_config) + config["architecture"] = "MixtureEmbModel" + config["extra_name"] = ( + f"SharedProb_AdaptiveDropout_NoProbConcat_prob_{prob}" + ) + config["shared_prob_gen"] = True + config["sigmoidal_prob"] = False + config["sigmoidal_embedding"] = False + config['training_intervention_prob'] = prob + config['concat_prob'] = False + config['emb_size'] = config['emb_size'] + mixed_emb_shared_prob_model, mixed_emb_shared_prob_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[prob][f'{split}'], + config, + mixed_emb_shared_prob_model, + mixed_emb_shared_prob_test_results, + ) + + # save results + joblib.dump(results, os.path.join(result_dir, f'results.joblib')) + + return results + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=( + 'Runs ablation study for RandInt in CUB dataset.' + ), + ) + parser.add_argument( + '--project_name', + default='', + help=( + "Project name used for Weights & Biases monitoring. If not " + "provided, then we will assume no W&B logging is used." + ), + metavar="name", + + ) + + parser.add_argument( + '--output_dir', + '-o', + default='results/cub_randint_ablation/', + help=( + "directory where we will dump our experiment's results. If not " + "given, then we will use ./results/cub_randint_ablation/." + ), + metavar="path", + + ) + parser.add_argument( + '--rerun', + '-r', + default=False, + action="store_true", + help=( + "If set, then we will force a rerun of the entire experiment even " + "if valid results are found in the provided output directory. " + "Note that this may overwrite and previous results, so use with " + "care." + ), + + ) + parser.add_argument( + '--activation_freq', + default=0, + help=( + 'how frequently, in terms of epochs, should we store the ' + 'embedding activations for our validation set. By default we ' + 'will not store any activations.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + '--single_frequency_epochs', + default=0, + help=( + 'how frequently, in terms of epochs, should we store the ' + 'embedding activations for our validation set. By default we will ' + 'not store any activations.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + '--num_workers', + default=8, + help=( + 'number of workers used for data feeders. Do not use more workers ' + 'than cores in the machine.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + "-d", + "--debug", + action="store_true", + default=False, + help="starts debug mode in our program.", + ) + parser.add_argument( + '-p', + '--param', + action='append', + nargs=2, + metavar=('param_name=value'), + help=( + 'Allows the passing of a config param that will overwrite ' + 'anything passed as part of the config file itself.' + ), + default=[], + ) + args = parser.parse_args() + if args.project_name: + # Lazy import to avoid importing unless necessary + import wandb + main( + rerun=args.rerun, + result_dir=args.output_dir, + project_name=args.project_name, + activation_freq=args.activation_freq, + num_workers=args.num_workers, + single_frequency_epochs=args.single_frequency_epochs, + global_params=args.param, + ) diff --git a/experiments/cub_subsample_experiment.py b/experiments/cub_subsample_experiment.py new file mode 100644 index 0000000..c834537 --- /dev/null +++ b/experiments/cub_subsample_experiment.py @@ -0,0 +1,460 @@ +import argparse +import copy +import joblib +import numpy as np +import os +import torch + +from data.CUB200.cub_loader import load_data, find_class_imbalance +from pathlib import Path +from pytorch_lightning import seed_everything + +import cem.experiments.cub_experiments as cub +import cem.train.training as training +import cem.train.utils as utils + + +def main( + rerun=False, + result_dir='results/cub_subsample/', + project_name='', + save_models=True, + activation_freq=0, + single_frequency_epochs=0, + global_params=None, + num_workers=8, +): + seed_everything(42) + # parameters for data, model, and training + og_config = dict( + cv=5, + max_epochs=300, + patience=15, + batch_size=128, + num_workers=num_workers, + emb_size=16, + extra_dims=0, + concept_loss_weight=5, + normalize_loss=False, + learning_rate=0.01, + weight_decay=4e-05, + scheduler_step=20, + weight_loss=True, + pretrain_model=True, + c_extractor_arch="resnet34", + optimizer="sgd", + bool=False, + early_stopping_monitor="val_loss", + early_stopping_mode="min", + early_stopping_delta=0.0, + corr_thresh=0.5, + dense_corr_thresh=0.25, + sampling_percent=1, + sampling_percents=[0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 1], + + momentum=0.9, + shared_prob_gen=False, + sigmoidal_prob=False, + sigmoidal_embedding=False, + training_intervention_prob=0.0, + embeding_activation=None, + concat_prob=False, + ) + + train_data_path = os.path.join(cub.BASE_DIR, 'train.pkl') + if og_config['weight_loss']: + og_imbalance = find_class_imbalance(train_data_path, True) + else: + og_imbalance = None + utils.extend_with_global_params(og_config, global_params or []) + + val_data_path = train_data_path.replace('train.pkl', 'val.pkl') + test_data_path = train_data_path.replace('train.pkl', 'test.pkl') + n_concepts, n_tasks = 112, 200 + + os.makedirs(result_dir, exist_ok=True) + joblib.dump( + og_config, + os.path.join(result_dir, f'experiment_config.joblib'), + ) + + if result_dir and activation_freq: + # Then let's save the testing data for further analysis later on + out_acts_save_dir = os.path.join(result_dir, "test_embedding_acts") + Path(out_acts_save_dir).mkdir(parents=True, exist_ok=True) + + results = {} + for sampling_percent in og_config['sampling_percents']: + print( + f"Training model by subsampling {sampling_percent *100}% of " + f"concepts" + ) + results[sampling_percent] = {} + new_n_concepts = int(np.ceil(n_concepts * sampling_percent)) + for split in range(og_config["cv"]): + print( + f'\tExperiment {split+1}/{og_config["cv"]} with sampling ' + f'rate {sampling_percent *100}% and {new_n_concepts} concepts' + ) + results[sampling_percent][f'{split}'] = {} + + # Do the subsampling + selected_concepts_file = os.path.join( + result_dir, + ( + f"selected_concepts_" + f"sampling_{sampling_percent}_fold_{split}.npy" + ), + ) + if (not rerun) and os.path.exists(selected_concepts_file): + selected_concepts = np.load(selected_concepts_file) + else: + if sampling_percent != 1: + selected_concepts = np.random.permutation( + n_concepts + )[:new_n_concepts] + else: + # Then simply select them all in their original order + selected_concepts = np.range(new_n_concepts) + np.save(selected_concepts_file, selected_concepts) + print("\t\tSelected concepts:", selected_concepts) + def subsample_transform(sample): + if isinstance(sample, list): + sample = np.array(sample) + return sample[selected_concepts] + + if og_config['weight_loss']: + imbalance = np.array(og_imbalance)[selected_concepts] + else: + imbalance = np.array(og_imbalance)[selected_concepts] + + train_dl = load_data( + pkl_paths=[train_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=CUB_DIR, + num_workers=og_config['num_workers'], + concept_transform=subsample_transform, + ) + val_dl = load_data( + pkl_paths=[val_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=CUB_DIR, + num_workers=og_config['num_workers'], + concept_transform=subsample_transform, + ) + test_dl = load_data( + pkl_paths=[test_data_path], + use_attr=True, + no_img=False, + batch_size=og_config['batch_size'], + uncertain_label=False, + n_class_attr=2, + image_dir='images', + resampling=False, + root_dir=CUB_DIR, + num_workers=og_config['num_workers'], + concept_transform=subsample_transform, + ) + + sample = next(iter(train_dl)) + print("Training sample shape is:", sample[0].shape) + print("Training label shape is:", sample[1].shape) + print("Training concept shape is:", sample[2].shape) + + + # train vanilla model with more capacity (i.e., no concept + # supervision) but with ReLU activation + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["extra_name"] = ( + f"NoConceptSupervisionReLU_ExtraCapacity_" + f"subsample_{sampling_percent}" + ) + config["sampling_percent"] = sampling_percent + config["bool"] = False + config["extra_dims"] = config['emb_size'] * new_n_concepts + config["bottleneck_nonlinear"] = "relu" + config["concept_loss_weight"] = 0 + extra_vanilla_relu_model, extra_vanilla_relu_test_results = \ + training.train_model( + n_concepts=new_n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[sampling_percent][f'{split}'], + config, + extra_vanilla_relu_model, + extra_vanilla_relu_test_results, + ) + + # fuzzy model + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["extra_name"] = f"Fuzzy_subsample_{sampling_percent}" + config["sampling_percent"] = sampling_percent + config["bool"] = False + config["extra_dims"] = 0 + config["sigmoidal_extra_capacity"] = False + config["sigmoidal_prob"] = True + extra_fuzzy_logit_model, extra_fuzzy_logit_test_results = \ + training.train_model( + n_concepts=new_n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[sampling_percent][f'{split}'], + config, + extra_fuzzy_logit_model, + extra_fuzzy_logit_test_results, + ) + + # train model *without* embeddings but with extra capacity. + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["bool"] = False + config["extra_dims"] = config['emb_size'] * new_n_concepts + config["sampling_percent"] = sampling_percent + config["extra_name"] = ( + f"FuzzyExtraCapacity_Logit_subsample_{sampling_percent}" + ) + config["sigmoidal_extra_capacity"] = False + config["sigmoidal_prob"] = False + extra_fuzzy_logit_model, extra_fuzzy_logit_test_results = \ + training.train_model( + n_concepts=new_n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[sampling_percent][f'{split}'], + config, + extra_fuzzy_logit_model, + extra_fuzzy_logit_test_results, + ) + + # train model *without* embeddings (concepts are just *Boolean* + # scalars) + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["extra_name"] = f"Bool_subsample_{sampling_percent}" + config["bool"] = True + config["sampling_percent"] = sampling_percent + config["selected_concepts"] = selected_concepts + bool_model, bool_test_results = training.train_model( + n_concepts=new_n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + imbalance=imbalance, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + save_model=save_models, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + ) + training.update_statistics( + results[sampling_percent][f'{split}'], + config, + bool_model, + bool_test_results, + save_model=save_models, + ) + + config = copy.deepcopy(og_config) + config["architecture"] = "MixtureEmbModel" + config["extra_name"] = ( + f"SharedProb_AdaptiveDropout_NoProbConcat_" + f"subsample_{sampling_percent}" + ) + config["sampling_percent"] = sampling_percent + config["shared_prob_gen"] = True + config["sigmoidal_prob"] = True + config["sigmoidal_embedding"] = False + config['training_intervention_prob'] = 0.25 + config['concat_prob'] = False + config['emb_size'] = config['emb_size'] + config["embeding_activation"] = "leakyrelu" + mixed_emb_shared_prob_model, mixed_emb_shared_prob_test_results = \ + training.train_model( + n_concepts=new_n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + imbalance=imbalance, + ) + training.update_statistics( + results[sampling_percent][f'{split}'], + config, + mixed_emb_shared_prob_model, + mixed_emb_shared_prob_test_results, + ) + + # save results + joblib.dump(results, os.path.join(result_dir, f'results.joblib')) + + return results + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=( + 'Runs concept subsampling experiment in CUB dataset.' + ), + ) + parser.add_argument( + '--project_name', + default='', + help=( + "Project name used for Weights & Biases monitoring. If not " + "provided, then we will assume no W&B logging is done." + ), + metavar="name", + + ) + + parser.add_argument( + '--output_dir', + '-o', + default='results/cub_subsample/', + help=( + "directory where we will dump our experiment's results. If not " + "given, then we will use ./results/cub_subsample/." + ), + metavar="path", + + ) + parser.add_argument( + '--rerun', + '-r', + default=False, + action="store_true", + help=( + "If set, then we will force a rerun of the entire experiment even " + "if valid results are found in the provided output directory. " + "Note that this may overwrite and previous results, so use with " + "care." + ), + + ) + parser.add_argument( + '--activation_freq', + default=0, + help=( + 'how frequently, in terms of epochs, should we store the ' + 'embedding activations for our validation set. By default we will ' + 'not store any activations.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + '--single_frequency_epochs', + default=0, + help=( + 'how frequently, in terms of epochs, should we store the ' + 'embedding activations for our validation set. By default we will ' + 'not store any activations.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + '--num_workers', + default=12, + help=( + 'number of workers used for data feeders. Do not use more workers ' + 'than cores in the machine.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + "-d", + "--debug", + action="store_true", + default=False, + help="starts debug mode in our program.", + ) + parser.add_argument( + '-p', + '--param', + action='append', + nargs=2, + metavar=('param_name=value'), + help=( + 'Allows the passing of a config param that will overwrite ' + 'anything passed as part of the config file itself.' + ), + default=[], + ) + args = parser.parse_args() + main( + rerun=args.rerun, + result_dir=args.output_dir, + project_name=args.project_name, + activation_freq=args.activation_freq, + num_workers=args.num_workers, + single_frequency_epochs=args.single_frequency_epochs, + global_params=args.param, + ) diff --git a/experiments/synthetic_datasets_experiments.py b/experiments/synthetic_datasets_experiments.py new file mode 100644 index 0000000..58ae727 --- /dev/null +++ b/experiments/synthetic_datasets_experiments.py @@ -0,0 +1,501 @@ +import argparse +import copy +import joblib +import numpy as np +import os +import pytorch_lightning as pl +import torch + +from pathlib import Path +from pytorch_lightning import seed_everything + +import cem.train.training as training +import cem.train.utils as utils + +################################################################################ +## DATASET GENERATORS +################################################################################ + + +def generate_xor_data(size): + # sample from normal distribution + x = np.random.uniform(0, 1, (size, 2)) + c = np.stack([ + x[:, 0] > 0.5, + x[:, 1] > 0.5, + ]).T + y = np.logical_xor(c[:, 0], c[:, 1]) + + x = torch.FloatTensor(x) + c = torch.FloatTensor(c) + y = torch.FloatTensor(y) + return x, c, y + + +def generate_trig_data(size): + h = np.random.normal(0, 2, (size, 3)) + x, y, z = h[:, 0], h[:, 1], h[:, 2] + + # raw features + input_features = np.stack([ + np.sin(x) + x, + np.cos(x) + x, + np.sin(y) + y, + np.cos(y) + y, + np.sin(z) + z, + np.cos(z) + z, + x ** 2 + y ** 2 + z ** 2, + ]).T + + # concetps + concetps = np.stack([ + x > 0, + y > 0, + z > 0, + ]).T + + # task + downstream_task = (x + y + z) > 1 + + input_features = torch.FloatTensor(input_features) + concetps = torch.FloatTensor(concetps) + downstream_task = torch.FloatTensor(downstream_task) + return input_features, concetps, downstream_task + + +def generate_dot_data(size): + # sample from normal distribution + emb_size = 2 + v1 = np.random.randn(size, emb_size) * 2 + v2 = np.ones(emb_size) + v3 = np.random.randn(size, emb_size) * 2 + v4 = -np.ones(emb_size) + x = np.hstack([v1+v3, v1-v3]) + c = np.stack([ + np.dot(v1, v2).ravel() > 0, + np.dot(v3, v4).ravel() > 0, + ]).T + y = ((v1*v3).sum(axis=-1) > 0).astype(np.int64) + + x = torch.FloatTensor(x) + c = torch.FloatTensor(c) + y = torch.Tensor(y) + return x, c, y + + +################################################################################ +## MAIN PROGRAM +################################################################################ + +def main( + dataset, + result_dir, + rerun=False, + project_name='', + activation_freq=0, + single_frequency_epochs=0, + global_params=None, +): + seed_everything(42) + # parameters for data, model, and training + og_config = dict( + cv=5, + dataset_size=3000, + max_epochs=500, + patience=15, + batch_size=256, + num_workers=8, + emb_size=128, + extra_dims=0, + concept_loss_weight=1, + normalize_loss=False, + learning_rate=0.01, + weight_decay=0, + scheduler_step=20, + weight_loss=False, + optimizer="adam", + bool=False, + early_stopping_monitor="val_loss", + early_stopping_mode="min", + early_stopping_delta=0.0, + masked=False, + check_val_every_n_epoch=30, + linear_c2y=True, + embeding_activation="leakyrelu", + + momentum=0.9, + shared_prob_gen=False, + sigmoidal_prob=False, + sigmoidal_embedding=False, + training_intervention_prob=0.0, + concat_prob=False, + ) + + if dataset == "xor": + generate_data = generate_xor_data + elif dataset in ["trig", "trigonometry"]: + generate_data = generate_trig_data + elif dataset in ["vector", "dot"]: + generate_data = generate_dot_data + else: + raise ValueError(f"Unsupported dataset {dataset}") + + utils.extend_with_global_params(og_config, global_params or []) + dataset_size = og_config['dataset_size'] + batch_size = og_config["batch_size"] + x, c, y = generate_data(int(dataset_size * 0.7)) + train_data = torch.utils.data.TensorDataset(x, y, c) + train_dl = torch.utils.data.DataLoader(train_data, batch_size=batch_size) + dataset = dataset.lower() + + x_test, c_test, y_test = generate_data(int(dataset_size * 0.2)) + test_data = torch.utils.data.TensorDataset(x_test, y_test, c_test) + test_dl = torch.utils.data.DataLoader(test_data, batch_size=batch_size) + + x_val, c_val, y_val = generate_data(int(dataset_size * 0.1)) + val_data = torch.utils.data.TensorDataset(x_val, y_val, c_val) + val_dl = torch.utils.data.DataLoader(val_data, batch_size=batch_size) + + if result_dir and activation_freq: + # Then let's save the testing data for further analysis later on + out_acts_save_dir = os.path.join(result_dir, "test_embedding_acts") + Path(out_acts_save_dir).mkdir(parents=True, exist_ok=True) + np.save(os.path.join(out_acts_save_dir, "x_test.npy"), x_test) + np.save(os.path.join(out_acts_save_dir, "y_test.npy"), y_test) + np.save(os.path.join(out_acts_save_dir, "c_test.npy"), c_test) + np.save(os.path.join(out_acts_save_dir, "x_val.npy"), x_val) + np.save(os.path.join(out_acts_save_dir, "y_val.npy"), y_val) + np.save(os.path.join(out_acts_save_dir, "c_val.npy"), c_val) + + sample = next(iter(train_dl)) + n_features, n_concepts, n_tasks = ( + sample[0].shape[-1], + sample[2].shape[-1], + 1, + ) + + # And make the concept extractor architecture + def c_extractor_arch(output_dim): + return torch.nn.Sequential(*[ + torch.nn.Linear(n_features, 128), + torch.nn.LeakyReLU(), + torch.nn.Linear(128, 128), + torch.nn.LeakyReLU(), + torch.nn.Linear(128, output_dim), + ]) + og_config['c_extractor_arch'] = c_extractor_arch + + print("Training sample shape is:", sample[0].shape) + print("Training label shape is:", sample[2].shape) + print("Training concept shape is:", sample[1].shape) + + os.makedirs(result_dir, exist_ok=True) + + results = {} + for split in range(og_config["cv"]): + print(f'Experiment {split+1}/{og_config["cv"]}') + results[f'{split}'] = {} + + # train model *without* embeddings (concepts are just *fuzzy* scalars) + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["bool"] = False + config["extra_name"] = "Fuzzy" + config["concept_loss_weight"] = config.get( + "cbm_concept_loss_weight", + config["concept_loss_weight"], + ) + fuzzy_model, fuzzy_test_results = training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + ) + training.update_statistics( + results[f'{split}'], + config, + fuzzy_model, + fuzzy_test_results, + ) + + # Trial period for mixture embedding model + config = copy.deepcopy(og_config) + config["architecture"] = "MixtureEmbModel" + config["extra_name"] = f"SharedProb_AdaptiveDropout_NoProbConcat" + config["shared_prob_gen"] = True + config["sigmoidal_prob"] = True + config["sigmoidal_embedding"] = False + config['training_intervention_prob'] = 0.25 + config['concat_prob'] = False + config['emb_size'] = config['emb_size'] + mixed_emb_shared_prob_model, mixed_emb_shared_prob_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + ) + training.update_statistics( + results[f'{split}'], + config, + mixed_emb_shared_prob_model, + mixed_emb_shared_prob_test_results, + ) + + # Trial period for mixture embedding model + config = copy.deepcopy(og_config) + config["architecture"] = "MixtureEmbModel" + config["extra_name"] = f"SharedProb_Adaptive_NoProbConcat" + config["shared_prob_gen"] = True + config["sigmoidal_prob"] = True + config["sigmoidal_embedding"] = False + config['training_intervention_prob'] = 0.0 + config['concat_prob'] = False + config['emb_size'] = config['emb_size'] + mixed_emb_shared_prob_model, mixed_emb_shared_prob_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + ) + training.update_statistics( + results[f'{split}'], + config, + mixed_emb_shared_prob_model, + mixed_emb_shared_prob_test_results, + ) + + # train model *without* embeddings but with extra capacity + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["bool"] = False + config["extra_dims"] = (config['emb_size'] - 1) * n_concepts + config["extra_name"] = "FuzzyExtraCapacity_LogitOnlyProb" + config["bottleneck_nonlinear"] = "leakyrelu" + config["sigmoidal_extra_capacity"] = False + config["sigmoidal_prob"] = True + extra_fuzzy_logit_model, extra_fuzzy_logit_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + ) + training.update_statistics( + results[f'{split}'], + config, + extra_fuzzy_logit_model, + extra_fuzzy_logit_test_results, + ) + + # train vanilla model with more capacity (i.e., no concept supervision) + # but with ReLU activation + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["bool"] = False + config["extra_dims"] = (config['emb_size'] - 1) * n_concepts + config["bottleneck_nonlinear"] = "leakyrelu" + config["extra_name"] = "NoConceptSupervisionReLU_ExtraCapacity" + config["concept_loss_weight"] = 0 + config["sigmoidal_extra_capacity"] = False + config["sigmoidal_prob"] = False + extra_vanilla_relu_model, extra_vanilla_relu_test_results = \ + training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + ) + training.update_statistics( + results[f'{split}'], + config, + extra_vanilla_relu_model, + extra_vanilla_relu_test_results, + ) + + # train model *without* embeddings (concepts are just *Boolean* scalars) + config = copy.deepcopy(og_config) + config["architecture"] = "ConceptBottleneckModel" + config["extra_name"] = "Bool" + config["bool"] = True + if "cbm_bool_concept_loss_weight" in config: + config["concept_loss_weight"] = config[ + "cbm_bool_concept_loss_weight" + ] + else: + config["concept_loss_weight"] = config.get( + "cbm_concept_loss_weight", + config["concept_loss_weight"], + ) + bool_model, bool_test_results = training.train_model( + n_concepts=n_concepts, + n_tasks=n_tasks, + config=config, + train_dl=train_dl, + val_dl=val_dl, + test_dl=test_dl, + split=split, + result_dir=result_dir, + rerun=rerun, + project_name=project_name, + seed=split, + activation_freq=activation_freq, + single_frequency_epochs=single_frequency_epochs, + ) + training.update_statistics( + results[f'{split}'], + config, + bool_model, + bool_test_results, + ) + + # save results + joblib.dump(results, os.path.join(result_dir, f'results.joblib')) + return results + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=( + 'Runs concept embedding experiment in our synthetic datasets.' + ), + ) + parser.add_argument( + 'dataset', + help=( + "Dataset to be used. One of xor, trig, dot." + ), + metavar="name", + + ) + parser.add_argument( + '--project_name', + default='', + help=( + "Project name used for Weights & Biases monitoring. If not " + "provided, then we will assume we will not be using wandb " + "for logging'." + ), + metavar="name", + + ) + + parser.add_argument( + '--output_dir', + '-o', + default='results/synthetic/', + help=( + "directory where we will dump our experiment's results. If not " + "given, then we will use results/synthetic/." + ), + metavar="path", + + ) + parser.add_argument( + '--rerun', + '-r', + default=False, + action="store_true", + help=( + "If set, then we will force a rerun of the entire experiment even " + "if valid results are found in the provided output directory. " + "Note that this may overwrite and previous results, so use with " + "care." + ), + + ) + parser.add_argument( + '--activation_freq', + default=0, + help=( + 'How frequently, in terms of epochs, should we store the ' + 'embedding activations for our validation set. By default we will ' + 'not store any activations.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + '--single_frequency_epochs', + default=0, + help=( + 'how many epochs we will monitor using an equivalent frequency of 1.' + ), + metavar='N', + type=int, + ) + parser.add_argument( + "-d", + "--debug", + action="store_true", + default=False, + help="starts debug mode in our program.", + ) + parser.add_argument( + '-p', + '--param', + action='append', + nargs=2, + metavar=('param_name=value'), + help=( + 'Allows the passing of a config param that will overwrite ' + 'anything passed as part of the config file itself.' + ), + default=[], + ) + args = parser.parse_args() + main( + dataset=args.dataset, + rerun=args.rerun, + result_dir=args.output_dir, + project_name=args.project_name, + activation_freq=args.activation_freq, + single_frequency_epochs=args.single_frequency_epochs, + global_params=args.param + ) diff --git a/figures/cem.png b/figures/cem.png new file mode 100644 index 0000000000000000000000000000000000000000..139a2ba29a61df3fe92f1dc6c3dabefa6bfc7a1d GIT binary patch literal 78124 zcmZs?WmH^E(=I%NyL)gaxJ%H%T@wNcK0$*7cLoo^HMqOG1cv|<+=6RxceihHzw0^Y ze!lbm+G}R@?ylZdU3FDm)nOWH@|b93XaE2JQ&B-i3jlxz0{}oR6eQR`89L;eum{jd zOCAKM93|fa0H^_qGEzG3hKB~o-UPA_zQ-Jq1fcJR35>60aSDp5X^A8?WS$liO=q!N zjfq)vjq{2j@1;s|oj*h700m!}+v#xN@^`LZyj(4MwwslY<61WzJ$4?rek~s#4)fOW zp77c`Nq(cIMnQoKfc)1(vV?k#Ik9L^8WAWZiGzcH@Lvz=AQeXc{~G^4Zx;kXaB8*S z3@KUv-x(n|JSp)1y~-yl3Ac0borDAP*Om%>Y>WAiX?G-W38ku!LWG15n)Wum)fuybt|oIEsq{dN;}haW?vw zAyI&H`@RNPL=cyYfx(YUKfK^g-n$c^aekg#YVulGH(mw4X-bqf7DvB;ao0 zx&%wdTtU}ZkWqj&+$~ub4j^>erR_x)umh(6jB^qn9UCfC!8={UN9O~gIfMju$^ZDs3D!gLoNXH%-a1w0Q$h+=197 zLlqQ(`^stt>KM;sMqd2d)dNR`+98Or8v_JP9&5A=>B)Ify& zUuGwnBAZPOA}VkU*X&J1G)~|D1WyPtYrSu!2DyG8k%?^?EPm^ywmeJG1-kOG1t0^L zn8<2*-hA)LU5{0Owq2DJr_A=6#tgdCd5_d6kj1|TL;}bWtED4#+iKsgY6qQN{A7!( zeho)n;2Pd}VD@ET;5q%zOhNZ~k($v(L{?S~`MQ%5$&<{Mg1Wz=yntEHjM=^ss-Qfj zt{e{Yh|g-4TSdMJ#()6gt3Vb&rB>n!=9(>l-t=QD<=cfCj@SN|()&NU0SbeZx_%O< z#tA%v(AJtVY{Sv5_6P|d7?xiLG1%c-I*khaBFISP9pf2!TtOL z2k!k-7uKWUz4EjV3d_z!l&Fv*iwhtaUI@Qy4Dg!y;_FWMxVWkG1sYl|o2;siUXc6{ zI~owrIX<#AaWIS&)r<23TO))BlE?wj(|kpEzA^CTKl~o z5kS0{pZG?9bzUpLYi$~L ztE;JnpSI{|+_(?sBZ4SEjY1AOnmLvpgZy zbFBgCf|h4j5m1#GfedBtQyJgUbp98Eobu#KEH$B!M|qHZPdlk(DY2+K=Yc6@?PLNw zopbS^eG_eL%-N@$LeTfgLppVR<}S@-jXr0!d2u@?fzV6Z1OBEe{jYLytqZSlH=+a2=BF+dbxI;%*)mAW8_>nSZULq34 zaok#=J{PD6wdBcbI7h_gu}w1b6Hvqp*2EL>r5prB!ZjGNdB6BxxWAW9lqj|!r3H(2 zQ&nKS?U9wl-XaN6Ue>Q0x<1X`WG#3a|K$fiaeKdj?tQekFfH`4zco)k>uWGhtwT!A zB$iY*06{6fJjmL(4VNd=_^dwS8I{c*u!{*P^+p2_pjk5Z%BJssCd!O(08-_vsk>__ zcSnd5UjInOi78^4wyci9a7C1)v5csQCs?`tNFm3s^MpO%2~PvYpi^esV^)rrOf?+; z9QOV^A-w2XWfhVoB9-b(UB$ar)GOcHmci%S(_I=k`tQiRJlbAjdC@A7B0&bU) zC{?E{T_v4#$7;LHv2yX1Jq>$>yTxOYYjEIG!Q3M8$|*7$y-7J87HasT*mjvJ#&_Rq zjdo^TLEB}~>q5t|I~aHS_z zm~{zH(=t7a_o1@ya&gU5#fd)k{kg%pMaB-D&u=Mm_n4lKvTWqKsVrZXgebi^)W;QR zg|YZF8P$dW8YD%i4KpI=zSrSJr|G=Yxy1EY4&{5DJv{B}7jo6av%?3XE%LZbH=evcec3d8#6V_P= zc1{O1hb$9GluCxI*G?|hw(rqAcU-dOI7bPOSw#Vb<}>jzUnm`Y)*jT@aduL79+;gJ zV9`to*Z&*@ML83;-q=1_a&8e?&VeQvkYxLC5vvfPX%WyjuYUpVfCl=aU-A7SLl4GY zD0DL0H2($qxMy)*Q_ES%CDno)hAyShLHgk^Rf)M^JV%gHWVx2Z&X|pU z=x3%m{KC&@MstGKC40H{x~y-R1Pu&Gu9L_Bc#cG9vlz6Ji+;AI3jT>%dpieSR;8|8 z?}!KH)ARy;BCO<#|MXxG z?d?z{pIVh{3sKgYIwjcdfA>$6d3$dl`54$;9Ke`7aEygkskF) z5zgREI6`DWwo6C02nj)#uTbyRhX;lWgDvqcw`9%PIp6zO4TtS}nLmA3HdWT}h+rhF zwN-jl#0q$V;+7RlDiTT|Dm zS|M3Dl({9E^|uT@r9F3w0yapTvbH~-HxkEO-Y!guq3W5OuwqMJKKDu!QtqE}#mc6= zBYKB?Q!_A4*4-;c0SSFT3?s(&*Q*bSIKbE$0bJQ7ZO%dKU=S)B2VoGO#-@9GI z4&N*e9erCoI@5x)Q*w&NU|Z9i)4BQR^$j4p6KV~Pk@ff$nbd%H2OUABo_EE?$4>Op zz@D1^cx2~G3i=6JB>F*m6Z-fC`Gg(`c;eXRvt{#j8q8H2+n(()kuFaU6d80tWgxww z=wlvHX8yZ)QM5mtCB2M)j;*%$kRxzC!kB0_NF0{FGOglU=M-0LdhfJZRKCNj{sY2+Kn)@huXJ3#lLb4hWbao_tXe*=9$OXUR}~` z@S-2&z0lF>-iJd3<7MM~(6n4JNe`x+RBR?m6F|sj6VDcx!uc_yxeaA2C@Dne9u%ku zh#a=1LmorIGX5DwFZ7fVX#I`_c$;l^F);O3n`6(ckA#*}gSHQzOw#HF^g7j3nKeHY zj!+*pFS28PWtug%=5iMjpzK0?tzXg+nkso6zn#T)h@4g8bIY6`hhPobw#nF`(U)== zPZ)|{TUb;*45M2d7RT{J>vB~_R|mn=C`t1XpXWMpLEUR&V_@v^aRaDpGGY9baq#I$ z#GWzyx|cHwd$jmry5yde56Uc`FV zitK|1MS}b#`8G`s)|xaX2X^ccW4M0r5IjSNHbcf>ZuuCiRVTFuEi5;@h>n@(t)5B` z*2kp*KdVplppXi7kq(l_J{u1+fpQ{#hV{HgoCrWkStd|UbN6WEcpnsTHF^2X+>9+) zSY>gs-$N$HpD9=0eG3Rs%)DmFetFe-K@a8&o%%VZOZ7yB+cIf{07D%2$r{qKp#GZ8 z5?=mh7+|{I;n8B_mm;`tmt|p?&pSa47cjY$gBEk=ZL8P--F>Q2NU`~xY2Q{x(%aEz z5(FsfTiWgE?V2g#e==Y$UYH7;!|qWSFV@WKmzE#QbJ*KMHfstxm4y~tu@z%I0HbG` z7Z*p}Qjc=%UxhL9dreys%RnFXkQGqsaVq&)uXo}|PFwgAr-M6eFVo`xEQdxk&-wV4 zu>4slWjCzHkiq*2&r4MJdhNl098@?9XQUbYL<2_rZKAIK`6hDe0_Hf1S@KYM&|zlu znEUA&mf1?O&`}2sOMuL0QzwSM_B1ns7d+;u;$=fX+o%p6tiiZEA_f-;(Gm@7axcVa zDf(YwyJdbq5%jevVBvF28ukdT#{%@OLbVBw0Zib#7r18IR^LK05L?JO-Q_`)neC*V zpIxac!xx(islZp5N{IB+23Sl;fkaVBGdj@^%skR`877QiK1L8XIu_Wm4+!f zD5Aq)+Eo)Dq)O&Ps|a|{XSH)e&mAFJIuh?}=?fd)!9$>M^pgfQ(CNFm_N5f;BsnFu zAfXQuLbFLl*xD?nby9ue=4vdpRqk)OR@9EVkQQ2vPIn23i-jA zu6u z5yPI0r&ajPqFUWMJWE^BJk?RLfpI%_lT~z4ez2$d4WHholz-JtqF1U!PVdR3PF=;?u5s$kc46_FuE|^k9u3 zkgFgLzh3`yLckanPE=lum|cQ}E~p?a8^ zE2qtA_db>2j4!2V@KSTbjqORN^|0uKkA0)R@5I@V{-OEoGHbT_RG2=RkK)hXYeZ#{ z4;RAp$q|pTCQ@lMR@&^=@RoM6nAykp4RGno&`cH0iTF`0R*YyLNr{o*CNav2{@yVO@GX48 zfh;hiZC;M$3WPcZ=yNuP0qt);Q^*7aFsV{EP#2?y zh&fhN-Gm0k%6xUvc<2Jic~dEHDU5ojD8?#iR)dY^kkN)jraKInXZ9HCTbi-lrP)aW zJ#pYWoY+Lx+p1g)<^3+-B0tq$Qi6=w4LA=NNoVd*t-kd?1$v!!Qv`;LU z9gWFXG-j49)d?Hbcjo!3`y3&X+f*1#d5MHYt4YrWMv|g_H_=G6w#HP;DSCbyj~4p%K2b z6FSk-{H2zCiM-w(nONQ~Q^eolr#i=PsW^5KZJYEvZbZ&|6U!_TUpc&9NhJO1O^YQ- zkNc{;z6tVhO+r#p2qOsr1ePUsxr>ZOMwsQJ4R3ZF=X_k zoJ6YQicn^HEo!r6dTw&B)MiIJ8!8$+w)-Eupg*Jh!K+RjPNTbS-wp_O2xdj(PB+OE zE?BG|>#dTmo9IdZ{OE(n?f4+tH6Coc)*Xh0^ylCyXMAUbO|$c4I_^QKe)J}gjEgdc zI~ytRgNEc*t4jJjvGrJk1m(z!g5#VWQHPhP@%n@O`a+h=iY~=v-^2v1%ks3PRi}h+ zVZ*n_jLqbqD#*^0p~{cH_=wpECRvQ94x6)Gx;k!7#Z?dQ-t3V0Koy8uIAfE!hb7>P zxndMOI679|j@lEA+C%|;=&o929*d9uEN!@7Xhhlw_y{Vd?_CVIEjw|ZHWq0URiC^O zDUO7>p~Bm1w+N@SR0E%`fjgC2Id(g#6H)9mkFMNL1lNh7(h0&O^>XsNo6m(|CC|fl zjUN}B2GYn&`J-~I({^~TF5I(~^gt)eIi4NUtSIRa$$Z>eV~iS@E3a@<-9&K*q7$f;K5)5BRD?|R5?1EYqJxe zbU|rNg7(dKw~U|u6S>pM$yi~R+t-D+?Gv$*9%-!Jl!(Y^?+(M8Y;MI6WyRBusEJmB zD*jwgbjHS%DWL(;{OC>6?Wa?}cq{g_j~I!`TMEdtC$wucZ;Ea%>0R>aJ-*z=>Pi?YWB32c+#xMr` zEM+cS>njOl%pvDwZ?0MXX@q=gB5_B0gVi3j*FoHf<=*@C0C+<-BRe@7D0R$&H=eg^ z&oYp8LK$f7faP(#+;JTsd3_nMqU-m;@I8h17>l0<7AnjS_$?luxX{YdZ0vF&?HTQ9@0`mSL2V4;jC*GBm6nUZY zKl61@%T!cb_WRQg_Nq}@*7#;5^-|&}$d|E~8_r1gC>(WO2eThi!d5x`1|tz3t4|Zc z8kZLcLK@fkP-EIC&f!r1aujQ?fE0%5aD*cd#oc*CQO`Wn$(quPvj3Ei)e#PH6^`yu ztZJ3Dm5J;p)WXB_v9is^;uS^&70FaZ&{_G)3SFb0y%!{Es=YS7q`H{kK+tpP-d%6! zq()*6G*dff&e(q>N2HUSVv&=||HX-2V>v3X&E*&Ls{0*ml|d%B4iwVB8miCb?BYfz@26Xo^JW zK3lsIquH!{iRJ(bB*H%p82i1FtQZRU7d#77&={Gs@vpe*u)K6QGgH23c|z(V2~L=Q zAW3yNP&au$WUL^X6+O@;w*Jf>>D^@L?WJ-aHg#Y}Y<-O_#%>>g7bECruFv-En-R(* zU{H^-dp;^dFid}OApK0be^uil)W&~v$J>@DuyQ0mog9w^xsy1LNP*g7cX2PhJiT0m zvr;4xYE)EGMIO;btFrCglQZ~(xO`FJcu;vHt^Z?GP=uxsX?`ez@(xx$s=x(yt*_kr zr&a|r7)}=%NEdSn_9c4gVQwl9bgAJ|z;3WtiSHc~3k3Z7%)<1S&|!jSUnck$p+h~4 zbNZ=vcV6WUjZM*{&+Djs!Ezi6?@x7Qcs zJc6&$1Kao7zQIzVO5J#=73m`i_{3^kl^Qc9Fl}h)d=Oj3HdKb0eRHej-&|LMJ;FB- zR5Y*A8k0a3eQ&^;^GYX%#<#yM3cHjOoeqrAFy?K?b}3x9lR9VY&6~MvHj)}mT`!IO}*|j=YP@oK$G7jc9`=u5rE#2 z@v;wRu>`RVBP*oBzTE#I1|@@N9a3ny4=Dp(#+NkG5EfcRQ6QsV6gs(55%zO^`fmj# z{GtphoK`unJe1EG>5Kt{21J3!nF|K=hC5)flp3WvhULoxk?HF}qDA8;uW!_d+uf4?r zy~Ztu3vy))%ND#Ez8w}HnDHsf*?|?jS++=Ea&5Vv799Lr-7+X}7~JYve2j%Rg)XLl zEUZ^8D;17N{XA1?JOgv_04vIq`O?b=2m1i%IUYr9x5ONg^cEwXe>^S9E$m>Ys9^ms zq^lAtY^ef-HqpO+=-<@a-(NIf9_0W3f3P{284!YE67f#Y-~TTY2F3g$|9@wa%1(fB zu@o;c=*TJ9JXD7^GQ~U6LRAHm z0{<-WC_`Ud@#YiPmV6-CiY3c?p_-g`4n? zH%I@Em2FY{{hU5X3NMKMP)V4J14rat{vVTPcKdsh-K04vh!AM;U?Xvb%O_d>vojb= z4C9XP%-F;T5C-94o_(;FS(ysPUuT!ocJcpUh4g)DfSVaC zfZH+*DJsl05{L^RieTKXvR8${0h9^^JRYM0#)xWG6Q_NA0DJ&0)Gi``An~~yE)`SL zKT+%-!d)`^uN(y2ErvK>9T0jVVlp(VJo6^xd9^<1Dp$rHI5&U%gy`GLwdR^kW4-vF zUVt5NQ6bf6l&kDZ<3;CDdwYF`&1n{Z42}e4P1|Jz1z-VZiJnah$r${6#vpt8=0pLJ3;Pb=P|}xRIpJ$K6B(%fbg90Z zXDHFmtj~Er?%5~86V=w2e%36pFQ-+uHm>>PV(v7Mr^XIZ_%6&fVSpJBqxM`mEtQ+- znZ6`9Bqn(E?gJxSbII7-*(bxMZk7R_sj0ZqES-z2+uNYLR<|R|^u&~u-8;7m5f{P7 z_UAFeELGnZMZhWx?as-?rpUh}gZnEPAa#irMTQ4%9R%Ian6^}WFwa<&_|nVzDJf!p zN3pZ$N51)vxlgmp$+lP9FjZyW3iRCT=@kBc1Qsvqk~0+?j{Vqs${!p=CWqv?^; zPb^%Hn9Zhn8dZ0{gkIv5FH$dO?dL1o)C`}gp@ndkGbrn{U^HOVEt8AsAh}+ z3Y@n?Z-<*!?g3^!a(`wH+=$2Z-Js{YkUAdz%8I_wecie z8d@6huvu2s`qANRzTAGWbtA@|4``5TlwjXjKrR;4m$seThiLo7JXCGlZAv`t^f~Va zn;b`ON-OmrjP=*m!BTY?XYOiQU=|>mB4=zo^@H6o{d}4hFX=1GGNon1iR8Yx_J@LD z-0wgrz@d!R?6ED4$87rPx23>-_$VfwFM^yi)1MHk-ky}Mb}=oR>pK=uwd<98wi6EwY={pwWP8| zX*4KNNZ~!X-1`wbPAU%wXq~*m&J}@?t&U{Mhl#L<9Dv zBZ9$YtA+Xxzf0eaIS7$g_R2>+K3rt^alEESl)iQCmmlg$k2xFP;sxlIS(bjnGP0`v z>l85lIt3&PkH4)k2qBUl)Zkh8WDLm9hz0H>qSxh5rT~6`_a2=@ii9Pj*xgd`$(Un< zy+M@--P(h0$KwY6&zW%Ic-faVwAM2HbZ^I&Dv!%7S3;W4nrDBKcr4bmz3PD zH?7`y-z-+y%syX?6|Fu$9z2-L-qP|hN1&4aBAI%x0`Gtbu?b2bG=XdpNoA)~QwNo& zW7Ioq<1m|{Bp`zd5gU&H$AGT2R|IS6gWxmtPy3tfCiN1%cTens%U&}+8C2mHJElG( zZ7y}O2CTX$deqs$zZ!InCN~0sx|auDc8datsrPm(E%KE%?(l+`3=@hzyg$G9bzfQRa*FOfQmRQ^eBiiV}wB%~GH#Fi% zh(b~w@C2*}I#5EWbl}64p%JgDTG6F{V4xRwUd;Fe6Tb1oiSL7Gus(&`x8#nMyMbn7xfyVMe>rA$|FuOSY8yegr z0_#CID6bGeY-{68-(FUX^~>Qz3_gRd7gvWS8m&UTqnpo5IBw5LPdN18ygwcPE}xO+ zV7MWvN;G?Hz#deN0{K3}u50A@D_^lUtF?xCtn(#Nmnh=yv(ocN6xovEgk#h*&$mm4 zXYGwcdKfeK5fHIH+wV)9un=Mr0yAc=S2$L%P_uxN0D(x$IWt0L0TJg?!{20~3dNM< zZ*Z!XpD6{7?q6N_W`wN@R(5gvvCG0v`gp@gpHP{HMZI>=(Ge+U@76aS)cMvr!@jwB zI_~CKoTZ6}9Pi%P#Eh^s!{f+ksuv_RCU$gqRzw^gd&1ZlRn3?44kKY-H)5gj>=B55 zJer5+~wSF3+ zUq-o*?EdoSTHO0k2!leD+2})bVZr+LK&aTu@0(Q#+-bFhlfjRGQAAijf)ijG@$?NO z5nu>cUFF?N*Z#NdNF5gKf2G3!hrc&U0)QqjXL)mTG`WhcA~B^gA3Y5QhhCfY0k5v5QDutI_)Lm_6ioFcUh?!iXpsHR^U#ppsXUp+r9? zGNaC1#1p#(mscW-EvQ_h9^?j8Y=VDp3RpxVHN!Y`+nTi(f1YD-<$0HLq?asV$2Mv+ zD^L~dCMH4NdVSP;IUx%F%gE68Brs9E{Q?VHF`s}67*`XmgA`Md{1T@B2L0Fom<=e9D(;0+UAiHk#p2CjUjw$d^Ip}# zj1PW5WkS*GyV1$xySKmhItK08(7HGP7#@m8Bm=BPqCh$yLYQp?;G(Fc;_$)353H7m zf(PS8`9v``>^K+v0=~@IpTQL}ns+u>jypP1hvi^4Pna81JQ_6tZ^08@w?2nz>B7xe z9lm=O)0O&T-qTgDl5z~{9(_$(u@1YE=d1qAr?HO!Q5U*!pN3ago4shoQzeMZI?2Kk zH(XQ5_)`O2{EGRvJAdG*$s3!81956WJM*Pytxdpc5pzOyQ}b<_1K(d;)Aj2YZUZ!X z?`ePV9b{G=_YPp0Omtp2SkDSrF4olDWx(EB$G$_^x1E+J zuCAscnjt?G--1nWWb|>Ic4F?4UhJS(G48-VgvC)#;D0Sev3{FMeQ{@3s#=d}1Ve5=XAW zptSha5duWIwVW*v+>`E?q|6NZyk_&nWe1&xJV6zp@1j?)kk%CZ*A;#R*9@*cdl>FG zlMus4$+s`rB}j=Lmge zXOR$ibH@7}KIEAKOR%>yS+{$Aw83}BB}umYZS)YNmrKGab}H>^?*~Q8Ov7ErjuVEp z0!34||GkH`q3kf<9i>2tUm&M^`>3d^p>mq8;uDhzssY+ z;ePFE0La08)y)5#F_C82?tO5$ z-KF{HsAjZu3sNs9@5JBeBnx&k?+8fTaLX_@Z$9WeYPP%Wdw+hfudII_7}G$ivuI-w z9Lb9Mk3qsQ>nHd3P#QlxSK0beJ~T~kFO^lcR^&blZH-H@rEtHYqA8m_1gHD$72H{` zoNn2d?e|UFQ*>=?PR2cs{;mOCEJA>!E^m`>`);o%{w(-x{v!C%u1@jFhq;smBpaCSy7 zTBQ;mi&kTP7xQe?#gm1T3n-}avFK;LwcSuJytWzt^wQfbT$MIKI%YkCx*p03(dQ17F#i4()-Pr{T-A@XJtB+8lwNwbz^u2XQDR0E?W6^|kB*76qx z(NqY>$c4O+y09loxZsFvIQqh$VXVR-ynazJatXGS=ridKW7@^9EFt(ILnh6b@F9!q zj=dRXLL*sR7~f5Nzi(~xYufc@d?K+U(X7$5qa&6}m~yb#B%85_vTRx(<;wkIYPoJr z(h`Du1?r;1_%0BpdL%QEqE`KIbB;`w@k&WqRDaQw^wC7h)!7Vo@}t6;?77lhI3s$%m~ zbR-py5&Sc7b3b+GhuQgt6AnIM8ZvI6Pd%wZ#f+NJIa_EH-Q3-Zx%+;h?nJw-ee-gH z7du1HHR;Avy&;`q>rKTg%{oVRQQzeuAhwjHHK8U{;>D3XZuV4@6s$F8_J3c{XApDO*7N#wlT z`SRz7JY{daW>ubM#zu{}?~4sQC512^?<#pDTFt;J=DWC*#NdvbCAXbFM!q@G+8Rp2 z4B&*4Lbzrv|J(9V2EeRBD$7)?lcokFNeNa)p{D)YhK}3IoB?a(!t}QT5W$fk88=f^ zWV=St;OMeW$rwE|$TEU4HoF1j*N*8i8J4xsgQeq-OAp(T?gt&w=!ZRjRy(_uv&5r3 zs-7twf@I&3vw^m8KTV`%j}D}GK(RhAyV$*wf2C6b1=u5g7q%%}Kc^8i>))YLntSN! zLR-I?&J=O{V|lEs8eVnA|KW6;y@}21IJ5QKLK3^$Hr{-ALG*UsVh^6I9_qK15QFPs z-9<8Zg~pE&4ew3`2=r_#P-%JLOn#3nwB*})W5M~4i1(4a8C!U~Y$I{OD!6y%|RWwRc$p-5ltCtZ zDX~8wV{@-dEh}vCsbY2PAV3NF=xq+=L`R}Q&yM}R!o3@N7C82!(-m-Iyz*oIea*#y z0t0yILJr=zqe1~M!baTdCc{X=`=G)Wj`U!x8wzP%OreY8c=!_Ql^<0x z2CPhYKG5b>1I#t;;7~Dn9crB5g9c2>w!6aUtj-(CRe`3)r0DzIJbvEqY`&iJgA$-z z9u5`|QDq&`|4t!b31L!xmyAX}c~P`2w%W(2b7jhib8*|=Y{H^LMm znd1-fsX0L*%+Aq!lZ{6ZvfrgHDhkCKN2jrLs5RW&t=Reb<(Zw%!qSUWnA`EHopJzM zq?y4F$gpM{Ib>Pq{s@64J|)!civo^+%2R_Pup>2ikD~N|=ub_D1FOC{7%DuQ$T)ql8zxlh!WgXJ!kcKZk_xF z4H-k(!%Sg-*4=K_i7A>SGBfg?R)BGI$EkC*8Yj#`M{0*sjBBg+tea0SU+&cJZRxu= zs}9Js_QLE(bEU(>mx6EY-i&qZ6}vum6rNM%3Amv^RtCM#ya6g|l)r*CHBx9&*JnqZ zQFE9ZWoF7VQ)@YpB2plEk?_T1*pcLX)=u&;`DqBA;za&Oe$>D6BMhEnNKy0Pq7?k1 zEqzxmkoTp1YLv?B#X^PApm1n)ady3<$xhUHch}#Gr4f9U($7b_i&v{m%x`l-l9nA_u;X zDw8-9w|sqzMIj2Yx9|;~ehP(rgs*J2oVHy+G2j@=k~@xlus7dh)d1Z1?gMw)Hy2cGmfc>u4$*0ZIDjBv?cl^zOv8={=9{5X+3(YA zgSxW0$D>vclf>ECcUw{RI-Kc-an{l|Bq-w2m|if_aif@SX_&x2bF{n*Ot6Z=8O@mr zEDB1fsj%R=$N_$-Rpp+#@J}D2J32Vs>$)L}xUUS>EfZL+N4lq=pZ zY~fZd@WvTO0WuAGn@K6SNg+x62pkC2T{D*VO%A*_%0evbqe%!ftm_=IOB;XWBOea0 zrID|H^I@p!``mE<0D0>ytWW>u6n*Ln4W?JK+l|IaN!IytQ}4lQEg;&WsNip_&pSx$ z0oK4SMZwpBUqN!H+E2UY4%fjL7{&l!_3?P@2Qf#`;p6dl zZ%+q_hI+et%XvHXt=DgPo^PMNr^;(#_jx=#tUZf;AR5hZ2{R#~xAg(t2Pka8=qsFS zSfRI0MfYoFV7a(;XVd&qh)|*)%``huw=`v_=-1ceWJ>WFA+}L?>Wf%2|wBWQ4BLx|GuYX~Uiz0wUB<5L^1UHa5)g#%UjwWXmLC^yb#tOAN ziYez`Fhl|np?9y{a7HW?p-RI83;>`!ik{+FL|3K-;{m(rq@5o$ec2*p;%ZkSo>R&l zc6Bi~swmfD)yCmOT0LWYD4MY?u)WpokN-fZmuwd@&6JOEnrK7N(u2Za{fPs;o|wj) zeucOycz0>5pS6U0n{J{GoBrySbR6abw*2JDl#<{+&(dWIKuqFG$+nKajS=ABt+9Rd zi>{k+=fkJo0rkxJFM|(e;|+RD)5ZO4=qp1jFV|~}n936`jf-d9tyNXtdnuu#moJ)* z1u!@ZBdK7m8_}P&0a)KOA8%t|8-=GqDHy?cP zF!5=QjW!%npncOCR`tS(k)?i#(ZVUb2oTRTsI155dx@d_OH}eT=!1Fn$g#8I@zg9m z>+!pk_Ys75ZSg=v&I-z9J;wJj+_2{RvEe)o{9c(^PtKfWqTb;&^N0?V`P(ML{<1M_ z#5~TGms8$d*Dv;boRApOkXP)l-u|RY(^@~pW7opvsWOi*K~>%WM7H>%2F2Dm<1B8gcta?F09jOK2k|&YF5^Q>2_RmS#P`gSe*c@eP+p~sFxuv zQZZ8Of4@++0vaJvuI-En_)uVd#d}QpsPOxM^T+6kydU2So#+J|yi%$x zn7cDJ9vw#e%19FVZ)_;BQEGTt?@9=$`0#&Vz>TDjPlBe8#7?n>MC1W_BzgHV+O<&M zI`>nReqBJ5oeU7?433cao&*quOcu%7Mh zYZq$wH*d(jcyw?*S&DiRjo}iEZEbr?i&Zj|7Yz*~%j~I*U$AmG0 zR0nsvGj>JaS@QZ8(U0=7iMzYw_!~MWxzEgAkz31t?y<#mg@;zhzTbydfuo_|@OPE> z4^?ACoG*9moMRpEU2iCZhJE#fkd3)l6S=FrzuFzOK42d75TaLa%^fFlN<4f`X1k9} z{m6_W!vQnws4O*14JK!LGf}HgaRa#4yQCH0R=+1xsc)(Y@1;E|W08+VyVk9N-X*MSlJjji(+ zF`7^Y(hV!(5h9Oi+0b1gG?V+dJx5VSr<%3zX^6lolsO-$FS4Z60NVPe;NGIb2w7Om zME0Z%M2c8V*M{*1FLjU`fEVbK9A?*^!q)L}tdLLf#yX^tDfa%$)jORk&Bf`MyK|bM z;o&Y)OuVz%%JKjdJ?3lP)UntIRGJH(DYDs;srAfPm04WPFKT z*~l#Wec!JTW2=LA@apr0@&gN!wAU*IC-nB~bKhBZa_4!7o$9N-$Bp2X0L-+RyqyZ? zBVJ|ZZM*fz7}mep)Mp&#lW>T^Vm5NHNtEA-1TRbns(=Q<-P; zeFzYSBk-mFTqx0i86+^PI+_Q96ge?&$V9R>7Ja&?+1%~Z)j`)g{eHE_9_xKJe){d6 z`ZN_q^K97iEgYFKyc_`hgIvgF^nBG_rkJoZg5B=1pGNdAbaAP@9 z9B#xFliPbF&to*dQVD=@LOMr{y#|^JgI9-t&IuazPgjw0jXxwY1Py2H;Y{1)_KV*A zG!ndNc_grJaAf0bv!y2LkXV7;IP$q^zddiO(JsnNmb&|?MjIKgDN-+WZ`yE(hPh+#V{L+|kdG<`CTw95V{hoYxZRG`C%l3?&RnOd%i zPmZOO&XN%Q)|z$98_1vZev7K~mJ2y@`wv&$4 zu_nL&J8!)+HS_V-sk$Fd?Pon}ueJ6*20lz()@j!0T()|%q2qaB!!hehP>4m=ktOok zGCpZ;;KBp{NW08#KKk(NJ429OHW*cfnv(igXsZb zwByb={VxJWzH&e3dJ;V4B=yRme6D*1+AEY+$Y~=~y+L<9z3A7KI5Yg%@pAuQES!A# z@(7Z$B9C>jm+j7d3Vbso_HpMlJkI%pyY~i6`>P%B$zE=DIt-fx=~_Lvt?X+*|1(~$ zI87FQU}r|tMUi4T(exBh8qR%7mY+-+OCYf4VWPKb2bZt8{L}NiR{yWh7s>^{BM5G0 zru6KT_|SR_Dj{K9Mg`6AjfE?RpywwqM~yzcp&fPZj@!r{xfEM8kNcIi8$D-*8u{(v z;@2Ops}1)=WkVdgd}XUkA^5jg-ks-!EWS@upMTfyYsesmXjG@>#vc^s2msU9`O{Ry zPU1-{=KgqvFEVO+cx6`B3LjZh*Ncgx56_p~t(6f#LnJu!cP=X_T)2Tt*TD!UJPF5& zB<<*vhakSVbnep=q!1-ubSaG-1I%jJ5#s@Myo^`C-E+SC^_p9h6u4LDXNGLuEp#9<%?4y8y8f%7 z$Hx6^A6F~-IXUX2qJl`@NMO4I9|4GW&edqoa>Ql45&tp?c z8B@hD#B3pr*j4x5>V5}1b!lno7H04(?k~LW6}erT}NzA==S$46a-q9N`GfSeB_uaJ_+_ zK=J(ghoohHvp#nY3qkja-6#7L-(u6WDnkr9@09m%`dvp|_3Zoxdq3psW0#yjer!da zq)GSN7sWQOx{dekFdA}fs6Nh|V+$+`vcd_`($HKWXP{~3Qc^a)LtdHo&YjPa$F^xQ zXY0tBPpuOQv`Rq#Ye0H|_kruhCpUEYN&v=EHPT+vxzQ^PBUva+S#q= zD`(Dr!g zHB!NC;LmT;eA)GQF&Ne}!|%v&sl~a$(+W|K*KZdctNiQ$5_rgfGo~%){jc*bR>VV4 zi_>R^$kozAu~5}=jd>DPw~5Q&;8Uz#p%N!9L8c_6=foCFc-o{MZh96+M#)C!`-cCB z@ud8bdtSMDoxkB3{V~tuKlf}bE&=gq^M;$u%F1Y4Dr{rLJenTl zW>@iKYx15n`|cCR)ovMthiP(EgDzH*I zU83k2RwiSC1icw1Ed)^+B4T*VIv?FWf+`vdTrE`&RXV036Jy?3g*AqyGG7rI7WZuz zB1b#APa>0I&Kf2EcL_r>SBg&=Ij=E)<7R{OWDP7ix8kjAGH(j5PH1X-hER>S^i< z-4YJ!mRFll1$m7*1(H+uejJcNHR&@^M(aD4`urs-?lpG+7L`(nJ>cKo;D zY7jlBz@aq{k1a)ft|opsbe+O%qJq<;j3`=%`Zpw-Onl{+O6YKj92Tw0@aUgbT&keP z3QQRs*kAZ8Mbp=MNGU|xom4|+f*}g5QJLCbPAcZk1j&?5zu$#6tha2;n z#iw&B&B&u%k}tFmqr6#FyUe}6w6cV#^DmowU6NC)xH1_Xi6l!yw}4YJk~*U@frUJX z4Q#OZ8&&H7jmgiXDRXlCY{#h<7~a+BtuG>RNm-v}J1=;>cpUTu4aEIj=kP^9gNcgs zpH3g*j<%VvEpI!|nJbg#vlL{(k-xvBu75Z`OT&Xyj#N}lZPA4?Jc}KNBqmWRXax7o zh>X6BiUp2$w5wgYI|>f_c78rdupZ$x9aa;)uHl*W zd-i9AEW%mEDGuMqo)%UHA~PduCbjkX4FZzBq6&qwzEIhUNkbW8p$)UX$t(pJDJ6iu zjrC`eF&fsSc+xsqiuGY8&Y$5~GEJ(7hIrE!yCO1a3Zb&}NIV9!QRylGDou*NHn){% z=@ixrZDlr_Bgc4RnenlULh{fo4zmb4|D@g=m}HA;Cd;uCBbvNfvZeKADsQowj#Z{^ z)}g`!kivxmknTL^T?QWMekl>wt`D554VtRSN|!11t|`S->l8i0mi%Ld%S0p=et=vu z<`)rL>{D;Cz)B(cAaxMFwJV+~gQ3fbz(`u1p?BLf`Sx`6_SYpLeI&HUzsHnW;8taM zingh&oJg@}v<{!_kc<>CKnw(v#}WeXM}OwWfHfjwV+!|u=>w^6#R)~|sq3~M5a34Y zt|!@Ho!z!yea0?hdiTEkR&`3FjJ+6i+H>ascI}p9@k&v~A<$oJwl~5% zGHq}f0^s?+ulM7IewbG_)rK;24mgfa7m5ns@vP;%0mlz|cuj_W!h}OJB$!TyV?+8w zy6jLepeM=~R<{2ImP9Ls@5SrizrQ=9~1!Qk9FV z#@NU?2o?js0c)~Rk7BQp!wQS{5PR|SR9;AfF||cLM&^)=%Np6lR3$M}9eiyqHcV91 z+-jIXvf{K@y-ywCkPnGup@U!z(|=$zq8`k$ zO@Frw!w4lnZsu~m_#x#oNm6_Q86|`l%1UF5mZLNs}8HVvm%$ zuk$8%R_H0e2SUYb+^~z{n=NVNC*&!=q`oS=`@ah5evMLcvo4ir2Q$UIY@u}g{114* zf(n?y+(lcxb<1QlYF4N7j&X0KYPj`d7N+h5k>V2zg z9;W`tA7vcIbHs9HU_`8Qr|91b{kT}fV|sP73jSrWYYD$^j7qUkJjs!u7B|UmN3qs8 zMXK=C^5af2E}yNNL$9pTglLjXSnj;_I8QUE|(np_nE2~R#pK~1Ez^#pVf`R)7I199+7+@ z<6DhYJl3<)`O%Vw6iKr(3vanIeEVs}(?GV!txyTuE5Xj!oy&jf3tUDj%JKmU@2HV7lXRKF}Mk5j7QwCA)aV-IoqFnI(5kUf% z&tRg&jh_N_;v`ftI}CX-m^KqF<7A-CHsjB2mSev1gA-}=`GPmmf(Q9M=e456qqge} z;LJTw1I@@5)18fxF0Y{*-(DB=c~ag(hVFe1tlV64(pR_ZP}xJq!`o)d!M1m4w{?*i zCy`1W$xkX&m`>r`ghab(7EW=h7iW+TB=)O;zPzkV2hUEg{B5N*VV&?~|r=u2Ik_hV_>E2w1kRE}-lD7Kb~={_9= z1%+siZU@)+CyD>^%g@`}w|zl4_oF`M0aCpmXRsec z&MRVG`EfWOY!#=Jq~xb~4_DHg9%Cp79dFN^-8}5<=$ZyQ_)UV}Vde6=RL5=7$B;y? zuB1@g=+|FQAIE9g^t`6~R~{B--SZU+#lgOr^2Ch`c--eSA_CDI50^S@^YNHq#`*9} zo;_PuEgYEvTt_z-D;=lzq)sq zESULwT%MP@)bpPwJ}17!W&9_;a{aHnrj*t$?T^f7$p**X!f}QMj9)JLsw06!yeZ>mgItok zE&rqKxM=q|-eTR(I9z6(WV74HTqc90gp?VRZXX8~DJyB?4wCl`u|Rlyx%0s{pkjk-ZtNn4=-*V@X;D_Vv5VaAz!#sD!&3A~`7R=P{*)Uod7zsbr z@&RTQ#6=5L!Q0r}g8uoJB;ObgQ7->+GhsqSR zQ)8tcLuy(+nDayba{jEB{$Lk*jXnK|ukq~t+p7uTE!83GrQv8*VmezLSvS$<)lZqx z9AHl3w_{Lpy&8Vu$G<1L6N4kg=iJ1e=zfU9mOvn(D2W#`nN51Az&`d@OPpFw#MhZ~ zShG5TE-q_OzXK{~scPWTITHYEQ*_}R6CCCbDajZG{p>fHcvIM z_TQLS{(R1@1}cGGuO#$5xafV_M^8mrY;{2OY`1j%cconDRbLUC+2|=xzrA@!;`t*& zh|$GDNDdVSH>%pdzYfHFgSf@(<0@Q{o2z;@9t>wMEaN*;w@vWnwS=4R8_15&EtaC1 z^=f;&^JUvVp@2UxMH${({PnP{z8l^^fHdYCv7u$~6}x`_5n51V@U6ySgGANf?84Y< zcfILSvow5KwYc`yg4o<#-+fnzIZJ5B$volqbj9^sjvx z7#d!H03|uhD^j!463}NUNN75GyL9(-G%yiEh`Wd$r8enQnQk=$M($}C8Q{MYy8dwM zs8Zaf=rjx+3-AS0yp}ecq0RG1JS~^C9Z4Z5&7|GSIWPnW;UCh8fB+B2+^RSwntS*pQI9aID_<07<+=d?bRQ~URTwfjN zaWfSm#3x^|-fkJ9K%cb^n+U72=XLnvDR>UjgX@ExoX963f(Y(yFGStti^igEJg9XK zt2j(K4JbgX7KEtA-+$^ljoG%?-e~`gKV--(Ov@e5JQMI%MmQar`6*Sr#qchm>a*fv zKkadV8Tg)ws5MUUWX8}NI`I%FY=zG+gL1g_w-|M{qLXa$z$$Ya+QXR!NX2a zY8z;`;ksS&Ndz-kAvHk9rsP4&9LAHrKH><~d_+_2^>?>0QCa%bB1@^R+-np}mge3(+L$`6SN29- zUO1TVXG6Nkr%1SPvLUeakF8zuWe5W5`ULhNMBS$uKWM1oPF(gJpITE0^9q1^!JA8 z_av48ash*pie*5wGiZw`67&qGesz$pXeA}QQ4gK0=6lE3_i|t%udD}@)ptD@?Vy|G8jY4ehlkNo76N>S$A`T=okiAZ zfPhi*n%eKo+DML<(EdZ4kj#te^eA|Qii#J>i>Vf;3LZKZOlWH2@9lKZNtYONA6XwW zS%Gw`d@jyrd2Bac#V%J8?@58!QK<=Jxrk~_aVF-_p3YzqMh1y&(8JCoe#dFf@}KN50}DoL_->>NbU9T`rE2anX@{hoSTJ9kdlArKM!AyQn;-MSGDh{YiOd zoe{i6ML{JL1+o$aq6ETdqph9Ni*)@Ji7>+>nV?b#P*_pXY#~o4C8ea?M=CkG%Z)m^sK21CW+HTthS zjW&ig3nF-r6URSzKIlgDy{(TzNA#pJ^Ivb`sybSJ5wRO34V{Yr=jMoI0?u?lFfrPG zZ{0V57DwDWD)x92o|i)RScufrJC8(X3woc9NN&x{^XA8By!=V4o@;QU&!?*9f4Ul> z{gp95m?6zq!%B+>2DxLl5Pu3UR!a#lAp}-=KMLxhg#Y-C|Drbg|L4182`RfTWHqmH z$<3KH2(J@|ujJ{l(i)F{czo{2%pwpHgiq7!q)-L$L38iiHg8;VSw{C4D5yXL{rDd} z0@)|Ow%JJIkA+;enPd-8H-Lc7xd>BaPI0v6huH+=RBV~03J-hRFGjLnxW9$h94SKl z&S_D^iGGJ9pRd+<%hMlV5l?oI&TM7x1){bDCiKIkOoB+UlGj|DKzE*c>Y*q` ztMS#OthgTNR+3R^vC9dk3e9pfYxOFS@;yn>FaU?;9F9VEeh?KWJ{d1LqVknzyY*7| z9>AY%;Sm++KD1}jEJ?lDFz+}v4A#K3@Z%5eK_dcslgn_99UtY9)&DWu`RK7t%%Qd5yPssg^VX-ZH0i>i|?bnc%)#;yZ9bX=p&ZR@j z3#TmK#bL-ANRDI#$et;8X8~%YW=d=w4~fHJ2`CQk#b_Av7PGOnd#<8EqRRnzmrFy8m+A%l z-pGAjdhg^hC4emOV)vEJbn&wkFD3L%(_gQdz-kQ5;#0`y1e8dkqvzJc{LP+Eh6Dws zQ2Ls%v_h-4s9^P{a7}?h*Q!t7CuRdU)^II`gs_A#A6Ow>54AAZsoUGsd;Yy2`?yLI zJ{M_FrjM7ryG8FyZ&OqD%cakZ4^giVPluvs2>;olPgo*YNhw1zo}>qsnvDw`PBD6l zciEDFRdfcF$@;J;6x(J07Xp4Luh6t9Ntqp<>*bDb>S^FR` z@mO)1>KL-Uemwd}kyVO_bw9p;iNd)lEGAl!U>QXi8i+kkKB>-?YRlzRiTsP#=dZWg zVM4(5$FC->W-wER7k*8zk(W?D=Ca$lsZ9)48RADz^n09;n^%g>4~(s0XkXFP|DD4c z02;3E+`Nw3p5uZER4j+dW|3{^deZ9kOxc4((#4dy)R{x-aJkX@=iy zsUI2nWl98cm+x@NQ+{ji@AJ?1YHiQ`7v$FCBkWdw-IattDeFM zor_q4=61-QPNyC3<KG))ctXq9>dulg0SL35KPfM8aW2!3jGS3ZceU+rdU!V&k3T zmYZMSlfe)KRNzJAm-U$v8B5nG*J?&#{p)aZ!n5pQ3;Wy)Cd^3;1S}7ewus+ znb3=WiZJVqY|XVkFtX^vM@V5W{-i*c+BwOsyRAOAzRBJ>1fKC{kND zYFnE8llaRnL*S}cc_mkbr1&&>($s7eI*ZSp4P_x+pOS>!kt{71fd{M&j^tpJV9dUx z7S+;^_jW@yJlA^K!1qAAMyFknB0>U>V^bLGZ=)v|%EI!%YvPvcB%xnOOOIjZBR2yg zp2_2^P$&o>6P*-hOLd(}-QXGLB1|9g%?fU9A`aha5?yJag^450Wn;U?a;pYc?qRbb zKxm!A#{G|WUbXP3^;n?4HC$tjf8{_gByWs5+p9@*t7)HBRe1Gt4RK@T8er4QbJkbY zR{kac`rJgV6B$Rn-F`?Tc_QB0N(FnRXqNW}YK1w83wxK>GI2~PLDhOXl-cG+@#aW=iwhR!4I-xyunLU zWvxTWq!BpnN3khEgq=dM#BP7jSU6b$G(EFY8e0004VpnyG~uvZf9-I?#NDSCoC3H< zn8+k$7CwUdQSm`6Vt6TEKXFP5(kcV83uUKziP1qURKyf9kQ)BI7KV_fLzc!TAj|mI zfY*sW(!6jcDVakuW1?|x1-Q27HnDAL7FeT&ftQDSdEWT1*7HTGziYFZx6~KQhwQnH zIJOIVbmnE9jT82IRS*DbcpWog$uYZ~4#Y5tm%m#F? zyuG4+#UUiFn1=SAz`GX2_C(0+Y zQ3e+4;l?gBAdIoR+FsGd8PB>yLvrR#aGDgyLsTTQr$ILgxx}sz#M~3o<9Ax^I$m$m}kWm(3Wfi$zsZO>KJ;KrT=6oXl zC)P786G|Viv7y(bs=7!%9(Rf`1c61i-fj6w>+yc29yn5ui@8dpuFNuM-87)gjFwKr zk6@0D(FeA6DzcYkdw7CbxOfmEm%LYqAvO@XS|Qym8!O_M4cjOVVii7b8)T@ln;Q2s z$6iY=x-gYj`S_`*q6FhbQE2n<0tQmB7?vmnwP+rqcp4#O3(BPgwAB)t?7WUGok0eG1e!Z5y5|0Id9j>W>Cr%!C zb`*50@DZANSBVfMNb)XBtO&@W5two60ZlzH4`U+PXgTHM$DyNNdz#|UR#(9w-}*xRDan}W%g4iqHDakD|Dm*3q2^9FESq`BDnjMx#n?! z$S6{B@D}CPDom9eR}%Q0uC6i+HZV3BI{ZH$!RC?GnqGu!S(xW2*jre*r*AVHiSr7? zm4(J$&&qGJysM|~ zJG%X%qMP*)5;AC*mbXm1-E-%!2&W+VodYLEm6klXW56g3(1*a7pyY(pDLh ztEyAo#=lB0aCst+D%z&>4 zGnMmuny!zF9Wwk!kF<0?tA;g;FY5~^ECZRRawy5buiDi^MMHLgYzDpD6N7Eb$V z!~n=VAXpmY@LuGubN8NaucVW?9GWSCxX~n#0>;tnV=8gTh_xO=%1539k2_kTaSd7d zZio_RmtCTtARo*H04`3!6)I3Bh+-VEW^PLNYrrye01z>68)nHhXtEAp*8~w=$O)ke z787g!QO5~l^f9pr19Tjsz6f7w#tHTyW^2PNnA(E1)7b)zSsM6Ip;3X0qY%RqcU5Q@ zB7uVqCOOn_M^)SN>Ot1V4AvMYJ&G&r%e&*U@4Q^3TPute)9P~OuI;jtx3fg}4X#be ze1!|v81xH~=yAr>V{7L@&75dVQ2;=);SE5+@akoN@l)C0(!%^~lk}8alo!bJ^#J?B z-o#Qb>^q=2L_aEgm*?N_ZriUs|larJ^a zm(4H-3|WXU4^lG6`5_NMacJP_xxx2-cRab;q&|IfM2Ov`quziB0>|PG3h=Q*-6eN%{fCkV>m%8h*57^u%>GqK9*drgf&` z7NXcks7Sff(8yINmJKq}!Qm<137v7+T^E15`& zJ{g)Y!*}RpFV-I~nLO=Vh-*ABm+MF&(jSwHue@QjUYmD)x~hc0!3U@+nXrqyffW|r zf7fM-!uMP-|V@PV5KSX@i^L#sFZ!Rr#rsjOBiBg4tbHvx(TOTuB9GaE+$s z2Y@~hRZ|K}E|8(%7jWh0g6`@sxkl)yQlwayg6XOcqWoi}25&5YhH)O#0Rf22{0r?m zJ<}1^awXx|ni%y6cu~tToqI%p!)Vw1`H~bV5p3$c^0TeM2+5VtYh1m;Jz5OmGy|c#mY+&~h;^ zUx-0#LxvI&Pt^}yDq*3Vx=h=>eO2CnC{)A3O=WePvKDpqgT`x-7rk$to-dH0Gcm~~ zr{jZ+2SVU5CIwn*QVO|I9$|fe$yt@SIX+AxHZ|fJBoIc^Vg?75Bj2O(AvtAn0sru1 z?x(?zG~=;p>;m(KczoyLGlsM0YqTBW{TO&$!DPKCu4N$g-ALd!eD38za25Pl9;c;s zn@}1r%q7wM`w(I%!9tk2h@2kKCRRM4f?|8E%CiMzqy7Y4U}$dQssObRwe#(J&cgs_ zte5!hm=OA6?o1$Su`lV1$$?KhaCQx=N;?w&L*Pl@AwsRf`W( zaRsjBMN^$Th^Z7~v1DElAmQR^l;AYlib;eItH4;TTPu-6$(hyDWbfUMiLr8o8`U>{ zTG3fM0|5|Hej`9m4Vox=E43CRUy1Uib$zRXU?sbCP>rU{O&QL-_5b zXIWrudSa_LE8F9@>~az|Z9V2?$6u(hQH;Ny7bfh!5ca|R`8=uAOM9J#H~Q;77NNy- z#a&Ox`EOxSGMNgEwY8}q@XF_}6@-n?*VB5Ran%lri)ejM9lfvyGnM(Yi|N=u9G{^F zT~deq&5=o>({nC`d~Ge)=qtH9tYvL zMP~R>&mxV0axetAc&mRFMFgcXFhu4Bn}`>;aW$iZ%euwos4}9;hr+N8yU`dKu%N9ZyCWK2TH|w2CRLMk18lWFeiiYX zFDZ?`oGlICu0fAL!!`4cUr;ThPNCWTy`uM3ZkLDA(6&Afo{8z-PB6gY02vdXwT{2C z(~Tcx)s3tKFM(9_1sKbk`{0~^v?*;F0whV3XQRY6xJ>R; z59*oy_-ld@*hcE3gayz`1*31^$a4INeUm{#lK9!s`aCH_snOphQ4ORc;}Rq`YJbnw zD6o;4Ua(}sX87{tBn5rXt4i9(Ln`R%hYqyGj(T0Hkl=almjLhKm;T9vNlajo8Wp=G zt5S$$ffOgglD7`hb4O5c1P8)dNTja-i$urno@r!pe|%0mj?3-2my+AGfpk5Xx7+ zBoT8irfTZ)xZ1w#X%nnk>iHeq0pxmbB3g+fIJDR+YBm)( zww0CPn{&NnkPT!<3Cz8N%F1%bkw~5G$W3AU#oYp z_a<-f<+|;lXKyDIgec3EdnkmSOG$BYtyMfPk|p!yn%WpK9B?56qERd}wi2;K_T68) zqLLYnXw= zCws5_ZE*Un(~2}2Bw$-M8#IKPfq6&}L?(lR05wdt$PCSD*C%t`*KwD-k}u%(>mm^JJW&)YuVOh@IbMBS zWoh(+voY8O2KW!}n=~l%C5F)^zOuNnfi1xSrXKS+7^;%cypY+@lC`$d`rSxWW+Tlr zjz|0#ds$QpW1NtdRIMswvRoX7|5zkkWQ>R%dtYWk#;T+Rd1AxEN%+S=PPT?9TDSKmCgi>-RF{N6jByT(7pHcB|fef0oaFfwS%^AkclENJ8I6nuk zhYIbo6I*=|-@{?VNB9%nR%5?5|KTv=6ybbw0yp z=?CQCGDU?o8q90$E46c*-Pa(?n^9l3clH>!A{vLl^qi@i9h|TU9xay^L(83$lVGYf zO{beBTy3ovf*9nl*S@8yueayPj)sR+5kDx1fa@Oz*&UH_3IaCYkD2#6_z=z|IPY40 zx()%vM8_%bsJE~W+7kf#nWXrCJLn(19xIQ3n$;(-lx+&aHL!jV0_kGraQzHa$F0=} zi!t-Xgv(YE!%opwzW*&~F`_aBwL*hrBjd`P)|IXFU*%(TN&L9A0xmetc8R7AXcd?ZygzY?%CxBcYR>tyAIq?+YzVd~;SXtRT7CeOY^M(7+K)SJ3KgRy+o;|k+TG3})RLv)m3R7Is1Cd%&fzFgZkEpX zwYQ+#SQdCuSZ<)@KJ@$7V{AO^c8=YPVf9ZHqr8t*zv5a`Y%QEto#aNG@U*{T5aI>G z>RrvV@1}iALXRG7wl8crKCa(red)IBY&4tq@dN-MSg+UbZJM30s0K8KaB^;ZgNGt@ zNoOY(iU1){@kHfM!tZm`3V#w+Ladi^q|#MoD~wE|#3J!I@l)@r^CEN8*(t{M^i3*kEa-m`$3#ytP1{YScKgxcX?a z=ntsKsA;Eh!nWec{%sms#?VPsC?9c*Saax2L1Z`=Vu@IviL&`q1I9iwR}ZaMz|aI}s)S={q-|AjP!T{=h~=7(6gokoTuPQw*uR$POnECQ(OHxBd&! zSxXqYu|P~6oqxTq>*5crzE0bfoA!>N-M1ueUPKgbp8$>Dc1@Gn=X}%};Bl?ydo(f}?37-uuj{j$QD4SXaGpfY#aNqnB_^dvw&@#XlO;o@7ECh?S zz8p|vb(Tx*sOInBWP42_9rsT?BpmI1kgKX~mL;BHjB17> zkClM{cb99BI$_Rn=}Id3HHO|m_ zBWkyACw;$k=C2)S_;q*t#X5zE6x|f!t}@<*&(N(rg5}f^l>IVrZI}T*U5~9c#_CEM zQ{q5;m32ymZ5=`7tV&<@*Quot$4X9TFz=P2u8J;#CmpSV z?}S~~6Sna%sEr3*t~Wc7xcmbwpc326j7c0PR=PDR66M7UvpA64UP#st4k8A83(6p< z+P({uaif%sYY-t~395PxG-Vz^;PAm0;2v@e7jQ`xkhXnPY!Yt8w0 z_a4X{{unN_$kuPh6SfpoT;I@IVR_20t_ktzFiwX?+Phhz|KU+}PVm}JozSli4&BRl z)jE&-crhnPw#wtjZ(;xXt8d2Nd2PK#2*O96<}LSsfwZ%6j)~#_dBbOrjjqTLA|{;b z1Xvsq;%xapr+Gcg^5OI$$g+BRf}Y4#V``qo$7K@fZ5Wc01f&x{dmH5$vL8v!KY8FO z*Va%r4a9M$r|z~7Gzt|CB>tf zM{XME$76!O5xz?Y6NQLv)q4K0e@}U&rs{sd!(tMGvFTBxKyJsve(ZwLM6^V%U7N2# zkRKq|l<9rp{UH)0BOf8MhV*L@mhG%5RSYRxwnL8WfGsSL6Psv(?Rj{KW^zL8*q-@a zl-Sjh8&0DOJX)Gn4N;}N%RjKHqCifz=d4iItjuovcP*-gOIzJzD;mSn$vD!%#WDVJ?kFI!G#trH3AmsE{!mZddN8r4IX43ZBPwTj{OsW*BZd$t*1JG|rv89xcr zw|m#S_+^YLo&kw#=IR{+8<(6i|BxIrU+I9%p`aK#bQnS6m6(_qDH(@1U=8IvgQjs} z79}5K$w7V%^q+b^M(@5~#->pxQ|?m_K)~U{8;W=g%VfA7|H#X<`RRx4k$8w-huDxW9W zQ(GQ{PtV|WeYbCir|h*bPpPy2H%#CD{eQtUG23!T%trz^^c!C!0`p&L&X1pr2&Ne< zbB9q#kFsO+-;?i@z8^}*Ly_ZQSKHpiv{;g6P|(zo1gJaut}=#aW=AoMZYUyRit7CR z{=JjIn3}i*9%)B(;s3?eJFr(8He156Z9D1M>e#kz+qSV|+qRRAjSf4uZ9Dn$p7YI| znfV9%+V#|3Rco!P#knaFAuVR4B)k+iiGv)s5@ND);izDO`K8AR%fSD|^JIe7Pp}T= z8_pB-&Buwyjv6i^dhlrHb7yiQ*bxE&6@-cWv`FckMJ#EnNli90Qq1X4<(2D4C5#cZ z%TgU^q7&4(A-vMN8Hu8f+ib~>1CLW$DHpW>+LymHUjaj^e8{=#I%j+R6CfT`0kNd* z9$Q}aV!B{`r$GZG;#X4*TSY97TO=|vc3OS?_CVSY5>H)X*mYI4XGp2-#1;gB?qoMf z+4C=}=?xeV8Ks_N(JmKi#?I@bER3%E6=O`uX|+UC&Ogs0BSEhcfau|7&wIz^Xa*Ah zLnk)hua}k>zoXR({*SNG6NehF?bP5LQ7m?-KHHyHP4PX%YF+51tFrd@$q$4RRsDZq zX-LW)U#n<&;#xjSys>WJ`4Csm~-0rnCjt={-8d{jlS7T66jf5qy85)nv zHXn|h5FL>S2l8GhUz3BAUBS~vEDo@~bs$SgfcRkp#zMm+G##!B=7*C-U}Ox!sHSIO zhxhn}d?uSG%Jxr?3rETFv}BMZHU#=QSX?!Rk%Mn<$nTFV1aTG8hb|{t0|MdCiRfwp$P{sDV%urE0dDV}~YvkQ%xr#GWivQZoe4)YHDy#UZ z(tRH$=R2>k+-#$bi^wFI@p!$A&T#AYm&$U(oORbE4FR#fZ{5%8fV{Uw@ZG4V9{(nY zC=!fuW_p>+c#+SUhsz=4*~ONC_7j29UfJnGC>4fyQ_jE2XA!@#>Y=KoF_ptTnk>K* zB@`$I@&&azsz}^T$g1~lgnH*kHo-G+ipMC&SHoYM{04-Cz0p>6Dc>%@BQ(VoPHtH^ z3yVHscGM|T;>+Z4L{YgO&j5kyPr}h=) z?tQAkWnst5QIdVbw*j$@V$7`Xm_O6iZ6~rN84;@L402fd8PD&42_xd_*h7bumH?tM zAv9XJ;`Axsj&QP?QP{d>Ts}i^ojWzW37()X&_|?mtf<&a$UXzIcO(!Er5_(%lJ~eu zrb?4&xt~6NWoy|SWUMEP3z%-5UF4VmKTdmeU^}%>)vS+gSzUg@2x^=297c1cwz0`L z|Di^+slb2{Z5Ig+E9+lre|D3xDO+b0Pzpd!#$Vy2bJ)0_)@cE!{#%CWV;?Na%{ zf9=3y_xJ(UBi?;qzLd9b1(eNoaIt2YMWhDI>khjqpL6#?u@Vu{0g`S5)IdxL$c#}i z*`{(HA0S}Pw18N!72`c(9Mfw3Tg;xU7kmVKY9SRD;Rk#-{wx0jdXaRAlEU5$9kQfx znYxwy-bLkdKPnR7Q;O4XVt7HvG1iv8yW`p;AwcyZ3KH6q)Vx3)lfZ$Pm`yB>KS z9=^RG4Ecj(aUP#AVy`nZsG58LK{W(40!29}AUOiTJV~NF z;3JxqM3?zAl)0D1Dvlz29Y>{j?B6pm`rna>JY1+!B~NLza(xB~E|-Szj4Kny`^6U5 zol&c&)8qY5zxqSDq-E-r1tM8rPu6T_dwHkyb=TcID<)R7CSQ; zBOWzUvPddWn8rT}tRn_HvXHrBLP-fjF;-UXqVAJBF_RrK!gSrotN(!`P-nVxLZiIO zJ@MI+7#e0%$U$GE=3dpLM6Wd|~DI5#rShy@|;~Pj9 z9~mgGj4mM1PZBtK{^?p*d%=zNsom@4v^xlx5^5ZYjQurad4Kc>k1%u3{OlmwttR3k zly@{R$F_A+rzfRw9{!E1FdO-hINDw*h3}OmA--sDL zNep(82K-7FcvT52XX{lI6uDh3N1nBkTg)aJ4@9^6r(Pb9Pc=~HVP8A43T)EhpGPh! z9shQH$;cAIU3i$aq|eoAd%K*j15?H9DKh+nYuX9dzw1K;hv3KAumh4?S{NX4dD#Hp#)H3AI39G%V3*N*S{-joQjgPAlqKjss za@6s<*l2fKj-NZf>(?gS`;!%3V7Lzx9M$KZK>OFjaYK5HH#-=y_p5$F;K`rigRu3Z z30;5krAx|qfh-;!?E>3rb4ZWOKkm9GTStO|EeYiX95@_ zG}45X)1vuafi}HK!5G#7csoVqH>ERR)Ef*;`TA*VP?;Gz;++--tcZZSKwv|>U& zcv`J?fCx%RgJvO$ruYH8WDIV@$Osd+r}MO$;en2GMV+J2aRZ^xVig^aOWpRv+I1vD zwhA)>R`uAl#-%Q76p_Ca?Y`j#@m@kAAiWB#)T)Lm$k`6N_U*&`yN+f%H$O8Gft-NS zYt`mG&++gf{F>Jzs* zZ@yopUpin?_@ZV@`r`d?HHw6f8gu;pj0dJRiS?nT{Nr8w_j>wtgZnkn4WEjQt=)DS zz;D<_5w1!Mdo^iO#}A|fL4+<)g0V>qsJgBc-^q^c>w$ka5@~t_3Wz`rOvlJRr|&S? zU=2|9~n7ydVB3UxWn8bMASg#t+w zXJ9297L6;H%nen^l@6l@H&5A8_u=U!5c=y$qXV(v4fS!uC{($!0SpwqM^kS!$KLJ(GbzThhoa2ru)^EEpQ~7(DeDtkotC-g zDI1^u2Hp=j*NP(-2Z>QZBQj()o<^Ol=v4})ITniRKM*}kNkc0j_6KwNU#mq556XDV zdV?3w7Kngrj*9dZ>!?&%Rpq%+S!qjWn9cQ-ug6n?`R&hklibP>D+f0wX)Qq{WS$q$ zH}+i~QEivI;t>O?;=|K=JU$DwD^{<@WOW3N!9+|XuqKyH zHC%;xtzBwFh8L>2c}s^n)3HHPmKzOftX-GSo^{HWD|^^egV7v^Prn;}DHUVL{38b6 z-&WKSIK#_A_iy{e$OeJxx_9_g)XUpkL`+Me=>407TvYI|Q7{`j0!BD=V%(YpETd*M zB^C8Rsp)xx=f0oG;U7b^sR+vjh_pCA6w+{gn4~}@jK$Wa!cF`dSX!Zke1@1GVI4PrapV1q;ZEPnD_r?qY&=;=iq4fm*$@grustkXUA5o| zaw35gLaHN9vb8#`$0%geL%KG6cs|dRfBEdo_}McbO{PcZVNRL2BJHl^{`t1Vo8*kB z;P=N?Ix~_5Uj5w9)%O{!34R+Io_ELTxBZLp1+T6-R2_az;5Ud3)QGp}K=c9)*;bvo z5$(lEry$P>)i;pI)8R+Y3oIY5O2Q8Qjn(L9)mu*I-ry%1U>(P*E zB5eO53y<~$7^**q5uq43iN9a$HA(+E*bK8ZS{vV|#Hg7<0lFzB-Jyv{<$)#E^% z`@D3&I}#Ovh5su;Ht)o_T!#zmx7lw%?ieJALtV^m$irG;-XU8Sg*wrR#uc7FcRz-A zH~g`l#$qi0cC^QnOmx^BUM|u-ZuM{-s5106rqgf6GY*Rz4%hdMfMrXVI+d@iE;FJR~PhAz;5>?}&($2$bvRX(l=64{6zqGJRl?SK)JKh!mk8=}+rxHTE-9%-P9dyw( zH0FnZ91CCm)sKu1Ev`ZaHV2h72h%x);O@xhX1^6$gUb@1p$Rf)AZj#U+x~N~$>9+_ z`BD&uP@Jjq8YP2y6z;kp;+63?#-QpN21E%qk%osAsr2T71Vs^$-XI86>CZ``iNV}% zQ_5+1(fOgQ*=kP?sA2@AYo?Y+AbW(G9v@mC6y@2mqrBu1Iawj*$9xyMpm1oxf#Al& zo1^?%UuLn@R!`hDHyiETOo>Nhd>pr-ZEm&3Hzp89t+=DY_Q!48_OHZqCcisw@e)jw zu^(bO>QbD4o{3fM+OIpIl{qG+D5hWx#_RFiRfUV}();i=m}##Tx&6>>FLI}K9*9|r z+e;zl+&g7x>Ep|M-fv=hi_PTavVU8B>cx!D$QU;VnutbW{^Q}TVTYd7mw^{#5hoB? zUH5UH#eLIL*vhF$YNm<2V&~*j4rWw3;GrAvP^VqGstRM0ulEp=Q zP7rGEW}YAuP&%JuEk=ds(N#)C=Oihd?bO$9wmAx@Y6eTx7jW@mRi5jjCzPR}0Wd4X zyMke#ir2flBSb~(!Yw2!ckcfQ*B=hWbUUgS5f~{gUwvb-_wS2`4t7Q7nYLr391k!! z@E8vO5UB)dZ$)4H(@)6C3;Q6IJ2;Mri2DPSqqQ001r6WPEOA5$r=;qz7!apKraCHR z#ZCt!rTP}1kjO$N;Q<9R=J&I-*Yo+Ijdp~Yzv?-4``SE%#dwG}-jDjs7dTVqKdomT z)I|St{6|GdN|l0{{YPlk|nESfo3=*y&6}*2xnoNq7sp5W!5127Y&L=y$RnD)W9MIJrvXavlEfASH^bqgKy`}zkl5dAKV7-ub?J` zli~V`3VX^sg|)MlUutt<8OwUBmu}6MvtTvaqk~2ua1h8(5;Qt?h*HrTGR-uT^g>HL zI%w#B6jUzEqyXa=gHQ|NwaaD#*zloZRn~zpmTQk4MVP{uZXtfil z>`Ayna~oIC`uHBH=^+@u&~6l{;t!7wj0D01S6m!O`LOwsQAg#6Fj;y8>h8Ubmg1V~ zJ-&>(-d8)vW~N3@4BHH6VtHqQ&T25X2NojriEU?aQrU6ZCH*2>NZSw3kXITlVYyt- zL+boLqj=rU*^01deVKCB*Ky@$-7jvv^8!XM`(Na@q5rU7{zSTzKqFheSNUx(vJU{f zdKg@XeRn>|kgzZmIK9c8{UB_*Rp(#eXQcgb!i#i&Gz~`+pR)%M{)YAm6;O-*M+}Pf zQ8q4sAW#bPj*5KJ0W=7?V2wdoB_ZQpa!9 z@8L1xqIl08B#dh9z9qlwv3JIYk%IP%O6k=KO8DCv%&mb3@cGqi`OR-MH9GAWQCv(x zo=DNp-t+I_ZqjQ}KRsZ2d>?3TyZ2Q57l$)Q@d~Ta9vXemJM?P(CSVo}8N{`febzu) z1#;Nz=>`Ke1e50ZM$z8goqjsb@4_nyE0wuFmZPaTEA?i$Z+Z@FnQaG8FN@$N>t9>^ zr|-+(8_@YVXQkz?DLIG%G9p10F;z)~fVbjDsQ+ZULnboRNWbcj)+?wQbk$PN@om?Y z3uF<_gudmr0WPxf5^Bsu2y%e=DpliFURHI%g>}gqP)3vb9#F4PHCi7w0Wlj8McaSw zXtdsQx$#qHqG-#T?G1s%=D?F)23mON4}&4+a|ENTFAe>;f60 zUFxqwx_Vd_9%dr~S|Mc{T9Ce36mH4ye*hT@7Nty=zH;%cq1UvhFu}`;k>BHE%CM^p zepo+b^yASW*>f0hgO8}ptj-#3Od#!N?SFWrosw z{p}x?Dg4mb<%Q9x5P7q+1f$_##EtE*qBy=n`uw%?YDwo`?>StN)13Y2zO?@dZtu)n zPJh~z-QdsLI%sPwwm*kSNGGqT)5#ITE582Gaeuo$xqUlbX5X}Z-D3$SB`y7*=_A9) zv(fg*o#l3e^gsilwD@y-sEzR^QTbh3#^aOMhxgUGdbNp5B@~ccTWQBykt8bv+9bdA zZL?;gGzKEyNNfxKbh!yrHv^x=24c(oAo5pjNF05=?`p~?1Cph3tDNpd)fBj0SXLG8 zi_%w~7sLzfXA;R@GP-uxo3`@_w zIj9Yq^SJX)o}hnyDGi>uyMwK8l@T+2Kby(>{4cppyyA$?H|AK!r#mZV%~s!`1Q((_z)>iuO*ZprXL@eF+~p0fb2= zKQZB=0W%2`6@;Rqn#iaIQl3GuMSOyHJCtdOUvlzmLXsv((n0tpW#6f~hAvJdyIB#m zY?jN=Sorto51)NffD>D<#bCRs2lZ4!>55|rycjQ9Nlm=Ej4J0@?|O|=Z2N|+`I|fa zZ_$aoKw9eZ-oSlbC@D2Hky>#8I7kMB+ecEkkPMPY)kU_Ec+$wYyqTF<5BA7kPF?V5 z-~LRmkL%F3o5kWf#?$x6_MHJ(jX%d@RpPtrztKNO_5AkGHEJ+;zW(jQt>}3ifLF_} z2k57Lc`1bgmPFW03A;a;q{EB1bnX4`{M&c-VC6b=l=l2yZ!u}N|L~}U(gF;5b``Mg zeP0%13zj!05BY$&TR($ct-pe*JyyeeW_}GCMfQoN!3p~aeclPy_LN(%KGl4eM|PC) z%NC6ruw}wycCtc z1$h(cwSK3++23-v%!DT(_KfuYL^z<0+9Rftj4#jc#cgCn?d6REkp1xC((^rcwvl~@ zo|_we{w!wd$$NraYq5=$yPV4Yrz@&Oyx(ace#+`2(K19MRBYisNNTof&4DsW+0svhh3jEHn-C4bUwh7*Rqm63C4+OAy!#}l zk`2(ih5RUuJy}F$!g z^C(XVpCGF`95ltxl9PiB+W%>IZ?>E-Z{${pI`~`ng8AZ25>O|113?KYaF=p4piv=I z3OsMSycO%~{bV>d(wk1wVzFgpv(=3K%rEDk7chm(vHx?7Wgt;D8d;w|7kITT&p-UO z4j{g^)vQcyvMv8gP{;R(3SGY zO=2^x&$Dvg*64N{0rog%HB!<>Ab(C~b!C3I|Fg!-keA+mC@C}ag#e(OB#(c1lH z7O+M6k?_sb&>f=O4HmB|7koDyJW-GYZO~Jw@4EwAZA5G>=*o^NMUq$hihjG$lPAFy zAu?`c&Pa5mih^mS2v{P-f=D47NVSW`)?=~YOt$6u0RX=GkzOu>pJD>)Uz;(ybT=71 zvv)_FR%veboi-&hMZ;40+V3+P?qBjXgxpP9(c>GtGO#U(V0>7S+?h#TOHBbbphBSVUBZq-^GgTq&fs_zbfcpP&iz$>A_B zAQQZChtpWs>w(L$$@h}XpncRYQ?1#Gx{OGutt?PF6~2L95lLGj;r1Igxe*Phw<7vI zE|&{splo=x`TBm>mq!yOg8FCZ5mT z#{No@dXP^U2#az7P+MM{cqxG%`s~T`7!>N+uXN&f^5oPv#UmwwP81)>!~rE#+3Kqp zd4P-%9E<@QOhhz5pp5|2aLjk6Nb&p#2^>_U@pm!FZ*yaq^$xdfoavpYg*nscF|||S|0zr2IyyR`?+1#6>j%Xev5kice;= zXpx$6EIXpskJ&x7>Rxt;091QS~4 zjO|5xqu(2tPCM=-X<$HkaqfE@pTR$4mMk;9@Xi}OXd^7=jsFim$cV<*zN|(kE`4l? zGrobpnm#%u7wZJh02JsIx>trT%|}qKW?RRN>f`Sg zm}~hDV0Qq{L_l?UJKD9+4g8Y*M(po-ifjdPT&@pA5m8Ztv!OI0Io)be0WtsY6K%)0 z>w~+qe8%qDH^zR^{LH|~l1c--;t5@pj457snc`82Rl}Ir7985vIxD3DE_y3^0QwT% z*#dTc!fF`gtM{_ezjp6>x=$phzhq|3g47xn;R~bzQ8rLx<+5y_{1hWm5RqmPdW2E1 zzLZWcb$RXn>1{mO2a0mL7Bun%p6a#@zn+&7%koqD*MGzZf!lrUpF6H&D zhKUY9e)lkaY(Qef(8b?Ohw=qp%|5i+UvwCBxuGcbz&4IzBJ0YIe#hFgouei7>1_P5 zQ4cbyj}@AQ^)1c{D$+%@xa1NAFaDsj>mL|>dcy3?lM;{3n`z&%`=*y zCy3i`@vG;McE*I*+POHAahT{B1@`CnOkeZ2hX`@~U9xhZV<3wSNr5Yh0%{8N=3}m; z@-?yG)34zSW~2>%_n8pPnIMbWb5rLmKzfK2v2T1o6-5B-40x7L;%fff&+3~-4c6w` z^#>A28$ShEEz|!3bP{^NEC1?_P#lc=;+8sFz5MnCx3uQi-&W<%LCQVp3HbsyTbaEM zZ*4Fl6$X zQNrd=gjIfahQS+S-m}r&9+SDtpSJ&H@}kC!Z|wK*px15u5k6!IBeCjM>1f~qC9ty) zG;ZKP{&2u7-ar241%&$3jqak@{S1wfHy)C>s5{|7%$fml_`mvWFbvsYWWev=(9lO- ze8%iHjqp0L1?71>Eq3gIFc#S2yA41b(q%#pBB!4;Xxmma_}KBC=}APmmt~50mq~c8|Xtj%ud! z`Ey+$T-{(zy5R_jouy-_)3M!}l(F)E-gP>2Jx$c%6Hc<5kL1;BC(VGx&#?)g z)*m1&(x!Kuz5l2C%oiuxK>lx3FXu(Hq}TsJ^!m#{|Gp9gdlPfESJ7C{#0<}W`V9;7 z@!Cx#hjTHE?V1%WnBh7caog{sSh-Lpd$Jm@dc7SUm)%x8+>LIoK^Hdl2+M=N0N&w6 zf*?^#FHF?=m!UzgT|a%wrTfqC3|H2nQVyMfs$2^SvtvF6Av#X%TpXWIs{yl!OJ4T~ zZ>IJ=t`y5(3k2*0EGIak!a!ij5Gz?vOyKfYO-I>~>pU7kpj&C8DbGpdDJf%gOcd|I zEJi`YeB_eYd>S+N;P4Zi{x)@B`9=bv_Cvi+>L(zrLwFidTBPu*-5?U_&iLgnh6?s$ zb07Gw`Wm*+8HUHvwro02@%>(U5DFNnhHt0Egn~ zMKYWYj2+C2AfQ|I^AcRRqM^iF25-6UZ(`l=Y@TeTb)m|2*XTD9#KmBT^UojkTO6G2 zpW2`$A*;s-Tz~oAY$N?cKg})!HUTq!fTT)Cl)nF;Lo+VZ|No&Gx=8}}Z3`#92MBMS z^e1kijNY?0E5o1qB=&;~0rP_NixY_u6}eddNq|3M`TZi9?K2a-upS-yYxU4ZZCMO7 z9b^ad5u#U#t&CbgjGSTwOOI3x_p@fXC*1cc@(0$NH(uH2b~nveZL_|;?UdikU>#nve6kU>SHkmmT%V=3k-4PI%n6-E3z|kX)YT z!I=N_<^w4Hq(b{a7Yc2Tv0eDGt&2*p{yMxgZ&^O6Re)T0uJiiOqWM!AvqIt0u!??( z8DXWQTcAMkqKIM*_8iLRd@_x=Bb( z%`Isi#d}GUqJHb~3pdo3=b+JF4LB@op)|I5)9Ie!Kq6dl^%^3ax_+sxSe7UT=(e17 zuKpVjEB6wovzgk{^T<(9iaWCP9jD!38fk?;_eW$;!386Cd{@hBr_!Q_X?liw2BUW$ zjt4AKi98bL3SQ|>77t-os@)JwY-}_x#>TLsMxG(Es3;Y1A0*4-j5|H1LvO?e^Pp9BSs)XK#U`6 zp)uIb{ViL^KS=%527+0OzMld6*PNJunBDL%@TN*2>`vTdQ?nJr z*z=ey`*u8iS+B@IMT4{5>JDjvM?L2<`9AbCf^C9P2FVlWIa`fRLmCL3S$KX~U)0-2 zBZpyEM5C>s6dPb~_9>yJg(BVT)1HQB(W1F3aqq785mLd;ZHwi8OAt+iO@4UEWwuRP z@cBfs_pF*v^-hGiZUWF4z6Z1stFLtU_ZXG@iqVk=W;Njv>6|gw_?`|T1LFF*wtab}zZhRtq%f2`w7B#~q0RL~^?zW)Pm*$p*)YZLAWy$}nFHDPuJ7p<%M!4?$8U zqmAJD6FNhINRh@SQJq2MP}LNwyViOBijkG@VK4g}|F2 z96l%70@9mE<9DQ-Uc7n$?AoI-?_VzWG6wdl)rBKeEW{XzTf2ha@f{gZps_weJ9|{d1KLPvx!Q9 zmAjb!;e6$Z&m9m_KlpxIJe%RXvjzgfh#nosn4(MWpg^{hd{=K^fUJz0a0Pl`CKpy* zumg@IqJadW?AQZrf&NtL8)VBIF7hW2p#7&CfPdGunm3o4BPAlWQ+T8=wX3MaF2Err z7h+pu^?!zmFpnQ1iiP9i3_yxkgeek@N3x_6ZBB=8vc!C<*$T?@|G=lguorVHzyx35 z|GBL6$?UB701IM-`g3l9u226+%&&k@lmJ{0eT5Z0+RU7i<9UY>&5*g!a-P23Z*u;` z{V)`;mOjs`++-oEqfbgFhn&sG{Uu`PXuJ~QhGbw`UrK5B(R65ja4;>b6HF^1hKXPf znlONFv24SMf||*Qv=0~-jHU_Ps1N>(Tig!Rd`AtsMd96I&CG9yvIDRw#{JFJMV9xY-T=8>4rsIAm62kv8gX2e-66#IR{i_&J+O|3qgxN`R z^1mrPbx8ao?CnW$8=Inf5KSl!&g6)1{uZbg3-OeEb%zmmGSqO(Gup6<=>&5hrMDlf zabXuqCAAcDsU-Bqvm}Q@L6fMC;n^~1wZdx51#-DawNW`}^%N)rLIu|E6ATma!f}J` zX+k@1_f`E@A(=Ba2}x`%aoPI}_q9K|z7Jb8ew)LShz@H&n%b| z4eytOdu&RcdPRz;XWoxskSKrzEwMhg3o(HueQ+k(ZZa%V1wTD- zW-E2UC~;IENT69J741^{MXbOdp&uek`wp$40Ye?hgwBiMj!-*sMZj|#htmc*$986J-bikzCRlva|4U!ANm<*G2Nr#gcRJb!e{+yrT?M{}(o zqu5SX+r(4i>LmXU#!uZdXR|u*t8b@j)L?vIdj>7u^7jQ|RTyL0J#$`#aYbEFs+0@+ zgDSVOBTv?}c9DQ1J2-*>qor4%bm>C)NdP1}{{7Tf!wkc33Oz94Od(OZ*1%IO6CN~+ zzql_v6($@Y^_Bafpp%t%N>*X)T$IjnayicNd8?W3rY!aN*OJKU7by&eA zG%tBU;JtD~!=N03AL2?RE20I&_U%yFToK3L4^Sk@X1z~;3X9gTY>_a%|OJ^bi}+_HlMxR z_j-*XvW;06rPDWuu&X&4?#Gkfo!4F7*Bv60)uF1%+^Q(I3IxS|G*z_Jo+fORm$8*l z!1>QBm*sz}(NI~ybY#X{c%0%sl_i2RLP73)n6IXG$pFXp;|+dyPhohr?VJr6JV-C%H84yC%DGDSG9+g;SZ#+;WyRv zI3_`wsK7;|azv-l?ZJFW8|lXvSHY;WC3YeuPHm2_)9^aIxR8Uvql(-Iq6_6~Fbb9_ zPODC~KqL)~eUQNsP0U;rMYd?lRQlA_E9@1t7y^oM171@*(R=bVRen4SD55H{}HT6oqz^gE0yu@!;o+ za7Uih&acynM=AH;v9C3&?NK7DskW#y6Tj*X+~>BJuFZXBykPAnj4xFoUBvJkNnBtK zEU2s{(g#LI&ktUWt)*3HuQhVp=4{07pM)K+w?IR5H;j_d60D5oe}y&H52{T`{w3)~ zCiZsUYHW@l&QqTn%s0<#wkCFgU9Z{x`ICF2o5cOx#)L6cDw<9OWi}jvD%|id=HyhK z>+IN#S-A5eHN=7P9Yn1}1}Fgub~KVdwms;lff4018G}=fF6q+HtEaeRlkD5vGh{*F)XMVa^p{>-P%9ENL0Med z344)&zrqa~LUxr7Ky_f}2erS^&Qx)OoDQwV4uRXF$o^XFM>;!k7V1C*ps!O1i4|n& zSSFa(XHX{3g;1#aL<7Pn=aSY|KkqZ+6F%SFzawvM?k(mks3yXaWwrm6+^0F7t%x?; zt-~%_Ie325XfMGcW;Ai=PO9r5J0vThsb@E8wXqC7oP^D;Z6+up4P*u=rmKk|X1Lxp zJihaTP~hKy;uV5ruXI&7O>8frPd{X}*1@EHi&iSH?YhCjq-m znFInNs0gyNy~ZRK277%Vn=M~CzqHzh%vfa_>A`^yxecZoSFtv}E5s*XAD?ruI6vr~ z-mL53M)nZLZMJH4Cxpq)=6y+Lw^|Y2(JHVb5 znD)Mu=&v7FbB2)*B9nchiRTEcEJ_)aHAKN3}up6>&_(yiSdf*$JW@f!dr}9ITGo-1bTOgioA-41(2TO z5-Y)HiHbkeRP1U~g524@mXI4SGzA4uL4?j&K^Uz`f_FeVXE+Si$+tq0=zcYfX0t_E2mGM@ZpAvVXk#wY*I(LdfU!%w!$qNiLF zF)vFTn~?EQ!>6nxn|RhDbm%nSSTOvacu5Tw2&z=E@z*&9lOu%{JB&KSAuiY4@p8NY z#>}svZ)<#RW_`y&@d?LiUB*(RI^(#hwly5Mtunwj2_IaZLtz+B1+^cXGdWH&X()}# zD&c?xFzOQ<6P_S&oQcLq+He3e3yk%F9uxgV|KWOuVXB0kc2~4rI#_ zU*T2Q;Ke=X2Z&UCy&zZeQhEeT8vDTgiB$E+D{W6tRus1w%e_a&!Hq;xXfkGn8cte= z`D%4CcWG*V$+rZKO63}q?jOELq)qsZ^+wFa%m;y2=%fO8XtN-~#z_*?yPdU9H1>25 zj|(BCMusjGy8Kp!FAWfve-Q1`;s#73`)l1AI_=n?#V#U6XM+j_I6xIl%lnES!Q{sfmv@y=V8&uC&skhLO( z$$@JJy|+H4e_RhkV|BtIB{55uC(LKFX5RjwXqDsZ!gaY~jZkUXNtym901=~BWazNB zKznMPrifB?~s4;Wr@_pcd{P^ZHq z2=;vvW>VE*-Tss=W3*|>-pjiUPNA`_oKCcN#JrJp``A9nXtn9COqp z!R0xizjnoU-JbE_AfNhAZ~?x{cKhwq5o75dIx!IgQzs&1XEBUf?Ho~e%gH=!N96OB zyNBojQjfvpkL~|U3&55C*O{S6Gnm<<-j~j<<_n`oHM6g6n=;vU6HQAVF;|*mo+Oo| z%nRtnM#Ra-xV^$kh*XlX@|l>;lx5NgtUcN%&-}^Wql0r{ox5WN2$DuROL7Wp!DH z+wX)B>}6@BrXE)g{jnYs^ELAL7Z6O-MK|bQbX{-|5MXYkF?yMqSgMQbJtE>Jf1`JA zvsB?y%JzRU`hOp%FY;P$p2MAs#6o zsqV#|J@&U=fFdQSUffn`!@)RhrvAKz%X)Y}rExZTvr&f>mm#loy8b3Jd6K)9;74g= z8XL|(6%;Ba4HnfuCfZ>Z5(C@yJFwN5G;7aLiZ@Z{-yB7xj=?dD2DWO+KxURagoxIn zL&bq1JIE!7UJX(y0;Wic^z@OL?K4sAw_tWO6;Y-}K`k_tkqjD<1(iI9bx+~uVMF0z zt(Yw%8~i*%f7$8)loNghsMIiDrBs7d$ZDS$j^uE2G7{XuaEUzTkmcMyCP8?C*|n*nqEBnf?x z;|tqiTh=TOOtw2UHNcYSh{3Ps9+JoT9ArBSQ{L?!q|iMc5VGez0ZyiscMhUr^(LXH zt)qa=@uA5|N+2r!#tj}lF#&6KVhLwLJ2TQNtsuoVDd=5d=V`XYvfn05Ov_%+gqbbw zlU|zqbsHf{B~?F(S=@~=4+Zv*&p~uECp#O?Q_rFs2&pl9rx-vrK^%>+Z^S@MYSePq z36I4VxGdslekL*L=UuH!P}%+>^5AF3JQCxD5&X9L%hukSIN5lq;9aBZ`H2&bAil}- zE;4=&-jwcl@)N(OEgcYE0y-wuo#+Cd^UKB5C(tr7vqpZWF9JprFqOL<2^SZ_S4ySu z`6;wTH&KVDP?}YcT$!*^B`}p2FQ9@7;Pv9=AsB?5;C@|C=vzrZQHgIhJ7~c!NTzD7 z3{CI}(O;!P#KXH$uxL@BmI35Snb^{qFX;!7d&bPT3sP8DCzujd(eA*3r+&TN|A_nL zQT<`TTK&lC)AiY|7LJ!I5t1BA*{C6!%P($iLclYDYivZBTqSle1I8f@B19{0AlDPF z@`wuMS&a~CpPEb-iRO~MNqc;q?fjGt?$*c)^`aFBxIs`-9 zT~PcgXzB?m(ij0S`E|IL($-AyMR){(8q|Sjx*jl*$#l8!(SvuxfQQmu>*r{+UcVrz(< z^e#T5#Pvgm;V+MS09`~rADYjP)`ZbkBtjbKx#YZ5P z1nlT|8xbijP?>8Y%vxPem_Q=+vS2YDnFUbEAxK<`hV}y^rzp&n3-{;_@iGl6fT0b= zP08w0q5!k|0=~I@3QbGH^!YH}J;DXHx8;8Ed{pv!Gmi;aCdiif?IHRzn%T=Is3jpK6)^^*Q#1IYu2nPcM$>Lu&dv2+k?6D z!v-{0nE|tm1{i2U{m97^xaN0yk{p(X)`bhM)WSq4J-JhWP=Q{i3AfnyS2F{}VK@^2 zI5T4Kxy#_k;c}LW7F&BJCP)Z*y~-MLRNGo=rX`S&sa#7Ie*!Y7b!F}Q$i;uVgqY-! z`9r;J|2=~AI+n;`lY_-UbhNHlEni^6|HL#uWzTb+(4&bJQ&eOv8@99_-Tbtw5>cD( zOuY!9aD=4#LLPe#t-9z>fpGzYACK(j+Vq%U_^USQFm2_>U(1i2W zuk6GAfbS*kFDP)Lp0^Rgux)4~2l01u&BFi10r+{0jtfJFd)a7FG>5KhUu^o|w^HSP za*81kn$`;)|3XA8K76`CKU0%Hp11clJ;b*Ec8Ch34r8rcZH%HO3iRhH23wcsUfQagOuN<>!6{YBjRw#Ax}k9lUmZf(HWj z9W$P~OX*Qa&P|bU&P|rS0Xq!U@4upEJq*a-d!&d(d=c=fz5%pyp}wyuN~G(;g#la3 zwVdZTr~~oG8N6T8#CN5cRl(hYq3>RBX_MLEX>GX0>Pc#gb6AnKUWT(jO|P%Ng=fLn zpF67BG}sH<#u74ohbQDh>o)&nuGaMpuX62E-%}O_pWkdu=<32YKr6J$%SN+m1wqSh zK|EuY8vNkE#%30&?llx6`jpMe!#T4O(%Le}p`jUm8L1};SI!lG>YhDZbOCS?s)!G} zx@v*&lXXH*`&4v(qcJ#~Z?r;(xvefOepJIkLTY6b)*g0$iX#WzJ0u@cqoX4rz}N%G z+wHW=5P+#xc!(J6>b6CE)xRK+HL-lmec-yzpj%=RF!Yv#j%m2`nJMv-^v7&{6D5VJ zm`hSrOWNr+=!D`im`Xf{Pkd@&9Vwu5x=QJYP z>D>^m@p^;vhIs@AywI%8L((l((-*9nIlsS3!N6#VAAM$}q8S1J*pWucaq(ja-HEhu zzS|L8dOM)K`rX0BnMCxj^kF9Tj&(~IV2T8aBd`#B{kKO=u0y{}zlEt?NGPKhRI3*Y z!ZPi@rL-`#4&!jdrKEDBU_u;nro48Nqy^{FO_XyU`!&R_O%0ENBN59dZl z!rqdsZNH5l-aKL8z99fDdCdw6!LXei-I)Oj3=!p=k$``x$OpK>Fd^|UIkx&^sYq>x z=rA!5DCDuBhxacsLSg4z(ms@b9emgJP?*f!fr+#+s5TpRHDckLf z5vKPkp`jV=CBwulWZf-X`RKdCD$P@1=eRDl7>&!GQ#)h#!jw-5)|3kgzK*>*O^Ecq zU%5vRcEk7kScZrN1?#vnd>vbmyi@-1*;S?Ys1JVY=mq~+#2B$P8s6tf^% zt;Iv=6&eMBCBJXQj@M<+o2!yc3$S0wTd(ir{rK23)C6-pUlCSvXIBVp;kdlJ2@fx& zWzy$8oCy5^N0UHN{B3``TSxliE*B1$XN;Lc#`VbPig> zR4{G^j)vdHDKy96p4G-HZC2kq-5%a?+@HTHfIo0rmktDOhVaRT{4HP8DtLx1w)Hw#G9s=h%(Azx z*hEC|cI}cPJFH4XERk;`6Lp2`Fz-(VgPTF40Lq^d3)9WMfCVx)$_5EUHd%d`Z(QP= z04#QAXC0vkl)mpbcxtspqV4nNmq>K57aaR}+9w!pAEU-%5iQd~+K0EXKB}wrY}a*{$>(h2={P7bGhav&Vu>Qk57T zogbw9#{z7@Fh%JlsuWxLKxtAAz1kn%m}*RuzlQw=K6Z>bS&mpV7>bYCU$WXwIF{to zg3^ZE`hEESasda|&8`KbNb|#*(<^!(Aruaj269)MC}>0?rFdpB=5aY0%_g9rBZYs; zdYeS&xhhRN9ZI73Zqui$Yvd^?e~a%Tn`ih$g>%G$$<0;sY$174CRPvYmA~W|Okd8- zmGiN^7(0Q`;SZbEYo4dT5>^r)E8TI-oUf%IE2sRG1wbl?h?bC!4omVD7cH$01c#{e zOXN!nN5O_apEf!r=jF+}dC}jg_ zY~{ScF0A*L8$g--aPf|{y?roKqM?!Ui9V|_O5uQYcfKatdc6_v=fc+Yqd`*z9x2!w z-5_(miliQvIO#w(s-@F_?UFAp__9*Zlm;PVmo@=k%^e-gU3 zWGpxPP!^E&*Fnv@D{8Ms>k`fIWJ*YJsYK7>WPaYNj>NxQn^U|ES)~;p5TGwf_KLOr zIOi?TyXJY=q_u?RnAIuk6y(O>3@vnbJWXcF=i z>d9t_=tQyb502aCzi*mJ0!hlHKfS*B=Pi|T_~p;mQmjH!f89RH`TDC=rDtdYji#!W zPOTGjE#ND=>7JXj7gp`VWV7Z?NCLfQu0SCAk_4-mlkAt$aJ1ujr#QlfW086$D=ZFp zHmm|t)Zp*Ym0uI~p@i5q$+yg4{?Q*0xS`{6Vo^3o)KDzcpi;-f&n3>1f2ntUKsK1? z9EL)E(T`}nVOXU6p@}{Vvm?6tl^MXZJ$<97yzqgFodQ5D8v>dy6)Wx(t9cHe0>fh9 z8)Tr-3-bB89|YFQ(7M;Sfpo*1tsOxPlIwJA} zh-w<<#}m_zBJK9UQEc18duX^7CMros9CE4ubnCVxm%}Vhr^tJIGp7j`WvD;NTA;Z+ zQ~~9?#PWBAYTb5p!&aE2WHYc+hdef0-zVeag@r!S^$!jfDjv;(oi`-Di&rs;cH>7K z*tvlgGXy#off%(4cpn2J=53!F5A$crKC`87pPz>)8|7MTs1@Tt1GQI`3JI_ecNfQH z@92@#=E|sd*Ox&I?N7qNk4%cCzk+asy&P`y(MG^4a??VJA0VA2Qn?+`@+gCauT`{z z4z^UyBE-lqFr3Nz|7Z;&vZf^`;QYTQO_Orc}eaHI6{fTo>0H!^GHc4=jvS;mB?Q?J}et|8e$f1S~MO!A<8u9Wv9J6`J=7k zJw3@w+E^%Q{h%NYQ7=Oz<@v#2BMIW4Jl zMy*^#bUR8_9+mxuKVC^_ocGUNMg+w2-W1xowXaC@=0B5MMY0E=3f15xY>}v}x6jXD zbz|6$J!F-lWPtYAW`(_%TY-FL1?LoicM)|gOjE8>VDVjV?^Gu^14={CbMLjtAPYSm z;*M4;ROaWhUj)zkJ$-d!b(gcfC1H!|&`CkmyQ_V_=`F;hjXl&CZSeTv8mQS;&DOAo z9bz+AGo9{G>g0W>9vcGjYSR~}Kd-0vEj`Nfo$5gkA}{h)VxhMWa(eC&4bre0IvfwO z0;(ELObg-+LJ0PAt|7$y;-(4|Ih5Qk55!|%+-?L5)E9q=JP34-M2Wve>v;cr8OmL4 zHM&gTcJs{U=}1+h)@MLD)#zTh%*)KCe0R=H%FYuAcSWIyDw*8``$p`lT1M|g;=U2% zA;~5;Y>8u+-vayRu%QYEwPvbq_x)z7>B4n!L`1}AP|R^EeQP|`br1CeeVQU{SFr8j z`>AdWAusU;sqG4nYwk==Sc|ZG% z7g);5dx;)8P7mHQfmM5<&C=~g=?k*)19?-$opr@Ph795A0ku1eHeq5zAMXA6=&mxB`@m@h30e2BSpX$AP&BO z+O-M;@1=@qaAk_OGzs5e8N-A|Jql1tNRxYhz_Y1F{_8^ncn4MIwSiHCe{ z7HlTFBJ;gJg#N9)m%zUZHK72sMnPN6X&m zDdqhShMCL=j6Jp_p))!QDlgyc8v4l7vv8ezGX*%);}U0IHFI!osY zaI~@*oM03U7C0ARc*)p0sRV+)5@669(>-|O)gR6ao^|&nP=>S~0{}EKt6Xn#|{rl26byu1yM(~V@ zjXe=fYE1%`>;ma<(T-|Rkr-Ljgm=Y;mJz5vneH#tvcb)is=KD3|79B9;Ij|JBkbDH zy@Ro}`U5kU?L5wgu}DIMb&an*5`D7U0sfKZJWUHjQN^cnP|rvXA0aIkpU<12og@pM z!%U%C13AF&qDX;*6;V%dykX-l-13a1&UkT(kvCuS$W|&bfxAL;*6ATus}MT7KK4mu z0{nn$1|Hc(ncYr)RzwfBfjAz{k7k50Y&=dj2lDK8NP*XS3fHI|E+{nD+W=@9>y&P% zHlun{BQOsIzZ3wzv|O~;xJ>Zo7IW77%c$U!qgTg@f$6EW-~uW1;9pAi!J5yOhGLE)Dsa_a$Bjq4m}u(cG9U z8bTCpC3+}YlTn0GnaQWQcmy-j+dmGP_P$V?pLsI7CDIS$Y`SpOJq5XZFa>926!ZyA zc@z{jTz@@}6p$jpHFu<&ITSyq*P>2DkgWYDR`(}C?5}5JFte(_QH8K_%b#oSxLoVM z)cUD8yF=LFrq$u{n(jgq_!peEG75u8vKHw&-XCctqEG;ZZ}YeMAdU=X?5|~9&oF!_ z!M%vaNp4Mt!q>L+j&6R+um%B4g~k~%Ivd_ZGTO?6%{j>|0JY%a{?-2FMlG2(hE=;FK6}AQT9Vp=L z9>>~cd#k;%QtFwNwj=J-V3 zbG_tpYTDA;bVMw6wW}N$t3n1%Mx}Xx+aP!%YpDHe*4)D7P6NCJSQPH=&R8x4`0v+aUca0G1^{=Bmuhq!mW+!x#)QHFam*y)IbD^1I56z#tX!@~4@iibj6%)i#3yxjQ zUhK#S4%zgJ^ZfArH7x6WBM=1{gLaC(nxnnpiVWBwjB0(PhK+x4xJ`9$8|i;XnEaD$ z+EMW)h$fM?SGOI@9s>Ut8c;~vmm(_85Lk40Fs^MV1fn`*#}cC@P_RRhOU=>J9xpGo zJr*R(ajfike5{KNIors3g8#Ja#v^omF>x{#P0RsSMO%}Ew0U{&+^K&Nzk3bnfYwqi z$4pAvDKs;$dhK*ONFxB6A!nY2O_2Tr7bIB!3oia)>Pp($Y!+iSJ$e*o4Ym?YYaAPy zy_iIrhHP5uWbUMNT2{P~RnCZ(>98FfjR+LCS~*uO&S<7Ul?^KC`3SBWC^N;3%fC+@ zx|iF2j|Zo)>C7}VrqWo1O4uEA2MskvY~TS*r`*JS(wz`wL897d3`%vm-$v}rlb(LI zv3b9T-spUY1!ob+pdG0;oMMAFi<9J>8*@bNB|RDDg^nZ^{n|7TMbo(+OCO3~veeL| zMTF+`TOV#WGwcg(w-btwdFQS?rA7c3N=Lx_FIix#ZGftc+ruJ>{<6cn$0#+Wo?q`n00b*Ex$B-Y(bKHLlBmyMX>C<$*1CBb zsedq~nh2PgaqLIZl!-rR^D{;aAn#Msl~CqOsKe2T5a zRN2A(i^yO$%s~^Kk+6&mtO>>ITMjkTt`>U{EJ{s1bgu)NrtmMv?S`H6_N^Ms`991+YKR=5U#3r0C11mISq8q;CMY{9 z=qnS8KEUAjUJx`TX0*}@HWfryCJAmNbO4v)acsLu{%)rY$~i_Dc-bhoY%C64bM z{6z>hPjuU-%@*Z_pAPV%H@8+A((Uupq;{Q~^-qI;y55az&xR9KPLEihK~cfO0Jqr@ z0Ql;IvFEVmVGQMmm5o(WL#`qcFd*lKi$anyQYoo7eJ(Q%Bt^V|1P}M?*`lv~y+L zmv_wZEbAuS(}}G4Ao|gd9e#%+$Zu;z1E7OMl#qM!q=pmpzURIo{bl~Kk7r_chtQg)7e7{ojA$~AN;d&tLLKiJC+2SGfml}Xf zPc|#avdy7iZaXmbAloaN|eNJ};mRI?uil?grJp&Kp|Hp6AU?Y4|<|S?0xe zpG-ipLq~geEv>PTQBc_++AG&-fK|yeK<-fWWBcmj-5I@4a7ku?ga22wWk|xz_A%5U zVsNiA5u`U%cOb+%q(96SL@=SNF@dR-Qyq$bBWov_Hi3QvCQ{DZ^nQ53)m$W9qC~2& zCTxjJUc5?&7j4Xk!)Z5U12Mbp(#$Rzu7BI{x>0z1!1*xW%#sZP!bK)d%GU(`vvy1@ z9N>Shh7CB_VJLJES%#*=27=8ZXv%LjJ(}^h0h9T3El!zyor-)Zx-x$&3&=({cRFc| z2{Q*X_slQ5aE^x%5&)2Gi$NjZe`)XNFEdw)<>Fq$TdxbLb=v#PSFWR|xSzz|t2CNR z6|BT&+LI>&2IKcKlz!3``M$>W&y&c;wnQhFCacoFZY$9Jj~3uic>dQ4qW*9A6RG|v z=2Yh8ej&Tj5a-sXI>%RG7VlpBzcDdYX>3YZ*rx8BzpEQ|Ozo+JK+0{zHxUUwEXMb{ zQ&YF3%XYiZ3v>oEM4z&fZu&h}TV3$v(|(3nG7Ekl`_@3T>HYL{04f$K{k<9&nUWxN zERXc%*BQQs9t)qM%D(8t*Yy3UgqeNztv;nk4}BFwl;}ZLcoy@I76b4!6lNjgVk+OJ zpI?;k#tH-8kjD4CIznoc^5+bC=S^+R38c=?-!fZXr0z&{5koH8e#1BnBXM{d#)4?c-iC zm|4Lza?+r;J?P94a^%HUpg<}fww3u*L!NJH;e=6p2nL?<2pRW%Dyc#Dpoc)#m&Yg5 zk8@EB(m+xN0dA?41owlv|DUQ==v)7zxBW`}`7bY;_`p~#;i=UU^x8HKf(o;P?a1a& z@aSlt)oA*ez&LAa#emOE7^&)xB3#3FA$C@xvv>W1o@2VTSx7bLmGJes%ec%4knnx% zkn0RjdeO2g!81Oqv3?WHq$f$KwAzo9zn<7`1-&*Vc_x;0KaE-C1tJNNK+(2UFG%OO zCILo^!$C8AoOfHkuw2D^#lqTK6RI)ekJ)D8E;z`s^c&8N)cSB3QGWlNXlW%g?(1PM zdXWZ;T%w1?7C)#|^Oj{{y;LNZ`DJ%AH#)-a;c^Jm7xmhih|>*oIDa#n#_ z*4sLY4vvDDwP5c@lx0`Sm+T%0Wx`;nld7a2lz`_aQJ^JFlH4GrAUUEg$k>9Vw2L}X z^o(@TPSaM+6ouGhp~6D+6_RXk4v(}7b9qAX&5A(5$V+K9cyxVdI$dr-9`Qv6%Iioz zB?ZbVCr+0+c-@|E!bA2zs@m17{)$u`qd0!s5Z{Xx zIb3t3XcvO^23WT!6W~GCuxK(XTuY=L=?FQyjuKZ54K7W+h4>pW)hAacNU1C<(fa*3 zDCe%+5l}3r$SS(+_gbpaYFT;Nks)exP58(t!o+M_RecFpLqTsoHoYxqk7l%M##KS#Q(Y+w+_|@hH^kfPzs-M;r zG!WN?}&0@7jdQ_|y=uPf!B{5GyJkvD@68 zeAva_)@i$P3r?VSfw;|TH|2$N*WAZ)p9XyNuu^y)K%v-XcFgX) zz6sp=nMamMSU$wHmQebLo2uW~^#cLFW(6}a>IlpSiL=-}6*=t`=w3H|$#I5W{Q0sK z4Ds6NASm4-h>V6x5xBrDNfC?9KLqoO8*vfi= zGzN%)!NrLN`1rPe?-C}Z?W%}zLmQu~NZ6v2a4_w{@^f#!_#ow~w7_sLM2*)I0+~?D z;8ab>(B)v6+=02d*~xxlqU5XmAA&&H0Gta=(u{{` zQ*0o3va4$!9ZKRb^ulbxAAM}gG(C-W(Bg#wR-JSl?l+pL)Yb!F`=2@tkyUr3Z=amR zR$s6}D~V$niXZ!+ud_W8`mn!4Z?>ledj;`@E;tDd7#C!%D9JLEh<=2=}f4FL1?Z>z^iIY&hXi1amb<3Gusu3mzO z7Awy(z@sonP%bPdYW{}!;EOm2Bq|1_>c}*Y*gA3Dc z43RZqB#?MmwJ?7xzfqoue?!Vws#A_Fq% zk)kWuxDsiLtvT@~@`#vZZA)@wIbIb#Fns&hQ2X|GW%4+m@}zfZ=t$I~gaJAV3}sa6 zwu%o5^i(l2+M`sF*PG*T>D;EWckehgh^2r5+THPuBpi^z+i6P2yalZVT*A}dvgTh~w)van2ZZL>22;hAbe7sJ)Fw1Bkg^GlqH zfM4*XJh;fRTOlyQbm1|fLVsV~_W-~}muLOcOu_DjP^Pgv{q<@MQOfdTD=JIuT0Z+* zN!%uaz!SK*-!Ir&!n-p+lLTdvJ}fmkP`?=@9Gj4=n^4ixI$_Ct!sZ7(`E!pIy%0k< ztzh5VPyI1pEcXGYZ-&qNN#lwR^Fib+-@chWAF8GEJKCo*629NB{tA4uW5l6j)%^W` ziEYk*HQoOuqJAL6Ci}z~qGa^qW$4IWeI9Y~k(d5X!8--R*j%bmNp4APkiW&`_iYbC zjKXZ3fVNIlOJB|US`h;=#jRWS2m85jG7}B{U5Y2A_-ykPF)u$*6=Ozq_D_lsm;6kv zEY)xi4leU_Pm#!{jwR`}NTK*3&xI{So!juCoO?G{j!S>bo`a4mtZ zT6U%s4VrpI$fPS3&M~)bQg;4AWi=xzbSAO6 zaJss%$C_;s4qcy1C*zlv1r=x%BsosUlo-gkEkuAO8Omhd!yVX%NDNb5X%jok86U(! z)oqMy@?=X(s7)U!$SxNVrP8y7+A!mL9nWO%G8rFUePPJwIZ0F|^dg&eKe+DAm3u3E z{&tDvH$e(4Ic`D!tCUQFs^@-DhhHAlUP+uK6tNQKW!dSEScIFBI6sQ-8owe~3N+S0 z$lypE0rdu1#ZmA)i!jWWg-3n=CF~k=xmjX(tgDA(!+IZ9M)tAuS0Gs7#kp5J1wQB#>T9}sAR9YNR}-6 z0#dg!3XewiMu$|FXT9;mLdx3fmjVRuQB%&KSxU9;fKyM72Z6O|xxDF0y{MSnD<0_R&D0Dx5#Pxa2b<|@lopVS{F0Jn%$gOwa0&xko zIXdj)J6YRi)<-LQG9e7}7DZMn~c_VrE3O&Om= znidU?t(caS5Ac!tF;(<^{IySCEvMEu_D&_5Kxc6DO{?~63P%<3bq{9X@4s8K;q6sa zWC|H+9nvQ9t`X2_02+e`do9KIeAzxas3eQQxtsX!ALds(4;&OtFU#!XZOc=Q?enr*3~(WK+E?)RGF6hyRs@l?nf=O#rKtt?^^9 zr8I~M14M5w^zH+kW}M~hOA8q{fy?Yw+J%5EBk5u5>)`I-Ip}aS`^!HcG=EIhwgu2c z-kc*X&N3T51_!}aP*8*OZx%bdo8Sb!vHfKBJq(SAbhGTf&=|PcyQRm53l~riva^?h z7fT9RCyIHiaNLclns>Q^k-Ob>H94p|Z|dsqmx|fv)l9{C%HOwwe#3=_yEr1_^wnvGcKLAy)4N%e(BXY8O9St{#%?KVQa{S#2IM<>=n8JNJDn)? z2?YmTZ}H*lBPQ%{X0Tl0P5O^nO$r3M5xF*P3|OlGjgF0uUtQ?a_IIJQOLV;yRDP

6|mn`X?VT^x5gQGLUX4Zq5f|C z=y*McaD5}0d|XWCy9uhvJ2>6EI;aTL%PHt0>x|M#JMaI^&3bU?6J3X}QdmG*O=D(b zmq%P1IMuOyI6OLnzw}Kclha8$nIKx(ddBNjC{aQiLZpl-Yhw+11h|CQ0Ha$Z#YTvy z5P)ak$GpJ!cf9i%7=MmUk-h6l&4dy{)3I_6w5G!Wc*!$L{|9>R$3Xr!PbCM1xvrk_ z{8-zVqqgtsuC4^)C)+n67XBBFhC9r?DQnc*d*EA#d3anr9ORhD|K8~Ia!-0 zyfc&d*<+&;b>kNhN4hx>8UyaR=#>=J!|9wz3?E;O1S=gG_6+Fn_2 z%&%n4fM0#-Aryv~MYqZ4D|AbdrOYs%zeW7930DGInqw0*7#jAJz`upacDLGgKXuXr z1WBJEOuvVJo+}ZRB&j{!cGW{;ywJ6jWI9`By+67en$6&x+f{P=yl~3eC2-$!jgt}9 z_4l)M9zF&KSQ#}#aD7Prziva0pqIbK(-u341*9?1_(IhR4F&yNT2!LRdmI}-%(m*_ zLFzT%Y~xOPPP&KjqLfP5^?is2yR5GzJweU>yReU(G9@VG<8C3h^lB#JzKq9}-?5Sq zEHiz&>Fy>7RlJW&d;p~qXQr<_nfthm+a0Qj;7;*0Zu$v-BPERu&pAV72$xKd*)$KmyQ(`ln zCNs%@&w!DY{G{0TYl$xFh|z!(xdk1P-N}7kZ2>($8;oHU#bRWaLGv0hJ)& zU;)Bei@w%@mBnXe>iXpbk^qQRC*qy7`KZY|LEdA&g#fQGw_Ruq#rghrl|!O3wMC&~cBFs0?tk z#}}w?Vvd@)bp}xi2zs9I9(OO3~xLUub;;RE-}{VV6x>*Y*Va!wQO{S*B4}cDqRoLdGy)Yy{EF zT=8@8X2Tg#SJ|JUe79e$d?P}oD!Zg>qj^h2Uv=ZktOrOT#fPKb=*c0#k~)0w-^8IT zQ-~|o0NylVgW3XZhoITjYl#^?=glU%zX>xB>unI0$zlKqd8*fy9%iQLbw7{e`aGut zRA@NSHx#*jt3RmkZUT-^7biR)_4`$TN_|yh3=VjqXBLB>LN(2~qtw0B@{d(7pBF;| z&5cj9#xjuJFWvAe|Fv~`?P4WGg$q`1ImvYz(20m$2Ox(+LVW>72a|(P9gRq_IL44! zWBa1xkEV-Jg+m*v0_sHxX#4J@=)+?=4=R4j*M8vb<8mAxl^$@H=@pu`(Zv4J{)Bj? zRz4rm?zQB6d7EKJJ7v=Bd^w&Gw&b`Wv?q_wK?#ccW^QR^4lbQigfq1%)qcDktTx%a z?&oxM-ZurwZl6AFyzW1Gk2GykQBlRyXEUN}3lf13Dm%6;U4>lL|DV9i&amUQiWU#8 zCRYv{Y7N}}9P4YrVf#pBevD=8nU6*KJ=me7@o_t?p%Od*nzm>1eX`}pMDH-AH0*G> zz&#JYW@5j*=Q6D9_JwAr+0!-2t$@++k3(xr=GMPa)FVF=Sb+|i#e2M#RG?I)9wQH_WO&JBabmFjf#v%5^f1D%*$JV|O3H$*ybsK1YDQ1d@9(sEP7mY36CKtFcF zrC^uZKwT09!@HJ;DMJ{;i?OS1j`@Jr4ck9G2@8LhErZSb&iJY3XI3Mj?VNr3{8Bj^ z7ps%04;tnlU~PL!EJs-%4eo!00>m2)Oz<(@6_($_@RVOOt#XqmxkX=Aj0t%KpB`e{ z0^<~92{XU|ps<;IYt3nB=WC{JDOl1dhR~dp09f9<8C=3@)jnR0lDt9+^7A;FOg}4i zJa5Lv;AE@PmI$?9K8%b-eeb~ib7F7>pP|4K8{ z;Nz1|mE%IYcy8As^Ho~*EBeq#iGa|0%tI#!t5MB4@HS-m85Io|I*JSGYXZ*E8BR3T zKfxgwZ-VI)$WF(Aap~7;7fZ6Ldp&GBPya9<(VZKmQuUI6Hc@IdEAq(Q)85l;i@Y!( zZs{bI2vq21>lgDA5h<-H@A{v6lSP+K($u@3n_CPYL)HF293{Aj0baRlIbX1fHlr?!cySmFYG6X875DO<; z5G~EcGBN}gL|pTlLZiwefak%(i1D+Cj!Z+}numk$1QYQs=gZl^4 zFgZAW8X|6{nPGb5HdGjCFST3hd8v3Qy=%HpZu4K0i{rK3xTn)I7~}cz@?_|aWDqNV zPc}{M4EbjIUdl*DXvdM^{IrI6h`EbFtd4`jn7q8z=OEiP$>$ld5IVWw6&hc)d3I+I zpIUCX&|o~7X%XSHT>l-2y$LDSr2xeT|5+dy2Lcu15vkj&_IW2eJJ>LINY`%&oAahe z0gjTX`3&ho7jkgJcrMh7*{4dB2ac0ev~NB zh4S+210-7EjO6x$)S27hhMUpt7~B`bfE6V7fAOOrTuht-#L>F9pV!;K?WTZ!ZQ7*1 zUv$=Wv%PG+Q0bNn-UHjQc^o{U{+)Q^j1oVe9$4m8wJPDw2+g}^8&UzY2#{M zNN)%})X#9iMi6CRUEA$%sq`*){p0ZT+2HlTUmy8qR6+X5_NW2ZaF2wMyxQO(K~1)# z<3Wqp;%>@N^pTF7uX#4+9THL`V4gvVGI>VV8ayhqLB2Fq^FvBFVx2B+SyjZ%F!FnP zTLQ)B3wRlTQ4X8IKOH>w7bw|bVxzBzA9PS!r!Zj=%bx_Fd0^19j7($@S7LaF3fwbX zz3(P6Y-pICym_8K5yoBPjBJ~?S?mA`FL>283}Lt6pQQEZVT2(da3o8z1S!l?3OhNM3RVvvfSM}azhiH^ZSe}moyt;ZD@~zxnsV4 zf(P*d@lzsa1NCgvC-CN5ZYkMHQ*zwfbCG(;8g?``*GtaFv#F2-P1~%jWJEyW zq%?&7Q`4QU`bS|_;B^W!tg7}THzPxq%0|t$_KCJ_&At-LN(t|SSgmRc3<%z}$c0^dNs3!ziy8An zBAbKdQnIN(Hv_#7Hn9bWFfdR9#Q(nh6C!ykD>`~;#c&RE3jlN~a~8-(^khlRM%)n;>%~-NokzXDHPe;)U&sY24ariWbH{ zLR!t;O@J^N2o2Y}9D+aCuFf&VsZYrGg9NST(wt1y4;#4G8q&}!haFF z5*I_S-KkqLD~5u!q9jMisKa3PzQH*16+*5vhGD}F##DGjXTQ0+6SP;bxXPM3Fz;C- zNbdsSw6}N66dO3rPkv3%W5VSs+X)Yn*d;HYZJ@wz1A&R*J{)GmZY>?f)35yWLUkCy zKd(F0u6P;h#Jc(_;$&Xr`GZ;PQkgXf<<~_;&bA;4q4QvR|JIhqgz6r?j@I*|o1Nv+ zdwN;rEK89E#PA9TWz(iEpyZy$in~WW=eArZ+L}EiO3Hi*7Rwu^0Otsa>-sfpUxLRl zz-rBX^n+uF`%n@~Pl!?EEQmZs7N&@YtGl_XOEae{9B&=%Oc(g`%V>d^KS-L@K0*J?Kgvg z;+Fy6P_{X(G!jgd;6HypI1AK(n&xdfD;kjxj_6e zCc025QogaebpJ;5?Y?kUUcF^!tzim%Mgjb|(xD?npnZX=`P7T>MJ!(u{L_Ig4xR)K zPN;e?5LGdfiiqKL-z6G%eo0MJ)0+0~dFi4~eBtC(tJUNO54NczuAQI9p0C;q`JlLy z`jW8H02x+1SEu5sci)9UX#(Cg@-0i4r|L1_ZNcPy?fJbknRW?uD2m#hrX4!|>~Niv z=oL$fj;_{L+^z>91ZsPOh9V+0dAAgcjVeC-5`5P}>ABRi%Uv>| zsMujW%g~eD-SEqx&NaD@M%zV zPLQAsdh)YdP9h5;47grvanOdA;|XMhu+ti^+WXq7IsMEQ{B*rp&$=fk0XDGltahgX zMZRNzL6F|vMJd_DD1*!#z);HSpXXfu^PKHJiBd@|I&bFas)tB z%VKwxqyV!~Bn}O|BJ}3J&bDa8NrEdDf?ex?y77HVD{kKU9kLWBONDU>cCdTc zH}$TN*q(hjPK;Q3ocz(eBnK{9>{*}cUMZ!9GGVc1FdNWUdI$xVLMlRC!UZ=>8f((r z%ruCRnqR}sd1NG}M+$&^>M?fMz@{7nhNeq@6-sRuNUvx7eTcv>~A{E)JSLK>;u`4q*fHFjOc>zSo@D@gxL6(nIoRMamL+Roz65h( z&>@5}Q7wP$V;8!-H`Rqd@mA@KWQ>j7v}_o@`rS76;a#l7MOCE@^ccDHjW}WtDZg&U zHIn4o7fq7mJf(l)IlCs8w&@%`7TzxFFIu~AC+Q0B*gZ{p@b?@3;U1d^Wsmt z^*&)#efA$fyd5ZBzBX7bw6MjzXD zl?@r*KdFRn)G_&@P#%yvfeRLH(fx){UwUJ~7aQjH5qbv&SxTu2fsnDcUm5#m@h;ny zdPkpBb=ef?6zKLb+;Vb3gz}e~KwIM4eGZiPH3+hzDEv%WzoytGy7AE^1Tz4^{_T#4 zN-{@APZU}H(fW#fCV1EzF%aN!r5WTpdBp=>pKAa9X7%jk*hQ%i!N=jX%~5~x>Yloh zlwI+Zixqx#U&KCgM=d8C%7J$EI%=TX{z;hUC(IyTzYwyl9JKneP+*~>MtWfB`sgnp_W6nA4 z58;U|poEi~Nx$Ku5b@^^#I!`yXJ6;z`rctu^f^%!0WFrVF~{;7Cz~g;>wMXx!AW<5 z6T;s87|$jO9y~4>HD8lJ>Xb8y=ygfAka;oxI80CV?CeWyfozL284P{0o7#w%v;Vvh zEfRz_!s{VPhs`Q}U>Hgx0_wkH$Kvf2wo+4JsBb@yF89vYLVWb~6Ed^*S4RpIdWCqS z^+##YkCnQyq+&6;-j<`qk#xk;rSLYkjfz`5pT8jXQkj|tvEF$S^A^d2Vym z>EvVeMqhiVbgH%S`c+<(0wlS7o`!&QFX;NNocX6{2k?t*Cxipiu^Fp$Nbfqq=etyV z{t+n$xTIaXZx$n#XHBs+&pK3qn5g^x-vJ>)RZ@T`;oNO*pVH>wd~k90D9;B9CL*L} z!4!-3Ea=-EK!wpAj9876upmVeyg`xy)UM}W zHePvA>3q9CtshXR=omyLN`?;|Z>5rg2`1nG)9V%HU=VLXf0&O#uD*ElJsw;V z@NOD__#!qV1&#v+ZB1qtP>?#5s&L&7dS5d!*M2NSAp?k@wggv*3N?kakv1>^S;)b% zYTC6=dZv>ID;i1+pCzqUc_$m;s@4d;jj@yJUM*1gw(s;+%^X6Oc`a{%c< zN>X9~1*Jneg`uRRrDR4F6r`jZ1wl%Lp*y5gX`}?CyCu&aAD{QU|MNTN+u_r&X79D` zxbFMD)>?bkey&VT!QjF%3t!K`1?4=a>6&vx z3elF^&=9aSR=n07E(ss~yQh$1AN=~GQEIjn!rxlZAq+SlFPNW35gT&<25RBd#^jc` z4on3at%qn{wjJD58gvZNh(STBCynA682UJeEZ6xmS-kNNNs#YbY<@+P!5RE{9K8u#=No~(R$a`IyLlUyW#oIxbDEf`ua^Nf@4NG zii7qjxxfmCo!P1_`e=17Jh_rIx30Gyp7vv9Gap!HH^)9HwSV^>43gX=rQVF>+a{CU z>`?Wj#HaKE$TQ#iUDy~G`BQQ*p6gRi&}pB>do051_oh5&-725h#$Rp@C2N&Gm$shF z>O-xH{B~5M8^eh^S2%L!c&P9rge)qv$?q<%mvdMTD~gnj5W%({Gb(*+U;ejEuZ>$& z=$X=1(i~THugN4q{ujowjQ(Eh2YBby_RMa+SiChQ*S>|#yFGy*UkwL-+ip%oG}mx6-{L!|sk`d;nz2lw%BIy41#eLVp!TAQtCbA}Z)C%0 zYVr{GOh$6>A&*PiBz@eJP0x88dwdM$Vd5B{&5Me=v@<25cS9s2I&xAG{drRn7ubBZ zjmVUEl@=%PnTB!5Sk!e>!#OhY!&DpSs-H8CT!mYnJP>K+%L{pN=VQo%?ibLDu(Np{ zFhkfYN@EbKJQRAL!xK!Q{l+TzR$EhCf zce>M){eyfaHmiJ=XHzMJvQ}SW&pA!MdZ{PJ=2oEg-2Qsm5*mvofCEJf_Sk<=^Al|M zbC0^YVa@CLgd9wW(-yAbNEG3dns^lyAY}V}xj{KmdT;C^opgKM zv0Z&Iy1hw+u1#u6&QI34%{Q%|U_$Eg$VPu)aZ8Z^4)?qt7*}{ab^kTxt^=40l3Z?( z+mLwQBfH=6U3}$GA$F*@{)SWc(^xM-58mkRx({Q68ZB4Wc4}97ItywkM-?;Se-8;EBQR^HQ8S>p#yEvZY<|eK8lcMNy zGFKE<$^8_jHh4QmHzg&>Q7d$Gi_#|fqRq2+0WaL(=loUR#MO}+F*ipXzp)Rkx<7CL zYe+=2;6Zu=MLJ1Uf<6J&1Cl!n-SlScHrRs(l|Mxb^kOND>nf$Qg}Raw4f0KE-0ile zTUXE+Q2JsoU3xuyLEBqz*^Ih7F_Uwb8MSY6uyG5zY$F$;H7OJjy29Y{F|y} zS-Y74j5n(z;pVxm046+5L&l=irg&ntx)79QHjoLE)>NJ3>1+p9TGda{mI+f_MQh=a zFk=2Hp0Vv4Z{(vY+NbsMCF1W;jC)0NB-q59Dl+ zWhh7~<_>Yfc~Yr;ztTzz{hrN4k7QEt(`49%njZRq7(9vlJ19GaiQK-g@ts{CX^|<1 zNq_B#j$YiY!h~)=BQWomPigO4zpST+q*q}J^Tej6hU*=4WY@Yg^V8#I>pu`2K$y1X zVi|R5Ier+=iowUnzr*D6ec~s_XGL!-v*~nH!4$6Gy%d+czE<}4QTI$o&R&_<=zNqs z5qCVkcg;+u$FFL3!g4nxb$yg?k6g}mF5_U&$MO4sEut%K`+3dByssUg9@=3qJ1qv| zzz12XWJaMdKhuCg54+*L#jx&=A;JXKCnpS%>!OZ7$E1E=&q{1H+!i1b3xkjnOk78{ zzHciN1aO-N5&rRkn9HcKfN~K^kiYwAFagf#=+1FO;5aoN0~d4rMK*ES>DCvy8SbB% z{I*{;GCrq+$ zqIN8qneJj|6+o8gsl1X0l)I@%KhaHDJFmNn7V7t2@2(V*<-HF$CYTOXv>D0Q2XhH5 zS$2MdnGzI)GY^v1jkTRj>ny5_wfF54BV?a{L!VJTMS-xAiO7BN3ALJ8!8JB}vIq2< zWAaYrzF_@OSEIMj#3_8lYMJu}ZGLXP0v_L)CMF{ZB+GkeE#&Jayi`-BxA*hDzJ-kY zK4Wd81cSe9V&}vr0fP65&zsh@C5gn@vQOJ#JV95_+OwwNidw$GH{yJboe~Ia2 ztR<8-m8iCAf9t#0sUQA2Gvt6qIHpls-|4AS>?$Ao$HX!;1>-Hr+S6i<^TW}=WZ6UsnN77kU`lv z(lDy8%sQ_{B>G~u7a#xBUn;8ZT>YyESw(9`FxTD)7C-seekDXgT#gYq_xi9Sn~6Sk z3>xb|WP_F#;hgIx`WP@5^wnyFq&S49L}s1*OCgpK%82^F@jRPwu^xD2PM9ytrcNxs zlWV6jvc=4)Wad(S-|f&}v~k3)i%|T+(|y`ND_Zw1tgK3roB6gA0fM3GD(q{Vm!T9; z3+Nv_4cdK0Q+gGYI@EaI<09SzXHLNU^-h%&>YM#NBx_bPfAkf7R~Q2m8d~RwF;3r^ z)H}_)88}eglpeKw_l;!{jsO7`OGD22(sN^kJfRjm6GDtQ+cc;QgBVjre-k1g7RE~V zddh8T5USjzCe$N66`yg#VE)gZI}|M2oa03*I5so zm6`;<9*Wddk}}GKxcm&>>#VX^?D+Co;@vyBOqF(jW6RE!%!M)BpprKMX6KOxw-Gko zU7b|o;@V@NzAsXt^(}MT5hnA55s^kN_Z#&^^|Wc~7L9}>aD>Hu#K|5xw1+*t*=zQ{^_PXIWcJ_XnEJn|acD z>tCLyPCv4mnKep#E1tP#Ra^1xTYo$XCc{&iO^*co5?VR+n{%NyBaLfhu5a4`0oJ;j~z?ZcCVhcaw=>YeWwxzwtdrS=&o zJ+eD2C&w*PtIMjKglAf7Pli~hFAZJa2K};mn<7aR4<}E5=Ix z1-;SNcXPmS&emaE((pBeG7Fi3<8)Oz>ajQ}MyQqb2`HxSv!iUeTnp5PN6lz7A zDk5hhrY_DO!sb$Z*4lhUY^po=ahtwg4v%*NZ(}whYo8D*_wdX|3s0F5h*QE;q5l3F zFY=XqrUP#Jd9Aq@dsiK;zA2eYjUXU6_~J1q>ttLyeRx*@SD5Z9{>?3KP?tq7*wB@9 z?0cec>>J|5+I_KZEC0f3_;QHLFIFBn5H1QpxX{dWDQPu(0nqu3E@$jEL8)c1I{M9w zW@!4re8Y}gi_OY@3JxSxUDF9$5Fq0D^rY&1+xO{7)~V7)KEfJi}X7 z0WZP{`j{J@vUJquG*M3`@U?gX_!UqBS6)a{<$q>pYT~!nOoHFF}P1g$5^RNoD(*(Spi2zSjv`gT|d@uq5u{Iq%E!HIUL%qi%UYwbDk_2 zRCCxbtU<1hPy?4M%^$1^{-XlM*`4gmdDn5?mO2_5+mAHI3|x#Dg>E;lv|OHlx!g#m z)l%U$Dc=@MIi?LeBVk|#2xi?l>p>}e`QdZdj9$OEJ{_{Ch)AJmU-X8r3osrHFkV3E zxS<{4s_A^lhC+yWO^Wz&HUX9bp}^9nqdJR_=`9bo+pcnd4w$ zHHix=8CU6WYnz-p2ZY0|^$vtZS3>e9VTq5C`d%*8gx@r|IAAe~aLuRrQ1Ik4C!evx z$J^Il9MH8ynM7j3$_#1$(vqV0n?KJAi?=skb7XSRNNzRCICM$--_Ua$YX^9BL%93* z>etQ(bE&+#xj)iGV^Do<4NtDuij=#hrKCFGwn1E_w{W=OU4xqy*Cud(o=aQmJ?7Hw7is*I*=~IKAiU>jX53F zr`q5+=fE#=+t1Sz$iIyPk9i2SkPvDBLFPo1SsilNI}i*gz*5Ax!$e`dH?l6xNX!syI#*Hqippq{ZmW@p^t#M(YPUj z7J9&}OWJSsbWh;tzm;B|o~MWfF8!WuU@#y^s0$==QP#-#dNC+_{VOs3E_Knh zjmL|-3@z|w4KwCc2<1>JkmP3l;T~3;IG=%Wm>S@eCJriT_qV38ghT&2)*o> zTQhjHb~O6cAVl_2v5HNq*VNI=-7#K~=w&MbESVJz08oHH8Al`UYhX6WPO*(G)6LT&K_C_0gjd^d zBJH-|k&ykVAZ&qiBrRHyhP@S!N@5aa-MMSLne$pSKYu5#nEL*SE!3leJ4zWasu7YM z;kRv883ED27mSufEw9w2J$#e;zQCSlilZz!Q#Y>W_sX~?QU0t$63obQaBMy{U-!9R zwQi@Bj$x~z)`5S_9UsyuQP=0r*NL+k$i=+j(kC5R+G? ze3I~`xp?7cz4%6-g_K_yic%RqG8SZ@U~Y66iQc7trD=Dk;L`}OlO378fZbuh`azWL^qqn~}`sp|+)5aYM7NjfGDRoQPPh-%al|>1@`nmKAKr#?+ zwzr)~yLi)tzp52SuQr|C@6leH zjz0Nc3B^t!Wnbg;LR+`>>0P#Z{N@mA&&(sRRu;fk-;Nu|kFM%L)M(ySIW0M5{X zP1|*6jJ1dqlcy|bB+xG3!?i{ocrCG?Qu0Rf;xZ2O>XM!M{gBoRYDY#A#*wzCYrdGA zQukkQ3@M))%Kz0MTGV^tGi>`dGZrc8F{=!RgFc%#gz*+n%@6lx0eR1=W{ev2sN38nl z%rSnlcE#sjN9SU%k%)F_z;Cwrw`pRRp=+uMl_!Q zJB;b=g4kR#5rCi*xrNXyLVDWLB@tgSG z9lKD7i+7jg%ig$ia-c> zCJE=L1fwFm53Jj9g*FlzjxtgCix$uGBhbZxk1u{$z)a&$KcUY(S=ySGu`ti?D2kl|I2fKH7 zRvTd$j(~)Gz6lhA^p65eYpS4Zu+zau74+u}pqd@L6&X}OEnwe@#=+H}UheYToB2^0 z$bPXW+s3j!)L7Y2u)}XrYWH(}RPp=q=I@e*^VO`hz6Uk<$X@D}F%&^6Ce6YD!LZWj zU-4cLT0*gsX_f-#Jj)PZ{!PVs)#Z!S&Wp4^}w#c-UyAE(W0)4Vj8|bj2KpDvu@9u+@NVh6+r-`3e zwkOj8*jnl65Qh3+I{mgX%vHz?PU6c`N#3iPftKR`8VfeaO7&Pt59 zn%r>$TnPu85lBdU7=-5*&e~aXXR!dDy*&KL&i|j>`akATluxw)6v+#wQ`$j?A1as8 zVPJV-1hi%PXFBI0p96qB98QcZe~Id&;7`B^1_!GX3PwAWnHU0;pc%8WgV*AzlZH~` zVp{ub`_@7xhuty`$C}6~+mG`t|90A`nGP^s?9`NGA`D*!yLoUb|AylSDsV4D+QPa>t(`(?3EiT8Xd#Sji0hhn{wR8< zd86fWJUHN4PHVL}PDkWgGP(7lS2;6s3-rX{*HOX|paVA)D5h7wOgW%u49$MmPO>Iu zDaiU3q1K7gYGICspE2Z}`=G%65{Nn3Z_;{ZX&)I{%L&1P6+8x~`r0|-c@t38e#swg z`3V4R^TnkN63IZD*n+(tc}vNF5io!?r+P*Umq0R`7vpGZV3IdbcL8tQxC9^K?7j{X z%Pax13Gw5S71u`D!_u8Da1ib1%YI9ZCobV@sAm#93SWhY8FXiqeI0M8pKoziK z)miUoMLN;EA4nicPSW#e&sX|SjAkLlaSA|5Bw4&j}D7Txq|KDfqqxTk<3Gom<`NEHwAThgJ-ne+3aG?>)X^2u-@#qZZ~v+NKp(pvKs?DcfhGa~;T9p1prx^P z0%kc8%?E$SKrUlS=r~6aIZ&`XP^Q4qPC2Xf=}ycTMI*&KvQm@uo8{a zdc5Z*yZ=uIyN`hZShl70(}BSrqNG75kcsu!3cHV(`7%a}r}==gs*=8fnE%9Z5HE{l zk?dlsWq+WFX{!ae*>E5Y8ufqHb3o!$41*n6sliY8#Scg+En>Yzc^I^tie{nMNd=(v`{K1uxV?SJlSJ^@_zr#u4|I>G*o z_k|eFg;#jHsgjbXxfs_m z&j31M#`l2@zBzeI=%h{#ueB2p6Udl^95+zYx5vL~}llT>R@ zautaCR?4dnRym%uyzMBt5d?mj%yqG9sYmsDQmSw#8{{dH`0H)>z!BZahkyKUuK`4v zhgmBj5M{DZXi2&?_$Ah}DaKqWtOvNetVr;(5yh|L*0Wbi<%nBqjfjr+^ZwzfULhs$ zsW`F|e!Wc% z^6$^K70zBDS}x&S)<(C!cIV5MzBz{9`~wsMS;d>LyHWn}LIZRJVzi6;YijN+GQfa= zhwLVR?$7k$oNKhCS(3=&WnEQ}KUuRG$UE89BF==|Mrie&i%{GoO(NDO;(4Bfk>jy! ztFhvD+M>Rn`zDY2r1`$00agFVXqgBA+)NF255UQM3R10hj{rFax_V4%vTw{Q7mMU% zH^0%G0U{^K%ApXJoDW_m=NwEOaeO|tx}?5ZMQN0!SjeL&ufo6+O_ zKHzffp)tnBTRG`q;I!+YgTY&A-TZGzR1AYRZ9#J&2{5!@TJrVuyHf#>a=1gJwBm8H z@13o8D=EDq{|Y#-0qE?rzCQtA*c=B@*(SOCO|Yh2>&k-KH9^4lvsxuNs1x)EoziM< z?qRb}HmQ{4YKJjGZhI8?P^?5;Y?5&uu|U?#iH0>IM0xWJOez6bV?uD=0Ko3^(HSQg z@R<$lyA}6F!dytxQLcsY3#3@7Z7I2g33Qf{$rGzfBfb?_b7PDtcCoiFJVAYoHTDOU zjo;GB6RDinKx5IV7s>v&Q;NJBp%0FRrjap`ZtrXr~(^ zFqDdUu#L9;BuuUfM$2yl`?X}}*bl4-SMyzKtaFRb36Hp?onLXFgb zK#3GgqRcC>1iwrBhH^ZU)YX;@kv8E(i}k35|GzpxLqj0;gjr-r1Mnu6;FqNCyGx?_ zoC^=|1Sh)b#l+^$xMTf_-1ljxBmOZF92|k6Z{^m8`~VX~Lx49q$NF@AWiG>=)B&lg z7&)rBh{tuyn2UvqC^J;HNJGSP<2)q%)Di#&@F$(Y*)geP4PPHfx4S$T8w+f#)T=lm z)sBUxE`Hf7c0!^{-ccudA3pob>oK_sQ?*9uiU|KLQ~$Wj0_f^$bj}H2vVY4D#*6=* zRsIbW;OkQiu!D}oO)!f^f58qk+dlv>s+itV@Lz4f$19BEEH3-UbpH3HKr=mN_N!(0 xh4){G|G5hp1dzPn20`eG|JnTiN9^Wnu%y+>leuqG%OK$I;eCyJpA{@${4XnUju!v` literal 0 HcmV?d00001